tigrcorn-certification 0.3.16.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tigrcorn_certification/__init__.py +55 -0
- tigrcorn_certification/aioquic_preflight.py +449 -0
- tigrcorn_certification/certification_env.py +419 -0
- tigrcorn_certification/conformance.py +42 -0
- tigrcorn_certification/explicit_surfaces.py +130 -0
- tigrcorn_certification/interop_runner.py +2017 -0
- tigrcorn_certification/perf_runner.py +725 -0
- tigrcorn_certification/py.typed +1 -0
- tigrcorn_certification/release_gates.py +1354 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/METADATA +242 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/RECORD +14 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/WHEEL +5 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/licenses/LICENSE +163 -0
- tigrcorn_certification-0.3.16.dev5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2017 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import platform
|
|
6
|
+
import re
|
|
7
|
+
import selectors
|
|
8
|
+
import shutil
|
|
9
|
+
import socket
|
|
10
|
+
import subprocess
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Iterable, Mapping, Sequence
|
|
17
|
+
|
|
18
|
+
from tigrcorn_config.observability_surface import QLOG_EXPERIMENTAL_SCHEMA_VERSION
|
|
19
|
+
from tigrcorn_transports.quic.packets import (
|
|
20
|
+
QuicLongHeaderPacket,
|
|
21
|
+
QuicRetryPacket,
|
|
22
|
+
QuicShortHeaderPacket,
|
|
23
|
+
QuicVersionNegotiationPacket,
|
|
24
|
+
decode_packet,
|
|
25
|
+
split_coalesced_packets,
|
|
26
|
+
)
|
|
27
|
+
from tigrcorn_core.version import __version__
|
|
28
|
+
|
|
29
|
+
DEFAULT_READY_TIMEOUT = 10.0
|
|
30
|
+
DEFAULT_RUN_TIMEOUT = 30.0
|
|
31
|
+
VALID_PROVENANCE_KINDS = {
|
|
32
|
+
'unspecified',
|
|
33
|
+
'same_stack_fixture',
|
|
34
|
+
'third_party_library',
|
|
35
|
+
'third_party_binary',
|
|
36
|
+
'package_owned',
|
|
37
|
+
}
|
|
38
|
+
VALID_EVIDENCE_TIERS = {'local_conformance', 'same_stack_replay', 'independent_certification', 'mixed'}
|
|
39
|
+
INTEROP_ARTIFACT_SCHEMA_VERSION = 1
|
|
40
|
+
QLOG_VERSION = '0.3'
|
|
41
|
+
INTEROP_BUNDLE_REQUIRED_FILES = (
|
|
42
|
+
'manifest.json',
|
|
43
|
+
'summary.json',
|
|
44
|
+
'index.json',
|
|
45
|
+
)
|
|
46
|
+
INTEROP_SCENARIO_REQUIRED_FILES = (
|
|
47
|
+
'summary.json',
|
|
48
|
+
'index.json',
|
|
49
|
+
'result.json',
|
|
50
|
+
'scenario.json',
|
|
51
|
+
'command.json',
|
|
52
|
+
'env.json',
|
|
53
|
+
'versions.json',
|
|
54
|
+
'wire_capture.json',
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(slots=True)
|
|
59
|
+
class InteropProcessSpec:
|
|
60
|
+
name: str
|
|
61
|
+
adapter: str
|
|
62
|
+
role: str
|
|
63
|
+
command: list[str]
|
|
64
|
+
env: dict[str, str] = field(default_factory=dict)
|
|
65
|
+
cwd: str | None = None
|
|
66
|
+
ready_pattern: str | None = None
|
|
67
|
+
ready_timeout: float = DEFAULT_READY_TIMEOUT
|
|
68
|
+
run_timeout: float = DEFAULT_RUN_TIMEOUT
|
|
69
|
+
version_command: list[str] | None = None
|
|
70
|
+
image: str | None = None
|
|
71
|
+
enabled: bool = True
|
|
72
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
73
|
+
provenance_kind: str = 'unspecified'
|
|
74
|
+
implementation_source: str | None = None
|
|
75
|
+
implementation_identity: str | None = None
|
|
76
|
+
implementation_version: str | None = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(slots=True)
|
|
80
|
+
class InteropScenario:
|
|
81
|
+
id: str
|
|
82
|
+
protocol: str
|
|
83
|
+
role: str
|
|
84
|
+
feature: str
|
|
85
|
+
peer: str
|
|
86
|
+
sut: InteropProcessSpec
|
|
87
|
+
peer_process: InteropProcessSpec
|
|
88
|
+
assertions: list[dict[str, Any]] = field(default_factory=list)
|
|
89
|
+
transport: str | None = None
|
|
90
|
+
ip_family: str = 'ipv4'
|
|
91
|
+
cipher_group: str | None = None
|
|
92
|
+
retry: bool = False
|
|
93
|
+
resumption: bool = False
|
|
94
|
+
zero_rtt: bool = False
|
|
95
|
+
key_update: bool = False
|
|
96
|
+
migration: bool = False
|
|
97
|
+
goaway: bool = False
|
|
98
|
+
qpack_blocking: bool = False
|
|
99
|
+
capture: dict[str, Any] = field(default_factory=dict)
|
|
100
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
101
|
+
evidence_tier: str = 'mixed'
|
|
102
|
+
enabled: bool = True
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def dimensions(self) -> dict[str, Any]:
|
|
106
|
+
return {
|
|
107
|
+
'protocol': self.protocol,
|
|
108
|
+
'role': self.role,
|
|
109
|
+
'feature': self.feature,
|
|
110
|
+
'peer': self.peer,
|
|
111
|
+
'cipher_group': self.cipher_group,
|
|
112
|
+
'ip_family': self.ip_family,
|
|
113
|
+
'retry': self.retry,
|
|
114
|
+
'resumption': self.resumption,
|
|
115
|
+
'zero_rtt': self.zero_rtt,
|
|
116
|
+
'key_update': self.key_update,
|
|
117
|
+
'migration': self.migration,
|
|
118
|
+
'goaway': self.goaway,
|
|
119
|
+
'qpack_blocking': self.qpack_blocking,
|
|
120
|
+
'evidence_tier': self.evidence_tier,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(slots=True)
|
|
125
|
+
class InteropMatrix:
|
|
126
|
+
name: str
|
|
127
|
+
scenarios: list[InteropScenario]
|
|
128
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def enabled_scenarios(self) -> list[InteropScenario]:
|
|
132
|
+
return [scenario for scenario in self.scenarios if scenario.enabled and scenario.sut.enabled and scenario.peer_process.enabled]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass(slots=True)
|
|
136
|
+
class InteropProcessResult:
|
|
137
|
+
name: str
|
|
138
|
+
adapter: str
|
|
139
|
+
role: str
|
|
140
|
+
exit_code: int | None
|
|
141
|
+
stdout_path: str
|
|
142
|
+
stderr_path: str
|
|
143
|
+
stdout_text: str = ''
|
|
144
|
+
stderr_text: str = ''
|
|
145
|
+
version: dict[str, Any] = field(default_factory=dict)
|
|
146
|
+
provenance: dict[str, Any] = field(default_factory=dict)
|
|
147
|
+
timed_out: bool = False
|
|
148
|
+
error: str | None = None
|
|
149
|
+
|
|
150
|
+
def to_observed(self) -> dict[str, Any]:
|
|
151
|
+
return {
|
|
152
|
+
'name': self.name,
|
|
153
|
+
'adapter': self.adapter,
|
|
154
|
+
'role': self.role,
|
|
155
|
+
'exit_code': self.exit_code,
|
|
156
|
+
'stdout_path': self.stdout_path,
|
|
157
|
+
'stderr_path': self.stderr_path,
|
|
158
|
+
'stdout_text': self.stdout_text,
|
|
159
|
+
'stderr_text': self.stderr_text,
|
|
160
|
+
'version': self.version,
|
|
161
|
+
'provenance': self.provenance,
|
|
162
|
+
'timed_out': self.timed_out,
|
|
163
|
+
'error': self.error,
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@dataclass(slots=True)
|
|
168
|
+
class InteropScenarioResult:
|
|
169
|
+
scenario_id: str
|
|
170
|
+
passed: bool
|
|
171
|
+
commit_hash: str
|
|
172
|
+
artifact_dir: str
|
|
173
|
+
assertions_failed: list[str] = field(default_factory=list)
|
|
174
|
+
error: str | None = None
|
|
175
|
+
sut: dict[str, Any] = field(default_factory=dict)
|
|
176
|
+
peer: dict[str, Any] = field(default_factory=dict)
|
|
177
|
+
transcript: dict[str, Any] = field(default_factory=dict)
|
|
178
|
+
negotiation: dict[str, Any] = field(default_factory=dict)
|
|
179
|
+
artifacts: dict[str, Any] = field(default_factory=dict)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(slots=True)
|
|
183
|
+
class InteropRunSummary:
|
|
184
|
+
matrix_name: str
|
|
185
|
+
commit_hash: str
|
|
186
|
+
artifact_root: str
|
|
187
|
+
total: int
|
|
188
|
+
passed: int
|
|
189
|
+
failed: int
|
|
190
|
+
skipped: int
|
|
191
|
+
scenarios: list[InteropScenarioResult]
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class InteropRunnerError(RuntimeError):
|
|
195
|
+
pass
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class _ManagedProcess:
|
|
199
|
+
def __init__(self, process: subprocess.Popen[Any], stdout_path: Path, stderr_path: Path) -> None:
|
|
200
|
+
self.process = process
|
|
201
|
+
self.stdout_path = stdout_path
|
|
202
|
+
self.stderr_path = stderr_path
|
|
203
|
+
|
|
204
|
+
def stop(self, *, timeout: float = 5.0) -> int | None:
|
|
205
|
+
if self.process.poll() is None:
|
|
206
|
+
try:
|
|
207
|
+
self.process.terminate()
|
|
208
|
+
except Exception:
|
|
209
|
+
pass
|
|
210
|
+
try:
|
|
211
|
+
self.process.wait(timeout=timeout)
|
|
212
|
+
except subprocess.TimeoutExpired:
|
|
213
|
+
try:
|
|
214
|
+
self.process.kill()
|
|
215
|
+
except Exception:
|
|
216
|
+
pass
|
|
217
|
+
try:
|
|
218
|
+
self.process.wait(timeout=timeout)
|
|
219
|
+
except subprocess.TimeoutExpired:
|
|
220
|
+
return None
|
|
221
|
+
return self.process.returncode
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class BasePeerAdapter:
|
|
225
|
+
def inspect_version(self, spec: InteropProcessSpec, *, env: Mapping[str, str], cwd: Path | None) -> dict[str, Any]:
|
|
226
|
+
raise NotImplementedError
|
|
227
|
+
|
|
228
|
+
def run_oneshot(
|
|
229
|
+
self,
|
|
230
|
+
spec: InteropProcessSpec,
|
|
231
|
+
*,
|
|
232
|
+
env: Mapping[str, str],
|
|
233
|
+
cwd: Path | None,
|
|
234
|
+
stdout_path: Path,
|
|
235
|
+
stderr_path: Path,
|
|
236
|
+
) -> InteropProcessResult:
|
|
237
|
+
raise NotImplementedError
|
|
238
|
+
|
|
239
|
+
def start_persistent(
|
|
240
|
+
self,
|
|
241
|
+
spec: InteropProcessSpec,
|
|
242
|
+
*,
|
|
243
|
+
env: Mapping[str, str],
|
|
244
|
+
cwd: Path | None,
|
|
245
|
+
stdout_path: Path,
|
|
246
|
+
stderr_path: Path,
|
|
247
|
+
) -> tuple[_ManagedProcess, InteropProcessResult]:
|
|
248
|
+
raise NotImplementedError
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
class SubprocessPeerAdapter(BasePeerAdapter):
|
|
252
|
+
def inspect_version(self, spec: InteropProcessSpec, *, env: Mapping[str, str], cwd: Path | None) -> dict[str, Any]:
|
|
253
|
+
executable = shutil.which(spec.command[0]) if spec.command else None
|
|
254
|
+
payload: dict[str, Any] = {
|
|
255
|
+
'command': list(spec.command),
|
|
256
|
+
'executable': executable,
|
|
257
|
+
}
|
|
258
|
+
if executable is not None:
|
|
259
|
+
try:
|
|
260
|
+
payload['executable_sha256'] = _sha256_path(Path(executable))
|
|
261
|
+
except Exception:
|
|
262
|
+
pass
|
|
263
|
+
if spec.version_command is not None:
|
|
264
|
+
try:
|
|
265
|
+
completed = subprocess.run(
|
|
266
|
+
spec.version_command,
|
|
267
|
+
cwd=str(cwd) if cwd is not None else None,
|
|
268
|
+
env=dict(env),
|
|
269
|
+
capture_output=True,
|
|
270
|
+
text=True,
|
|
271
|
+
timeout=min(spec.run_timeout, 15.0),
|
|
272
|
+
)
|
|
273
|
+
payload['version_command'] = list(spec.version_command)
|
|
274
|
+
payload['version_exit_code'] = completed.returncode
|
|
275
|
+
payload['version_stdout'] = completed.stdout.strip()
|
|
276
|
+
payload['version_stderr'] = completed.stderr.strip()
|
|
277
|
+
except Exception as exc:
|
|
278
|
+
payload['version_error'] = str(exc)
|
|
279
|
+
return payload
|
|
280
|
+
|
|
281
|
+
def run_oneshot(
|
|
282
|
+
self,
|
|
283
|
+
spec: InteropProcessSpec,
|
|
284
|
+
*,
|
|
285
|
+
env: Mapping[str, str],
|
|
286
|
+
cwd: Path | None,
|
|
287
|
+
stdout_path: Path,
|
|
288
|
+
stderr_path: Path,
|
|
289
|
+
) -> InteropProcessResult:
|
|
290
|
+
with stdout_path.open('w', encoding='utf-8', errors='replace') as stdout_handle, stderr_path.open('w', encoding='utf-8', errors='replace') as stderr_handle:
|
|
291
|
+
try:
|
|
292
|
+
completed = subprocess.run(
|
|
293
|
+
spec.command,
|
|
294
|
+
cwd=str(cwd) if cwd is not None else None,
|
|
295
|
+
env=dict(env),
|
|
296
|
+
stdout=stdout_handle,
|
|
297
|
+
stderr=stderr_handle,
|
|
298
|
+
text=True,
|
|
299
|
+
timeout=spec.run_timeout,
|
|
300
|
+
)
|
|
301
|
+
return InteropProcessResult(
|
|
302
|
+
name=spec.name,
|
|
303
|
+
adapter=spec.adapter,
|
|
304
|
+
role=spec.role,
|
|
305
|
+
exit_code=completed.returncode,
|
|
306
|
+
stdout_path=str(stdout_path),
|
|
307
|
+
stderr_path=str(stderr_path),
|
|
308
|
+
stdout_text=stdout_path.read_text(encoding='utf-8', errors='replace') if stdout_path.exists() else '',
|
|
309
|
+
stderr_text=stderr_path.read_text(encoding='utf-8', errors='replace') if stderr_path.exists() else '',
|
|
310
|
+
)
|
|
311
|
+
except subprocess.TimeoutExpired:
|
|
312
|
+
return InteropProcessResult(
|
|
313
|
+
name=spec.name,
|
|
314
|
+
adapter=spec.adapter,
|
|
315
|
+
role=spec.role,
|
|
316
|
+
exit_code=None,
|
|
317
|
+
stdout_path=str(stdout_path),
|
|
318
|
+
stderr_path=str(stderr_path),
|
|
319
|
+
stdout_text=stdout_path.read_text(encoding='utf-8', errors='replace') if stdout_path.exists() else '',
|
|
320
|
+
stderr_text=stderr_path.read_text(encoding='utf-8', errors='replace') if stderr_path.exists() else '',
|
|
321
|
+
timed_out=True,
|
|
322
|
+
error=f'{spec.name} timed out after {spec.run_timeout:.3f}s',
|
|
323
|
+
)
|
|
324
|
+
except Exception as exc:
|
|
325
|
+
return InteropProcessResult(
|
|
326
|
+
name=spec.name,
|
|
327
|
+
adapter=spec.adapter,
|
|
328
|
+
role=spec.role,
|
|
329
|
+
exit_code=None,
|
|
330
|
+
stdout_path=str(stdout_path),
|
|
331
|
+
stderr_path=str(stderr_path),
|
|
332
|
+
stdout_text=stdout_path.read_text(encoding='utf-8', errors='replace') if stdout_path.exists() else '',
|
|
333
|
+
stderr_text=stderr_path.read_text(encoding='utf-8', errors='replace') if stderr_path.exists() else '',
|
|
334
|
+
error=str(exc),
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
def start_persistent(
|
|
338
|
+
self,
|
|
339
|
+
spec: InteropProcessSpec,
|
|
340
|
+
*,
|
|
341
|
+
env: Mapping[str, str],
|
|
342
|
+
cwd: Path | None,
|
|
343
|
+
stdout_path: Path,
|
|
344
|
+
stderr_path: Path,
|
|
345
|
+
) -> tuple[_ManagedProcess, InteropProcessResult]:
|
|
346
|
+
stdout_handle = stdout_path.open('w', encoding='utf-8', errors='replace')
|
|
347
|
+
stderr_handle = stderr_path.open('w', encoding='utf-8', errors='replace')
|
|
348
|
+
try:
|
|
349
|
+
process = subprocess.Popen(
|
|
350
|
+
spec.command,
|
|
351
|
+
cwd=str(cwd) if cwd is not None else None,
|
|
352
|
+
env=dict(env),
|
|
353
|
+
stdout=stdout_handle,
|
|
354
|
+
stderr=stderr_handle,
|
|
355
|
+
text=True,
|
|
356
|
+
)
|
|
357
|
+
finally:
|
|
358
|
+
stdout_handle.close()
|
|
359
|
+
stderr_handle.close()
|
|
360
|
+
managed = _ManagedProcess(process, stdout_path, stderr_path)
|
|
361
|
+
error = _wait_for_server_ready(
|
|
362
|
+
spec=spec,
|
|
363
|
+
process=process,
|
|
364
|
+
env=env,
|
|
365
|
+
stdout_path=stdout_path,
|
|
366
|
+
stderr_path=stderr_path,
|
|
367
|
+
)
|
|
368
|
+
result = InteropProcessResult(
|
|
369
|
+
name=spec.name,
|
|
370
|
+
adapter=spec.adapter,
|
|
371
|
+
role=spec.role,
|
|
372
|
+
exit_code=process.returncode,
|
|
373
|
+
stdout_path=str(stdout_path),
|
|
374
|
+
stderr_path=str(stderr_path),
|
|
375
|
+
stdout_text=stdout_path.read_text(encoding='utf-8', errors='replace') if stdout_path.exists() else '',
|
|
376
|
+
stderr_text=stderr_path.read_text(encoding='utf-8', errors='replace') if stderr_path.exists() else '',
|
|
377
|
+
error=error,
|
|
378
|
+
)
|
|
379
|
+
if error is not None:
|
|
380
|
+
managed.stop(timeout=1.0)
|
|
381
|
+
result.exit_code = process.returncode
|
|
382
|
+
result.stdout_text = stdout_path.read_text(encoding='utf-8', errors='replace') if stdout_path.exists() else ''
|
|
383
|
+
result.stderr_text = stderr_path.read_text(encoding='utf-8', errors='replace') if stderr_path.exists() else ''
|
|
384
|
+
return managed, result
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
class DockerPeerAdapter(SubprocessPeerAdapter):
|
|
388
|
+
def inspect_version(self, spec: InteropProcessSpec, *, env: Mapping[str, str], cwd: Path | None) -> dict[str, Any]:
|
|
389
|
+
payload = super().inspect_version(spec, env=env, cwd=cwd)
|
|
390
|
+
if spec.image is not None:
|
|
391
|
+
payload['image'] = spec.image
|
|
392
|
+
try:
|
|
393
|
+
completed = subprocess.run(
|
|
394
|
+
['docker', 'image', 'inspect', '--format', '{{json .RepoDigests}}', spec.image],
|
|
395
|
+
capture_output=True,
|
|
396
|
+
text=True,
|
|
397
|
+
timeout=15.0,
|
|
398
|
+
)
|
|
399
|
+
payload['image_inspect_exit_code'] = completed.returncode
|
|
400
|
+
payload['image_repo_digests'] = completed.stdout.strip()
|
|
401
|
+
payload['docker_stderr'] = completed.stderr.strip()
|
|
402
|
+
except Exception as exc:
|
|
403
|
+
payload['image_inspect_error'] = str(exc)
|
|
404
|
+
return payload
|
|
405
|
+
|
|
406
|
+
def _docker_command(self, spec: InteropProcessSpec) -> list[str]:
|
|
407
|
+
if spec.image is None:
|
|
408
|
+
raise InteropRunnerError('docker adapter requires an image')
|
|
409
|
+
command = ['docker', 'run', '--rm']
|
|
410
|
+
for key, value in spec.env.items():
|
|
411
|
+
command.extend(['-e', f'{key}={value}'])
|
|
412
|
+
command.append(spec.image)
|
|
413
|
+
command.extend(spec.command)
|
|
414
|
+
return command
|
|
415
|
+
|
|
416
|
+
def run_oneshot(
|
|
417
|
+
self,
|
|
418
|
+
spec: InteropProcessSpec,
|
|
419
|
+
*,
|
|
420
|
+
env: Mapping[str, str],
|
|
421
|
+
cwd: Path | None,
|
|
422
|
+
stdout_path: Path,
|
|
423
|
+
stderr_path: Path,
|
|
424
|
+
) -> InteropProcessResult:
|
|
425
|
+
docker_spec = InteropProcessSpec(
|
|
426
|
+
name=spec.name,
|
|
427
|
+
adapter=spec.adapter,
|
|
428
|
+
role=spec.role,
|
|
429
|
+
command=self._docker_command(spec),
|
|
430
|
+
env={},
|
|
431
|
+
cwd=spec.cwd,
|
|
432
|
+
ready_pattern=spec.ready_pattern,
|
|
433
|
+
ready_timeout=spec.ready_timeout,
|
|
434
|
+
run_timeout=spec.run_timeout,
|
|
435
|
+
version_command=spec.version_command,
|
|
436
|
+
image=spec.image,
|
|
437
|
+
enabled=spec.enabled,
|
|
438
|
+
metadata=dict(spec.metadata),
|
|
439
|
+
)
|
|
440
|
+
return super().run_oneshot(docker_spec, env=env, cwd=cwd, stdout_path=stdout_path, stderr_path=stderr_path)
|
|
441
|
+
|
|
442
|
+
def start_persistent(
|
|
443
|
+
self,
|
|
444
|
+
spec: InteropProcessSpec,
|
|
445
|
+
*,
|
|
446
|
+
env: Mapping[str, str],
|
|
447
|
+
cwd: Path | None,
|
|
448
|
+
stdout_path: Path,
|
|
449
|
+
stderr_path: Path,
|
|
450
|
+
) -> tuple[_ManagedProcess, InteropProcessResult]:
|
|
451
|
+
docker_spec = InteropProcessSpec(
|
|
452
|
+
name=spec.name,
|
|
453
|
+
adapter=spec.adapter,
|
|
454
|
+
role=spec.role,
|
|
455
|
+
command=self._docker_command(spec),
|
|
456
|
+
env={},
|
|
457
|
+
cwd=spec.cwd,
|
|
458
|
+
ready_pattern=spec.ready_pattern,
|
|
459
|
+
ready_timeout=spec.ready_timeout,
|
|
460
|
+
run_timeout=spec.run_timeout,
|
|
461
|
+
version_command=spec.version_command,
|
|
462
|
+
image=spec.image,
|
|
463
|
+
enabled=spec.enabled,
|
|
464
|
+
metadata=dict(spec.metadata),
|
|
465
|
+
)
|
|
466
|
+
return super().start_persistent(docker_spec, env=env, cwd=cwd, stdout_path=stdout_path, stderr_path=stderr_path)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
_ADAPTERS: dict[str, type[BasePeerAdapter]] = {
|
|
470
|
+
'subprocess': SubprocessPeerAdapter,
|
|
471
|
+
'docker': DockerPeerAdapter,
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
class _PacketTraceWriter:
|
|
476
|
+
def __init__(self, path: Path) -> None:
|
|
477
|
+
self.path = path
|
|
478
|
+
self._lock = threading.Lock()
|
|
479
|
+
self._handle = path.open('w', encoding='utf-8')
|
|
480
|
+
|
|
481
|
+
def write(self, *, direction: str, transport: str, local: tuple[str, int], remote: tuple[str, int], payload: bytes) -> None:
|
|
482
|
+
record = {
|
|
483
|
+
'timestamp': time.time(),
|
|
484
|
+
'direction': direction,
|
|
485
|
+
'transport': transport,
|
|
486
|
+
'local': {'host': local[0], 'port': local[1]},
|
|
487
|
+
'remote': {'host': remote[0], 'port': remote[1]},
|
|
488
|
+
'length': len(payload),
|
|
489
|
+
'payload_hex': payload.hex(),
|
|
490
|
+
}
|
|
491
|
+
with self._lock:
|
|
492
|
+
self._handle.write(json.dumps(record, sort_keys=True) + '\n')
|
|
493
|
+
self._handle.flush()
|
|
494
|
+
|
|
495
|
+
def close(self) -> None:
|
|
496
|
+
try:
|
|
497
|
+
self._handle.close()
|
|
498
|
+
except Exception:
|
|
499
|
+
pass
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
class _TCPRelay(threading.Thread):
|
|
503
|
+
def __init__(self, source: socket.socket, sink: socket.socket, writer: _PacketTraceWriter, *, direction: str, local: tuple[str, int], remote: tuple[str, int]) -> None:
|
|
504
|
+
super().__init__(daemon=True)
|
|
505
|
+
self.source = source
|
|
506
|
+
self.sink = sink
|
|
507
|
+
self.writer = writer
|
|
508
|
+
self.direction = direction
|
|
509
|
+
self.local = local
|
|
510
|
+
self.remote = remote
|
|
511
|
+
|
|
512
|
+
def run(self) -> None:
|
|
513
|
+
try:
|
|
514
|
+
while True:
|
|
515
|
+
chunk = self.source.recv(65535)
|
|
516
|
+
if not chunk:
|
|
517
|
+
break
|
|
518
|
+
self.writer.write(direction=self.direction, transport='tcp', local=self.local, remote=self.remote, payload=chunk)
|
|
519
|
+
self.sink.sendall(chunk)
|
|
520
|
+
except OSError:
|
|
521
|
+
pass
|
|
522
|
+
finally:
|
|
523
|
+
try:
|
|
524
|
+
self.sink.shutdown(socket.SHUT_WR)
|
|
525
|
+
except OSError:
|
|
526
|
+
pass
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
class TCPRecordProxy:
|
|
530
|
+
def __init__(self, *, listen_host: str, listen_port: int, target_host: str, target_port: int, packet_trace_path: Path, ip_family: str = 'ipv4') -> None:
|
|
531
|
+
family = socket.AF_INET6 if ip_family == 'ipv6' else socket.AF_INET
|
|
532
|
+
self.listen_host = listen_host
|
|
533
|
+
self.listen_port = listen_port
|
|
534
|
+
self.target_host = target_host
|
|
535
|
+
self.target_port = target_port
|
|
536
|
+
self._server = socket.socket(family, socket.SOCK_STREAM)
|
|
537
|
+
self._server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
538
|
+
if family == socket.AF_INET6:
|
|
539
|
+
self._server.bind((listen_host, listen_port, 0, 0))
|
|
540
|
+
else:
|
|
541
|
+
self._server.bind((listen_host, listen_port))
|
|
542
|
+
self._server.listen(5)
|
|
543
|
+
self._server.settimeout(0.2)
|
|
544
|
+
self._writer = _PacketTraceWriter(packet_trace_path)
|
|
545
|
+
self._stop = threading.Event()
|
|
546
|
+
self._thread = threading.Thread(target=self._accept_loop, daemon=True)
|
|
547
|
+
self._connections: list[tuple[socket.socket, socket.socket]] = []
|
|
548
|
+
|
|
549
|
+
def start(self) -> None:
|
|
550
|
+
self._thread.start()
|
|
551
|
+
|
|
552
|
+
def _accept_loop(self) -> None:
|
|
553
|
+
while not self._stop.is_set():
|
|
554
|
+
try:
|
|
555
|
+
client_sock, _client_addr = self._server.accept()
|
|
556
|
+
except TimeoutError:
|
|
557
|
+
continue
|
|
558
|
+
except OSError:
|
|
559
|
+
break
|
|
560
|
+
try:
|
|
561
|
+
server_sock = socket.create_connection((self.target_host, self.target_port), timeout=5.0)
|
|
562
|
+
except OSError:
|
|
563
|
+
client_sock.close()
|
|
564
|
+
continue
|
|
565
|
+
self._connections.append((client_sock, server_sock))
|
|
566
|
+
local_client = _normalize_sockaddr(client_sock.getsockname())
|
|
567
|
+
remote_server = _normalize_sockaddr(server_sock.getpeername())
|
|
568
|
+
local_server = _normalize_sockaddr(server_sock.getsockname())
|
|
569
|
+
remote_client = _normalize_sockaddr(client_sock.getpeername())
|
|
570
|
+
c2s = _TCPRelay(client_sock, server_sock, self._writer, direction='client_to_server', local=local_client, remote=remote_server)
|
|
571
|
+
s2c = _TCPRelay(server_sock, client_sock, self._writer, direction='server_to_client', local=local_server, remote=remote_client)
|
|
572
|
+
c2s.start()
|
|
573
|
+
s2c.start()
|
|
574
|
+
|
|
575
|
+
def close(self) -> None:
|
|
576
|
+
self._stop.set()
|
|
577
|
+
try:
|
|
578
|
+
self._server.close()
|
|
579
|
+
except OSError:
|
|
580
|
+
pass
|
|
581
|
+
self._thread.join(timeout=1.0)
|
|
582
|
+
for left, right in self._connections:
|
|
583
|
+
try:
|
|
584
|
+
left.close()
|
|
585
|
+
except OSError:
|
|
586
|
+
pass
|
|
587
|
+
try:
|
|
588
|
+
right.close()
|
|
589
|
+
except OSError:
|
|
590
|
+
pass
|
|
591
|
+
self._writer.close()
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
class UDPRecordProxy:
|
|
595
|
+
def __init__(self, *, listen_host: str, listen_port: int, target_host: str, target_port: int, packet_trace_path: Path, ip_family: str = 'ipv4') -> None:
|
|
596
|
+
family = socket.AF_INET6 if ip_family == 'ipv6' else socket.AF_INET
|
|
597
|
+
self.listen_host = listen_host
|
|
598
|
+
self.listen_port = listen_port
|
|
599
|
+
self.target_host = target_host
|
|
600
|
+
self.target_port = target_port
|
|
601
|
+
self._downstream = socket.socket(family, socket.SOCK_DGRAM)
|
|
602
|
+
self._upstream = socket.socket(family, socket.SOCK_DGRAM)
|
|
603
|
+
if family == socket.AF_INET6:
|
|
604
|
+
self._downstream.bind((listen_host, listen_port, 0, 0))
|
|
605
|
+
self._upstream.bind((listen_host, 0, 0, 0))
|
|
606
|
+
else:
|
|
607
|
+
self._downstream.bind((listen_host, listen_port))
|
|
608
|
+
self._upstream.bind((listen_host, 0))
|
|
609
|
+
self._downstream.setblocking(False)
|
|
610
|
+
self._upstream.setblocking(False)
|
|
611
|
+
self._writer = _PacketTraceWriter(packet_trace_path)
|
|
612
|
+
self._stop = threading.Event()
|
|
613
|
+
self._thread = threading.Thread(target=self._loop, daemon=True)
|
|
614
|
+
self._last_client: tuple[str, int] | None = None
|
|
615
|
+
|
|
616
|
+
def start(self) -> None:
|
|
617
|
+
self._thread.start()
|
|
618
|
+
|
|
619
|
+
def _loop(self) -> None:
|
|
620
|
+
selector = selectors.DefaultSelector()
|
|
621
|
+
selector.register(self._downstream, selectors.EVENT_READ)
|
|
622
|
+
selector.register(self._upstream, selectors.EVENT_READ)
|
|
623
|
+
target = (self.target_host, self.target_port)
|
|
624
|
+
try:
|
|
625
|
+
while not self._stop.is_set():
|
|
626
|
+
events = selector.select(timeout=0.2)
|
|
627
|
+
for key, _mask in events:
|
|
628
|
+
if key.fileobj is self._downstream:
|
|
629
|
+
try:
|
|
630
|
+
payload, addr = self._downstream.recvfrom(65535)
|
|
631
|
+
except OSError:
|
|
632
|
+
continue
|
|
633
|
+
self._last_client = _normalize_sockaddr(addr)
|
|
634
|
+
self._writer.write(
|
|
635
|
+
direction='client_to_server',
|
|
636
|
+
transport='udp',
|
|
637
|
+
local=_normalize_sockaddr(self._downstream.getsockname()),
|
|
638
|
+
remote=(self.target_host, self.target_port),
|
|
639
|
+
payload=payload,
|
|
640
|
+
)
|
|
641
|
+
try:
|
|
642
|
+
self._upstream.sendto(payload, target)
|
|
643
|
+
except OSError:
|
|
644
|
+
continue
|
|
645
|
+
elif key.fileobj is self._upstream:
|
|
646
|
+
try:
|
|
647
|
+
payload, _addr = self._upstream.recvfrom(65535)
|
|
648
|
+
except OSError:
|
|
649
|
+
continue
|
|
650
|
+
if self._last_client is None:
|
|
651
|
+
continue
|
|
652
|
+
self._writer.write(
|
|
653
|
+
direction='server_to_client',
|
|
654
|
+
transport='udp',
|
|
655
|
+
local=_normalize_sockaddr(self._upstream.getsockname()),
|
|
656
|
+
remote=self._last_client,
|
|
657
|
+
payload=payload,
|
|
658
|
+
)
|
|
659
|
+
try:
|
|
660
|
+
self._downstream.sendto(payload, self._last_client)
|
|
661
|
+
except OSError:
|
|
662
|
+
continue
|
|
663
|
+
finally:
|
|
664
|
+
selector.close()
|
|
665
|
+
|
|
666
|
+
def close(self) -> None:
|
|
667
|
+
self._stop.set()
|
|
668
|
+
self._thread.join(timeout=1.0)
|
|
669
|
+
try:
|
|
670
|
+
self._downstream.close()
|
|
671
|
+
except OSError:
|
|
672
|
+
pass
|
|
673
|
+
try:
|
|
674
|
+
self._upstream.close()
|
|
675
|
+
except OSError:
|
|
676
|
+
pass
|
|
677
|
+
self._writer.close()
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
class ExternalInteropRunner:
|
|
681
|
+
def __init__(self, *, matrix: InteropMatrix, artifact_root: str | Path, source_root: str | Path | None = None) -> None:
|
|
682
|
+
for scenario in matrix.scenarios:
|
|
683
|
+
_validate_scenario_provenance(scenario)
|
|
684
|
+
self.matrix = matrix
|
|
685
|
+
self.artifact_root = Path(artifact_root)
|
|
686
|
+
self.source_root = Path(source_root) if source_root is not None else Path.cwd()
|
|
687
|
+
self.commit_hash = detect_source_revision(self.source_root)
|
|
688
|
+
self.environment_manifest = build_environment_manifest(self.source_root, commit_hash=self.commit_hash)
|
|
689
|
+
|
|
690
|
+
def run(self, *, scenario_ids: Iterable[str] | None = None, strict: bool = False) -> InteropRunSummary:
|
|
691
|
+
selected = set(scenario_ids or ())
|
|
692
|
+
scenarios = self.matrix.enabled_scenarios
|
|
693
|
+
if selected:
|
|
694
|
+
scenarios = [scenario for scenario in scenarios if scenario.id in selected]
|
|
695
|
+
run_root = self.artifact_root / self.commit_hash / self.matrix.name
|
|
696
|
+
run_root.mkdir(parents=True, exist_ok=True)
|
|
697
|
+
bundle_kind = str(self.matrix.metadata.get('bundle_kind', self.matrix.metadata.get('evidence_tier', 'mixed')) or 'mixed')
|
|
698
|
+
wrapper_families = dict(self.matrix.metadata.get('phase9b_wrapper_families', {}))
|
|
699
|
+
_write_json(
|
|
700
|
+
run_root / 'manifest.json',
|
|
701
|
+
{
|
|
702
|
+
'matrix_name': self.matrix.name,
|
|
703
|
+
'bundle_kind': bundle_kind,
|
|
704
|
+
'artifact_schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
705
|
+
'required_bundle_files': list(INTEROP_BUNDLE_REQUIRED_FILES),
|
|
706
|
+
'required_scenario_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
707
|
+
'commit_hash': self.commit_hash,
|
|
708
|
+
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
709
|
+
'dimensions': summarize_matrix_dimensions(self.matrix),
|
|
710
|
+
'environment': self.environment_manifest,
|
|
711
|
+
'matrix_sha256': _sha256_bytes(json.dumps(_matrix_to_json(self.matrix), sort_keys=True).encode('utf-8')),
|
|
712
|
+
'wrapper_families': wrapper_families,
|
|
713
|
+
},
|
|
714
|
+
)
|
|
715
|
+
results: list[InteropScenarioResult] = []
|
|
716
|
+
passed = 0
|
|
717
|
+
failed = 0
|
|
718
|
+
skipped = len([scenario for scenario in self.matrix.scenarios if scenario not in scenarios])
|
|
719
|
+
for scenario in scenarios:
|
|
720
|
+
result = self._run_scenario(scenario, run_root)
|
|
721
|
+
results.append(result)
|
|
722
|
+
if result.passed:
|
|
723
|
+
passed += 1
|
|
724
|
+
else:
|
|
725
|
+
failed += 1
|
|
726
|
+
if strict:
|
|
727
|
+
break
|
|
728
|
+
summary = InteropRunSummary(
|
|
729
|
+
matrix_name=self.matrix.name,
|
|
730
|
+
commit_hash=self.commit_hash,
|
|
731
|
+
artifact_root=str(run_root),
|
|
732
|
+
total=len(results),
|
|
733
|
+
passed=passed,
|
|
734
|
+
failed=failed,
|
|
735
|
+
skipped=skipped,
|
|
736
|
+
scenarios=results,
|
|
737
|
+
)
|
|
738
|
+
root_index_payload = {
|
|
739
|
+
'schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
740
|
+
'bundle_kind': bundle_kind,
|
|
741
|
+
'required_bundle_files': list(INTEROP_BUNDLE_REQUIRED_FILES),
|
|
742
|
+
'required_scenario_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
743
|
+
'matrix_name': summary.matrix_name,
|
|
744
|
+
'commit_hash': summary.commit_hash,
|
|
745
|
+
'artifact_root': summary.artifact_root,
|
|
746
|
+
'total': summary.total,
|
|
747
|
+
'passed': summary.passed,
|
|
748
|
+
'failed': summary.failed,
|
|
749
|
+
'skipped': summary.skipped,
|
|
750
|
+
'wrapper_families': wrapper_families,
|
|
751
|
+
'scenarios': [
|
|
752
|
+
{
|
|
753
|
+
'id': item.scenario_id,
|
|
754
|
+
'passed': item.passed,
|
|
755
|
+
'artifact_dir': item.artifact_dir,
|
|
756
|
+
'assertions_failed': item.assertions_failed,
|
|
757
|
+
'error': item.error,
|
|
758
|
+
'summary_path': str(Path(item.artifact_dir) / 'summary.json'),
|
|
759
|
+
'index_path': str(Path(item.artifact_dir) / 'index.json'),
|
|
760
|
+
'result_path': str(Path(item.artifact_dir) / 'result.json'),
|
|
761
|
+
}
|
|
762
|
+
for item in summary.scenarios
|
|
763
|
+
],
|
|
764
|
+
}
|
|
765
|
+
_write_json(run_root / 'index.json', root_index_payload)
|
|
766
|
+
_write_json(
|
|
767
|
+
run_root / 'summary.json',
|
|
768
|
+
{
|
|
769
|
+
'schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
770
|
+
'bundle_kind': bundle_kind,
|
|
771
|
+
'matrix_name': summary.matrix_name,
|
|
772
|
+
'commit_hash': summary.commit_hash,
|
|
773
|
+
'artifact_root': summary.artifact_root,
|
|
774
|
+
'total': summary.total,
|
|
775
|
+
'passed': summary.passed,
|
|
776
|
+
'failed': summary.failed,
|
|
777
|
+
'skipped': summary.skipped,
|
|
778
|
+
'scenario_ids': [item.scenario_id for item in summary.scenarios],
|
|
779
|
+
'required_bundle_files': list(INTEROP_BUNDLE_REQUIRED_FILES),
|
|
780
|
+
'required_scenario_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
781
|
+
'wrapper_families': wrapper_families,
|
|
782
|
+
},
|
|
783
|
+
)
|
|
784
|
+
return summary
|
|
785
|
+
|
|
786
|
+
def _run_scenario(self, scenario: InteropScenario, run_root: Path) -> InteropScenarioResult:
|
|
787
|
+
scenario_root = run_root / _safe_name(scenario.id)
|
|
788
|
+
if scenario_root.exists():
|
|
789
|
+
shutil.rmtree(scenario_root)
|
|
790
|
+
scenario_root.mkdir(parents=True, exist_ok=True)
|
|
791
|
+
transport = scenario.transport or _default_transport_for_protocol(scenario.protocol)
|
|
792
|
+
socket_type = socket.SOCK_DGRAM if transport == 'udp' else socket.SOCK_STREAM
|
|
793
|
+
bind_host = '::1' if scenario.ip_family == 'ipv6' else '127.0.0.1'
|
|
794
|
+
bind_port = _reserve_port(bind_host, socket_type)
|
|
795
|
+
proxy_port = _reserve_distinct_port(bind_host, socket_type, {bind_port})
|
|
796
|
+
packet_trace_path = scenario_root / 'packet_trace.jsonl'
|
|
797
|
+
qlog_path = scenario_root / 'qlog.json'
|
|
798
|
+
sut_stdout_path = scenario_root / 'sut_stdout.log'
|
|
799
|
+
sut_stderr_path = scenario_root / 'sut_stderr.log'
|
|
800
|
+
peer_stdout_path = scenario_root / 'peer_stdout.log'
|
|
801
|
+
peer_stderr_path = scenario_root / 'peer_stderr.log'
|
|
802
|
+
sut_transcript_path = scenario_root / 'sut_transcript.json'
|
|
803
|
+
peer_transcript_path = scenario_root / 'peer_transcript.json'
|
|
804
|
+
sut_negotiation_path = scenario_root / 'sut_negotiation.json'
|
|
805
|
+
peer_negotiation_path = scenario_root / 'peer_negotiation.json'
|
|
806
|
+
connect_host = bind_host
|
|
807
|
+
connect_port = bind_port
|
|
808
|
+
proxy: TCPRecordProxy | UDPRecordProxy | None = None
|
|
809
|
+
if scenario.capture.get('proxy', True):
|
|
810
|
+
if transport == 'udp':
|
|
811
|
+
proxy = UDPRecordProxy(
|
|
812
|
+
listen_host=bind_host,
|
|
813
|
+
listen_port=proxy_port,
|
|
814
|
+
target_host=bind_host,
|
|
815
|
+
target_port=bind_port,
|
|
816
|
+
packet_trace_path=packet_trace_path,
|
|
817
|
+
ip_family=scenario.ip_family,
|
|
818
|
+
)
|
|
819
|
+
else:
|
|
820
|
+
proxy = TCPRecordProxy(
|
|
821
|
+
listen_host=bind_host,
|
|
822
|
+
listen_port=proxy_port,
|
|
823
|
+
target_host=bind_host,
|
|
824
|
+
target_port=bind_port,
|
|
825
|
+
packet_trace_path=packet_trace_path,
|
|
826
|
+
ip_family=scenario.ip_family,
|
|
827
|
+
)
|
|
828
|
+
proxy.start()
|
|
829
|
+
connect_port = proxy_port
|
|
830
|
+
else:
|
|
831
|
+
packet_trace_path.touch()
|
|
832
|
+
context = {
|
|
833
|
+
'bind_host': bind_host,
|
|
834
|
+
'bind_port': str(bind_port),
|
|
835
|
+
'target_host': connect_host,
|
|
836
|
+
'target_port': str(connect_port),
|
|
837
|
+
'artifact_dir': str(scenario_root),
|
|
838
|
+
'packet_trace_path': str(packet_trace_path),
|
|
839
|
+
'qlog_path': str(qlog_path),
|
|
840
|
+
'scenario_id': scenario.id,
|
|
841
|
+
'matrix_name': self.matrix.name,
|
|
842
|
+
'commit_hash': self.commit_hash,
|
|
843
|
+
'protocol': scenario.protocol,
|
|
844
|
+
'feature': scenario.feature,
|
|
845
|
+
'role': scenario.role,
|
|
846
|
+
'ip_family': scenario.ip_family,
|
|
847
|
+
'cipher_group': scenario.cipher_group or '',
|
|
848
|
+
'retry': scenario.retry,
|
|
849
|
+
'resumption': scenario.resumption,
|
|
850
|
+
'zero_rtt': scenario.zero_rtt,
|
|
851
|
+
'key_update': scenario.key_update,
|
|
852
|
+
'migration': scenario.migration,
|
|
853
|
+
'goaway': scenario.goaway,
|
|
854
|
+
'qpack_blocking': scenario.qpack_blocking,
|
|
855
|
+
}
|
|
856
|
+
sut_spec = _materialize_process_spec(scenario.sut, context)
|
|
857
|
+
peer_spec = _materialize_process_spec(scenario.peer_process, context)
|
|
858
|
+
sut_env = _build_process_env(self.source_root, sut_spec, sut_transcript_path, sut_negotiation_path, context)
|
|
859
|
+
peer_env = _build_process_env(self.source_root, peer_spec, peer_transcript_path, peer_negotiation_path, context)
|
|
860
|
+
sut_cwd = Path(sut_spec.cwd) if sut_spec.cwd is not None else self.source_root
|
|
861
|
+
peer_cwd = Path(peer_spec.cwd) if peer_spec.cwd is not None else self.source_root
|
|
862
|
+
sut_adapter = _instantiate_adapter(sut_spec.adapter)
|
|
863
|
+
peer_adapter = _instantiate_adapter(peer_spec.adapter)
|
|
864
|
+
sut_version = sut_adapter.inspect_version(sut_spec, env=sut_env, cwd=sut_cwd)
|
|
865
|
+
peer_version = peer_adapter.inspect_version(peer_spec, env=peer_env, cwd=peer_cwd)
|
|
866
|
+
sut_result: InteropProcessResult | None = None
|
|
867
|
+
peer_result: InteropProcessResult | None = None
|
|
868
|
+
server_handle: _ManagedProcess | None = None
|
|
869
|
+
error: str | None = None
|
|
870
|
+
try:
|
|
871
|
+
if sut_spec.role == 'server' and peer_spec.role == 'client':
|
|
872
|
+
server_handle, sut_result = sut_adapter.start_persistent(
|
|
873
|
+
sut_spec,
|
|
874
|
+
env=sut_env,
|
|
875
|
+
cwd=sut_cwd,
|
|
876
|
+
stdout_path=sut_stdout_path,
|
|
877
|
+
stderr_path=sut_stderr_path,
|
|
878
|
+
)
|
|
879
|
+
sut_result.version = sut_version
|
|
880
|
+
sut_result.provenance = _build_provenance_payload(sut_spec, sut_version)
|
|
881
|
+
if sut_result.error is None:
|
|
882
|
+
peer_result = peer_adapter.run_oneshot(
|
|
883
|
+
peer_spec,
|
|
884
|
+
env=peer_env,
|
|
885
|
+
cwd=peer_cwd,
|
|
886
|
+
stdout_path=peer_stdout_path,
|
|
887
|
+
stderr_path=peer_stderr_path,
|
|
888
|
+
)
|
|
889
|
+
peer_result.version = peer_version
|
|
890
|
+
peer_result.provenance = _build_provenance_payload(peer_spec, peer_version)
|
|
891
|
+
else:
|
|
892
|
+
error = sut_result.error
|
|
893
|
+
elif sut_spec.role == 'client' and peer_spec.role == 'server':
|
|
894
|
+
server_handle, peer_result = peer_adapter.start_persistent(
|
|
895
|
+
peer_spec,
|
|
896
|
+
env=peer_env,
|
|
897
|
+
cwd=peer_cwd,
|
|
898
|
+
stdout_path=peer_stdout_path,
|
|
899
|
+
stderr_path=peer_stderr_path,
|
|
900
|
+
)
|
|
901
|
+
peer_result.version = peer_version
|
|
902
|
+
peer_result.provenance = _build_provenance_payload(peer_spec, peer_version)
|
|
903
|
+
if peer_result.error is None:
|
|
904
|
+
sut_result = sut_adapter.run_oneshot(
|
|
905
|
+
sut_spec,
|
|
906
|
+
env=sut_env,
|
|
907
|
+
cwd=sut_cwd,
|
|
908
|
+
stdout_path=sut_stdout_path,
|
|
909
|
+
stderr_path=sut_stderr_path,
|
|
910
|
+
)
|
|
911
|
+
sut_result.version = sut_version
|
|
912
|
+
sut_result.provenance = _build_provenance_payload(sut_spec, sut_version)
|
|
913
|
+
else:
|
|
914
|
+
error = peer_result.error
|
|
915
|
+
else:
|
|
916
|
+
raise InteropRunnerError('exactly one side must be server and the other must be client')
|
|
917
|
+
except Exception as exc:
|
|
918
|
+
error = str(exc)
|
|
919
|
+
finally:
|
|
920
|
+
time.sleep(0.1)
|
|
921
|
+
if server_handle is not None:
|
|
922
|
+
exit_code = server_handle.stop(timeout=2.0)
|
|
923
|
+
if sut_result is not None and sut_spec.role == 'server':
|
|
924
|
+
sut_result.exit_code = exit_code
|
|
925
|
+
sut_result.stdout_text = sut_stdout_path.read_text(encoding='utf-8', errors='replace') if sut_stdout_path.exists() else ''
|
|
926
|
+
sut_result.stderr_text = sut_stderr_path.read_text(encoding='utf-8', errors='replace') if sut_stderr_path.exists() else ''
|
|
927
|
+
if peer_result is not None and peer_spec.role == 'server':
|
|
928
|
+
peer_result.exit_code = exit_code
|
|
929
|
+
peer_result.stdout_text = peer_stdout_path.read_text(encoding='utf-8', errors='replace') if peer_stdout_path.exists() else ''
|
|
930
|
+
peer_result.stderr_text = peer_stderr_path.read_text(encoding='utf-8', errors='replace') if peer_stderr_path.exists() else ''
|
|
931
|
+
if proxy is not None:
|
|
932
|
+
proxy.close()
|
|
933
|
+
if sut_result is None:
|
|
934
|
+
sut_result = InteropProcessResult(
|
|
935
|
+
name=sut_spec.name,
|
|
936
|
+
adapter=sut_spec.adapter,
|
|
937
|
+
role=sut_spec.role,
|
|
938
|
+
exit_code=None,
|
|
939
|
+
stdout_path=str(sut_stdout_path),
|
|
940
|
+
stderr_path=str(sut_stderr_path),
|
|
941
|
+
error='sut did not run',
|
|
942
|
+
version=sut_version,
|
|
943
|
+
provenance=_build_provenance_payload(sut_spec, sut_version),
|
|
944
|
+
)
|
|
945
|
+
if peer_result is None:
|
|
946
|
+
peer_result = InteropProcessResult(
|
|
947
|
+
name=peer_spec.name,
|
|
948
|
+
adapter=peer_spec.adapter,
|
|
949
|
+
role=peer_spec.role,
|
|
950
|
+
exit_code=None,
|
|
951
|
+
stdout_path=str(peer_stdout_path),
|
|
952
|
+
stderr_path=str(peer_stderr_path),
|
|
953
|
+
error='peer did not run',
|
|
954
|
+
version=peer_version,
|
|
955
|
+
provenance=_build_provenance_payload(peer_spec, peer_version),
|
|
956
|
+
)
|
|
957
|
+
if error is None:
|
|
958
|
+
error = sut_result.error or peer_result.error
|
|
959
|
+
sut_transcript = _load_json_if_present(sut_transcript_path)
|
|
960
|
+
peer_transcript = _load_json_if_present(peer_transcript_path)
|
|
961
|
+
sut_negotiation = _load_json_if_present(sut_negotiation_path)
|
|
962
|
+
peer_negotiation = _load_json_if_present(peer_negotiation_path)
|
|
963
|
+
if sut_transcript is None and sut_spec.role == 'server':
|
|
964
|
+
sut_transcript = _synthesize_sut_transcript(
|
|
965
|
+
scenario=scenario,
|
|
966
|
+
sut_spec=sut_spec,
|
|
967
|
+
sut_result=sut_result,
|
|
968
|
+
peer_transcript=peer_transcript,
|
|
969
|
+
)
|
|
970
|
+
_write_json(sut_transcript_path, sut_transcript)
|
|
971
|
+
if sut_negotiation is None and sut_spec.role == 'server':
|
|
972
|
+
sut_negotiation = _synthesize_sut_negotiation(
|
|
973
|
+
scenario=scenario,
|
|
974
|
+
sut_spec=sut_spec,
|
|
975
|
+
sut_result=sut_result,
|
|
976
|
+
peer_negotiation=peer_negotiation,
|
|
977
|
+
peer_transcript=peer_transcript,
|
|
978
|
+
source_root=self.source_root,
|
|
979
|
+
)
|
|
980
|
+
_write_json(sut_negotiation_path, sut_negotiation)
|
|
981
|
+
if transport == 'udp' and scenario.protocol in {'quic', 'quic-tls', 'http3'}:
|
|
982
|
+
generate_observer_qlog(
|
|
983
|
+
packet_trace_path=packet_trace_path,
|
|
984
|
+
qlog_path=qlog_path,
|
|
985
|
+
title=scenario.id,
|
|
986
|
+
protocol=scenario.protocol,
|
|
987
|
+
ip_family=scenario.ip_family,
|
|
988
|
+
negotiation=(sut_negotiation if isinstance(sut_negotiation, dict) else None) or (peer_negotiation if isinstance(peer_negotiation, dict) else None),
|
|
989
|
+
error=error,
|
|
990
|
+
)
|
|
991
|
+
artifacts = {
|
|
992
|
+
'packet_trace': _artifact_metadata(packet_trace_path),
|
|
993
|
+
'qlog': _artifact_metadata(qlog_path),
|
|
994
|
+
'sut_transcript': _artifact_metadata(sut_transcript_path),
|
|
995
|
+
'peer_transcript': _artifact_metadata(peer_transcript_path),
|
|
996
|
+
'sut_negotiation': _artifact_metadata(sut_negotiation_path),
|
|
997
|
+
'peer_negotiation': _artifact_metadata(peer_negotiation_path),
|
|
998
|
+
}
|
|
999
|
+
observed = {
|
|
1000
|
+
'scenario': {'id': scenario.id, **scenario.dimensions, 'metadata': scenario.metadata},
|
|
1001
|
+
'sut': sut_result.to_observed(),
|
|
1002
|
+
'peer': peer_result.to_observed(),
|
|
1003
|
+
'transcript': {'sut': sut_transcript, 'peer': peer_transcript},
|
|
1004
|
+
'negotiation': {'sut': sut_negotiation, 'peer': peer_negotiation},
|
|
1005
|
+
'artifacts': artifacts,
|
|
1006
|
+
}
|
|
1007
|
+
failed_assertions = evaluate_assertions(scenario.assertions, observed)
|
|
1008
|
+
passed = error is None and not failed_assertions
|
|
1009
|
+
result = InteropScenarioResult(
|
|
1010
|
+
scenario_id=scenario.id,
|
|
1011
|
+
passed=passed,
|
|
1012
|
+
commit_hash=self.commit_hash,
|
|
1013
|
+
artifact_dir=str(scenario_root),
|
|
1014
|
+
assertions_failed=failed_assertions,
|
|
1015
|
+
error=error,
|
|
1016
|
+
sut=sut_result.to_observed(),
|
|
1017
|
+
peer=peer_result.to_observed(),
|
|
1018
|
+
transcript={'sut': sut_transcript, 'peer': peer_transcript},
|
|
1019
|
+
negotiation={'sut': sut_negotiation, 'peer': peer_negotiation},
|
|
1020
|
+
artifacts=artifacts,
|
|
1021
|
+
)
|
|
1022
|
+
_write_json(
|
|
1023
|
+
scenario_root / 'result.json',
|
|
1024
|
+
{
|
|
1025
|
+
'scenario_id': result.scenario_id,
|
|
1026
|
+
'passed': result.passed,
|
|
1027
|
+
'commit_hash': result.commit_hash,
|
|
1028
|
+
'artifact_dir': result.artifact_dir,
|
|
1029
|
+
'assertions_failed': result.assertions_failed,
|
|
1030
|
+
'error': result.error,
|
|
1031
|
+
'sut': result.sut,
|
|
1032
|
+
'peer': result.peer,
|
|
1033
|
+
'transcript': result.transcript,
|
|
1034
|
+
'negotiation': result.negotiation,
|
|
1035
|
+
'artifacts': result.artifacts,
|
|
1036
|
+
},
|
|
1037
|
+
)
|
|
1038
|
+
_write_json(
|
|
1039
|
+
scenario_root / 'scenario.json',
|
|
1040
|
+
{
|
|
1041
|
+
'id': scenario.id,
|
|
1042
|
+
'dimensions': scenario.dimensions,
|
|
1043
|
+
'assertions': scenario.assertions,
|
|
1044
|
+
'capture': scenario.capture,
|
|
1045
|
+
'metadata': scenario.metadata,
|
|
1046
|
+
'sut': _spec_to_json(scenario.sut),
|
|
1047
|
+
'peer_process': _spec_to_json(scenario.peer_process),
|
|
1048
|
+
},
|
|
1049
|
+
)
|
|
1050
|
+
_write_json(
|
|
1051
|
+
scenario_root / 'command.json',
|
|
1052
|
+
{
|
|
1053
|
+
'scenario_id': scenario.id,
|
|
1054
|
+
'sut': {
|
|
1055
|
+
'adapter': sut_spec.adapter,
|
|
1056
|
+
'command': sut_spec.command,
|
|
1057
|
+
'version_command': sut_spec.version_command,
|
|
1058
|
+
'cwd': str(sut_cwd),
|
|
1059
|
+
},
|
|
1060
|
+
'peer': {
|
|
1061
|
+
'adapter': peer_spec.adapter,
|
|
1062
|
+
'command': peer_spec.command,
|
|
1063
|
+
'version_command': peer_spec.version_command,
|
|
1064
|
+
'cwd': str(peer_cwd),
|
|
1065
|
+
},
|
|
1066
|
+
},
|
|
1067
|
+
)
|
|
1068
|
+
_write_json(
|
|
1069
|
+
scenario_root / 'env.json',
|
|
1070
|
+
{
|
|
1071
|
+
'scenario_id': scenario.id,
|
|
1072
|
+
'shared_context': context,
|
|
1073
|
+
'sut': {
|
|
1074
|
+
'cwd': str(sut_cwd),
|
|
1075
|
+
'env': _snapshot_interop_env(sut_env, sut_spec),
|
|
1076
|
+
},
|
|
1077
|
+
'peer': {
|
|
1078
|
+
'cwd': str(peer_cwd),
|
|
1079
|
+
'env': _snapshot_interop_env(peer_env, peer_spec),
|
|
1080
|
+
},
|
|
1081
|
+
},
|
|
1082
|
+
)
|
|
1083
|
+
_write_json(
|
|
1084
|
+
scenario_root / 'versions.json',
|
|
1085
|
+
{
|
|
1086
|
+
'scenario_id': scenario.id,
|
|
1087
|
+
'sut': sut_version,
|
|
1088
|
+
'peer': peer_version,
|
|
1089
|
+
'sut_provenance': sut_result.provenance,
|
|
1090
|
+
'peer_provenance': peer_result.provenance,
|
|
1091
|
+
},
|
|
1092
|
+
)
|
|
1093
|
+
_write_json(
|
|
1094
|
+
scenario_root / 'wire_capture.json',
|
|
1095
|
+
{
|
|
1096
|
+
'scenario_id': scenario.id,
|
|
1097
|
+
'transport': transport,
|
|
1098
|
+
'capture': scenario.capture,
|
|
1099
|
+
'packet_trace': artifacts['packet_trace'],
|
|
1100
|
+
'qlog': artifacts['qlog'],
|
|
1101
|
+
'logs': {
|
|
1102
|
+
'sut_stdout': _artifact_metadata(sut_stdout_path),
|
|
1103
|
+
'sut_stderr': _artifact_metadata(sut_stderr_path),
|
|
1104
|
+
'peer_stdout': _artifact_metadata(peer_stdout_path),
|
|
1105
|
+
'peer_stderr': _artifact_metadata(peer_stderr_path),
|
|
1106
|
+
},
|
|
1107
|
+
'transcripts': {
|
|
1108
|
+
'sut_transcript': artifacts['sut_transcript'],
|
|
1109
|
+
'peer_transcript': artifacts['peer_transcript'],
|
|
1110
|
+
},
|
|
1111
|
+
'negotiation': {
|
|
1112
|
+
'sut_negotiation': artifacts['sut_negotiation'],
|
|
1113
|
+
'peer_negotiation': artifacts['peer_negotiation'],
|
|
1114
|
+
},
|
|
1115
|
+
},
|
|
1116
|
+
)
|
|
1117
|
+
_write_json(
|
|
1118
|
+
scenario_root / 'summary.json',
|
|
1119
|
+
{
|
|
1120
|
+
'schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
1121
|
+
'scenario_id': scenario.id,
|
|
1122
|
+
'protocol': scenario.protocol,
|
|
1123
|
+
'feature': scenario.feature,
|
|
1124
|
+
'peer': scenario.peer,
|
|
1125
|
+
'role': scenario.role,
|
|
1126
|
+
'evidence_tier': scenario.evidence_tier,
|
|
1127
|
+
'passed': result.passed,
|
|
1128
|
+
'error': result.error,
|
|
1129
|
+
'assertions_failed': result.assertions_failed,
|
|
1130
|
+
'required_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
1131
|
+
},
|
|
1132
|
+
)
|
|
1133
|
+
_write_json(
|
|
1134
|
+
scenario_root / 'index.json',
|
|
1135
|
+
{
|
|
1136
|
+
'schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
1137
|
+
'scenario_id': scenario.id,
|
|
1138
|
+
'artifact_dir': str(scenario_root),
|
|
1139
|
+
'passed': result.passed,
|
|
1140
|
+
'error': result.error,
|
|
1141
|
+
'required_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
1142
|
+
'artifact_files': {},
|
|
1143
|
+
'result_path': str(scenario_root / 'result.json'),
|
|
1144
|
+
'summary_path': str(scenario_root / 'summary.json'),
|
|
1145
|
+
},
|
|
1146
|
+
)
|
|
1147
|
+
artifact_inventory = {
|
|
1148
|
+
name: _artifact_metadata(scenario_root / name)
|
|
1149
|
+
for name in INTEROP_SCENARIO_REQUIRED_FILES
|
|
1150
|
+
}
|
|
1151
|
+
artifact_inventory.update(
|
|
1152
|
+
{
|
|
1153
|
+
'packet_trace.jsonl': _artifact_metadata(packet_trace_path),
|
|
1154
|
+
'qlog.json': _artifact_metadata(qlog_path),
|
|
1155
|
+
'sut_stdout.log': _artifact_metadata(sut_stdout_path),
|
|
1156
|
+
'sut_stderr.log': _artifact_metadata(sut_stderr_path),
|
|
1157
|
+
'peer_stdout.log': _artifact_metadata(peer_stdout_path),
|
|
1158
|
+
'peer_stderr.log': _artifact_metadata(peer_stderr_path),
|
|
1159
|
+
'sut_transcript.json': _artifact_metadata(sut_transcript_path),
|
|
1160
|
+
'peer_transcript.json': _artifact_metadata(peer_transcript_path),
|
|
1161
|
+
'sut_negotiation.json': _artifact_metadata(sut_negotiation_path),
|
|
1162
|
+
'peer_negotiation.json': _artifact_metadata(peer_negotiation_path),
|
|
1163
|
+
}
|
|
1164
|
+
)
|
|
1165
|
+
_write_json(
|
|
1166
|
+
scenario_root / 'summary.json',
|
|
1167
|
+
{
|
|
1168
|
+
'schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
1169
|
+
'scenario_id': scenario.id,
|
|
1170
|
+
'protocol': scenario.protocol,
|
|
1171
|
+
'feature': scenario.feature,
|
|
1172
|
+
'peer': scenario.peer,
|
|
1173
|
+
'role': scenario.role,
|
|
1174
|
+
'evidence_tier': scenario.evidence_tier,
|
|
1175
|
+
'passed': result.passed,
|
|
1176
|
+
'error': result.error,
|
|
1177
|
+
'assertions_failed': result.assertions_failed,
|
|
1178
|
+
'required_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
1179
|
+
'artifact_files': artifact_inventory,
|
|
1180
|
+
},
|
|
1181
|
+
)
|
|
1182
|
+
_write_json(
|
|
1183
|
+
scenario_root / 'index.json',
|
|
1184
|
+
{
|
|
1185
|
+
'schema_version': INTEROP_ARTIFACT_SCHEMA_VERSION,
|
|
1186
|
+
'scenario_id': scenario.id,
|
|
1187
|
+
'artifact_dir': str(scenario_root),
|
|
1188
|
+
'passed': result.passed,
|
|
1189
|
+
'error': result.error,
|
|
1190
|
+
'required_files': list(INTEROP_SCENARIO_REQUIRED_FILES),
|
|
1191
|
+
'artifact_files': artifact_inventory,
|
|
1192
|
+
'result_path': str(scenario_root / 'result.json'),
|
|
1193
|
+
'summary_path': str(scenario_root / 'summary.json'),
|
|
1194
|
+
},
|
|
1195
|
+
)
|
|
1196
|
+
return result
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
# ----- Public helpers ----------------------------------------------------
|
|
1200
|
+
|
|
1201
|
+
def load_external_matrix(path: str | Path) -> InteropMatrix:
|
|
1202
|
+
payload = json.loads(Path(path).read_text(encoding='utf-8'))
|
|
1203
|
+
matrix_payload = payload.get('matrix', payload)
|
|
1204
|
+
metadata = dict(matrix_payload.get('metadata', {}))
|
|
1205
|
+
default_evidence_tier = str(metadata.get('evidence_tier', 'mixed'))
|
|
1206
|
+
scenarios = [_load_scenario(entry, default_evidence_tier=default_evidence_tier) for entry in matrix_payload.get('scenarios', [])]
|
|
1207
|
+
return InteropMatrix(
|
|
1208
|
+
name=matrix_payload['name'],
|
|
1209
|
+
scenarios=scenarios,
|
|
1210
|
+
metadata=metadata,
|
|
1211
|
+
)
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
def summarize_matrix_dimensions(matrix: InteropMatrix) -> dict[str, list[Any]]:
|
|
1216
|
+
keys = [
|
|
1217
|
+
'protocol', 'role', 'feature', 'peer', 'cipher_group', 'ip_family', 'retry', 'resumption', 'zero_rtt', 'key_update', 'migration', 'goaway', 'qpack_blocking', 'evidence_tier'
|
|
1218
|
+
]
|
|
1219
|
+
dimensions: dict[str, set[Any]] = {key: set() for key in keys}
|
|
1220
|
+
for scenario in matrix.scenarios:
|
|
1221
|
+
for key, value in scenario.dimensions.items():
|
|
1222
|
+
dimensions[key].add(value)
|
|
1223
|
+
return {key: sorted(values) for key, values in dimensions.items()}
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
def detect_source_revision(source_root: str | Path) -> str:
|
|
1228
|
+
env_commit = os.environ.get('TIGRCORN_COMMIT_HASH') or os.environ.get('GIT_COMMIT')
|
|
1229
|
+
if env_commit:
|
|
1230
|
+
return env_commit
|
|
1231
|
+
try:
|
|
1232
|
+
completed = subprocess.run(
|
|
1233
|
+
['git', '-C', str(Path(source_root)), 'rev-parse', 'HEAD'],
|
|
1234
|
+
capture_output=True,
|
|
1235
|
+
text=True,
|
|
1236
|
+
timeout=5.0,
|
|
1237
|
+
)
|
|
1238
|
+
if completed.returncode == 0 and completed.stdout.strip():
|
|
1239
|
+
return completed.stdout.strip()
|
|
1240
|
+
except Exception:
|
|
1241
|
+
pass
|
|
1242
|
+
return f'tree-{hash_source_tree(source_root)[:16]}'
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
|
|
1246
|
+
def build_environment_manifest(source_root: str | Path, *, commit_hash: str | None = None) -> dict[str, Any]:
|
|
1247
|
+
source_root = Path(source_root)
|
|
1248
|
+
return {
|
|
1249
|
+
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
1250
|
+
'python': {
|
|
1251
|
+
'version': platform.python_version(),
|
|
1252
|
+
'implementation': platform.python_implementation(),
|
|
1253
|
+
'executable': os.sys.executable,
|
|
1254
|
+
},
|
|
1255
|
+
'platform': {
|
|
1256
|
+
'system': platform.system(),
|
|
1257
|
+
'release': platform.release(),
|
|
1258
|
+
'machine': platform.machine(),
|
|
1259
|
+
'platform': platform.platform(),
|
|
1260
|
+
},
|
|
1261
|
+
'tigrcorn': {
|
|
1262
|
+
'version': __version__,
|
|
1263
|
+
'commit_hash': commit_hash or detect_source_revision(source_root),
|
|
1264
|
+
'source_tree_sha256': hash_source_tree(source_root),
|
|
1265
|
+
},
|
|
1266
|
+
'tools': {
|
|
1267
|
+
'git': _probe_command(['git', '--version']),
|
|
1268
|
+
'docker': _probe_command(['docker', '--version']),
|
|
1269
|
+
'curl': _probe_command(['curl', '--version']),
|
|
1270
|
+
'openssl': _probe_command(['openssl', 'version']),
|
|
1271
|
+
},
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
def hash_source_tree(source_root: str | Path) -> str:
|
|
1277
|
+
source_root = Path(source_root)
|
|
1278
|
+
entries: list[tuple[str, str]] = []
|
|
1279
|
+
skipped_prefixes = (
|
|
1280
|
+
('docs', 'review', 'conformance', 'releases'),
|
|
1281
|
+
('.artifacts',),
|
|
1282
|
+
('.tmp',),
|
|
1283
|
+
('dist',),
|
|
1284
|
+
)
|
|
1285
|
+
for root, _dirs, filenames in os.walk(source_root):
|
|
1286
|
+
root_path = Path(root)
|
|
1287
|
+
if '.git' in root_path.parts or '__pycache__' in root_path.parts:
|
|
1288
|
+
continue
|
|
1289
|
+
relative_parts = root_path.relative_to(source_root).parts if root_path != source_root else ()
|
|
1290
|
+
if any(part.startswith('tmp') for part in relative_parts):
|
|
1291
|
+
continue
|
|
1292
|
+
if any(relative_parts[:len(prefix)] == prefix for prefix in skipped_prefixes):
|
|
1293
|
+
continue
|
|
1294
|
+
for filename in sorted(filenames):
|
|
1295
|
+
path = root_path / filename
|
|
1296
|
+
if path.suffix in {'.pyc', '.pyo'} or not path.is_file():
|
|
1297
|
+
continue
|
|
1298
|
+
entries.append((str(path.relative_to(source_root)), _sha256_path(path)))
|
|
1299
|
+
return _sha256_bytes(json.dumps(entries, separators=(',', ':')).encode('utf-8'))
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
|
|
1303
|
+
def evaluate_assertions(assertions: list[dict[str, Any]], observed: dict[str, Any]) -> list[str]:
|
|
1304
|
+
failures: list[str] = []
|
|
1305
|
+
for index, assertion in enumerate(assertions):
|
|
1306
|
+
path = assertion.get('path')
|
|
1307
|
+
if not isinstance(path, str):
|
|
1308
|
+
failures.append(f'assertion[{index}] missing path')
|
|
1309
|
+
continue
|
|
1310
|
+
try:
|
|
1311
|
+
actual = _resolve_path(observed, path)
|
|
1312
|
+
except KeyError:
|
|
1313
|
+
failures.append(f'assertion[{index}] path not found: {path}')
|
|
1314
|
+
continue
|
|
1315
|
+
if 'equals' in assertion and actual != assertion['equals']:
|
|
1316
|
+
failures.append(f'assertion[{index}] {path} expected {assertion["equals"]!r}, got {actual!r}')
|
|
1317
|
+
if 'not_equals' in assertion and actual == assertion['not_equals']:
|
|
1318
|
+
failures.append(f'assertion[{index}] {path} unexpectedly equals {assertion["not_equals"]!r}')
|
|
1319
|
+
if 'contains' in assertion:
|
|
1320
|
+
expected = assertion['contains']
|
|
1321
|
+
if isinstance(actual, (str, bytes)):
|
|
1322
|
+
if expected not in actual:
|
|
1323
|
+
failures.append(f'assertion[{index}] {path} does not contain {expected!r}')
|
|
1324
|
+
elif isinstance(actual, Mapping):
|
|
1325
|
+
if expected not in actual:
|
|
1326
|
+
failures.append(f'assertion[{index}] {path} missing key {expected!r}')
|
|
1327
|
+
elif isinstance(actual, Iterable):
|
|
1328
|
+
if expected not in actual:
|
|
1329
|
+
failures.append(f'assertion[{index}] {path} does not contain item {expected!r}')
|
|
1330
|
+
else:
|
|
1331
|
+
failures.append(f'assertion[{index}] {path} is not containable')
|
|
1332
|
+
if 'regex' in assertion and not re.search(str(assertion['regex']), str(actual)):
|
|
1333
|
+
failures.append(f'assertion[{index}] {path} does not match /{assertion["regex"]}/')
|
|
1334
|
+
if 'greater_or_equal' in assertion and actual < assertion['greater_or_equal']:
|
|
1335
|
+
failures.append(f'assertion[{index}] {path} expected >= {assertion["greater_or_equal"]!r}, got {actual!r}')
|
|
1336
|
+
if 'less_or_equal' in assertion and actual > assertion['less_or_equal']:
|
|
1337
|
+
failures.append(f'assertion[{index}] {path} expected <= {assertion["less_or_equal"]!r}, got {actual!r}')
|
|
1338
|
+
if 'in' in assertion and actual not in assertion['in']:
|
|
1339
|
+
failures.append(f'assertion[{index}] {path} expected one of {assertion["in"]!r}, got {actual!r}')
|
|
1340
|
+
return failures
|
|
1341
|
+
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def generate_observer_qlog(
|
|
1345
|
+
*,
|
|
1346
|
+
packet_trace_path: str | Path,
|
|
1347
|
+
qlog_path: str | Path,
|
|
1348
|
+
title: str,
|
|
1349
|
+
protocol: str,
|
|
1350
|
+
ip_family: str,
|
|
1351
|
+
negotiation: dict[str, Any] | None = None,
|
|
1352
|
+
error: str | None = None,
|
|
1353
|
+
) -> None:
|
|
1354
|
+
trace_path = Path(packet_trace_path)
|
|
1355
|
+
records: list[dict[str, Any]] = []
|
|
1356
|
+
if trace_path.exists():
|
|
1357
|
+
for line in trace_path.read_text(encoding='utf-8').splitlines():
|
|
1358
|
+
line = line.strip()
|
|
1359
|
+
if not line:
|
|
1360
|
+
continue
|
|
1361
|
+
records.append(json.loads(line))
|
|
1362
|
+
if not records:
|
|
1363
|
+
_write_json(
|
|
1364
|
+
Path(qlog_path),
|
|
1365
|
+
{
|
|
1366
|
+
'qlog_version': QLOG_VERSION,
|
|
1367
|
+
'schema_version': QLOG_EXPERIMENTAL_SCHEMA_VERSION,
|
|
1368
|
+
'traces': [],
|
|
1369
|
+
},
|
|
1370
|
+
)
|
|
1371
|
+
return
|
|
1372
|
+
base_time = float(records[0]['timestamp'])
|
|
1373
|
+
events: list[list[Any]] = [
|
|
1374
|
+
[
|
|
1375
|
+
0.0,
|
|
1376
|
+
'connectivity',
|
|
1377
|
+
'connection_started',
|
|
1378
|
+
{
|
|
1379
|
+
'ip_version': 'ipv6' if ip_family == 'ipv6' else 'ipv4',
|
|
1380
|
+
'protocol': protocol,
|
|
1381
|
+
'server': {'host': 'redacted', 'port': 'redacted'},
|
|
1382
|
+
},
|
|
1383
|
+
]
|
|
1384
|
+
]
|
|
1385
|
+
if negotiation:
|
|
1386
|
+
events.append([0.0, 'transport', 'parameters_set', dict(negotiation)])
|
|
1387
|
+
for record in records:
|
|
1388
|
+
payload = bytes.fromhex(record['payload_hex'])
|
|
1389
|
+
packets = [_describe_quic_packet(chunk) for chunk in _split_observed_packets(payload)]
|
|
1390
|
+
packets = [item for item in packets if item is not None]
|
|
1391
|
+
if not packets:
|
|
1392
|
+
packets = [{'packet_type': 'unknown', 'length': len(payload)}]
|
|
1393
|
+
else:
|
|
1394
|
+
packets = [_redact_qlog_packet(item) for item in packets]
|
|
1395
|
+
events.append([
|
|
1396
|
+
round((float(record['timestamp']) - base_time) * 1000.0, 3),
|
|
1397
|
+
'transport',
|
|
1398
|
+
'packet_received' if record['direction'] == 'client_to_server' else 'packet_sent',
|
|
1399
|
+
{
|
|
1400
|
+
'direction': record['direction'],
|
|
1401
|
+
'length': record['length'],
|
|
1402
|
+
'packets': packets,
|
|
1403
|
+
},
|
|
1404
|
+
])
|
|
1405
|
+
if error:
|
|
1406
|
+
events.append([round((float(records[-1]['timestamp']) - base_time) * 1000.0, 3), 'transport', 'connection_closed', {'error': error}])
|
|
1407
|
+
_write_json(
|
|
1408
|
+
Path(qlog_path),
|
|
1409
|
+
{
|
|
1410
|
+
'qlog_version': QLOG_VERSION,
|
|
1411
|
+
'schema_version': QLOG_EXPERIMENTAL_SCHEMA_VERSION,
|
|
1412
|
+
'traces': [
|
|
1413
|
+
{
|
|
1414
|
+
'vantage_point': {'type': 'network', 'name': 'tigrcorn-interop-runner'},
|
|
1415
|
+
'title': title,
|
|
1416
|
+
'common_fields': {
|
|
1417
|
+
'protocol_type': 'QUIC',
|
|
1418
|
+
'tigrcorn_qlog': {
|
|
1419
|
+
'experimental': True,
|
|
1420
|
+
'schema_version': QLOG_EXPERIMENTAL_SCHEMA_VERSION,
|
|
1421
|
+
'redaction': {
|
|
1422
|
+
'network_endpoints': 'redacted',
|
|
1423
|
+
'connection_ids': 'redacted',
|
|
1424
|
+
'payload_bytes': 'omitted',
|
|
1425
|
+
},
|
|
1426
|
+
},
|
|
1427
|
+
},
|
|
1428
|
+
'events': events,
|
|
1429
|
+
}
|
|
1430
|
+
],
|
|
1431
|
+
},
|
|
1432
|
+
)
|
|
1433
|
+
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def run_external_matrix(
|
|
1437
|
+
matrix_path: str | Path,
|
|
1438
|
+
*,
|
|
1439
|
+
artifact_root: str | Path,
|
|
1440
|
+
source_root: str | Path | None = None,
|
|
1441
|
+
scenario_ids: Iterable[str] | None = None,
|
|
1442
|
+
strict: bool = False,
|
|
1443
|
+
) -> InteropRunSummary:
|
|
1444
|
+
matrix = load_external_matrix(matrix_path)
|
|
1445
|
+
runner = ExternalInteropRunner(matrix=matrix, artifact_root=artifact_root, source_root=source_root)
|
|
1446
|
+
return runner.run(scenario_ids=scenario_ids, strict=strict)
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
# ----- Internal helpers --------------------------------------------------
|
|
1450
|
+
|
|
1451
|
+
def _load_scenario(entry: dict[str, Any], *, default_evidence_tier: str = 'mixed') -> InteropScenario:
|
|
1452
|
+
evidence_tier = str(entry.get('evidence_tier', default_evidence_tier))
|
|
1453
|
+
if evidence_tier not in VALID_EVIDENCE_TIERS:
|
|
1454
|
+
raise InteropRunnerError(f'invalid evidence_tier: {evidence_tier!r}')
|
|
1455
|
+
scenario = InteropScenario(
|
|
1456
|
+
id=entry['id'],
|
|
1457
|
+
protocol=entry['protocol'],
|
|
1458
|
+
role=entry['role'],
|
|
1459
|
+
feature=entry['feature'],
|
|
1460
|
+
peer=entry['peer'],
|
|
1461
|
+
sut=_load_process_spec(entry['sut']),
|
|
1462
|
+
peer_process=_load_process_spec(entry['peer_process']),
|
|
1463
|
+
assertions=[dict(item) for item in entry.get('assertions', [])],
|
|
1464
|
+
transport=entry.get('transport'),
|
|
1465
|
+
ip_family=entry.get('ip_family', 'ipv4'),
|
|
1466
|
+
cipher_group=entry.get('cipher_group'),
|
|
1467
|
+
retry=bool(entry.get('retry', False)),
|
|
1468
|
+
resumption=bool(entry.get('resumption', False)),
|
|
1469
|
+
zero_rtt=bool(entry.get('zero_rtt', False)),
|
|
1470
|
+
key_update=bool(entry.get('key_update', False)),
|
|
1471
|
+
migration=bool(entry.get('migration', False)),
|
|
1472
|
+
goaway=bool(entry.get('goaway', False)),
|
|
1473
|
+
qpack_blocking=bool(entry.get('qpack_blocking', False)),
|
|
1474
|
+
capture=dict(entry.get('capture', {})),
|
|
1475
|
+
metadata=dict(entry.get('metadata', {})),
|
|
1476
|
+
evidence_tier=evidence_tier,
|
|
1477
|
+
enabled=bool(entry.get('enabled', True)),
|
|
1478
|
+
)
|
|
1479
|
+
_validate_scenario_provenance(scenario)
|
|
1480
|
+
return scenario
|
|
1481
|
+
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
def _load_process_spec(entry: dict[str, Any]) -> InteropProcessSpec:
|
|
1485
|
+
command = entry.get('command')
|
|
1486
|
+
if not isinstance(command, list) or not all(isinstance(item, str) for item in command):
|
|
1487
|
+
raise InteropRunnerError('process command must be a list of strings')
|
|
1488
|
+
version_command = entry.get('version_command')
|
|
1489
|
+
if version_command is not None and (not isinstance(version_command, list) or not all(isinstance(item, str) for item in version_command)):
|
|
1490
|
+
raise InteropRunnerError('version_command must be a list of strings when provided')
|
|
1491
|
+
spec = InteropProcessSpec(
|
|
1492
|
+
name=entry['name'],
|
|
1493
|
+
adapter=entry.get('adapter', 'subprocess'),
|
|
1494
|
+
role=entry['role'],
|
|
1495
|
+
command=list(command),
|
|
1496
|
+
env={str(key): str(value) for key, value in dict(entry.get('env', {})).items()},
|
|
1497
|
+
cwd=entry.get('cwd'),
|
|
1498
|
+
ready_pattern=entry.get('ready_pattern'),
|
|
1499
|
+
ready_timeout=float(entry.get('ready_timeout', DEFAULT_READY_TIMEOUT)),
|
|
1500
|
+
run_timeout=float(entry.get('run_timeout', DEFAULT_RUN_TIMEOUT)),
|
|
1501
|
+
version_command=list(version_command) if version_command is not None else None,
|
|
1502
|
+
image=entry.get('image'),
|
|
1503
|
+
enabled=bool(entry.get('enabled', True)),
|
|
1504
|
+
metadata=dict(entry.get('metadata', {})),
|
|
1505
|
+
provenance_kind=str(entry.get('provenance_kind', 'unspecified')),
|
|
1506
|
+
implementation_source=entry.get('implementation_source'),
|
|
1507
|
+
implementation_identity=entry.get('implementation_identity'),
|
|
1508
|
+
implementation_version=entry.get('implementation_version'),
|
|
1509
|
+
)
|
|
1510
|
+
_validate_process_provenance(spec)
|
|
1511
|
+
return spec
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
|
|
1515
|
+
def _validate_process_provenance(spec: InteropProcessSpec) -> None:
|
|
1516
|
+
if spec.provenance_kind not in VALID_PROVENANCE_KINDS:
|
|
1517
|
+
raise InteropRunnerError(f'invalid provenance_kind for {spec.name}: {spec.provenance_kind!r}')
|
|
1518
|
+
if spec.provenance_kind != 'unspecified' and not spec.implementation_identity:
|
|
1519
|
+
raise InteropRunnerError(f'implementation_identity is required for {spec.name} when provenance_kind is {spec.provenance_kind!r}')
|
|
1520
|
+
if spec.provenance_kind in {'third_party_library', 'third_party_binary'} and not spec.implementation_source:
|
|
1521
|
+
raise InteropRunnerError(f'implementation_source is required for third-party peer {spec.name}')
|
|
1522
|
+
|
|
1523
|
+
|
|
1524
|
+
|
|
1525
|
+
def _validate_scenario_provenance(scenario: InteropScenario) -> None:
|
|
1526
|
+
if scenario.evidence_tier not in VALID_EVIDENCE_TIERS:
|
|
1527
|
+
raise InteropRunnerError(f'invalid evidence_tier for {scenario.id}: {scenario.evidence_tier!r}')
|
|
1528
|
+
if scenario.evidence_tier == 'independent_certification':
|
|
1529
|
+
peer_kind = scenario.peer_process.provenance_kind
|
|
1530
|
+
if peer_kind not in {'third_party_library', 'third_party_binary'}:
|
|
1531
|
+
raise InteropRunnerError(
|
|
1532
|
+
f'independent_certification scenario {scenario.id} requires a third-party peer, not {peer_kind!r}'
|
|
1533
|
+
)
|
|
1534
|
+
if not scenario.peer_process.implementation_identity or not scenario.peer_process.implementation_source:
|
|
1535
|
+
raise InteropRunnerError(
|
|
1536
|
+
f'independent_certification scenario {scenario.id} requires peer implementation_identity and implementation_source'
|
|
1537
|
+
)
|
|
1538
|
+
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def _build_provenance_payload(spec: InteropProcessSpec, version: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
|
1542
|
+
payload: dict[str, Any] = {
|
|
1543
|
+
'kind': spec.provenance_kind,
|
|
1544
|
+
'implementation_source': spec.implementation_source,
|
|
1545
|
+
'implementation_identity': spec.implementation_identity,
|
|
1546
|
+
'implementation_version': spec.implementation_version,
|
|
1547
|
+
}
|
|
1548
|
+
if version:
|
|
1549
|
+
observed = version.get('version_stdout') or version.get('stdout')
|
|
1550
|
+
if observed:
|
|
1551
|
+
payload['observed_version_output'] = observed
|
|
1552
|
+
return payload
|
|
1553
|
+
|
|
1554
|
+
|
|
1555
|
+
def _instantiate_adapter(name: str) -> BasePeerAdapter:
|
|
1556
|
+
try:
|
|
1557
|
+
return _ADAPTERS[name]()
|
|
1558
|
+
except KeyError as exc:
|
|
1559
|
+
raise InteropRunnerError(f'unknown interop adapter: {name}') from exc
|
|
1560
|
+
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
def _resolve_process_command(command: Sequence[str]) -> list[str]:
|
|
1564
|
+
resolved = list(command)
|
|
1565
|
+
if not resolved:
|
|
1566
|
+
return resolved
|
|
1567
|
+
executable = resolved[0]
|
|
1568
|
+
if executable == '/opt/pyvenv/bin/python':
|
|
1569
|
+
resolved[0] = os.environ.get('TIGRCORN_INTEROP_PYTHON', os.sys.executable)
|
|
1570
|
+
return resolved
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
def _materialize_process_spec(spec: InteropProcessSpec, context: Mapping[str, str]) -> InteropProcessSpec:
|
|
1575
|
+
return InteropProcessSpec(
|
|
1576
|
+
name=_apply_template(spec.name, context),
|
|
1577
|
+
adapter=spec.adapter,
|
|
1578
|
+
role=spec.role,
|
|
1579
|
+
command=_resolve_process_command([_apply_template(item, context) for item in spec.command]),
|
|
1580
|
+
env={key: _apply_template(value, context) for key, value in spec.env.items()},
|
|
1581
|
+
cwd=_apply_template(spec.cwd, context) if spec.cwd is not None else None,
|
|
1582
|
+
ready_pattern=_apply_template(spec.ready_pattern, context) if spec.ready_pattern is not None else None,
|
|
1583
|
+
ready_timeout=spec.ready_timeout,
|
|
1584
|
+
run_timeout=spec.run_timeout,
|
|
1585
|
+
version_command=_resolve_process_command([_apply_template(item, context) for item in spec.version_command]) if spec.version_command is not None else None,
|
|
1586
|
+
image=_apply_template(spec.image, context) if spec.image is not None else None,
|
|
1587
|
+
enabled=spec.enabled,
|
|
1588
|
+
metadata=dict(spec.metadata),
|
|
1589
|
+
provenance_kind=spec.provenance_kind,
|
|
1590
|
+
implementation_source=_apply_template(spec.implementation_source, context) if spec.implementation_source is not None else None,
|
|
1591
|
+
implementation_identity=_apply_template(spec.implementation_identity, context) if spec.implementation_identity is not None else None,
|
|
1592
|
+
implementation_version=_apply_template(spec.implementation_version, context) if spec.implementation_version is not None else None,
|
|
1593
|
+
)
|
|
1594
|
+
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
def _build_process_env(source_root: Path, spec: InteropProcessSpec, transcript_path: Path, negotiation_path: Path, context: Mapping[str, str]) -> dict[str, str]:
|
|
1598
|
+
env = dict(os.environ)
|
|
1599
|
+
env.update(spec.env)
|
|
1600
|
+
pythonpath_parts = [str(source_root / 'src'), str(source_root)]
|
|
1601
|
+
if env.get('PYTHONPATH'):
|
|
1602
|
+
pythonpath_parts.append(env['PYTHONPATH'])
|
|
1603
|
+
env['PYTHONPATH'] = os.pathsep.join(pythonpath_parts)
|
|
1604
|
+
env['PYTHONUNBUFFERED'] = '1'
|
|
1605
|
+
env['INTEROP_BIND_HOST'] = context['bind_host']
|
|
1606
|
+
env['INTEROP_BIND_PORT'] = context['bind_port']
|
|
1607
|
+
env['INTEROP_TARGET_HOST'] = context['target_host']
|
|
1608
|
+
env['INTEROP_TARGET_PORT'] = context['target_port']
|
|
1609
|
+
env['INTEROP_ARTIFACT_DIR'] = context['artifact_dir']
|
|
1610
|
+
env['INTEROP_PACKET_TRACE_PATH'] = context['packet_trace_path']
|
|
1611
|
+
env['INTEROP_QLOG_PATH'] = context['qlog_path']
|
|
1612
|
+
env['INTEROP_TRANSCRIPT_PATH'] = str(transcript_path)
|
|
1613
|
+
env['INTEROP_NEGOTIATION_PATH'] = str(negotiation_path)
|
|
1614
|
+
env['INTEROP_SCENARIO_ID'] = context['scenario_id']
|
|
1615
|
+
env['INTEROP_MATRIX_NAME'] = context['matrix_name']
|
|
1616
|
+
env['INTEROP_COMMIT_HASH'] = context['commit_hash']
|
|
1617
|
+
env['INTEROP_PROTOCOL'] = context['protocol']
|
|
1618
|
+
env['INTEROP_FEATURE'] = context['feature']
|
|
1619
|
+
env['INTEROP_ROLE'] = spec.role
|
|
1620
|
+
env['INTEROP_IP_FAMILY'] = context['ip_family']
|
|
1621
|
+
if context.get('retry'):
|
|
1622
|
+
env['INTEROP_ENABLE_RETRY'] = '1'
|
|
1623
|
+
if context.get('resumption'):
|
|
1624
|
+
env['INTEROP_ENABLE_RESUMPTION'] = '1'
|
|
1625
|
+
if context.get('zero_rtt'):
|
|
1626
|
+
env['INTEROP_ENABLE_ZERO_RTT'] = '1'
|
|
1627
|
+
if context.get('key_update'):
|
|
1628
|
+
env['INTEROP_ENABLE_KEY_UPDATE'] = '1'
|
|
1629
|
+
if context.get('migration'):
|
|
1630
|
+
env['INTEROP_ENABLE_MIGRATION'] = '1'
|
|
1631
|
+
if context.get('goaway'):
|
|
1632
|
+
env['INTEROP_ENABLE_GOAWAY'] = '1'
|
|
1633
|
+
if context.get('qpack_blocking'):
|
|
1634
|
+
env['INTEROP_ENABLE_QPACK_BLOCKING'] = '1'
|
|
1635
|
+
if context.get('cipher_group'):
|
|
1636
|
+
env['INTEROP_CIPHER_GROUP'] = context['cipher_group']
|
|
1637
|
+
return env
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
|
|
1641
|
+
def _snapshot_interop_env(env: Mapping[str, str], spec: InteropProcessSpec) -> dict[str, str]:
|
|
1642
|
+
explicit_keys = set(spec.env)
|
|
1643
|
+
return {
|
|
1644
|
+
key: str(value)
|
|
1645
|
+
for key, value in sorted(env.items())
|
|
1646
|
+
if key.startswith('INTEROP_') or key in explicit_keys
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def _wait_for_server_ready(*, spec: InteropProcessSpec, process: subprocess.Popen[Any], env: Mapping[str, str], stdout_path: Path, stderr_path: Path) -> str | None:
|
|
1651
|
+
bind_host = env.get('INTEROP_BIND_HOST')
|
|
1652
|
+
bind_port = int(env['INTEROP_BIND_PORT']) if env.get('INTEROP_BIND_PORT') and env['INTEROP_BIND_PORT'].isdigit() else None
|
|
1653
|
+
transport = 'udp' if env.get('INTEROP_PROTOCOL') in {'quic', 'quic-tls', 'http3'} else 'tcp'
|
|
1654
|
+
ready_regex = re.compile(spec.ready_pattern) if spec.ready_pattern is not None else None
|
|
1655
|
+
deadline = time.monotonic() + spec.ready_timeout
|
|
1656
|
+
while time.monotonic() < deadline:
|
|
1657
|
+
if process.poll() is not None:
|
|
1658
|
+
return f'{spec.name} exited before becoming ready'
|
|
1659
|
+
if ready_regex is not None:
|
|
1660
|
+
stdout_text = stdout_path.read_text(encoding='utf-8', errors='replace') if stdout_path.exists() else ''
|
|
1661
|
+
stderr_text = stderr_path.read_text(encoding='utf-8', errors='replace') if stderr_path.exists() else ''
|
|
1662
|
+
if ready_regex.search(stdout_text) or ready_regex.search(stderr_text):
|
|
1663
|
+
return None
|
|
1664
|
+
if bind_host is not None and bind_port is not None and _probe_server_port(bind_host, bind_port, transport):
|
|
1665
|
+
return None
|
|
1666
|
+
if transport == 'udp' and ready_regex is None and time.monotonic() + 0.0 >= deadline - spec.ready_timeout + 0.2:
|
|
1667
|
+
return None
|
|
1668
|
+
time.sleep(0.05)
|
|
1669
|
+
return f'{spec.name} did not become ready within {spec.ready_timeout:.3f}s'
|
|
1670
|
+
|
|
1671
|
+
|
|
1672
|
+
|
|
1673
|
+
def _probe_server_port(host: str, port: int, transport: str) -> bool:
|
|
1674
|
+
if transport != 'tcp':
|
|
1675
|
+
return False
|
|
1676
|
+
family = socket.AF_INET6 if ':' in host else socket.AF_INET
|
|
1677
|
+
try:
|
|
1678
|
+
with socket.socket(family, socket.SOCK_STREAM) as probe:
|
|
1679
|
+
probe.settimeout(0.1)
|
|
1680
|
+
probe.connect((host, port))
|
|
1681
|
+
return True
|
|
1682
|
+
except OSError:
|
|
1683
|
+
return False
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
|
|
1687
|
+
def _resolve_path(payload: Any, path: str) -> Any:
|
|
1688
|
+
current = payload
|
|
1689
|
+
for part in path.split('.'):
|
|
1690
|
+
if isinstance(current, Mapping):
|
|
1691
|
+
if part not in current:
|
|
1692
|
+
raise KeyError(path)
|
|
1693
|
+
current = current[part]
|
|
1694
|
+
elif isinstance(current, list) and part.isdigit():
|
|
1695
|
+
index = int(part)
|
|
1696
|
+
try:
|
|
1697
|
+
current = current[index]
|
|
1698
|
+
except IndexError as exc:
|
|
1699
|
+
raise KeyError(path) from exc
|
|
1700
|
+
else:
|
|
1701
|
+
raise KeyError(path)
|
|
1702
|
+
return current
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
|
|
1706
|
+
def _default_transport_for_protocol(protocol: str) -> str:
|
|
1707
|
+
return 'udp' if protocol in {'quic', 'quic-tls', 'http3'} else 'tcp'
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
|
|
1711
|
+
def _reserve_port(host: str, socktype: int) -> int:
|
|
1712
|
+
family = socket.AF_INET6 if ':' in host else socket.AF_INET
|
|
1713
|
+
with socket.socket(family, socktype) as sock:
|
|
1714
|
+
if family == socket.AF_INET6:
|
|
1715
|
+
sock.bind((host, 0, 0, 0))
|
|
1716
|
+
else:
|
|
1717
|
+
sock.bind((host, 0))
|
|
1718
|
+
return int(sock.getsockname()[1])
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
def _reserve_distinct_port(host: str, socktype: int, forbidden: set[int]) -> int:
|
|
1723
|
+
for _ in range(128):
|
|
1724
|
+
port = _reserve_port(host, socktype)
|
|
1725
|
+
if port not in forbidden:
|
|
1726
|
+
return port
|
|
1727
|
+
raise InteropRunnerError('unable to reserve a distinct port for the interop runner')
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
|
|
1731
|
+
def _normalize_sockaddr(addr: Any) -> tuple[str, int]:
|
|
1732
|
+
if isinstance(addr, tuple) and len(addr) >= 2:
|
|
1733
|
+
return str(addr[0]), int(addr[1])
|
|
1734
|
+
raise InteropRunnerError(f'unsupported socket address: {addr!r}')
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
|
|
1738
|
+
def _apply_template(value: str, context: Mapping[str, str]) -> str:
|
|
1739
|
+
try:
|
|
1740
|
+
return value.format_map(context)
|
|
1741
|
+
except KeyError:
|
|
1742
|
+
return value
|
|
1743
|
+
|
|
1744
|
+
|
|
1745
|
+
|
|
1746
|
+
def _artifact_metadata(path: Path) -> dict[str, Any]:
|
|
1747
|
+
return {
|
|
1748
|
+
'path': str(path),
|
|
1749
|
+
'exists': path.exists(),
|
|
1750
|
+
'size': path.stat().st_size if path.exists() else 0,
|
|
1751
|
+
'sha256': _sha256_path(path) if path.exists() else None,
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
|
|
1756
|
+
def _probe_command(command: list[str]) -> dict[str, Any]:
|
|
1757
|
+
executable = shutil.which(command[0])
|
|
1758
|
+
payload: dict[str, Any] = {'command': command, 'executable': executable, 'available': executable is not None}
|
|
1759
|
+
if executable is None:
|
|
1760
|
+
return payload
|
|
1761
|
+
try:
|
|
1762
|
+
completed = subprocess.run(command, capture_output=True, text=True, timeout=5.0)
|
|
1763
|
+
payload['exit_code'] = completed.returncode
|
|
1764
|
+
payload['stdout'] = completed.stdout.strip()
|
|
1765
|
+
payload['stderr'] = completed.stderr.strip()
|
|
1766
|
+
except Exception as exc:
|
|
1767
|
+
payload['error'] = str(exc)
|
|
1768
|
+
return payload
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
|
|
1772
|
+
def _load_json_if_present(path: Path) -> Any:
|
|
1773
|
+
if not path.exists():
|
|
1774
|
+
return None
|
|
1775
|
+
text = path.read_text(encoding='utf-8').strip()
|
|
1776
|
+
if not text:
|
|
1777
|
+
return None
|
|
1778
|
+
return json.loads(text)
|
|
1779
|
+
|
|
1780
|
+
|
|
1781
|
+
|
|
1782
|
+
def _extract_cli_option(command: Sequence[str], flag: str) -> str | None:
|
|
1783
|
+
for index, item in enumerate(command):
|
|
1784
|
+
if item == flag and index + 1 < len(command):
|
|
1785
|
+
return command[index + 1]
|
|
1786
|
+
return None
|
|
1787
|
+
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def _resolve_cli_path(value: str | None, source_root: Path) -> str | None:
|
|
1791
|
+
if value in (None, ''):
|
|
1792
|
+
return None
|
|
1793
|
+
root = source_root.resolve()
|
|
1794
|
+
path = Path(value)
|
|
1795
|
+
if not path.is_absolute():
|
|
1796
|
+
path = (root / path).resolve()
|
|
1797
|
+
if path.exists() and (path == root or root in path.parents):
|
|
1798
|
+
return str(path.relative_to(root))
|
|
1799
|
+
return str(path)
|
|
1800
|
+
|
|
1801
|
+
|
|
1802
|
+
|
|
1803
|
+
def _synthesize_sut_transcript(
|
|
1804
|
+
*,
|
|
1805
|
+
scenario: InteropScenario,
|
|
1806
|
+
sut_spec: InteropProcessSpec,
|
|
1807
|
+
sut_result: InteropProcessResult,
|
|
1808
|
+
peer_transcript: Any,
|
|
1809
|
+
) -> dict[str, Any]:
|
|
1810
|
+
peer_request = peer_transcript.get('request') if isinstance(peer_transcript, dict) else None
|
|
1811
|
+
peer_response = peer_transcript.get('response') if isinstance(peer_transcript, dict) else None
|
|
1812
|
+
return {
|
|
1813
|
+
'observation_model': 'interop_runner_synthesized_from_peer_observation',
|
|
1814
|
+
'scenario_id': scenario.id,
|
|
1815
|
+
'protocol': scenario.protocol,
|
|
1816
|
+
'feature': scenario.feature,
|
|
1817
|
+
'role': 'server',
|
|
1818
|
+
'request': peer_request,
|
|
1819
|
+
'response': peer_response,
|
|
1820
|
+
'server_process': {
|
|
1821
|
+
'name': sut_spec.name,
|
|
1822
|
+
'adapter': sut_spec.adapter,
|
|
1823
|
+
'role': sut_spec.role,
|
|
1824
|
+
'implementation_source': sut_result.provenance.get('implementation_source'),
|
|
1825
|
+
'implementation_identity': sut_result.provenance.get('implementation_identity'),
|
|
1826
|
+
'implementation_version': sut_result.provenance.get('implementation_version'),
|
|
1827
|
+
'exit_code': sut_result.exit_code,
|
|
1828
|
+
'stdout_path': sut_result.stdout_path,
|
|
1829
|
+
'stderr_path': sut_result.stderr_path,
|
|
1830
|
+
},
|
|
1831
|
+
'derived_from_peer_transcript': isinstance(peer_transcript, dict),
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
|
|
1836
|
+
def _synthesize_sut_negotiation(
|
|
1837
|
+
*,
|
|
1838
|
+
scenario: InteropScenario,
|
|
1839
|
+
sut_spec: InteropProcessSpec,
|
|
1840
|
+
sut_result: InteropProcessResult,
|
|
1841
|
+
peer_negotiation: Any,
|
|
1842
|
+
peer_transcript: Any,
|
|
1843
|
+
source_root: Path,
|
|
1844
|
+
) -> dict[str, Any]:
|
|
1845
|
+
peer_map = peer_negotiation if isinstance(peer_negotiation, dict) else {}
|
|
1846
|
+
peer_response = peer_transcript.get('response') if isinstance(peer_transcript, dict) else {}
|
|
1847
|
+
response_extension_header = peer_map.get('response_extension_header')
|
|
1848
|
+
if response_extension_header in (None, '') and isinstance(peer_response, dict):
|
|
1849
|
+
response_extension_header = peer_response.get('extension_header')
|
|
1850
|
+
negotiated_extensions = list(peer_map.get('negotiated_extensions') or [])
|
|
1851
|
+
if not negotiated_extensions and isinstance(response_extension_header, str) and response_extension_header.lower().startswith('permessage-deflate'):
|
|
1852
|
+
negotiated_extensions = ['PerMessageDeflate']
|
|
1853
|
+
ssl_certfile = _resolve_cli_path(_extract_cli_option(sut_spec.command, '--ssl-certfile'), source_root)
|
|
1854
|
+
ssl_keyfile = _resolve_cli_path(_extract_cli_option(sut_spec.command, '--ssl-keyfile'), source_root)
|
|
1855
|
+
return {
|
|
1856
|
+
'observation_model': 'interop_runner_synthesized_from_peer_observation',
|
|
1857
|
+
'scenario_id': scenario.id,
|
|
1858
|
+
'protocol': peer_map.get('protocol') or scenario.protocol,
|
|
1859
|
+
'feature': scenario.feature,
|
|
1860
|
+
'role': 'server',
|
|
1861
|
+
'implementation': sut_result.provenance.get('implementation_source') or sut_spec.implementation_source or sut_spec.name,
|
|
1862
|
+
'implementation_source': sut_result.provenance.get('implementation_source'),
|
|
1863
|
+
'implementation_identity': sut_result.provenance.get('implementation_identity'),
|
|
1864
|
+
'implementation_version': sut_result.provenance.get('implementation_version'),
|
|
1865
|
+
'handshake_complete': peer_map.get('handshake_complete'),
|
|
1866
|
+
'compression_requested': peer_map.get('compression_requested'),
|
|
1867
|
+
'response_extension_header': response_extension_header,
|
|
1868
|
+
'negotiated_extensions': negotiated_extensions,
|
|
1869
|
+
'connect_protocol_enabled': peer_map.get('connect_protocol_enabled'),
|
|
1870
|
+
'settings_enable_connect_protocol': peer_map.get('settings_enable_connect_protocol'),
|
|
1871
|
+
'certificate_inputs': {
|
|
1872
|
+
'server_certfile': {
|
|
1873
|
+
'path': ssl_certfile,
|
|
1874
|
+
'exists': bool(ssl_certfile),
|
|
1875
|
+
},
|
|
1876
|
+
'server_keyfile': {
|
|
1877
|
+
'path': ssl_keyfile,
|
|
1878
|
+
'exists': bool(ssl_keyfile),
|
|
1879
|
+
},
|
|
1880
|
+
},
|
|
1881
|
+
'derived_from_peer_negotiation': isinstance(peer_negotiation, dict),
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
|
|
1886
|
+
def _write_json(path: Path, payload: Any) -> None:
|
|
1887
|
+
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n', encoding='utf-8')
|
|
1888
|
+
|
|
1889
|
+
|
|
1890
|
+
|
|
1891
|
+
def _safe_name(value: str) -> str:
|
|
1892
|
+
return re.sub(r'[^A-Za-z0-9_.-]+', '-', value).strip('-') or 'scenario'
|
|
1893
|
+
|
|
1894
|
+
|
|
1895
|
+
|
|
1896
|
+
def _sha256_bytes(data: bytes) -> str:
|
|
1897
|
+
import hashlib
|
|
1898
|
+
|
|
1899
|
+
return hashlib.sha256(data).hexdigest()
|
|
1900
|
+
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
def _sha256_path(path: Path) -> str:
|
|
1904
|
+
import hashlib
|
|
1905
|
+
|
|
1906
|
+
digest = hashlib.sha256()
|
|
1907
|
+
with path.open('rb') as handle:
|
|
1908
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b''):
|
|
1909
|
+
digest.update(chunk)
|
|
1910
|
+
return digest.hexdigest()
|
|
1911
|
+
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def _split_observed_packets(payload: bytes) -> list[bytes]:
|
|
1915
|
+
try:
|
|
1916
|
+
return split_coalesced_packets(payload, destination_connection_id_length=8)
|
|
1917
|
+
except Exception:
|
|
1918
|
+
return [payload]
|
|
1919
|
+
|
|
1920
|
+
|
|
1921
|
+
|
|
1922
|
+
def _describe_quic_packet(payload: bytes) -> dict[str, Any] | None:
|
|
1923
|
+
try:
|
|
1924
|
+
packet = decode_packet(payload, destination_connection_id_length=8)
|
|
1925
|
+
except Exception:
|
|
1926
|
+
return None
|
|
1927
|
+
description: dict[str, Any] = {'length': len(payload)}
|
|
1928
|
+
if isinstance(packet, QuicLongHeaderPacket):
|
|
1929
|
+
description['packet_type'] = packet.packet_type.name.lower()
|
|
1930
|
+
description['version'] = packet.version
|
|
1931
|
+
description['dcid'] = packet.destination_connection_id.hex()
|
|
1932
|
+
description['scid'] = packet.source_connection_id.hex()
|
|
1933
|
+
description['packet_number'] = int.from_bytes(packet.packet_number, 'big')
|
|
1934
|
+
elif isinstance(packet, QuicRetryPacket):
|
|
1935
|
+
description['packet_type'] = 'retry'
|
|
1936
|
+
description['version'] = packet.version
|
|
1937
|
+
description['dcid'] = packet.destination_connection_id.hex()
|
|
1938
|
+
description['scid'] = packet.source_connection_id.hex()
|
|
1939
|
+
elif isinstance(packet, QuicVersionNegotiationPacket):
|
|
1940
|
+
description['packet_type'] = 'version_negotiation'
|
|
1941
|
+
description['versions'] = list(packet.supported_versions)
|
|
1942
|
+
description['dcid'] = packet.destination_connection_id.hex()
|
|
1943
|
+
description['scid'] = packet.source_connection_id.hex()
|
|
1944
|
+
elif isinstance(packet, QuicShortHeaderPacket):
|
|
1945
|
+
description['packet_type'] = '1rtt'
|
|
1946
|
+
description['dcid'] = packet.destination_connection_id.hex()
|
|
1947
|
+
description['packet_number'] = int.from_bytes(packet.packet_number, 'big')
|
|
1948
|
+
description['key_phase'] = packet.key_phase
|
|
1949
|
+
else:
|
|
1950
|
+
return None
|
|
1951
|
+
return description
|
|
1952
|
+
|
|
1953
|
+
|
|
1954
|
+
def _redact_qlog_packet(payload: dict[str, Any]) -> dict[str, Any]:
|
|
1955
|
+
redacted = dict(payload)
|
|
1956
|
+
for key in ('dcid', 'scid'):
|
|
1957
|
+
if key in redacted:
|
|
1958
|
+
redacted[key] = 'redacted'
|
|
1959
|
+
return redacted
|
|
1960
|
+
|
|
1961
|
+
|
|
1962
|
+
|
|
1963
|
+
def _matrix_to_json(matrix: InteropMatrix) -> dict[str, Any]:
|
|
1964
|
+
return {
|
|
1965
|
+
'name': matrix.name,
|
|
1966
|
+
'metadata': matrix.metadata,
|
|
1967
|
+
'scenarios': [
|
|
1968
|
+
{
|
|
1969
|
+
'id': scenario.id,
|
|
1970
|
+
'protocol': scenario.protocol,
|
|
1971
|
+
'role': scenario.role,
|
|
1972
|
+
'feature': scenario.feature,
|
|
1973
|
+
'peer': scenario.peer,
|
|
1974
|
+
'transport': scenario.transport,
|
|
1975
|
+
'ip_family': scenario.ip_family,
|
|
1976
|
+
'cipher_group': scenario.cipher_group,
|
|
1977
|
+
'retry': scenario.retry,
|
|
1978
|
+
'resumption': scenario.resumption,
|
|
1979
|
+
'zero_rtt': scenario.zero_rtt,
|
|
1980
|
+
'key_update': scenario.key_update,
|
|
1981
|
+
'migration': scenario.migration,
|
|
1982
|
+
'goaway': scenario.goaway,
|
|
1983
|
+
'qpack_blocking': scenario.qpack_blocking,
|
|
1984
|
+
'capture': scenario.capture,
|
|
1985
|
+
'metadata': scenario.metadata,
|
|
1986
|
+
'evidence_tier': scenario.evidence_tier,
|
|
1987
|
+
'assertions': scenario.assertions,
|
|
1988
|
+
'sut': _spec_to_json(scenario.sut),
|
|
1989
|
+
'peer_process': _spec_to_json(scenario.peer_process),
|
|
1990
|
+
'enabled': scenario.enabled,
|
|
1991
|
+
}
|
|
1992
|
+
for scenario in matrix.scenarios
|
|
1993
|
+
],
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
|
|
1997
|
+
|
|
1998
|
+
def _spec_to_json(spec: InteropProcessSpec) -> dict[str, Any]:
|
|
1999
|
+
return {
|
|
2000
|
+
'name': spec.name,
|
|
2001
|
+
'adapter': spec.adapter,
|
|
2002
|
+
'role': spec.role,
|
|
2003
|
+
'command': spec.command,
|
|
2004
|
+
'env': spec.env,
|
|
2005
|
+
'cwd': spec.cwd,
|
|
2006
|
+
'ready_pattern': spec.ready_pattern,
|
|
2007
|
+
'ready_timeout': spec.ready_timeout,
|
|
2008
|
+
'run_timeout': spec.run_timeout,
|
|
2009
|
+
'version_command': spec.version_command,
|
|
2010
|
+
'image': spec.image,
|
|
2011
|
+
'enabled': spec.enabled,
|
|
2012
|
+
'metadata': spec.metadata,
|
|
2013
|
+
'provenance_kind': spec.provenance_kind,
|
|
2014
|
+
'implementation_source': spec.implementation_source,
|
|
2015
|
+
'implementation_identity': spec.implementation_identity,
|
|
2016
|
+
'implementation_version': spec.implementation_version,
|
|
2017
|
+
}
|