wiretruth 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wiretruth/__init__.py +3 -0
- wiretruth/__main__.py +4 -0
- wiretruth/checks.py +504 -0
- wiretruth/cli.py +417 -0
- wiretruth/envs.py +120 -0
- wiretruth/formats.py +1128 -0
- wiretruth/grading.py +164 -0
- wiretruth/lint.py +275 -0
- wiretruth/providers/__init__.py +133 -0
- wiretruth/providers/openai_chat.py +363 -0
- wiretruth/recorder.py +481 -0
- wiretruth/replay.py +391 -0
- wiretruth/results.py +216 -0
- wiretruth/runner.py +353 -0
- wiretruth/schemas/bug-classes.schema.json +29 -0
- wiretruth/schemas/call.schema.json +113 -0
- wiretruth/schemas/common.schema.json +98 -0
- wiretruth/schemas/fixture.schema.json +243 -0
- wiretruth/schemas/observation.schema.json +191 -0
- wiretruth/schemas/results.schema.json +91 -0
- wiretruth/schemas/scenario.schema.json +91 -0
- wiretruth/schemas/shim-manifest.schema.json +125 -0
- wiretruth/schemas/upstream.schema.json +60 -0
- wiretruth/secret_scan.py +190 -0
- wiretruth-0.1.0.dist-info/METADATA +158 -0
- wiretruth-0.1.0.dist-info/RECORD +30 -0
- wiretruth-0.1.0.dist-info/WHEEL +5 -0
- wiretruth-0.1.0.dist-info/entry_points.txt +2 -0
- wiretruth-0.1.0.dist-info/licenses/LICENSE +21 -0
- wiretruth-0.1.0.dist-info/top_level.txt +1 -0
wiretruth/cli.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
"""Command-line entry point.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 ok, 1 check failure (invalid suite data, grade mismatch), 2 usage or setup error.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import contextlib
|
|
8
|
+
import os
|
|
9
|
+
import signal
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from wiretruth import __version__
|
|
16
|
+
from wiretruth.envs import (
|
|
17
|
+
RunError,
|
|
18
|
+
find_library,
|
|
19
|
+
installed_packages,
|
|
20
|
+
list_envs,
|
|
21
|
+
prune_envs,
|
|
22
|
+
resolve_interpreter,
|
|
23
|
+
)
|
|
24
|
+
from wiretruth.formats import FormatError, load_fixture, load_shim_manifest
|
|
25
|
+
from wiretruth.lint import SetupError, lint
|
|
26
|
+
from wiretruth.providers import PROVIDERS
|
|
27
|
+
from wiretruth.recorder import (
|
|
28
|
+
RecordError,
|
|
29
|
+
api_key,
|
|
30
|
+
describe_request,
|
|
31
|
+
destination,
|
|
32
|
+
find_scenarios,
|
|
33
|
+
record_scenario,
|
|
34
|
+
summary,
|
|
35
|
+
)
|
|
36
|
+
from wiretruth.replay import HOST, ReplayServer
|
|
37
|
+
from wiretruth.results import (
|
|
38
|
+
collect_results,
|
|
39
|
+
load_results_grades,
|
|
40
|
+
package_changes,
|
|
41
|
+
results_path,
|
|
42
|
+
summary_lines,
|
|
43
|
+
write_results,
|
|
44
|
+
)
|
|
45
|
+
from wiretruth.runner import DEFAULT_JOBS, compare_grades, load_expected_grades, run_suite
|
|
46
|
+
|
|
47
|
+
EXIT_CHECK = 1
|
|
48
|
+
EXIT_USAGE = 2
|
|
49
|
+
|
|
50
|
+
COMMANDS = {
|
|
51
|
+
"run": "run shims against replayed fixtures and grade the results",
|
|
52
|
+
"serve": "serve one fixture on 127.0.0.1 for manual reproduction",
|
|
53
|
+
"record": "record provider responses into fixtures (needs API keys)",
|
|
54
|
+
"lint": "validate suite data and results",
|
|
55
|
+
"envs": "list, build or prune per-library environments",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
60
|
+
parser = argparse.ArgumentParser(
|
|
61
|
+
prog="wiretruth",
|
|
62
|
+
description="Replay real provider responses through LLM SDKs and grade what they report.",
|
|
63
|
+
)
|
|
64
|
+
parser.add_argument("--version", action="version", version=f"wiretruth {__version__}")
|
|
65
|
+
subparsers = parser.add_subparsers(dest="command", metavar="COMMAND")
|
|
66
|
+
for name, help_text in COMMANDS.items():
|
|
67
|
+
subparser = subparsers.add_parser(name, help=help_text, description=help_text)
|
|
68
|
+
if name == "lint":
|
|
69
|
+
subparser.add_argument(
|
|
70
|
+
"path",
|
|
71
|
+
nargs="?",
|
|
72
|
+
default=".",
|
|
73
|
+
help="repository root that holds suite/ and results/ (default: current directory)",
|
|
74
|
+
)
|
|
75
|
+
elif name == "run":
|
|
76
|
+
shim = subparser.add_mutually_exclusive_group(required=True)
|
|
77
|
+
shim.add_argument("--shim", type=Path, help="a shim directory with wiretruth-shim.json")
|
|
78
|
+
shim.add_argument("--lib", metavar="NAME", help="a library shim under shims/*/NAME/")
|
|
79
|
+
subparser.add_argument(
|
|
80
|
+
"--scenario",
|
|
81
|
+
default="*",
|
|
82
|
+
help="a scenario id, or a glob such as 'truncation.*' (default: every scenario; "
|
|
83
|
+
"read from suite/ in the current directory)",
|
|
84
|
+
)
|
|
85
|
+
subparser.add_argument(
|
|
86
|
+
"--provider", choices=sorted(PROVIDERS), help="only this wire API (default: all)"
|
|
87
|
+
)
|
|
88
|
+
subparser.add_argument(
|
|
89
|
+
"--jobs",
|
|
90
|
+
type=int,
|
|
91
|
+
default=DEFAULT_JOBS,
|
|
92
|
+
help=f"cells to run at once (default: {DEFAULT_JOBS})",
|
|
93
|
+
)
|
|
94
|
+
results = subparser.add_mutually_exclusive_group()
|
|
95
|
+
results.add_argument(
|
|
96
|
+
"--write-results",
|
|
97
|
+
action="store_true",
|
|
98
|
+
help="write results/NAME/VERSION.json for a library shim (runs every cell)",
|
|
99
|
+
)
|
|
100
|
+
results.add_argument(
|
|
101
|
+
"--check-against",
|
|
102
|
+
type=Path,
|
|
103
|
+
nargs="?",
|
|
104
|
+
const=True,
|
|
105
|
+
metavar="FILE",
|
|
106
|
+
help="exit 1 if a grade or requirement status differs from this results file "
|
|
107
|
+
"(default: results/NAME/VERSION.json; runs every cell)",
|
|
108
|
+
)
|
|
109
|
+
results.add_argument(
|
|
110
|
+
"--check-grades",
|
|
111
|
+
type=Path,
|
|
112
|
+
metavar="FILE",
|
|
113
|
+
help="exit 1 if a grade or requirement status differs from this file "
|
|
114
|
+
"(runs every cell)",
|
|
115
|
+
)
|
|
116
|
+
elif name == "envs":
|
|
117
|
+
action = subparser.add_mutually_exclusive_group()
|
|
118
|
+
action.add_argument("--list", action="store_true", help="list environments (default)")
|
|
119
|
+
action.add_argument("--prune", action="store_true", help="remove stale environments")
|
|
120
|
+
action.add_argument(
|
|
121
|
+
"--build", metavar="NAME", help="build or reuse a library environment"
|
|
122
|
+
)
|
|
123
|
+
elif name == "serve":
|
|
124
|
+
subparser.add_argument(
|
|
125
|
+
"fixture",
|
|
126
|
+
help="a fixture file, or a fixture id such as openai-chat/truncation.text "
|
|
127
|
+
"(read from suite/fixtures/ in the current directory)",
|
|
128
|
+
)
|
|
129
|
+
subparser.add_argument(
|
|
130
|
+
"--port", type=int, default=0, help=f"port on {HOST} (default: a free one)"
|
|
131
|
+
)
|
|
132
|
+
elif name == "record":
|
|
133
|
+
subparser.add_argument(
|
|
134
|
+
"--provider", required=True, choices=sorted(PROVIDERS), help="the wire API"
|
|
135
|
+
)
|
|
136
|
+
subparser.add_argument(
|
|
137
|
+
"--scenario",
|
|
138
|
+
required=True,
|
|
139
|
+
help="a scenario id, or a glob such as 'truncation.*' "
|
|
140
|
+
"(read from suite/scenarios/ in the current directory)",
|
|
141
|
+
)
|
|
142
|
+
subparser.add_argument(
|
|
143
|
+
"--model", help="the model to request (default: the provider's current one)"
|
|
144
|
+
)
|
|
145
|
+
subparser.add_argument(
|
|
146
|
+
"--dry-run",
|
|
147
|
+
action="store_true",
|
|
148
|
+
help="print each request, with the key masked, and send nothing",
|
|
149
|
+
)
|
|
150
|
+
subparser.add_argument(
|
|
151
|
+
"--replace",
|
|
152
|
+
action="store_true",
|
|
153
|
+
help="overwrite an existing fixture instead of keeping it as a dated variant",
|
|
154
|
+
)
|
|
155
|
+
subparser.add_argument(
|
|
156
|
+
"--keep-bytes",
|
|
157
|
+
action="store_true",
|
|
158
|
+
help="store the body as the byte chunks read, instead of events or JSON",
|
|
159
|
+
)
|
|
160
|
+
return parser
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _plural(count: int, noun: str) -> str:
|
|
164
|
+
return f"{count} {noun}{'' if count == 1 else 's'}"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def run_lint(root: Path) -> int:
|
|
168
|
+
try:
|
|
169
|
+
report = lint(root)
|
|
170
|
+
except SetupError as error:
|
|
171
|
+
print(f"wiretruth lint: {error}", file=sys.stderr)
|
|
172
|
+
return EXIT_USAGE
|
|
173
|
+
for note in report.notes:
|
|
174
|
+
print(f"wiretruth lint: {note}", file=sys.stderr)
|
|
175
|
+
for problem in report.problems:
|
|
176
|
+
print(problem)
|
|
177
|
+
if report.problems:
|
|
178
|
+
failing = len({problem.file for problem in report.problems})
|
|
179
|
+
print(
|
|
180
|
+
f"wiretruth lint: {_plural(len(report.problems), 'error')} in "
|
|
181
|
+
f"{failing} of {_plural(report.files, 'file')}"
|
|
182
|
+
)
|
|
183
|
+
return EXIT_CHECK
|
|
184
|
+
print(f"wiretruth lint: {_plural(report.files, 'file')}, no errors")
|
|
185
|
+
return 0
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def run_serve(target: str, port: int) -> int:
|
|
189
|
+
path = Path(target)
|
|
190
|
+
if not path.is_file():
|
|
191
|
+
path = Path("suite", "fixtures", f"{target}.json")
|
|
192
|
+
if not path.is_file():
|
|
193
|
+
print(f"wiretruth serve: {target}: no such file, and no fixture {path}", file=sys.stderr)
|
|
194
|
+
return EXIT_USAGE
|
|
195
|
+
try:
|
|
196
|
+
fixture = load_fixture(path)
|
|
197
|
+
except FormatError as error:
|
|
198
|
+
print(error, file=sys.stderr)
|
|
199
|
+
return EXIT_CHECK
|
|
200
|
+
server = ReplayServer(fixture, port=port)
|
|
201
|
+
# Python leaves SIGINT ignored when it starts with it ignored (a background job, some wrappers).
|
|
202
|
+
previous = signal.signal(signal.SIGINT, signal.default_int_handler)
|
|
203
|
+
try:
|
|
204
|
+
# A second Ctrl-C while the server drains still ends in the summary below.
|
|
205
|
+
with contextlib.suppress(KeyboardInterrupt), contextlib.ExitStack() as stack:
|
|
206
|
+
try:
|
|
207
|
+
stack.enter_context(server)
|
|
208
|
+
except (OSError, OverflowError) as error:
|
|
209
|
+
print(f"wiretruth serve: can't listen on {HOST}:{port}: {error}", file=sys.stderr)
|
|
210
|
+
return EXIT_USAGE
|
|
211
|
+
print(f"wiretruth serve: {fixture.id} on {server.url}")
|
|
212
|
+
print(f"{fixture.provider} base URL: {server.base_urls()[fixture.provider]}")
|
|
213
|
+
print("Press Ctrl-C to stop.", flush=True)
|
|
214
|
+
while True:
|
|
215
|
+
time.sleep(60) # interruptible on every platform, unlike a bare lock wait
|
|
216
|
+
finally:
|
|
217
|
+
signal.signal(signal.SIGINT, signal.SIG_DFL if previous is None else previous)
|
|
218
|
+
log = server.request_log
|
|
219
|
+
print(f"wiretruth serve: stopped after {_plural(len(log), 'request')}")
|
|
220
|
+
count = len(fixture.exchanges)
|
|
221
|
+
for request in log:
|
|
222
|
+
if request.exchange is None:
|
|
223
|
+
answer = "no matching exchange (404)"
|
|
224
|
+
else:
|
|
225
|
+
sent = "sent in full" if request.response_complete else "not sent in full"
|
|
226
|
+
answer = f"exchange {request.exchange + 1} of {count}, {sent}"
|
|
227
|
+
print(f" {request.method} {request.path}: {answer}")
|
|
228
|
+
return 0
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def run_record(args: argparse.Namespace) -> int:
|
|
232
|
+
provider = PROVIDERS[args.provider]
|
|
233
|
+
model = args.model or provider.DEFAULT_MODEL
|
|
234
|
+
root = Path()
|
|
235
|
+
# Everything that can fail before a request is checked first, for every scenario, so a
|
|
236
|
+
# mistake in the last one doesn't come after paying for the others.
|
|
237
|
+
try:
|
|
238
|
+
scenarios, skipped = find_scenarios(root, args.scenario, provider.ID)
|
|
239
|
+
for scenario_id in skipped:
|
|
240
|
+
print(f"wiretruth record: skipped {scenario_id}: it doesn't list {provider.ID}")
|
|
241
|
+
requests = [provider.build_request(s.call_for(provider.ID), model) for s in scenarios]
|
|
242
|
+
if args.dry_run:
|
|
243
|
+
for scenario, request in zip(scenarios, requests, strict=True):
|
|
244
|
+
print(f"# {scenario.id}")
|
|
245
|
+
print(describe_request(provider, request, provider.API_BASE))
|
|
246
|
+
return 0
|
|
247
|
+
key = api_key(provider, os.environ)
|
|
248
|
+
for scenario in scenarios:
|
|
249
|
+
destination(root, provider.ID, scenario.id, replace=args.replace)
|
|
250
|
+
except FormatError as error:
|
|
251
|
+
print(error, file=sys.stderr)
|
|
252
|
+
return EXIT_CHECK
|
|
253
|
+
except (RecordError, ValueError) as error:
|
|
254
|
+
print(f"wiretruth record: {error}", file=sys.stderr)
|
|
255
|
+
return EXIT_USAGE
|
|
256
|
+
# A scenario that fails doesn't stop the others: each recording that works is kept.
|
|
257
|
+
code = 0
|
|
258
|
+
for scenario in scenarios:
|
|
259
|
+
try:
|
|
260
|
+
recording = record_scenario(
|
|
261
|
+
provider,
|
|
262
|
+
scenario,
|
|
263
|
+
model=model,
|
|
264
|
+
key=key,
|
|
265
|
+
root=root,
|
|
266
|
+
base_url=provider.API_BASE,
|
|
267
|
+
env=os.environ,
|
|
268
|
+
replace=args.replace,
|
|
269
|
+
keep_bytes=args.keep_bytes,
|
|
270
|
+
)
|
|
271
|
+
except FormatError as error:
|
|
272
|
+
print(error, file=sys.stderr, flush=True)
|
|
273
|
+
code = EXIT_CHECK
|
|
274
|
+
continue
|
|
275
|
+
except (RecordError, ValueError, OSError) as error:
|
|
276
|
+
print(f"wiretruth record: {scenario.id}: {error}", file=sys.stderr, flush=True)
|
|
277
|
+
code = code or EXIT_USAGE
|
|
278
|
+
continue
|
|
279
|
+
print(f"wrote {summary(recording)}", flush=True)
|
|
280
|
+
for warning in recording.warnings:
|
|
281
|
+
print(f" warning: {warning}", flush=True)
|
|
282
|
+
return code
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def run_shim(args: argparse.Namespace) -> int:
|
|
286
|
+
used = {
|
|
287
|
+
"--write-results": args.write_results,
|
|
288
|
+
"--check-against": args.check_against is not None,
|
|
289
|
+
"--check-grades": args.check_grades is not None,
|
|
290
|
+
}
|
|
291
|
+
flag = next((name for name, on in used.items() if on), None)
|
|
292
|
+
if flag is not None and (args.scenario != "*" or args.provider is not None):
|
|
293
|
+
# The file lists every cell, so a filtered run would report the rest as missing.
|
|
294
|
+
print(
|
|
295
|
+
f"wiretruth run: {flag} runs every cell; drop --scenario and --provider",
|
|
296
|
+
file=sys.stderr,
|
|
297
|
+
)
|
|
298
|
+
return EXIT_USAGE
|
|
299
|
+
if args.write_results and args.lib is None:
|
|
300
|
+
print(
|
|
301
|
+
"wiretruth run: --write-results needs --lib; results files are for library shims",
|
|
302
|
+
file=sys.stderr,
|
|
303
|
+
)
|
|
304
|
+
return EXIT_USAGE
|
|
305
|
+
saved = None
|
|
306
|
+
try:
|
|
307
|
+
expected = None if args.check_grades is None else load_expected_grades(args.check_grades)
|
|
308
|
+
shim_dir = find_library(Path(), args.lib) if args.lib is not None else args.shim
|
|
309
|
+
manifest = load_shim_manifest(shim_dir / "wiretruth-shim.json")
|
|
310
|
+
version = manifest.packages[manifest.primary_package]
|
|
311
|
+
if args.check_against is not None:
|
|
312
|
+
path = (
|
|
313
|
+
results_path(Path(), manifest.name, version)
|
|
314
|
+
if args.check_against is True
|
|
315
|
+
else args.check_against
|
|
316
|
+
)
|
|
317
|
+
if not path.is_file():
|
|
318
|
+
raise RunError(f"{path}: no such results file; write it with --write-results")
|
|
319
|
+
try:
|
|
320
|
+
saved = load_results_grades(path)
|
|
321
|
+
except FormatError as error:
|
|
322
|
+
print(error, file=sys.stderr)
|
|
323
|
+
return EXIT_CHECK
|
|
324
|
+
if (saved.name, saved.version) != (manifest.name, version):
|
|
325
|
+
raise RunError(
|
|
326
|
+
f"{path}: results are for {saved.name} {saved.version}; "
|
|
327
|
+
f"expected {manifest.name} {version}"
|
|
328
|
+
)
|
|
329
|
+
expected = saved.grades
|
|
330
|
+
report = run_suite(
|
|
331
|
+
Path(),
|
|
332
|
+
shim_dir,
|
|
333
|
+
pattern=args.scenario,
|
|
334
|
+
provider=args.provider,
|
|
335
|
+
jobs=args.jobs,
|
|
336
|
+
)
|
|
337
|
+
except (ValueError, RecordError, OSError) as error:
|
|
338
|
+
print(f"wiretruth run: {error}", file=sys.stderr)
|
|
339
|
+
return EXIT_USAGE
|
|
340
|
+
for line in summary_lines(report, manifest):
|
|
341
|
+
print(line)
|
|
342
|
+
if args.write_results:
|
|
343
|
+
harness = sum(cell.grade.grade == "HARNESS" for cell in report.cells)
|
|
344
|
+
if harness:
|
|
345
|
+
print(
|
|
346
|
+
f"wiretruth run: --write-results: {_plural(harness, 'HARNESS cell')}; "
|
|
347
|
+
"fix the shim or the harness first",
|
|
348
|
+
file=sys.stderr,
|
|
349
|
+
)
|
|
350
|
+
return EXIT_CHECK
|
|
351
|
+
try:
|
|
352
|
+
path = write_results(Path(), collect_results(Path(), report, manifest))
|
|
353
|
+
except FormatError as error:
|
|
354
|
+
print(error, file=sys.stderr)
|
|
355
|
+
return EXIT_CHECK
|
|
356
|
+
except (RunError, OSError) as error:
|
|
357
|
+
print(f"wiretruth run: {error}", file=sys.stderr)
|
|
358
|
+
return EXIT_USAGE
|
|
359
|
+
print(f"wrote {path}")
|
|
360
|
+
if saved is not None:
|
|
361
|
+
try:
|
|
362
|
+
interpreter = resolve_interpreter(Path(), manifest)
|
|
363
|
+
if interpreter is not None:
|
|
364
|
+
changes = package_changes(saved.packages, installed_packages(interpreter))
|
|
365
|
+
if changes:
|
|
366
|
+
print(f"packages differ from {path}: {', '.join(changes)}")
|
|
367
|
+
except (RunError, OSError) as error:
|
|
368
|
+
print(f"wiretruth run: {error}", file=sys.stderr)
|
|
369
|
+
return EXIT_USAGE
|
|
370
|
+
if expected is not None:
|
|
371
|
+
differences = compare_grades(report.cells, expected)
|
|
372
|
+
for difference in differences:
|
|
373
|
+
print(difference)
|
|
374
|
+
if differences:
|
|
375
|
+
return EXIT_CHECK
|
|
376
|
+
return 0
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def run_envs(args: argparse.Namespace) -> int:
|
|
380
|
+
try:
|
|
381
|
+
if args.build is not None:
|
|
382
|
+
shim_dir = find_library(Path(), args.build)
|
|
383
|
+
manifest = load_shim_manifest(shim_dir / "wiretruth-shim.json")
|
|
384
|
+
interpreter = resolve_interpreter(Path(), manifest)
|
|
385
|
+
if interpreter is None:
|
|
386
|
+
raise RunError("only Python library environments can be built")
|
|
387
|
+
print(interpreter)
|
|
388
|
+
elif args.prune:
|
|
389
|
+
for path in prune_envs(Path()):
|
|
390
|
+
print(f"removed {path}")
|
|
391
|
+
else:
|
|
392
|
+
for path, current in list_envs(Path()):
|
|
393
|
+
print(f"{'current' if current else 'stale'} {path}")
|
|
394
|
+
except (ValueError, OSError) as error:
|
|
395
|
+
print(f"wiretruth envs: {error}", file=sys.stderr)
|
|
396
|
+
return EXIT_USAGE
|
|
397
|
+
return 0
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
401
|
+
parser = build_parser()
|
|
402
|
+
args = parser.parse_args(argv)
|
|
403
|
+
if args.command is None:
|
|
404
|
+
parser.print_help(sys.stderr)
|
|
405
|
+
return EXIT_USAGE
|
|
406
|
+
if args.command == "lint":
|
|
407
|
+
return run_lint(Path(args.path))
|
|
408
|
+
if args.command == "serve":
|
|
409
|
+
return run_serve(args.fixture, args.port)
|
|
410
|
+
if args.command == "record":
|
|
411
|
+
return run_record(args)
|
|
412
|
+
if args.command == "run":
|
|
413
|
+
return run_shim(args)
|
|
414
|
+
if args.command == "envs":
|
|
415
|
+
return run_envs(args)
|
|
416
|
+
print(f"wiretruth {args.command}: not implemented yet", file=sys.stderr)
|
|
417
|
+
return EXIT_USAGE
|
wiretruth/envs.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Cached Python environments for pinned library shims."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from wiretruth.formats import ShimManifest, load_shim_manifest
|
|
11
|
+
|
|
12
|
+
MARKER = "wiretruth-install.json"
|
|
13
|
+
PYTHON = "3.12"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RunError(ValueError):
|
|
17
|
+
"""The run cannot start because its inputs or environment are unusable."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def find_library(root: Path, name: str) -> Path:
|
|
21
|
+
matches = [
|
|
22
|
+
path.parent
|
|
23
|
+
for path in (root / "shims").glob("*/*/wiretruth-shim.json")
|
|
24
|
+
if path.parent.name == name
|
|
25
|
+
]
|
|
26
|
+
if not matches:
|
|
27
|
+
raise RunError(f"no library shim named {name!r} under {root / 'shims'}")
|
|
28
|
+
if len(matches) != 1:
|
|
29
|
+
raise RunError(f"multiple library shims named {name!r} under {root / 'shims'}")
|
|
30
|
+
return matches[0]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _env_dir(root: Path, manifest: ShimManifest) -> Path:
|
|
34
|
+
version = manifest.packages[manifest.primary_package]
|
|
35
|
+
return root.resolve() / ".wiretruth" / "envs" / "py" / f"{manifest.name}-{version}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _python(path: Path) -> Path:
|
|
39
|
+
return path / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _pins(manifest: ShimManifest) -> list[str]:
|
|
43
|
+
return [f"{name}=={version}" for name, version in sorted(manifest.packages.items())]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _marker(manifest: ShimManifest) -> bytes:
|
|
47
|
+
return json.dumps({"python": PYTHON, "packages": _pins(manifest)}).encode()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _matches(path: Path, manifest: ShimManifest) -> bool:
|
|
51
|
+
marker = path / MARKER
|
|
52
|
+
return _python(path).is_file() and marker.is_file() and marker.read_bytes() == _marker(manifest)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def resolve_interpreter(root: Path, manifest: ShimManifest) -> str | None:
|
|
56
|
+
if manifest.language != "python":
|
|
57
|
+
return None
|
|
58
|
+
if manifest.primary_package == "wiretruth":
|
|
59
|
+
return sys.executable
|
|
60
|
+
path = _env_dir(root, manifest)
|
|
61
|
+
interpreter = str(_python(path))
|
|
62
|
+
if _matches(path, manifest):
|
|
63
|
+
return interpreter
|
|
64
|
+
if shutil.which("uv") is None:
|
|
65
|
+
raise RunError("uv is not on PATH; install uv to build library environments")
|
|
66
|
+
if path.exists():
|
|
67
|
+
shutil.rmtree(path)
|
|
68
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
for command in (
|
|
70
|
+
["uv", "venv", "--python", PYTHON, str(path)],
|
|
71
|
+
# "--": a package name from the manifest can never become a uv option.
|
|
72
|
+
["uv", "pip", "install", "--python", interpreter, "--", *_pins(manifest)],
|
|
73
|
+
):
|
|
74
|
+
result = subprocess.run(command, stdout=sys.stderr, check=False)
|
|
75
|
+
if result.returncode:
|
|
76
|
+
raise RunError(f"{' '.join(command)} exited with code {result.returncode}")
|
|
77
|
+
(path / MARKER).write_bytes(_marker(manifest))
|
|
78
|
+
return interpreter
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def parse_installed_packages(output: str) -> dict[str, str]:
|
|
82
|
+
packages = {}
|
|
83
|
+
for line in output.splitlines():
|
|
84
|
+
name, separator, version = line.partition("==")
|
|
85
|
+
if not separator or not name.strip() or not version.strip():
|
|
86
|
+
raise RunError("uv pip freeze: expected name==version on every line")
|
|
87
|
+
packages[name.strip()] = version.strip()
|
|
88
|
+
return packages
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def installed_packages(interpreter: str) -> dict[str, str]:
|
|
92
|
+
command = ["uv", "pip", "freeze", "--python", interpreter]
|
|
93
|
+
try:
|
|
94
|
+
result = subprocess.run(command, capture_output=True, text=True, check=False)
|
|
95
|
+
except OSError as error:
|
|
96
|
+
raise RunError(f"uv pip freeze: {error}") from error
|
|
97
|
+
if result.returncode:
|
|
98
|
+
raise RunError(f"{' '.join(command)} exited with code {result.returncode}")
|
|
99
|
+
return parse_installed_packages(result.stdout)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def list_envs(root: Path) -> tuple[tuple[Path, bool], ...]:
|
|
103
|
+
"""List cached directories and whether their installed pins match a current manifest."""
|
|
104
|
+
manifests = [load_shim_manifest(path) for path in (root / "shims").rglob("wiretruth-shim.json")]
|
|
105
|
+
current = {
|
|
106
|
+
_env_dir(root, manifest)
|
|
107
|
+
for manifest in manifests
|
|
108
|
+
if manifest.language == "python"
|
|
109
|
+
and manifest.primary_package != "wiretruth"
|
|
110
|
+
and _matches(_env_dir(root, manifest), manifest)
|
|
111
|
+
}
|
|
112
|
+
parent = root.resolve() / ".wiretruth" / "envs" / "py"
|
|
113
|
+
return tuple((path, path in current) for path in sorted(parent.glob("*")) if path.is_dir())
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def prune_envs(root: Path) -> tuple[Path, ...]:
|
|
117
|
+
stale = tuple(path for path, current in list_envs(root) if not current)
|
|
118
|
+
for path in stale:
|
|
119
|
+
shutil.rmtree(path)
|
|
120
|
+
return stale
|