testuary 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testuary/__init__.py +10 -0
- testuary/__main__.py +1075 -0
- testuary/_chat_log.py +256 -0
- testuary/_chat_models.py +234 -0
- testuary/_extensions.py +63 -0
- testuary/_model_access.py +129 -0
- testuary/_phase_keys.py +13 -0
- testuary/_progress.py +614 -0
- testuary/_templates/__init__.py +7 -0
- testuary/_templates/examples/README.txt +26 -0
- testuary/_templates/examples/controls.ini +8 -0
- testuary/_templates/examples/hints.json +27 -0
- testuary/_templates/examples/language-profile.json +17 -0
- testuary/_templates/examples/models.ini +12 -0
- testuary/_templates/examples/testuary_example_chat.py +15 -0
- testuary/_templates/hints.json +31 -0
- testuary/_templates/models.ini +93 -0
- testuary/_test_output.py +740 -0
- testuary/_words.py +202 -0
- testuary/agent/__init__.py +10 -0
- testuary/agent/_base.py +2270 -0
- testuary/agent/_context.py +417 -0
- testuary/agent/_edit_tool.py +733 -0
- testuary/agent/_helpers.py +1231 -0
- testuary/agent/_langgraph.py +1847 -0
- testuary/agent/_limits.py +214 -0
- testuary/agent/coverage_agent.py +1896 -0
- testuary/agent/mutation_agent.py +2134 -0
- testuary/config.ini +747 -0
- testuary/config.py +980 -0
- testuary/consolidation.py +4737 -0
- testuary/demo/__init__.py +61 -0
- testuary/demo/backend.py +493 -0
- testuary/demo/cli.py +1226 -0
- testuary/demo/data/LICENSE-tenacity.txt +212 -0
- testuary/demo/data/NOTICE.md +23 -0
- testuary/demo/data/java/LICENSE-jsonassert.txt +213 -0
- testuary/demo/data/java/NOTICE.md +25 -0
- testuary/demo/data/java/manifest.json +206 -0
- testuary/demo/data/java/models.ini +4 -0
- testuary/demo/data/java/preset.ini +22 -0
- testuary/demo/data/java/project.json +6 -0
- testuary/demo/data/java/scenario.json.gz +0 -0
- testuary/demo/data/manifest.json +215 -0
- testuary/demo/data/models.ini +4 -0
- testuary/demo/data/preset.ini +23 -0
- testuary/demo/data/project.json +7 -0
- testuary/demo/data/scenario.json.gz +0 -0
- testuary/demo/data/toy/NOTICE.md +18 -0
- testuary/demo/data/toy/manifest.json +52 -0
- testuary/demo/data/toy/models.ini +4 -0
- testuary/demo/data/toy/preset.ini +19 -0
- testuary/demo/data/toy/project.json +6 -0
- testuary/demo/data/toy/scenario.json.gz +0 -0
- testuary/demo/data/tutorial/LICENSE-tenacity.txt +212 -0
- testuary/demo/data/tutorial/NOTICE.md +27 -0
- testuary/demo/data/tutorial/hints.json +7 -0
- testuary/demo/data/tutorial/manifest.json +222 -0
- testuary/demo/data/tutorial/models.ini +4 -0
- testuary/demo/data/tutorial/preset.ini +24 -0
- testuary/demo/data/tutorial/project.json +7 -0
- testuary/demo/data/tutorial/prompt_profile.json +17 -0
- testuary/demo/data/tutorial/scenario.json.gz +0 -0
- testuary/demo/pacing.py +251 -0
- testuary/demo/recorded.py +1015 -0
- testuary/demo/scenario.py +500 -0
- testuary/demo/setup_demo.py +356 -0
- testuary/evidence.py +927 -0
- testuary/exits.py +107 -0
- testuary/generation_evidence.py +669 -0
- testuary/hints.py +311 -0
- testuary/langadapter_profile.json +86 -0
- testuary/languages/README.md +238 -0
- testuary/languages/__init__.py +580 -0
- testuary/languages/generic.py +1830 -0
- testuary/languages/mutation.py +1315 -0
- testuary/languages/preparation.py +372 -0
- testuary/loadable.py +164 -0
- testuary/measurement.py +1789 -0
- testuary/model_requests.py +803 -0
- testuary/models.py +1034 -0
- testuary/normal_run.py +92 -0
- testuary/orchestrator.py +2699 -0
- testuary/phase_state.py +1290 -0
- testuary/presentation/__init__.py +8 -0
- testuary/presentation/event_file.py +98 -0
- testuary/presentation/events.py +286 -0
- testuary/presentation/measurement.py +173 -0
- testuary/presentation/readiness.py +462 -0
- testuary/presentation/terminal.py +582 -0
- testuary/presentation/view.py +873 -0
- testuary/project.py +826 -0
- testuary/prompts/__init__.py +798 -0
- testuary/prompts/test_coverage.json +747 -0
- testuary/prompts/test_initial.json +24 -0
- testuary/prompts/test_mutation.json +777 -0
- testuary/py.typed +0 -0
- testuary/readiness.py +1047 -0
- testuary/reduction.py +919 -0
- testuary/results.py +1247 -0
- testuary/settings.py +1807 -0
- testuary/setup.py +1698 -0
- testuary/starter_salvage.py +621 -0
- testuary/starter_selection.py +243 -0
- testuary/starters.py +2986 -0
- testuary/target_phases.py +3261 -0
- testuary/tutorial/__init__.py +480 -0
- testuary/web/__init__.py +471 -0
- testuary/web/__main__.py +6 -0
- testuary/web/_data.py +925 -0
- testuary/web/_jobs.py +979 -0
- testuary/web/_launch.py +86 -0
- testuary/web/_registry.py +344 -0
- testuary/web/_results.py +1195 -0
- testuary/web/_server.py +1281 -0
- testuary/web/extensions.py +174 -0
- testuary/web/static/app.js +3545 -0
- testuary/web/static/icon.svg +1 -0
- testuary/web/static/index.html +34 -0
- testuary/web/static/style.css +498 -0
- testuary-0.2.0.data/data/share/doc/testuary/CHANGELOG.md +524 -0
- testuary-0.2.0.data/data/share/doc/testuary/LICENSE +21 -0
- testuary-0.2.0.data/data/share/doc/testuary/README.md +147 -0
- testuary-0.2.0.data/data/share/doc/testuary/THIRD_PARTY_NOTICES.md +51 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/contributing.md +65 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/explanation/code-organization.md +194 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/explanation/measurement-and-recovery.md +245 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/extending/hints.md +100 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/extending/index.md +39 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/extending/languages.md +326 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/extending/model-provider.md +290 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/extending/prompt-profile.md +138 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/extending/web.md +811 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/getting-started.md +390 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/cluster.md +317 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/configuration.md +377 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/demonstrations.md +193 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/machine.md +96 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/output.md +194 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/terminal.md +369 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/guide/web.md +502 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/index.md +125 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/models-and-prompts.md +301 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/reference/api.md +241 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/reference/cli.md +337 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/reference/index.md +9 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/reference/result-files.md +644 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/reference/settings.md +783 -0
- testuary-0.2.0.data/data/share/doc/testuary/docs/reference/tested-environments.md +108 -0
- testuary-0.2.0.data/data/share/doc/testuary/models.example.ini +8 -0
- testuary-0.2.0.dist-info/METADATA +217 -0
- testuary-0.2.0.dist-info/RECORD +159 -0
- testuary-0.2.0.dist-info/WHEEL +4 -0
- testuary-0.2.0.dist-info/entry_points.txt +3 -0
- testuary-0.2.0.dist-info/licenses/LICENSE +21 -0
- testuary-0.2.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +51 -0
- testuary-0.2.0.dist-info/licenses/src/testuary/demo/data/LICENSE-tenacity.txt +212 -0
- testuary-0.2.0.dist-info/licenses/src/testuary/demo/data/java/LICENSE-jsonassert.txt +213 -0
- testuary-0.2.0.dist-info/licenses/src/testuary/demo/data/tutorial/LICENSE-tenacity.txt +212 -0
testuary/__main__.py
ADDED
|
@@ -0,0 +1,1075 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import importlib
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from testuary import project as project_module
|
|
9
|
+
from testuary import results
|
|
10
|
+
from testuary import exits
|
|
11
|
+
from testuary import languages
|
|
12
|
+
from testuary import orchestrator
|
|
13
|
+
from testuary.orchestrator import log_print
|
|
14
|
+
from testuary.settings import _CHOICES as _SETTING_CHOICES
|
|
15
|
+
from testuary.settings import _OptionParser
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# The command line: argument parsing and the subcommands. The per-target
|
|
19
|
+
# flow lives in orchestrator.
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _provision_parser() -> argparse.ArgumentParser:
|
|
23
|
+
return argparse.ArgumentParser(
|
|
24
|
+
allow_abbrev=False,
|
|
25
|
+
prog="testuary provision",
|
|
26
|
+
description="Install the runtime the settings of the project in this directory ask for "
|
|
27
|
+
"(such as a Python version with its requirements, made with uv in "
|
|
28
|
+
"Testuary's cache), so that a run need not do it. May download.",
|
|
29
|
+
epilog="Exit status: 0 when the runtime the settings ask for is ready, or when they "
|
|
30
|
+
"ask for none, 1 when the settings cannot be read or the runtime cannot be "
|
|
31
|
+
"installed, 2 when an option is refused.")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cmd_provision(argv) -> int:
|
|
35
|
+
"""Prepare and print the runtime each installed language's settings
|
|
36
|
+
request for the project in this directory, as its setup context describes
|
|
37
|
+
it (such as a Python version to provision). This command allows
|
|
38
|
+
environment preparation to finish before generation begins."""
|
|
39
|
+
import sys as _sys
|
|
40
|
+
|
|
41
|
+
_provision_parser().parse_args(argv)
|
|
42
|
+
from langadapter import names
|
|
43
|
+
|
|
44
|
+
from testuary import config
|
|
45
|
+
from testuary.settings import resolve_settings
|
|
46
|
+
|
|
47
|
+
prepared = []
|
|
48
|
+
try:
|
|
49
|
+
settings = resolve_settings(load_extensions=True)
|
|
50
|
+
with config.use_run_settings(settings):
|
|
51
|
+
for language in names():
|
|
52
|
+
described = languages.setup_context(language, settings).interpreter
|
|
53
|
+
request = getattr(described, "provisioning", None)
|
|
54
|
+
if request is not None:
|
|
55
|
+
prepared.append((language, project_module.provision_runtime(language, request)))
|
|
56
|
+
except (ValueError, RuntimeError) as exc:
|
|
57
|
+
print(exc, file=_sys.stderr)
|
|
58
|
+
return 1
|
|
59
|
+
for language, runtime in prepared:
|
|
60
|
+
print(f"{language}: {runtime} (ready)")
|
|
61
|
+
if not prepared:
|
|
62
|
+
print("No runtime to install: the project's settings ask for none.")
|
|
63
|
+
return 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _init_parser() -> argparse.ArgumentParser:
|
|
67
|
+
parser = argparse.ArgumentParser(
|
|
68
|
+
allow_abbrev=False,
|
|
69
|
+
prog="testuary init", description="Copy example files: models.ini to your user directory and "
|
|
70
|
+
"hints.json to the project. Existing files are kept.",
|
|
71
|
+
epilog="Exit status: 0 when the files are copied or exist already, 1 when a packaged "
|
|
72
|
+
"example is missing or a copy fails, 2 when an option is refused.")
|
|
73
|
+
parser.add_argument(
|
|
74
|
+
"--examples", action="store_true",
|
|
75
|
+
help="Copy the extension examples (a client factory, a language-specific prompt "
|
|
76
|
+
"profile and per-language hints) into ./testuary-examples instead.")
|
|
77
|
+
return parser
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _cmd_init(argv=()) -> int:
|
|
81
|
+
"""``testuary init``: copy the bundled starters from
|
|
82
|
+
``testuary/_templates/``: ``models.ini`` into the user directory and
|
|
83
|
+
``hints.json`` into the project directory, found from the current directory
|
|
84
|
+
upward as a run finds it, where a run's default hints file is read. With
|
|
85
|
+
``--examples`` it copies the extension examples instead.
|
|
86
|
+
|
|
87
|
+
Existing files are never overwritten (the user's models.ini may hold
|
|
88
|
+
real secrets). Skipped files are reported.
|
|
89
|
+
|
|
90
|
+
Returns the exit code: 0 on success (or 'all already exist'),
|
|
91
|
+
1 when a bundled template is missing or a copy fails.
|
|
92
|
+
"""
|
|
93
|
+
import sys as _sys
|
|
94
|
+
from testuary.settings import file_locations, project_locations
|
|
95
|
+
from testuary.settings import _bundled_template_path
|
|
96
|
+
|
|
97
|
+
if _init_parser().parse_args(argv).examples:
|
|
98
|
+
return _copy_examples(Path.cwd() / "testuary-examples")
|
|
99
|
+
models = file_locations().models
|
|
100
|
+
copies = [
|
|
101
|
+
("models.ini", models),
|
|
102
|
+
("hints.json", project_locations(Path.cwd()).project / "hints.json"),
|
|
103
|
+
]
|
|
104
|
+
rc = 0
|
|
105
|
+
for bundled_name, target in copies:
|
|
106
|
+
if target.exists():
|
|
107
|
+
print(f" skip: {target.name} already exists in {target.parent}")
|
|
108
|
+
continue
|
|
109
|
+
src = _bundled_template_path(bundled_name)
|
|
110
|
+
if src is None:
|
|
111
|
+
print(
|
|
112
|
+
f" error: bundled template {bundled_name} not found in the "
|
|
113
|
+
"installed testuary package",
|
|
114
|
+
file=_sys.stderr,
|
|
115
|
+
)
|
|
116
|
+
rc = 1
|
|
117
|
+
continue
|
|
118
|
+
try:
|
|
119
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
120
|
+
_write_new(target, src.read_bytes(), private=bundled_name == "models.ini")
|
|
121
|
+
print(f" wrote: {target}")
|
|
122
|
+
except FileExistsError:
|
|
123
|
+
print(f" skip: {target.name} already exists in {target.parent}")
|
|
124
|
+
except OSError as exc:
|
|
125
|
+
print(f" error copying {bundled_name} to {target}: {exc}", file=_sys.stderr)
|
|
126
|
+
rc = 1
|
|
127
|
+
if rc == 0:
|
|
128
|
+
print(
|
|
129
|
+
f"\nAdd a section for your model to {models}. Its name is the alias for "
|
|
130
|
+
"--model. Then run testuary check --model ALIAS in your project directory."
|
|
131
|
+
)
|
|
132
|
+
return rc
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _write_new(target: Path, data: bytes, *, private: bool) -> None:
|
|
136
|
+
"""Create ``target`` with ``data``, never over an existing file, even one
|
|
137
|
+
made since it was looked for (FileExistsError). A private file, the
|
|
138
|
+
model catalogue into which keys are typed, is readable by its owner
|
|
139
|
+
only on POSIX, from its creation on."""
|
|
140
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
|
|
141
|
+
descriptor = os.open(target, flags, 0o600 if private else 0o666)
|
|
142
|
+
with os.fdopen(descriptor, "wb") as stream:
|
|
143
|
+
stream.write(data)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _copy_examples(target_root: Path) -> int:
|
|
147
|
+
"""Copy the bundled extension examples below ``target_root``. An existing
|
|
148
|
+
file is left unchanged. Returns 0, or 1 when a file could not be written."""
|
|
149
|
+
import sys as _sys
|
|
150
|
+
from testuary.settings import _bundled_template_path
|
|
151
|
+
|
|
152
|
+
examples = _bundled_template_path("examples")
|
|
153
|
+
if examples is None:
|
|
154
|
+
print(" error: the bundled examples were not found in the installed testuary package",
|
|
155
|
+
file=_sys.stderr)
|
|
156
|
+
return 1
|
|
157
|
+
rc = 0
|
|
158
|
+
for source in sorted(path for path in examples.rglob("*") if path.is_file()):
|
|
159
|
+
relative = source.relative_to(examples)
|
|
160
|
+
if "__pycache__" in relative.parts:
|
|
161
|
+
continue
|
|
162
|
+
target = target_root / relative
|
|
163
|
+
if target.exists():
|
|
164
|
+
print(f" skip: {target} already exists")
|
|
165
|
+
continue
|
|
166
|
+
try:
|
|
167
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
168
|
+
shutil.copy2(source, target)
|
|
169
|
+
print(f" wrote: {target}")
|
|
170
|
+
except OSError as exc:
|
|
171
|
+
print(f" error copying {relative} to {target}: {exc}", file=_sys.stderr)
|
|
172
|
+
rc = 1
|
|
173
|
+
if rc == 0:
|
|
174
|
+
print(f"\nSee {target_root / 'README.txt'} to check a project with the examples.")
|
|
175
|
+
return rc
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _check_run_options(args) -> Optional[set]:
|
|
179
|
+
"""Check the run options together (``orchestrator.check_run_options``),
|
|
180
|
+
set the per-phase reuse booleans the pipeline reads (``args.reuse_starter``
|
|
181
|
+
.. ``args.reuse_finishing``) and return the selected languages, None for
|
|
182
|
+
every registered one. Raises ValueError naming what is wrong."""
|
|
183
|
+
languages = (tuple(results.parse_csv_field(args.languages))
|
|
184
|
+
if args.languages else None)
|
|
185
|
+
reuse, languages = orchestrator.check_run_options(
|
|
186
|
+
runs=args.runs, run=args.run, reuse=tuple(results.parse_csv_field(args.reuse or "")),
|
|
187
|
+
resume=args.resume, stop_after_phase=args.stop_after_phase, languages=languages or None)
|
|
188
|
+
args.reuse = ",".join(reuse) or None
|
|
189
|
+
for phase in reuse:
|
|
190
|
+
setattr(args, f"reuse_{phase}", True)
|
|
191
|
+
return set(languages) if languages else None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _reuse_text(text: str) -> str:
|
|
195
|
+
"""``--reuse``'s text, refused when it names no phase."""
|
|
196
|
+
if not results.parse_csv_field(text):
|
|
197
|
+
raise argparse.ArgumentTypeError(f"no phase named. Choose from {', '.join(orchestrator.PHASES)}")
|
|
198
|
+
return text
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _cmd_new_language(argv) -> int:
|
|
202
|
+
"""LangAdapter's scaffold for a new language, written for Testuary:
|
|
203
|
+
``langadapter new-language`` with ``--for testuary`` and the same
|
|
204
|
+
arguments, so a language started from Testuary lands where one started
|
|
205
|
+
from LangAdapter does. Its usage and help name ``testuary
|
|
206
|
+
new-language``."""
|
|
207
|
+
from langadapter import cli
|
|
208
|
+
|
|
209
|
+
argv = list(argv)
|
|
210
|
+
if "--for" not in argv and not any(item.startswith("--for=") for item in argv):
|
|
211
|
+
argv += ["--for", "testuary"]
|
|
212
|
+
return cli.main(["new-language", *argv], prog="testuary")
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def version_lines() -> str:
|
|
216
|
+
"""The Testuary and LangAdapter versions in use, one per line: each
|
|
217
|
+
package's own ``__version__``, else its installed metadata. The package's
|
|
218
|
+
own value holds for an editable install too, whose metadata keeps the
|
|
219
|
+
version it was installed at."""
|
|
220
|
+
from importlib import metadata
|
|
221
|
+
|
|
222
|
+
lines = []
|
|
223
|
+
for name in ("testuary", "langadapter"):
|
|
224
|
+
try:
|
|
225
|
+
version = getattr(importlib.import_module(name), "__version__", None)
|
|
226
|
+
except ImportError:
|
|
227
|
+
version = None
|
|
228
|
+
if not version:
|
|
229
|
+
try:
|
|
230
|
+
version = metadata.version(name)
|
|
231
|
+
except metadata.PackageNotFoundError:
|
|
232
|
+
version = "not installed"
|
|
233
|
+
lines.append(f"{name} {version}")
|
|
234
|
+
return "\n".join(lines)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class _VersionAction(argparse.Action):
|
|
238
|
+
"""Print the versions and exit, as argparse's version action does, but
|
|
239
|
+
keeping one line per package."""
|
|
240
|
+
|
|
241
|
+
def __init__(self, option_strings, dest=argparse.SUPPRESS, default=argparse.SUPPRESS,
|
|
242
|
+
help="Print the installed Testuary and LangAdapter versions and exit."):
|
|
243
|
+
super().__init__(option_strings, dest=dest, default=default, nargs=0, help=help)
|
|
244
|
+
|
|
245
|
+
def __call__(self, parser, namespace, values, option_string=None):
|
|
246
|
+
print(version_lines(), flush=True)
|
|
247
|
+
parser.exit(0)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _primary_parser() -> argparse.ArgumentParser:
|
|
251
|
+
parser = argparse.ArgumentParser(
|
|
252
|
+
allow_abbrev=False,
|
|
253
|
+
prog="testuary", description="Generate and measure tests for selected targets.",
|
|
254
|
+
epilog="generate (the default in a project) writes and measures tests, and its "
|
|
255
|
+
"flags also work without 'generate'. check checks the setup without "
|
|
256
|
+
"contacting a model (the quick check). setup asks for model access, what "
|
|
257
|
+
"runs in parallel on this machine and a project. init copies example files. "
|
|
258
|
+
"provision installs the runtime the project asks for. new-language writes a "
|
|
259
|
+
"new language's file. tutorial teaches the extension points offline. "
|
|
260
|
+
"Use 'testuary [FILES]' in a project to run its [project] source_files, "
|
|
261
|
+
"or the targets named, and 'testuary --demo' for the offline demonstration.",
|
|
262
|
+
)
|
|
263
|
+
parser.add_argument(
|
|
264
|
+
"command", nargs="?", choices=("generate", "check", "init", "provision", "new-language", "setup", "tutorial"),
|
|
265
|
+
help="Select a command, each described below. COMMAND --help lists its options.",
|
|
266
|
+
)
|
|
267
|
+
parser.add_argument("--version", action=_VersionAction)
|
|
268
|
+
return parser
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
#: The demonstration's options, by destination: check refuses them, and a
|
|
272
|
+
#: demonstration takes them while it names every other option it overrides.
|
|
273
|
+
DEMO_OPTIONS = frozenset({"demo", "demo_language", "demo_data", "demo_speed", "demo_duration", "demo_step"})
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _build_parser(*, prog: str = "testuary", checking: bool = False) -> argparse.ArgumentParser:
|
|
277
|
+
"""Construct the pipeline argparse parser.
|
|
278
|
+
|
|
279
|
+
A separate function so tests and the project-command parsing can
|
|
280
|
+
introspect every registered flag without running the pipeline.
|
|
281
|
+
The subcommand keywords (``generate``, ``check``, ``init``, ``provision``,
|
|
282
|
+
``new-language``, ``setup``, ``tutorial``) are handled before this parser sees argv and
|
|
283
|
+
so are not registered here.
|
|
284
|
+
"""
|
|
285
|
+
parser = _OptionParser(
|
|
286
|
+
prog=prog, allow_abbrev=False,
|
|
287
|
+
description="Generate test suites for the selected targets with a language model. A "
|
|
288
|
+
"run asks the model for several independent test suites (the starter "
|
|
289
|
+
"suites) and combines them, improves the result in a coverage phase and a "
|
|
290
|
+
"mutation phase, and measures it in Finishing.")
|
|
291
|
+
if checking:
|
|
292
|
+
parser.description = ("Check the setup without contacting a model or installing anything "
|
|
293
|
+
"(the quick check). --verify also starts the tools and tries "
|
|
294
|
+
"the project in a copy.")
|
|
295
|
+
parser.add_argument(
|
|
296
|
+
"--verify", action="store_true",
|
|
297
|
+
help="Also start the tools a run needs once and try the project in a copy as a run "
|
|
298
|
+
"does, to see that they work (the tool check). Python imports pytest, "
|
|
299
|
+
"coverage and the mutation tool with the project's interpreter and collects "
|
|
300
|
+
"the tests. Java runs java and javac, resolves the build, compiles and finds "
|
|
301
|
+
"the tests, and starts the measurement tools a run needs. The first run tries "
|
|
302
|
+
"the build of a project with its own build wrapper, which may download its "
|
|
303
|
+
"build tool. The check runs no test, installs nothing and contacts no model. "
|
|
304
|
+
"What it cannot try is reported as not tried.",
|
|
305
|
+
)
|
|
306
|
+
parser.add_argument("--json", dest="json_output", action="store_true",
|
|
307
|
+
help="Print the report as JSON instead of text (API keys removed).")
|
|
308
|
+
parser.add_argument(
|
|
309
|
+
"files", nargs="*", default=[], metavar="FILE",
|
|
310
|
+
help=("Targets to run, relative to the directory the command starts in. "
|
|
311
|
+
"Without them, the project's [project] source_files. --source-files "
|
|
312
|
+
"names them relative to the source directory instead."),
|
|
313
|
+
)
|
|
314
|
+
parser.add_argument(
|
|
315
|
+
"--demo",
|
|
316
|
+
action="store_true",
|
|
317
|
+
help=(
|
|
318
|
+
"Play back one recorded run offline: every model answer and "
|
|
319
|
+
"measurement comes from a recording shipped with Testuary, so it "
|
|
320
|
+
"needs no model, network, project or test tools. It uses the "
|
|
321
|
+
"recorded settings and lists any of yours it overrides. The run "
|
|
322
|
+
"directory, the display options, the event file and the "
|
|
323
|
+
"demonstration's language are yours."
|
|
324
|
+
),
|
|
325
|
+
)
|
|
326
|
+
parser.add_argument(
|
|
327
|
+
"--demo-language", default="python", metavar="LANGUAGE",
|
|
328
|
+
help="Language of the offline demonstration: python (the default), java, or "
|
|
329
|
+
"another language whose demonstration data come with Testuary, such as toy. "
|
|
330
|
+
"Requires --demo.",
|
|
331
|
+
)
|
|
332
|
+
parser.add_argument(
|
|
333
|
+
"--demo-data", default=None, metavar="NAME",
|
|
334
|
+
help="Demonstration data that come with Testuary, played in place of a "
|
|
335
|
+
"language's own, such as a run recorded for a tutorial. Another name is "
|
|
336
|
+
"refused with the names Testuary has. Requires --demo, without --demo-language.",
|
|
337
|
+
)
|
|
338
|
+
parser.add_argument(
|
|
339
|
+
"--demo-speed", default="auto", metavar="SPEED",
|
|
340
|
+
help="Pace of the demonstration's playback: auto (paced on an interactive "
|
|
341
|
+
"terminal, instant when output is redirected, in tests or in continuous "
|
|
342
|
+
"integration), instant, or a factor of the configured length's speed such "
|
|
343
|
+
"as 2 (half as long) or 0.5 (twice as long). Pacing changes no value or "
|
|
344
|
+
"record. Requires --demo.",
|
|
345
|
+
)
|
|
346
|
+
parser.add_argument(
|
|
347
|
+
"--demo-playback-seconds", metavar="SECONDS", dest="demo_duration",
|
|
348
|
+
help="Length of a paced playback in seconds, at --demo-speed 1, instead of "
|
|
349
|
+
"TESTUARY_DEMO_PLAYBACK_SECONDS or [demo] playback_seconds. The "
|
|
350
|
+
"demonstrations guide describes the length. Requires --demo.",
|
|
351
|
+
)
|
|
352
|
+
parser.add_argument(
|
|
353
|
+
"--demo-step", action="store_true",
|
|
354
|
+
help="Advance the demonstration's playback one step at a time when Enter is "
|
|
355
|
+
"pressed, for screenshots. Requires --demo and an interactive terminal.",
|
|
356
|
+
)
|
|
357
|
+
if checking:
|
|
358
|
+
# check refuses the demonstration's options, so its help leaves them out.
|
|
359
|
+
for action in parser._actions:
|
|
360
|
+
if action.dest in DEMO_OPTIONS:
|
|
361
|
+
action.help = argparse.SUPPRESS
|
|
362
|
+
parser.add_argument(
|
|
363
|
+
"--display", choices=("auto", "compact", "detailed", "plain", "debug"), default="auto",
|
|
364
|
+
help="How the run is shown: auto (compact on a terminal, plain otherwise), compact (one "
|
|
365
|
+
"updating screen), plain (one line per change), detailed (also every step and the "
|
|
366
|
+
"full setup report), debug (also every log line and internal event names).",
|
|
367
|
+
)
|
|
368
|
+
parser.add_argument(
|
|
369
|
+
"--no-color", action="store_true",
|
|
370
|
+
help="Disable terminal colors.",
|
|
371
|
+
)
|
|
372
|
+
if not checking:
|
|
373
|
+
parser.add_argument(
|
|
374
|
+
"--events", metavar="FILE",
|
|
375
|
+
help="Also write the run's progress to FILE as JSON lines, for programs that show "
|
|
376
|
+
"a run (the terminal guide describes the records). The directory must exist, "
|
|
377
|
+
"and an existing "
|
|
378
|
+
"file is replaced. Works with --demo.",
|
|
379
|
+
)
|
|
380
|
+
parser.add_argument(
|
|
381
|
+
"--config",
|
|
382
|
+
action="append",
|
|
383
|
+
help=(
|
|
384
|
+
"Read settings from this INI file in place of the config.ini in "
|
|
385
|
+
"your user directory. Repeat it for several files: later files win. "
|
|
386
|
+
"The configuration guide describes the layers that override it."
|
|
387
|
+
),
|
|
388
|
+
)
|
|
389
|
+
parser.add_argument(
|
|
390
|
+
"--no-project-settings",
|
|
391
|
+
action="store_true",
|
|
392
|
+
help=(
|
|
393
|
+
"Read no project's settings file (.testuary/project.ini), here or in a "
|
|
394
|
+
"directory above: for runs whose settings are all given in their own "
|
|
395
|
+
"settings files. The working directory is then the project directory."
|
|
396
|
+
),
|
|
397
|
+
)
|
|
398
|
+
parser.add_argument(
|
|
399
|
+
"--home",
|
|
400
|
+
metavar="DIR",
|
|
401
|
+
help=(
|
|
402
|
+
"The user directory that holds models.ini and the global config.ini, "
|
|
403
|
+
"in place of TESTUARY_HOME or the platform's user directory."
|
|
404
|
+
),
|
|
405
|
+
)
|
|
406
|
+
parser.add_argument(
|
|
407
|
+
"--model",
|
|
408
|
+
help="Override [api] model: a model alias of models.ini, or a model name [api] provider serves.",
|
|
409
|
+
)
|
|
410
|
+
parser.add_argument(
|
|
411
|
+
"--proxy-port",
|
|
412
|
+
type=int,
|
|
413
|
+
help=(
|
|
414
|
+
"Use a model server on this port of this machine: sets "
|
|
415
|
+
"[api] api_url to http://localhost:<port>/v1, which wins over "
|
|
416
|
+
"the endpoint of the models.ini alias and of the provider "
|
|
417
|
+
"section. --api-url wins over it."
|
|
418
|
+
),
|
|
419
|
+
)
|
|
420
|
+
parser.add_argument(
|
|
421
|
+
"--api-url",
|
|
422
|
+
help=(
|
|
423
|
+
"Override [api] api_url with a full base URL, for example "
|
|
424
|
+
"http://localhost:8010/v1. It wins over --proxy-port and over "
|
|
425
|
+
"the endpoint of the models.ini alias and of the provider "
|
|
426
|
+
"section."
|
|
427
|
+
),
|
|
428
|
+
)
|
|
429
|
+
parser.add_argument(
|
|
430
|
+
"--prompt-profile",
|
|
431
|
+
help="A JSON file that changes the prompts sent to the model (examples: testuary init "
|
|
432
|
+
"--examples). Read when the command starts.",
|
|
433
|
+
)
|
|
434
|
+
parser.add_argument(
|
|
435
|
+
"--hints-file",
|
|
436
|
+
dest="hints_file",
|
|
437
|
+
help=(
|
|
438
|
+
"Hints file to read from the working directory, in place of "
|
|
439
|
+
"[main] hints_file. The hints guide says when a file is optional "
|
|
440
|
+
"and what refuses a run."
|
|
441
|
+
),
|
|
442
|
+
)
|
|
443
|
+
parser.add_argument(
|
|
444
|
+
"--reuse-starter-suites",
|
|
445
|
+
dest="reuse_starter_suites",
|
|
446
|
+
action="store_true",
|
|
447
|
+
default=None,
|
|
448
|
+
help=(
|
|
449
|
+
"Run the starter phase again from the separate starter suites an "
|
|
450
|
+
"earlier run saved: check and combine them again, and generate only "
|
|
451
|
+
"a missing or empty suite again, which sends model requests (without "
|
|
452
|
+
"saved suites, every one). --reuse starter instead keeps the "
|
|
453
|
+
"combined result. Sets [starter] reuse_suites."
|
|
454
|
+
),
|
|
455
|
+
)
|
|
456
|
+
parser.add_argument(
|
|
457
|
+
"--phase-scores",
|
|
458
|
+
dest="phase_scores",
|
|
459
|
+
choices=_SETTING_CHOICES["phase_scores"],
|
|
460
|
+
default=None,
|
|
461
|
+
help=(
|
|
462
|
+
"What tests.csv reports for the delivered suites of the starter "
|
|
463
|
+
"and coverage phases, read from what those phases recorded: "
|
|
464
|
+
"full reports their line and branch coverage and their "
|
|
465
|
+
"mutation score over the full catalogue, coverage omits the mutation "
|
|
466
|
+
"score, off omits these columns. Nothing is measured again for "
|
|
467
|
+
"them. Overrides [finishing] phase_scores."
|
|
468
|
+
),
|
|
469
|
+
)
|
|
470
|
+
parser.add_argument(
|
|
471
|
+
"--project-name",
|
|
472
|
+
help=(
|
|
473
|
+
"The project's name in results and records (${project}). Overrides "
|
|
474
|
+
"[project] name. Empty uses the source directory's name."
|
|
475
|
+
),
|
|
476
|
+
)
|
|
477
|
+
parser.add_argument(
|
|
478
|
+
"--source-dir",
|
|
479
|
+
dest="source_dir",
|
|
480
|
+
help="The source directory, whose files are tested. Without it, the project directory: the one "
|
|
481
|
+
"whose .testuary directory is found from the current directory upward, else the current "
|
|
482
|
+
"directory. A relative path is taken from the current directory. In container mode give "
|
|
483
|
+
"the absolute path inside the container.",
|
|
484
|
+
)
|
|
485
|
+
parser.add_argument(
|
|
486
|
+
"--python-executable",
|
|
487
|
+
help="The project's Python interpreter ([project] python_executable). It installs nothing.",
|
|
488
|
+
)
|
|
489
|
+
parser.add_argument("--java-home", help="The project's JDK home or java executable ([project] java_home).")
|
|
490
|
+
parser.add_argument("--build-java-home", help="The build tool's JDK home ([project] build_java_home).")
|
|
491
|
+
parser.add_argument(
|
|
492
|
+
"--output-dir",
|
|
493
|
+
dest="output_dir",
|
|
494
|
+
help=(
|
|
495
|
+
"The run directory: the results of each target and all logs go "
|
|
496
|
+
"below it. A relative path is taken from the current directory. "
|
|
497
|
+
"Overrides [project] output_dir, "
|
|
498
|
+
"which may contain ${model} and ${project} placeholders."
|
|
499
|
+
),
|
|
500
|
+
)
|
|
501
|
+
parser.add_argument(
|
|
502
|
+
"--python-import-root",
|
|
503
|
+
help=(
|
|
504
|
+
"The directory from which the project's Python package is imported "
|
|
505
|
+
"([project] python_import_root). pytest still runs in the source "
|
|
506
|
+
"directory, with the project's own configuration."
|
|
507
|
+
),
|
|
508
|
+
)
|
|
509
|
+
parser.add_argument(
|
|
510
|
+
"--languages",
|
|
511
|
+
help=(
|
|
512
|
+
"Refuse the run if a target is in a language outside this "
|
|
513
|
+
f"list (installed: {', '.join(languages.available_language_names())}). "
|
|
514
|
+
"Each target's language comes from its file extension. Without this "
|
|
515
|
+
"option every installed language is allowed."
|
|
516
|
+
),
|
|
517
|
+
)
|
|
518
|
+
parser.add_argument(
|
|
519
|
+
"--targets",
|
|
520
|
+
type=str,
|
|
521
|
+
help=(
|
|
522
|
+
"Run only these targets of the --source-files list, by position, "
|
|
523
|
+
"one after another: one position (`7`), a range (`1-5`), or an "
|
|
524
|
+
"open range (`5-` for the fifth to the end, `-5` for the first "
|
|
525
|
+
"five). Default: the whole list. "
|
|
526
|
+
"Outside container mode each target runs in its own copy of the "
|
|
527
|
+
"source directory, so one target's edits cannot change the next "
|
|
528
|
+
"one's measurements."
|
|
529
|
+
),
|
|
530
|
+
)
|
|
531
|
+
parser.add_argument(
|
|
532
|
+
"--source-files",
|
|
533
|
+
default=None,
|
|
534
|
+
dest="source_files",
|
|
535
|
+
help=(
|
|
536
|
+
"Comma list of targets to generate tests for, in the "
|
|
537
|
+
"order they run. A relative path is taken from the source "
|
|
538
|
+
"directory. Each target must exist and be in an installed language. "
|
|
539
|
+
"Without --targets the run covers the whole list."
|
|
540
|
+
),
|
|
541
|
+
)
|
|
542
|
+
parser.add_argument(
|
|
543
|
+
"--concurrent-requests",
|
|
544
|
+
dest="concurrent_requests",
|
|
545
|
+
type=int,
|
|
546
|
+
help=(
|
|
547
|
+
"How many model requests for the starter suites run at once "
|
|
548
|
+
"(concurrent requests), in place of [starter] concurrent_requests "
|
|
549
|
+
"and concurrent_requests_ollama. Fewer put less load on the "
|
|
550
|
+
"model server and shorten each request, which helps models "
|
|
551
|
+
"that write long answers."
|
|
552
|
+
),
|
|
553
|
+
)
|
|
554
|
+
stop_phases = orchestrator.available_phase_names()
|
|
555
|
+
parser.add_argument(
|
|
556
|
+
"--stop-after-phase",
|
|
557
|
+
type=lambda text: text.strip().lower(),
|
|
558
|
+
metavar="{" + ",".join(stop_phases) + "}",
|
|
559
|
+
default=None,
|
|
560
|
+
help=(
|
|
561
|
+
"Stop each target after this phase has saved its result, to try "
|
|
562
|
+
f"a phase alone: {', '.join(stop_phases)}. The default runs "
|
|
563
|
+
"all four phases. Gives no final scores, and each target is "
|
|
564
|
+
"reported as stopped."
|
|
565
|
+
),
|
|
566
|
+
)
|
|
567
|
+
parser.add_argument(
|
|
568
|
+
"--reuse",
|
|
569
|
+
type=_reuse_text,
|
|
570
|
+
default=None,
|
|
571
|
+
help=(
|
|
572
|
+
"Use the saved results of these phases from the earlier run in "
|
|
573
|
+
"this run directory instead of running them again: "
|
|
574
|
+
f"{', '.join(orchestrator.PHASES)}. The results guide says what "
|
|
575
|
+
"each one reuses. Example: --reuse coverage."
|
|
576
|
+
),
|
|
577
|
+
)
|
|
578
|
+
# Every per-phase reuse boolean exists on args whether or not --reuse
|
|
579
|
+
# names its phase, so the pipeline's reads never depend on the flags
|
|
580
|
+
# given.
|
|
581
|
+
parser.set_defaults(
|
|
582
|
+
reuse_starter=False, reuse_coverage=False, reuse_mutation=False,
|
|
583
|
+
reuse_finishing=False,
|
|
584
|
+
)
|
|
585
|
+
parser.add_argument(
|
|
586
|
+
"--resume",
|
|
587
|
+
action="store_true",
|
|
588
|
+
dest="resume",
|
|
589
|
+
help=(
|
|
590
|
+
"Continue an interrupted run in the same run directory: each target "
|
|
591
|
+
"and repetition continues from the first phase that did not "
|
|
592
|
+
"finish. Finished phases (starter, coverage, mutation, finishing) "
|
|
593
|
+
"are reused if their saved results and the target's source are "
|
|
594
|
+
"unchanged. Combines only with --reuse starter."
|
|
595
|
+
),
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
# -----------------------------------------------------------------
|
|
599
|
+
# Runs. --runs N runs a target's test pipeline N times, as runs r0 to
|
|
600
|
+
# r(N-1), each in its own directory results/<workspace>/r<k>/
|
|
601
|
+
# (project.run_dir). --run K runs only rK.
|
|
602
|
+
# -----------------------------------------------------------------
|
|
603
|
+
parser.add_argument(
|
|
604
|
+
"--runs", type=int, default=1, dest="runs", metavar="N",
|
|
605
|
+
help=(
|
|
606
|
+
"Run each target's four phases N times, as repetitions r0 to "
|
|
607
|
+
"r(N-1), each from a fresh copy of the project and in its own "
|
|
608
|
+
"directory. N must be 1 with --stop-after-phase."
|
|
609
|
+
),
|
|
610
|
+
)
|
|
611
|
+
parser.add_argument(
|
|
612
|
+
"--run", type=int, default=None, dest="run", metavar="K",
|
|
613
|
+
help=(
|
|
614
|
+
"Run only repetition K (rK) of each target, the same one --runs "
|
|
615
|
+
"makes when N is above K, for example to add a missing repetition."
|
|
616
|
+
),
|
|
617
|
+
)
|
|
618
|
+
return parser
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _is_check_command() -> bool:
|
|
622
|
+
import sys
|
|
623
|
+
|
|
624
|
+
return sys.argv[1:2] == ["check"]
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _resumable(argv) -> bool:
|
|
628
|
+
"""Whether --resume continues ``argv``: a run, neither a demonstration
|
|
629
|
+
nor one whose --reuse selection goes beyond starter."""
|
|
630
|
+
argv = list(argv)
|
|
631
|
+
if argv[:1] in (["check"], ["init"], ["provision"], ["new-language"], ["tutorial"],
|
|
632
|
+
["setup"]) or "--demo" in argv:
|
|
633
|
+
return False
|
|
634
|
+
reuse = []
|
|
635
|
+
for index, item in enumerate(argv):
|
|
636
|
+
# As argparse does, the last --reuse wins.
|
|
637
|
+
if item == "--reuse" and index + 1 < len(argv):
|
|
638
|
+
reuse = argv[index + 1].split(",")
|
|
639
|
+
elif item.startswith("--reuse="):
|
|
640
|
+
reuse = item.split("=", 1)[1].split(",")
|
|
641
|
+
return not {unit.strip().lower() for unit in reuse if unit.strip()} - {"starter"}
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _interrupt_advice(argv) -> str:
|
|
645
|
+
"""What the person can do after interrupting ``argv``. Only commands that
|
|
646
|
+
can continue are told how."""
|
|
647
|
+
argv = list(argv)
|
|
648
|
+
if argv[:1] == ["check"]:
|
|
649
|
+
return "No setup was changed."
|
|
650
|
+
if argv[:1] in (["init"], ["provision"], ["new-language"]):
|
|
651
|
+
return "Run the command again when ready."
|
|
652
|
+
if argv[:1] == ["tutorial"]:
|
|
653
|
+
return "Start the tutorial again with a new or empty --output-dir."
|
|
654
|
+
if "--demo" in argv:
|
|
655
|
+
return ("Start the demonstration again with a new or empty --output-dir. "
|
|
656
|
+
"A demonstration cannot be resumed.")
|
|
657
|
+
if argv[:1] == ["setup"]:
|
|
658
|
+
return "Run the command again when ready."
|
|
659
|
+
if not _resumable(argv):
|
|
660
|
+
return ("Finished results and saved model answers are kept. Continue with the "
|
|
661
|
+
"same command (--resume does not combine with this --reuse selection).")
|
|
662
|
+
return ("Finished results and saved model answers are kept. Continue with the "
|
|
663
|
+
"same command and --resume.")
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def main():
|
|
667
|
+
"""The process entry: an interrupted command ends with a short report and
|
|
668
|
+
exit status 130 instead of a traceback."""
|
|
669
|
+
import sys as _sys
|
|
670
|
+
|
|
671
|
+
try:
|
|
672
|
+
return run_command()
|
|
673
|
+
except KeyboardInterrupt as stop:
|
|
674
|
+
remains = _retry_interrupted_cleanup(stop)
|
|
675
|
+
# The display has already reported the cancellation. A Python
|
|
676
|
+
# traceback would add nothing for the person who pressed Ctrl+C.
|
|
677
|
+
print("Stopped by you. " + _interrupt_advice(_sys.argv[1:]), file=_sys.stderr, flush=True)
|
|
678
|
+
if remains is not None:
|
|
679
|
+
print(remains, file=_sys.stderr, flush=True)
|
|
680
|
+
return exits.INTERRUPTED
|
|
681
|
+
except orchestrator._PhaseStopNotReached as exc:
|
|
682
|
+
# The run's log names why the phase did not reach the stop.
|
|
683
|
+
print(f"Stopped: {exc}. The run's log says why.",
|
|
684
|
+
file=_sys.stderr, flush=True)
|
|
685
|
+
return 1
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _retry_interrupted_cleanup(stop: BaseException) -> Optional[str]:
|
|
689
|
+
"""Close once more the mutation session whose own cleanup an interrupt
|
|
690
|
+
cut short, as a second Ctrl+C during the first one's cleanup does: the
|
|
691
|
+
interrupt carries the session as its cleanup handle. Returns a line
|
|
692
|
+
naming what remains when that close fails too, else None. The exit stays
|
|
693
|
+
the interrupted one either way. From here on a further Ctrl+C is
|
|
694
|
+
ignored: it would cut this cleanup short, or kill the exit itself."""
|
|
695
|
+
import signal
|
|
696
|
+
import threading
|
|
697
|
+
|
|
698
|
+
from testuary.languages.mutation import close_cancelled
|
|
699
|
+
|
|
700
|
+
if threading.current_thread() is threading.main_thread():
|
|
701
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
702
|
+
|
|
703
|
+
failures = []
|
|
704
|
+
handle = getattr(stop, "mutation_cleanup_handle", None)
|
|
705
|
+
if handle is not None:
|
|
706
|
+
try:
|
|
707
|
+
handle.close()
|
|
708
|
+
except BaseException as exc:
|
|
709
|
+
failures.append(exc)
|
|
710
|
+
# A session the stop cancelled whose close the interrupt skipped.
|
|
711
|
+
failures += close_cancelled()
|
|
712
|
+
if failures:
|
|
713
|
+
exc = failures[0]
|
|
714
|
+
return (f"The mutation tool may still be running after the interruption "
|
|
715
|
+
f"({type(exc).__name__}: {exc}). Look for leftover cosmic-ray or Java "
|
|
716
|
+
"processes.")
|
|
717
|
+
return None
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def run_command():
|
|
721
|
+
"""Run the command in ``sys.argv`` and return its status.
|
|
722
|
+
|
|
723
|
+
An interruption propagates, so an in-process caller such as the
|
|
724
|
+
demonstration records it as a cancellation. ``main`` reports it for the
|
|
725
|
+
process entry.
|
|
726
|
+
"""
|
|
727
|
+
checking = _is_check_command()
|
|
728
|
+
try:
|
|
729
|
+
return _main()
|
|
730
|
+
except Exception as exc:
|
|
731
|
+
# A read-only inspection must not write a previous generation's log,
|
|
732
|
+
# and a requested stop that was not reached is no program failure.
|
|
733
|
+
if checking or isinstance(exc, orchestrator._PhaseStopNotReached):
|
|
734
|
+
raise
|
|
735
|
+
from testuary.model_requests import failure_traceback
|
|
736
|
+
try:
|
|
737
|
+
log_print(
|
|
738
|
+
"FATAL: uncaught exception ended the run:\n"
|
|
739
|
+
+ failure_traceback(exc),
|
|
740
|
+
level=0,
|
|
741
|
+
)
|
|
742
|
+
except Exception:
|
|
743
|
+
pass
|
|
744
|
+
raise
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def _resume_command(argv):
|
|
748
|
+
"""The exact command that continues ``argv``, or None where --resume does
|
|
749
|
+
not apply (a demonstration, or a --reuse selection beyond starter)."""
|
|
750
|
+
import os
|
|
751
|
+
import shlex
|
|
752
|
+
import subprocess
|
|
753
|
+
|
|
754
|
+
argv = list(argv)
|
|
755
|
+
if not _resumable(argv):
|
|
756
|
+
return None
|
|
757
|
+
command = ["testuary", *argv] + ([] if "--resume" in argv else ["--resume"])
|
|
758
|
+
return subprocess.list2cmdline(command) if os.name == "nt" else shlex.join(command)
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def _present_plan(plan, presenter, events=None) -> None:
|
|
762
|
+
"""The run's plan to the event file, if any, and to the display."""
|
|
763
|
+
_record(presenter, events, "plan", plan)
|
|
764
|
+
show = getattr(presenter, "plan", None)
|
|
765
|
+
if show is not None:
|
|
766
|
+
show(plan)
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
def _present_completion(completion, presenter, events=None):
|
|
770
|
+
from dataclasses import replace
|
|
771
|
+
import sys as _sys
|
|
772
|
+
|
|
773
|
+
from testuary.presentation.terminal import present_completion
|
|
774
|
+
|
|
775
|
+
completion = replace(completion, resume_command=_resume_command(_sys.argv[1:]))
|
|
776
|
+
try:
|
|
777
|
+
from testuary.presentation.terminal import write_completion
|
|
778
|
+
|
|
779
|
+
# With the command that continues the run, as the console shows it.
|
|
780
|
+
write_completion(completion, provenance=presenter.provenance,
|
|
781
|
+
clean=getattr(presenter, "clean", None))
|
|
782
|
+
except Exception as exc:
|
|
783
|
+
log_print(f"WARNING: the end-of-run summary could not be written ({type(exc).__name__}: {exc})")
|
|
784
|
+
_record(presenter, events, "completion", completion)
|
|
785
|
+
present_completion(completion, presenter=presenter)
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def _event_file_path(parser, value) -> str:
|
|
789
|
+
"""``--events`` as an absolute path, checked before anything runs. The
|
|
790
|
+
demonstration plays its run in a directory of its own, so a relative path is
|
|
791
|
+
resolved here. The write is tried by appending, so a command refused
|
|
792
|
+
after this check leaves an earlier file as it was. The run replaces the
|
|
793
|
+
file when it starts."""
|
|
794
|
+
try:
|
|
795
|
+
path = orchestrator._writable_event_file(value)
|
|
796
|
+
except ValueError as exc:
|
|
797
|
+
parser.error(f"--events: {exc}")
|
|
798
|
+
try:
|
|
799
|
+
with open(path, "a", encoding="utf-8"):
|
|
800
|
+
pass
|
|
801
|
+
except OSError as exc:
|
|
802
|
+
parser.error(f"--events: cannot write {path}: {exc.strerror or exc}")
|
|
803
|
+
return str(path)
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _display_lock(presenter):
|
|
807
|
+
import threading
|
|
808
|
+
|
|
809
|
+
return getattr(presenter, "_lock", None) or threading.RLock()
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def _observe(presenter, events):
|
|
813
|
+
"""The run's event sink. With an event file each event is written there,
|
|
814
|
+
then shown, both under the display's lock, so the file holds the events in
|
|
815
|
+
the order the display shows them. Without one the display is the sink."""
|
|
816
|
+
if events is None:
|
|
817
|
+
return presenter
|
|
818
|
+
lock = _display_lock(presenter)
|
|
819
|
+
|
|
820
|
+
def observe(event):
|
|
821
|
+
with lock:
|
|
822
|
+
events(event)
|
|
823
|
+
presenter(event)
|
|
824
|
+
|
|
825
|
+
return observe
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
def _record(presenter, events, kind, value) -> None:
|
|
829
|
+
"""Write the setup report or the completion to the event file, if any,
|
|
830
|
+
under the display's lock, just before the display shows it."""
|
|
831
|
+
if events is not None:
|
|
832
|
+
with _display_lock(presenter):
|
|
833
|
+
getattr(events, kind)(value)
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _project_settings_here() -> bool:
|
|
837
|
+
"""Whether the directory the command starts in belongs to a project with a
|
|
838
|
+
settings file, found as a run finds it."""
|
|
839
|
+
from testuary.settings import project_locations
|
|
840
|
+
|
|
841
|
+
try:
|
|
842
|
+
return project_locations(Path.cwd()).project_settings.is_file()
|
|
843
|
+
except (OSError, ValueError):
|
|
844
|
+
return False
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _project_files(named, source_files, settings, *, start):
|
|
848
|
+
"""The source files a command runs, relative to the source directory, and
|
|
849
|
+
whether they are the project's own list. They are the files named on
|
|
850
|
+
the command line, relative to ``start``, else ``--source-files`` as
|
|
851
|
+
given, else the project's ``[project] source_files``, and none when it
|
|
852
|
+
lists none: the run then refuses itself, after the checks before it. A
|
|
853
|
+
named file that does not exist, is a directory, lies outside the source
|
|
854
|
+
directory, or is of another language than ``[project] language``, is
|
|
855
|
+
refused, unless that language is unchecked: an extension the settings
|
|
856
|
+
name may add it, which the tool check and a run check after loading."""
|
|
857
|
+
from testuary.settings import ConfigurationError
|
|
858
|
+
|
|
859
|
+
project = settings.sections["project"]
|
|
860
|
+
if named and source_files:
|
|
861
|
+
raise ConfigurationError("name the targets once: as FILE arguments or with --source-files")
|
|
862
|
+
if source_files:
|
|
863
|
+
return tuple(results.parse_csv_field(str(source_files))), False
|
|
864
|
+
if not named:
|
|
865
|
+
listed = tuple(results.parse_csv_field(str(project.get("source_files") or "")))
|
|
866
|
+
return listed, bool(listed)
|
|
867
|
+
root_text = str(project.get("source_dir") or "").strip()
|
|
868
|
+
root = Path(root_text) if root_text else settings.project_dir
|
|
869
|
+
root = (root if root.is_absolute() else settings.base_dir / root).resolve()
|
|
870
|
+
language = str(project.get("language") or "").strip().lower()
|
|
871
|
+
if "[project] language" in settings.unchecked:
|
|
872
|
+
language = ""
|
|
873
|
+
files = []
|
|
874
|
+
for name in named:
|
|
875
|
+
path = Path(name)
|
|
876
|
+
path = (path if path.is_absolute() else Path(start) / path).resolve()
|
|
877
|
+
try:
|
|
878
|
+
relative = path.relative_to(root).as_posix()
|
|
879
|
+
except ValueError:
|
|
880
|
+
raise ConfigurationError(
|
|
881
|
+
f"{name} lies outside the source directory {root}") from None
|
|
882
|
+
if path.is_dir():
|
|
883
|
+
raise ConfigurationError(f"{name} is a directory, not a source file")
|
|
884
|
+
if not path.is_file():
|
|
885
|
+
raise ConfigurationError(f"{name} does not exist")
|
|
886
|
+
if language and languages.detect_language(str(path)) != language:
|
|
887
|
+
raise ConfigurationError(f"{name} is not a {language} source file")
|
|
888
|
+
files.append(relative)
|
|
889
|
+
return tuple(files), False
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def _run_cli_invocation(args, selected_languages, *, presenter, checking=False, event_sink=None,
|
|
893
|
+
events=None):
|
|
894
|
+
"""One resolved CLI invocation and one scoped admission-report boundary."""
|
|
895
|
+
from testuary.presentation import present_readiness
|
|
896
|
+
from testuary.readiness import _configuration_error_report
|
|
897
|
+
from testuary.settings import ConfigurationError, resolve_settings
|
|
898
|
+
|
|
899
|
+
settings = None
|
|
900
|
+
refusal = None
|
|
901
|
+
try:
|
|
902
|
+
try:
|
|
903
|
+
# A run and the tool check load the extensions, the quick check none.
|
|
904
|
+
settings = resolve_settings(config_files=args.config, home=getattr(args, "home", None),
|
|
905
|
+
project_settings=not getattr(args, "no_project_settings", False),
|
|
906
|
+
load_extensions=not checking or bool(getattr(args, "verify", False)),
|
|
907
|
+
overrides=orchestrator._from_working_dir(
|
|
908
|
+
orchestrator._cli_settings_overrides(args)))
|
|
909
|
+
except (ValueError, TypeError) as exc:
|
|
910
|
+
raise ConfigurationError(str(exc)) from exc
|
|
911
|
+
redact_with = getattr(presenter, "redact_with", None)
|
|
912
|
+
if redact_with is not None:
|
|
913
|
+
# The run's own keys are removed exactly, also where no pattern
|
|
914
|
+
# recognizes them, as the check's report removes them.
|
|
915
|
+
from testuary.settings import _redact_diagnostic
|
|
916
|
+
|
|
917
|
+
resolved = settings
|
|
918
|
+
redact_with(lambda text: _redact_diagnostic(text, resolved))
|
|
919
|
+
if not getattr(args, "demo", False):
|
|
920
|
+
files, from_project = _project_files(getattr(args, "files", None) or [], args.source_files,
|
|
921
|
+
settings, start=Path.cwd())
|
|
922
|
+
args.source_files = files
|
|
923
|
+
if from_project and not checking and len(files) > 1:
|
|
924
|
+
print("\n".join([f"Project {settings.project_dir.name}: {len(files)} targets, "
|
|
925
|
+
"each a full run:",
|
|
926
|
+
*(f" {index}. {path}" for index, path in enumerate(files, 1))]),
|
|
927
|
+
flush=True)
|
|
928
|
+
if checking:
|
|
929
|
+
try:
|
|
930
|
+
request = orchestrator._run_request(args)
|
|
931
|
+
except (ValueError, TypeError) as exc:
|
|
932
|
+
raise ConfigurationError(str(exc)) from exc
|
|
933
|
+
report = orchestrator.check(request, settings=settings, verify=args.verify)
|
|
934
|
+
else:
|
|
935
|
+
invocation = orchestrator._without_cli_configuration(args)
|
|
936
|
+
from testuary._progress import demonstrating
|
|
937
|
+
|
|
938
|
+
def pre_run(report):
|
|
939
|
+
_record(presenter, events, "readiness", report)
|
|
940
|
+
present_readiness(report, presenter=presenter, before_run=True, playback=demonstrating())
|
|
941
|
+
|
|
942
|
+
invocation._pre_run = pre_run
|
|
943
|
+
invocation._on_plan = lambda plan: _present_plan(plan, presenter, events)
|
|
944
|
+
invocation._on_complete = lambda completion: _present_completion(
|
|
945
|
+
completion, presenter, events)
|
|
946
|
+
sink = presenter if event_sink is None else event_sink
|
|
947
|
+
return orchestrator.run_pipeline(invocation, selected_languages, settings=settings, event_sink=sink)
|
|
948
|
+
except ConfigurationError as exc:
|
|
949
|
+
report = _configuration_error_report(
|
|
950
|
+
exc, settings=settings, scope="verification" if checking and args.verify else "inspection",
|
|
951
|
+
before_run=not checking)
|
|
952
|
+
refusal = exc
|
|
953
|
+
if checking and args.json_output:
|
|
954
|
+
import json
|
|
955
|
+
|
|
956
|
+
print(json.dumps(report.as_dict(), ensure_ascii=True, allow_nan=False))
|
|
957
|
+
else:
|
|
958
|
+
# A refused generate ran no check: its report is the setup before the run.
|
|
959
|
+
options = {} if checking else {"before_run": True}
|
|
960
|
+
if not checking:
|
|
961
|
+
_record(presenter, events, "readiness", report)
|
|
962
|
+
present_readiness(report, presenter=presenter, **options)
|
|
963
|
+
if refusal is not None:
|
|
964
|
+
raise SystemExit(report.exit_code) from refusal
|
|
965
|
+
return report.exit_code
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _main():
|
|
969
|
+
# Template commands parse their arguments before accessing the filesystem.
|
|
970
|
+
import sys as _sys
|
|
971
|
+
|
|
972
|
+
# Use UTF-8 for parent-process output as well as subprocess output.
|
|
973
|
+
# Replacement handling prevents unsupported terminal characters from
|
|
974
|
+
# interrupting a run.
|
|
975
|
+
for _stream in (_sys.stdout, _sys.stderr):
|
|
976
|
+
try:
|
|
977
|
+
_stream.reconfigure(encoding="utf-8", errors="replace")
|
|
978
|
+
except (AttributeError, OSError):
|
|
979
|
+
# A replaced stream may not be a text wrapper, and non-TTY
|
|
980
|
+
# streams (pipes, files) may reject the call. Either case
|
|
981
|
+
# leaves the original stream in effect.
|
|
982
|
+
pass
|
|
983
|
+
|
|
984
|
+
argv = _sys.argv[1:]
|
|
985
|
+
if not argv:
|
|
986
|
+
if not _project_settings_here():
|
|
987
|
+
_primary_parser().print_help()
|
|
988
|
+
return 2
|
|
989
|
+
# In a project, the bare command runs its source files.
|
|
990
|
+
argv = ["generate"]
|
|
991
|
+
if argv in (["--help"], ["-h"], ["--version"]):
|
|
992
|
+
_primary_parser().parse_args(argv)
|
|
993
|
+
|
|
994
|
+
# The subcommands with their own parsers. Setup and the tutorial load
|
|
995
|
+
# their modules only when chosen.
|
|
996
|
+
subcommands = {
|
|
997
|
+
"init": _cmd_init, "provision": _cmd_provision, "new-language": _cmd_new_language,
|
|
998
|
+
"setup": lambda rest: importlib.import_module("testuary.setup").main(rest),
|
|
999
|
+
"tutorial": lambda rest: importlib.import_module("testuary.tutorial").main(rest),
|
|
1000
|
+
}
|
|
1001
|
+
if argv[0] in subcommands:
|
|
1002
|
+
_sys.exit(subcommands[argv[0]](argv[1:]))
|
|
1003
|
+
|
|
1004
|
+
explicit_generate = argv[0] == "generate"
|
|
1005
|
+
checking = argv[0] == "check"
|
|
1006
|
+
if explicit_generate or checking:
|
|
1007
|
+
argv = argv[1:]
|
|
1008
|
+
parser = _build_parser(prog="testuary check" if checking else "testuary generate" if explicit_generate else "testuary",
|
|
1009
|
+
checking=checking)
|
|
1010
|
+
if not argv and not _project_settings_here():
|
|
1011
|
+
parser.print_usage()
|
|
1012
|
+
print("No project here and no targets named: run testuary setup, or name the targets.")
|
|
1013
|
+
return 2
|
|
1014
|
+
args = parser.parse_args(argv)
|
|
1015
|
+
if getattr(args, "events", None) is not None:
|
|
1016
|
+
args.events = _event_file_path(parser, args.events)
|
|
1017
|
+
|
|
1018
|
+
demo_options = (args.demo_language != "python" or args.demo_data is not None
|
|
1019
|
+
or args.demo_speed != "auto" or args.demo_duration is not None
|
|
1020
|
+
or args.demo_step)
|
|
1021
|
+
if checking and (args.demo or demo_options):
|
|
1022
|
+
parser.error("check does not run a demonstration. Use --demo separately")
|
|
1023
|
+
if args.demo and args.demo_data is not None and args.demo_language != "python":
|
|
1024
|
+
parser.error("--demo-data selects the data, whose language they name. "
|
|
1025
|
+
"Drop --demo-language")
|
|
1026
|
+
# Refused options stop before any configuration is read, with the
|
|
1027
|
+
# parser's status 2. check refuses them in its own report, which --json
|
|
1028
|
+
# prints as JSON.
|
|
1029
|
+
if not checking:
|
|
1030
|
+
try:
|
|
1031
|
+
selected_languages = _check_run_options(args)
|
|
1032
|
+
except ValueError as exc:
|
|
1033
|
+
parser.error(str(exc))
|
|
1034
|
+
if args.demo:
|
|
1035
|
+
# Before any configuration is read: the demonstration runs the
|
|
1036
|
+
# recorded configuration in its own workspace.
|
|
1037
|
+
from testuary.demo import cli as demo_cli
|
|
1038
|
+
from testuary._progress import demonstration_display
|
|
1039
|
+
|
|
1040
|
+
with demonstration_display():
|
|
1041
|
+
_sys.exit(demo_cli.main(args, parser))
|
|
1042
|
+
|
|
1043
|
+
if args.demo_language != "python":
|
|
1044
|
+
parser.error("--demo-language requires --demo. A normal run takes each target's "
|
|
1045
|
+
"language from its file extension.")
|
|
1046
|
+
if args.demo_data is not None:
|
|
1047
|
+
parser.error("--demo-data requires --demo")
|
|
1048
|
+
if args.demo_speed != "auto" or args.demo_duration is not None or args.demo_step:
|
|
1049
|
+
parser.error("--demo-speed, --demo-playback-seconds and --demo-step require --demo")
|
|
1050
|
+
|
|
1051
|
+
if checking:
|
|
1052
|
+
from testuary.presentation import TerminalPresenter
|
|
1053
|
+
|
|
1054
|
+
if args.json_output:
|
|
1055
|
+
return _run_cli_invocation(args, None, presenter=None, checking=True)
|
|
1056
|
+
with TerminalPresenter(args.display, no_color=args.no_color) as presenter:
|
|
1057
|
+
return _run_cli_invocation(args, None, presenter=presenter, checking=True)
|
|
1058
|
+
|
|
1059
|
+
from contextlib import nullcontext
|
|
1060
|
+
|
|
1061
|
+
from testuary.presentation import EventFile, TerminalPresenter
|
|
1062
|
+
|
|
1063
|
+
events_path = vars(args).pop("events", None)
|
|
1064
|
+
with TerminalPresenter(args.display, no_color=args.no_color) as presenter, \
|
|
1065
|
+
(EventFile(events_path, clean=getattr(presenter, "clean", None))
|
|
1066
|
+
if events_path else nullcontext()) as events:
|
|
1067
|
+
args._diagnostic_writer = presenter.write_diagnostic
|
|
1068
|
+
return _run_cli_invocation(args, selected_languages, presenter=presenter,
|
|
1069
|
+
event_sink=_observe(presenter, events), events=events)
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
if __name__ == "__main__":
|
|
1073
|
+
# The pipeline's return is the exit status, as the installed
|
|
1074
|
+
# ``testuary`` script uses it: 1 when a target failed to execute.
|
|
1075
|
+
raise SystemExit(main())
|