testuary 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testuary-0.2.0/.gitignore +14 -0
- testuary-0.2.0/CHANGELOG.md +524 -0
- testuary-0.2.0/CITATION.cff +21 -0
- testuary-0.2.0/LICENSE +21 -0
- testuary-0.2.0/PKG-INFO +217 -0
- testuary-0.2.0/README.md +147 -0
- testuary-0.2.0/THIRD_PARTY_NOTICES.md +51 -0
- testuary-0.2.0/docs/contributing.md +65 -0
- testuary-0.2.0/docs/explanation/code-organization.md +194 -0
- testuary-0.2.0/docs/explanation/measurement-and-recovery.md +245 -0
- testuary-0.2.0/docs/extending/hints.md +100 -0
- testuary-0.2.0/docs/extending/index.md +39 -0
- testuary-0.2.0/docs/extending/languages.md +326 -0
- testuary-0.2.0/docs/extending/model-provider.md +290 -0
- testuary-0.2.0/docs/extending/prompt-profile.md +138 -0
- testuary-0.2.0/docs/extending/web.md +811 -0
- testuary-0.2.0/docs/getting-started.md +390 -0
- testuary-0.2.0/docs/guide/cluster.md +317 -0
- testuary-0.2.0/docs/guide/configuration.md +377 -0
- testuary-0.2.0/docs/guide/demonstrations.md +193 -0
- testuary-0.2.0/docs/guide/machine.md +96 -0
- testuary-0.2.0/docs/guide/output.md +194 -0
- testuary-0.2.0/docs/guide/terminal.md +369 -0
- testuary-0.2.0/docs/guide/web.md +502 -0
- testuary-0.2.0/docs/index.md +125 -0
- testuary-0.2.0/docs/models-and-prompts.md +301 -0
- testuary-0.2.0/docs/reference/api.md +241 -0
- testuary-0.2.0/docs/reference/cli.md +337 -0
- testuary-0.2.0/docs/reference/index.md +9 -0
- testuary-0.2.0/docs/reference/result-files.md +644 -0
- testuary-0.2.0/docs/reference/settings.md +783 -0
- testuary-0.2.0/docs/reference/tested-environments.md +108 -0
- testuary-0.2.0/examples/quickstart/README.md +112 -0
- testuary-0.2.0/examples/quickstart/java-project/.testuary/project.ini +18 -0
- testuary-0.2.0/examples/quickstart/java-project/pom.xml +25 -0
- testuary-0.2.0/examples/quickstart/java-project/src/main/java/example/Shipping.java +36 -0
- testuary-0.2.0/examples/quickstart/python-project/.testuary/project.ini +18 -0
- testuary-0.2.0/examples/quickstart/python-project/shipping.py +25 -0
- testuary-0.2.0/examples/quickstart/run_demos.ps1 +15 -0
- testuary-0.2.0/examples/quickstart/run_demos.py +39 -0
- testuary-0.2.0/examples/quickstart/run_demos.sh +12 -0
- testuary-0.2.0/examples/quickstart/start_web.py +22 -0
- testuary-0.2.0/models.example.ini +8 -0
- testuary-0.2.0/pyproject.toml +202 -0
- testuary-0.2.0/src/testuary/__init__.py +10 -0
- testuary-0.2.0/src/testuary/__main__.py +1075 -0
- testuary-0.2.0/src/testuary/_chat_log.py +256 -0
- testuary-0.2.0/src/testuary/_chat_models.py +234 -0
- testuary-0.2.0/src/testuary/_extensions.py +63 -0
- testuary-0.2.0/src/testuary/_model_access.py +129 -0
- testuary-0.2.0/src/testuary/_phase_keys.py +13 -0
- testuary-0.2.0/src/testuary/_progress.py +614 -0
- testuary-0.2.0/src/testuary/_templates/__init__.py +7 -0
- testuary-0.2.0/src/testuary/_templates/examples/README.txt +26 -0
- testuary-0.2.0/src/testuary/_templates/examples/controls.ini +8 -0
- testuary-0.2.0/src/testuary/_templates/examples/hints.json +27 -0
- testuary-0.2.0/src/testuary/_templates/examples/language-profile.json +17 -0
- testuary-0.2.0/src/testuary/_templates/examples/models.ini +12 -0
- testuary-0.2.0/src/testuary/_templates/examples/testuary_example_chat.py +15 -0
- testuary-0.2.0/src/testuary/_templates/hints.json +31 -0
- testuary-0.2.0/src/testuary/_templates/models.ini +93 -0
- testuary-0.2.0/src/testuary/_test_output.py +740 -0
- testuary-0.2.0/src/testuary/_words.py +202 -0
- testuary-0.2.0/src/testuary/agent/__init__.py +10 -0
- testuary-0.2.0/src/testuary/agent/_base.py +2270 -0
- testuary-0.2.0/src/testuary/agent/_context.py +417 -0
- testuary-0.2.0/src/testuary/agent/_edit_tool.py +733 -0
- testuary-0.2.0/src/testuary/agent/_helpers.py +1231 -0
- testuary-0.2.0/src/testuary/agent/_langgraph.py +1847 -0
- testuary-0.2.0/src/testuary/agent/_limits.py +214 -0
- testuary-0.2.0/src/testuary/agent/coverage_agent.py +1896 -0
- testuary-0.2.0/src/testuary/agent/mutation_agent.py +2134 -0
- testuary-0.2.0/src/testuary/config.ini +747 -0
- testuary-0.2.0/src/testuary/config.py +980 -0
- testuary-0.2.0/src/testuary/consolidation.py +4737 -0
- testuary-0.2.0/src/testuary/demo/__init__.py +61 -0
- testuary-0.2.0/src/testuary/demo/backend.py +493 -0
- testuary-0.2.0/src/testuary/demo/cli.py +1226 -0
- testuary-0.2.0/src/testuary/demo/data/LICENSE-tenacity.txt +212 -0
- testuary-0.2.0/src/testuary/demo/data/NOTICE.md +23 -0
- testuary-0.2.0/src/testuary/demo/data/java/LICENSE-jsonassert.txt +213 -0
- testuary-0.2.0/src/testuary/demo/data/java/NOTICE.md +25 -0
- testuary-0.2.0/src/testuary/demo/data/java/manifest.json +206 -0
- testuary-0.2.0/src/testuary/demo/data/java/models.ini +4 -0
- testuary-0.2.0/src/testuary/demo/data/java/preset.ini +22 -0
- testuary-0.2.0/src/testuary/demo/data/java/project.json +6 -0
- testuary-0.2.0/src/testuary/demo/data/java/scenario.json.gz +0 -0
- testuary-0.2.0/src/testuary/demo/data/manifest.json +215 -0
- testuary-0.2.0/src/testuary/demo/data/models.ini +4 -0
- testuary-0.2.0/src/testuary/demo/data/preset.ini +23 -0
- testuary-0.2.0/src/testuary/demo/data/project.json +7 -0
- testuary-0.2.0/src/testuary/demo/data/scenario.json.gz +0 -0
- testuary-0.2.0/src/testuary/demo/data/toy/NOTICE.md +18 -0
- testuary-0.2.0/src/testuary/demo/data/toy/manifest.json +52 -0
- testuary-0.2.0/src/testuary/demo/data/toy/models.ini +4 -0
- testuary-0.2.0/src/testuary/demo/data/toy/preset.ini +19 -0
- testuary-0.2.0/src/testuary/demo/data/toy/project.json +6 -0
- testuary-0.2.0/src/testuary/demo/data/toy/scenario.json.gz +0 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/LICENSE-tenacity.txt +212 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/NOTICE.md +27 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/hints.json +7 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/manifest.json +222 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/models.ini +4 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/preset.ini +24 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/project.json +7 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/prompt_profile.json +17 -0
- testuary-0.2.0/src/testuary/demo/data/tutorial/scenario.json.gz +0 -0
- testuary-0.2.0/src/testuary/demo/pacing.py +251 -0
- testuary-0.2.0/src/testuary/demo/recorded.py +1015 -0
- testuary-0.2.0/src/testuary/demo/scenario.py +500 -0
- testuary-0.2.0/src/testuary/demo/setup_demo.py +356 -0
- testuary-0.2.0/src/testuary/evidence.py +927 -0
- testuary-0.2.0/src/testuary/exits.py +107 -0
- testuary-0.2.0/src/testuary/generation_evidence.py +669 -0
- testuary-0.2.0/src/testuary/hints.py +311 -0
- testuary-0.2.0/src/testuary/langadapter_profile.json +86 -0
- testuary-0.2.0/src/testuary/languages/README.md +238 -0
- testuary-0.2.0/src/testuary/languages/__init__.py +580 -0
- testuary-0.2.0/src/testuary/languages/generic.py +1830 -0
- testuary-0.2.0/src/testuary/languages/mutation.py +1315 -0
- testuary-0.2.0/src/testuary/languages/preparation.py +372 -0
- testuary-0.2.0/src/testuary/loadable.py +164 -0
- testuary-0.2.0/src/testuary/measurement.py +1789 -0
- testuary-0.2.0/src/testuary/model_requests.py +803 -0
- testuary-0.2.0/src/testuary/models.py +1034 -0
- testuary-0.2.0/src/testuary/normal_run.py +92 -0
- testuary-0.2.0/src/testuary/orchestrator.py +2699 -0
- testuary-0.2.0/src/testuary/phase_state.py +1290 -0
- testuary-0.2.0/src/testuary/presentation/__init__.py +8 -0
- testuary-0.2.0/src/testuary/presentation/event_file.py +98 -0
- testuary-0.2.0/src/testuary/presentation/events.py +286 -0
- testuary-0.2.0/src/testuary/presentation/measurement.py +173 -0
- testuary-0.2.0/src/testuary/presentation/readiness.py +462 -0
- testuary-0.2.0/src/testuary/presentation/terminal.py +582 -0
- testuary-0.2.0/src/testuary/presentation/view.py +873 -0
- testuary-0.2.0/src/testuary/project.py +826 -0
- testuary-0.2.0/src/testuary/prompts/__init__.py +798 -0
- testuary-0.2.0/src/testuary/prompts/test_coverage.json +747 -0
- testuary-0.2.0/src/testuary/prompts/test_initial.json +24 -0
- testuary-0.2.0/src/testuary/prompts/test_mutation.json +777 -0
- testuary-0.2.0/src/testuary/py.typed +0 -0
- testuary-0.2.0/src/testuary/readiness.py +1047 -0
- testuary-0.2.0/src/testuary/reduction.py +919 -0
- testuary-0.2.0/src/testuary/results.py +1247 -0
- testuary-0.2.0/src/testuary/settings.py +1807 -0
- testuary-0.2.0/src/testuary/setup.py +1698 -0
- testuary-0.2.0/src/testuary/starter_salvage.py +621 -0
- testuary-0.2.0/src/testuary/starter_selection.py +243 -0
- testuary-0.2.0/src/testuary/starters.py +2986 -0
- testuary-0.2.0/src/testuary/target_phases.py +3261 -0
- testuary-0.2.0/src/testuary/tutorial/__init__.py +480 -0
- testuary-0.2.0/src/testuary/web/__init__.py +471 -0
- testuary-0.2.0/src/testuary/web/__main__.py +6 -0
- testuary-0.2.0/src/testuary/web/_data.py +925 -0
- testuary-0.2.0/src/testuary/web/_jobs.py +979 -0
- testuary-0.2.0/src/testuary/web/_launch.py +86 -0
- testuary-0.2.0/src/testuary/web/_registry.py +344 -0
- testuary-0.2.0/src/testuary/web/_results.py +1195 -0
- testuary-0.2.0/src/testuary/web/_server.py +1281 -0
- testuary-0.2.0/src/testuary/web/extensions.py +174 -0
- testuary-0.2.0/src/testuary/web/static/app.js +3545 -0
- testuary-0.2.0/src/testuary/web/static/icon.svg +1 -0
- testuary-0.2.0/src/testuary/web/static/index.html +34 -0
- testuary-0.2.0/src/testuary/web/static/style.css +498 -0
- testuary-0.2.0/tools/language_page.py +93 -0
- testuary-0.2.0/tools/reference_pages.py +299 -0
|
@@ -0,0 +1,524 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes are listed here, newest first.
|
|
4
|
+
|
|
5
|
+
## 0.2.0
|
|
6
|
+
|
|
7
|
+
First public release. Testuary writes unit test suites for Python and Java
|
|
8
|
+
code, and for C code on Linux, with a language model. It runs the generated
|
|
9
|
+
tests, measures their line and branch coverage and their mutation score,
|
|
10
|
+
asks the model to close the gaps within the limits you set, removes tests
|
|
11
|
+
that add no coverage and catch no extra mutant, and delivers the resulting
|
|
12
|
+
suite with its measurements. Language support comes from LangAdapter 0.1.0,
|
|
13
|
+
released with it as a separate package.
|
|
14
|
+
|
|
15
|
+
### New in this release
|
|
16
|
+
|
|
17
|
+
- Java projects built with Maven or Gradle and tested with JUnit 4, 5 or 6,
|
|
18
|
+
with JaCoCo for coverage and PIT for mutation.
|
|
19
|
+
- C files on Linux, through LangAdapter's C adapter, with gcov for coverage
|
|
20
|
+
and Mull for mutation. Testuary writes their tests in the language the
|
|
21
|
+
adapter names for them: C with utest.h, or Python for a C file built into
|
|
22
|
+
a Python extension. Its prompts and the Results view's place for a
|
|
23
|
+
delivered test follow the tests' language.
|
|
24
|
+
- Language support in LangAdapter, with a way to add a language. Every
|
|
25
|
+
language's runtime is found through its adapter in the same way. Without
|
|
26
|
+
a selected interpreter or `python_version`, a Python project runs with
|
|
27
|
+
its only environment, its `.venv` or the one activated in the shell. An
|
|
28
|
+
environment Testuary runs in outside the project is set aside, so the
|
|
29
|
+
project's `.venv` runs the tests. A `[project]` setting that two
|
|
30
|
+
installed languages declare takes the kind, default and host-path rule of
|
|
31
|
+
the first of them by name.
|
|
32
|
+
- Commands to set up and check a project before a run: `testuary init`,
|
|
33
|
+
`testuary setup`, `testuary check`, `testuary new-language` and
|
|
34
|
+
`testuary tutorial`.
|
|
35
|
+
- Settings in your user directory and in a `.testuary` directory in each
|
|
36
|
+
project, with a reference of every setting.
|
|
37
|
+
- Several repetitions of each file with `--runs`, each repetition's results
|
|
38
|
+
in its own directory, and a summary across them.
|
|
39
|
+
- Progress views in the terminal, and an end-of-run summary saved with the
|
|
40
|
+
results.
|
|
41
|
+
- An offline demonstration that plays back a recorded run without a model:
|
|
42
|
+
`testuary --demo`. It refuses a wrong combination of run options, such as
|
|
43
|
+
`--runs 2 --run 1`, with exit status 2 before the demonstration starts.
|
|
44
|
+
- A quickstart directory, `examples/quickstart` in the repository and the
|
|
45
|
+
source distribution: scripts that play the demonstrations and start the
|
|
46
|
+
web interface, and two small example projects to run with your own model.
|
|
47
|
+
- Prompt profiles and hints files, which change what the model is told. A
|
|
48
|
+
prompt profile may name every placeholder the prompts fill, among them
|
|
49
|
+
`$lang_test_decl_full`, `$lang_test_collection_rule` and
|
|
50
|
+
`$language_level`.
|
|
51
|
+
- A local web interface, started with `testuary-web`. It shows the
|
|
52
|
+
projects, models and settings, tests a model's connection, starts runs
|
|
53
|
+
and checks, follows a run's progress and shows its results and the
|
|
54
|
+
demonstrations. A test file of a delivered suite can be copied from the
|
|
55
|
+
results into its project. A project can be added from its GitHub HTTPS
|
|
56
|
+
address, which git clones into a directory you choose. The `[web]`
|
|
57
|
+
settings choose its port, 8765 by default, and who may use it, every
|
|
58
|
+
program on the computer by default.
|
|
59
|
+
|
|
60
|
+
What the model sees:
|
|
61
|
+
|
|
62
|
+
- The file under test is read as UTF-8 with undecodable bytes replaced and
|
|
63
|
+
CRLF or CR read as LF, in a local and a container run alike, so the model
|
|
64
|
+
and the measurement see the same text in both.
|
|
65
|
+
- Each loop starts by reading the tests, then gets the current measurement,
|
|
66
|
+
then edits. A tool that is not available yet says what makes it
|
|
67
|
+
available.
|
|
68
|
+
- The rules for valid tests are each language's own and name what a test
|
|
69
|
+
may read. Python tests may read a module's `__dict__` and `vars()`.
|
|
70
|
+
- A test that the time limit cut off is named, and the model can fix it
|
|
71
|
+
without losing its other changes.
|
|
72
|
+
- The tool calls of one answer run one after another, in order, so that
|
|
73
|
+
several edits in one answer all take effect. One edit call can carry a
|
|
74
|
+
list of edits.
|
|
75
|
+
- A tool that fails for a temporary reason gets an answer, and the loop
|
|
76
|
+
continues.
|
|
77
|
+
- Text that is cut for length is marked, and every such limit is a setting.
|
|
78
|
+
- A write outside the loop's test output directory is refused, and the
|
|
79
|
+
refusal the model receives names the field it checks, `test_output_dir`.
|
|
80
|
+
- After a passing normal test run, a run under the coverage tool that
|
|
81
|
+
fails, ends without a test result or does not finish is the tool's
|
|
82
|
+
refusal, in both loops. The reply to the loop's measurement names the
|
|
83
|
+
tests that stopped the tool where it can, and it counts as a failed test
|
|
84
|
+
run. Only a normal test run that does not finish is a timeout of the
|
|
85
|
+
suite.
|
|
86
|
+
- A passing test run of a Python file can carry a STANDS ALONE note: the
|
|
87
|
+
tests that left the module under test or a library setting changed,
|
|
88
|
+
and with `report_reverse_order_failures = true` in `[coverage]` or
|
|
89
|
+
`[mutation]` the tests that fail when the file runs as delivered in
|
|
90
|
+
reverse order. After a note that names tests, the model can change the
|
|
91
|
+
tests before it measures, and a loop at its goal ends only after a
|
|
92
|
+
passing run without such a note, within its limits.
|
|
93
|
+
|
|
94
|
+
What is measured:
|
|
95
|
+
|
|
96
|
+
- By default each starter suite gets its own mutation score from a run of
|
|
97
|
+
that suite alone. `[starter] suite_mutation` can derive the starter
|
|
98
|
+
suites' scores from one run of all of them instead, and
|
|
99
|
+
`[main] reduced_suite_mutation` the reduced suite's score at a phase end
|
|
100
|
+
from the merged suite's run.
|
|
101
|
+
- With `per_mutant_timeout_seconds = auto`, the per-mutant time limit for
|
|
102
|
+
C is `per_mutant_timeout_factor` times the mutation tool's own run of the
|
|
103
|
+
passing suite, each test in a process of its own as a mutant's tests
|
|
104
|
+
run. A mutant that no test kills runs all its tests and survives, in a
|
|
105
|
+
large suite too. A fixed value bounds all of a mutant's tests together.
|
|
106
|
+
Logs state a C time limit per mutant.
|
|
107
|
+
- One normal test run decides whether tests pass. For Python it is the
|
|
108
|
+
run as delivered: each test file in its own session, with fixed seeds
|
|
109
|
+
set when the session starts and nothing reset between tests.
|
|
110
|
+
- Tests are removed only where no model is active. The starter suites'
|
|
111
|
+
cleanup removes failing tests. When a suite passes its normal run and a
|
|
112
|
+
coverage or mutation measurement then fails and names tests, in the
|
|
113
|
+
starter phase or at a phase end, Testuary removes the named tests, runs
|
|
114
|
+
the suite again and records each removal, as
|
|
115
|
+
[Tests removed after a failing measurement](https://k-kammler.github.io/testuary/explanation/measurement-and-recovery/#tests-removed-after-a-failing-measurement)
|
|
116
|
+
describes. A loop removes no test from the suite it inherits, and at the
|
|
117
|
+
end of each phase Testuary chooses among the suites that pass with
|
|
118
|
+
complete numbers.
|
|
119
|
+
- A starter suite that the coverage or mutation tool cannot measure first
|
|
120
|
+
loses the tests the failure names. When that does not help, it is
|
|
121
|
+
generated again within its attempts, or left out.
|
|
122
|
+
- Finishing, the fourth phase, takes the delivered suite's coverage from
|
|
123
|
+
the end of the phase that measured it and its mutation results from the
|
|
124
|
+
end of the mutation phase, and measures only what those did not. With
|
|
125
|
+
`[finishing] measure_mutation = false`, Finishing records no mutation
|
|
126
|
+
score and judges no mutation minimum, also when the mutation phase
|
|
127
|
+
measured the delivered suite.
|
|
128
|
+
- Every measurement runs in a private copy of the project.
|
|
129
|
+
- Every mutant has an identifier that stays the same from run to run.
|
|
130
|
+
- A Python test's coverage includes its fixtures' setup and teardown. A
|
|
131
|
+
fixture shared by several tests counts for the first of them, and its
|
|
132
|
+
teardown for the last.
|
|
133
|
+
- Per-test times are precise, and the reduction does not charge a run's
|
|
134
|
+
one-time setup to its first test.
|
|
135
|
+
- Of several suites with equal scores, the end of a phase keeps the
|
|
136
|
+
reduced suite, or else the fastest. When their times do not tell them
|
|
137
|
+
apart, or a test fails in a timing run, the earlier one stays. A tie that
|
|
138
|
+
no candidate's time could break records the input order as its basis,
|
|
139
|
+
and the results page says so.
|
|
140
|
+
- Candidate suites are merged by LangAdapter, which checks the merged file
|
|
141
|
+
with the project's own Python or Java tools. When it refuses a pair,
|
|
142
|
+
nothing is merged and the better single candidate is kept. A merge whose
|
|
143
|
+
tools give no answer is an error, not a refusal. A Python merge keeps one
|
|
144
|
+
copy of a top-level statement both suites hold identically, such as a
|
|
145
|
+
repeated docstring or import guard, where one copy cannot change what a
|
|
146
|
+
test sees, and a definition dropped as shared takes its blank lines along.
|
|
147
|
+
- When the merged suite's per-test evidence covers other lines or branches
|
|
148
|
+
than its own coverage run, as C's tests can when each runs alone, the
|
|
149
|
+
run's coverage stays the suite's coverage, the reduction works on the
|
|
150
|
+
per-test evidence, and the selection audit records both sides
|
|
151
|
+
(`merged_evidence_difference`). A starter suite delivered alone takes the
|
|
152
|
+
coverage of its run under the delivered name
|
|
153
|
+
(`delivered_coverage_difference`). A candidate whose per-test evidence disagrees with its own coverage run is
|
|
154
|
+
scored on that run, without the evidence, with the disagreement as its
|
|
155
|
+
reason.
|
|
156
|
+
- An exact reduction whose time limit runs out before a tier of its
|
|
157
|
+
tie-break records the failure `time_limit_exhausted` with that tier. A
|
|
158
|
+
CBC that cannot run ends the exact reduction with the failure
|
|
159
|
+
`solver_error` and the tier it stopped in, and the phase end writes
|
|
160
|
+
`consolidation_failure.json`. The recorded error names the error type,
|
|
161
|
+
without the solver's message, which can hold paths of your machine.
|
|
162
|
+
- A final mutation measurement that did not finish reports no score.
|
|
163
|
+
- A mutation measurement cut by its time limit counts its untested mutants
|
|
164
|
+
as not killed, so its score is a lower bound. The mutation phase works
|
|
165
|
+
with that score, and a cut final mutation measurement records it beside
|
|
166
|
+
its missing score.
|
|
167
|
+
- When a loop's test run, a mutation measurement or the final coverage run
|
|
168
|
+
reaches its time limit, the run's summary and the end-of-run summary
|
|
169
|
+
carry a warning that names the setting and its value.
|
|
170
|
+
|
|
171
|
+
How runs behave:
|
|
172
|
+
|
|
173
|
+
- A temporary failure of the model service costs no model request.
|
|
174
|
+
- A run ends with exit status 1 when a file failed.
|
|
175
|
+
- A run is refused before any work, as a refused setting is, when its
|
|
176
|
+
required tools are missing or when `--reuse` finds saved work missing for
|
|
177
|
+
a target. It ends with exit status 2, and a program that calls
|
|
178
|
+
`run_pipeline` receives `testuary.settings.ConfigurationError`. For
|
|
179
|
+
missing tools the setup report names what is missing and how
|
|
180
|
+
to fix it, `--events` records that report, and the run's last event reads
|
|
181
|
+
"Setup refused".
|
|
182
|
+
- A target outside the source directory is refused before the run, with or
|
|
183
|
+
without `--resume`, as a file no language handles is. A target whose file
|
|
184
|
+
name holds a comma runs as one target. A target's test file is the one
|
|
185
|
+
its name gives (`test_<stem>.py`, `<Stem>Test.java`), so a reused run
|
|
186
|
+
never takes another target's file from the shared tests directory.
|
|
187
|
+
- A resumed run reuses its saved starter suites, and a resume with changed
|
|
188
|
+
measurement settings is refused. A resume compares the SHA-256 of a model
|
|
189
|
+
provider's file or a language's file and refuses one that changed, naming
|
|
190
|
+
it. A change to one of Finishing's own settings, such as
|
|
191
|
+
`[finishing] measure_lint`, is no refusal: the resume measures Finishing
|
|
192
|
+
again, with no model request.
|
|
193
|
+
- `--resume` judges each repetition's phases against the canonical source
|
|
194
|
+
that repetition's phase recorded when it started: `phase_state.json`
|
|
195
|
+
holds `target_file_path` and `target_source_sha256` in each repetition's
|
|
196
|
+
block of a phase. A repetition made from an earlier source counts as
|
|
197
|
+
unfinished, also after a later repetition ran from the edited source.
|
|
198
|
+
- A resume or reuse knows the model service by its provider and model
|
|
199
|
+
name. Another address, such as a local proxy on another port, is a
|
|
200
|
+
warning.
|
|
201
|
+
- A resumed or reused run that keeps its earlier `tests.csv` row and
|
|
202
|
+
measures mutation in Finishing again writes each of Finishing's metrics
|
|
203
|
+
columns afresh, empty where the new measurement gave no value, and every
|
|
204
|
+
write with a phase result writes `error` afresh. An earlier attempt's
|
|
205
|
+
lower bound, untested count, kill counts or error text does not stay
|
|
206
|
+
beside the new result in `tests.csv` or `summary.json`. A write that
|
|
207
|
+
measured no mutation, such as a timing pass, keeps the earlier scores.
|
|
208
|
+
- A loop that ends at its request, token or time limit names that limit,
|
|
209
|
+
such as TOKEN LIMIT REACHED, an ordinary end, not FAILED.
|
|
210
|
+
- The coverage and the mutation phase give a stop the same status. A
|
|
211
|
+
coverage phase that `max_final_answers_without_measurement` stopped, with
|
|
212
|
+
the stop reason `final_answers_without_measurement`, records that it
|
|
213
|
+
stopped before its goal, as the mutation phase does: its suite is green
|
|
214
|
+
and measured, and its target's row holds the mutation phase's results.
|
|
215
|
+
Repeated test timeouts end either phase before its goal when the suite
|
|
216
|
+
it delivers is measured, else with failing tests. A mutation phase whose
|
|
217
|
+
suite runs longer than its time limit before any edit records
|
|
218
|
+
`skipped_inherited_slow` ("skipped: the suite it received runs longer
|
|
219
|
+
than its time limit"), a failed phase with its reason.
|
|
220
|
+
- A run whose phase named by `--stop-after-phase` is not reached ends with
|
|
221
|
+
its `Stopped:` line, without a traceback.
|
|
222
|
+
- A command refused after its arguments are read leaves an earlier
|
|
223
|
+
`--events` file as it was.
|
|
224
|
+
|
|
225
|
+
What a run records:
|
|
226
|
+
|
|
227
|
+
- A mutation run that the mutation tool's origin check stops, because a
|
|
228
|
+
test fails without a mutant, is logged with its stopping tests and
|
|
229
|
+
recorded in the phase end's `consolidation_record.json` as
|
|
230
|
+
`mutation_origin_stops`.
|
|
231
|
+
- Each phase's `<phase>_duration_s` counts only the work the row keeps: an
|
|
232
|
+
attempt that crashed, or that failed in a phase other than the starter
|
|
233
|
+
phase, counts for nothing there, while `duration_s_target` and the
|
|
234
|
+
`time_*` columns keep a failed attempt's time. `duration_s_target` and
|
|
235
|
+
every `time_*` column are empty when the repetition crashed, or when a
|
|
236
|
+
resume reused a phase or the saved starter suites of a stopped or crashed
|
|
237
|
+
repetition, and `starter_duration_s` too in the second case, since the
|
|
238
|
+
time of that work is not known.
|
|
239
|
+
- `merged_tests` in `consolidation_record.json` and `coverage_merged_tests`
|
|
240
|
+
in `tests.csv` are empty where no merged suite was counted.
|
|
241
|
+
`mutation_merged_tests` and `mutation_delivered_tests` hold 0 for an
|
|
242
|
+
empty count.
|
|
243
|
+
- A phase end's `consolidation_record.json` tells a reduced suite that was
|
|
244
|
+
never built (status `absent`) from one that does not pass (`failing`) or
|
|
245
|
+
was not fully measured (`incomplete`), each with its reason. An absent
|
|
246
|
+
one records no tests, removed tests or source hash, and the results page
|
|
247
|
+
shows it as not built. A merged suite whose run wrote no exact coverage
|
|
248
|
+
matrix is recorded as `incomplete` with that reason.
|
|
249
|
+
- An inherited phase end's `consolidation_record.json` holds its keys in
|
|
250
|
+
the order of a measured phase end's record, followed by `inherited_from`,
|
|
251
|
+
`inherited_artifacts`, `measured` and `measurement_calls`.
|
|
252
|
+
`reduced_coverage_matrix.json` holds the keys of
|
|
253
|
+
`merged_coverage_matrix.json` in their order, followed by `kept_tests`
|
|
254
|
+
and `derive_kept_tests`, and a time map with no measured time is null.
|
|
255
|
+
`selected_coverage_matrix.json` holds per-test times and solver costs
|
|
256
|
+
only for a merged or reduced winner, and a candidate scored on one
|
|
257
|
+
coverage run records the target's statement and branch totals that run
|
|
258
|
+
measured.
|
|
259
|
+
- A phase end's selection audit names the axes the selection compared in
|
|
260
|
+
`selector`, records `runtime_tie_break_candidate_order` as null when no
|
|
261
|
+
timing tie-break ran, and records an input suite that does not pass with
|
|
262
|
+
status `incomplete`. The candidates' times in the audit are their tests'
|
|
263
|
+
measured times summed, or null where a test was not timed.
|
|
264
|
+
- A phase end's `collision_resolution` lists in `renamed_names` the names
|
|
265
|
+
the merge's collision rename gave, when it gave any, and `rename_twins`
|
|
266
|
+
counts the copies of a test that the rename kept beside the original.
|
|
267
|
+
`consolidation_record.json` keeps `collision_resolution` when the merged
|
|
268
|
+
suite fails validation.
|
|
269
|
+
- A phase end's `reduction_solver_metadata` names CBC by its version
|
|
270
|
+
alone, without the solver's path on your machine. With
|
|
271
|
+
`reduction_solver = greedy` it records the measured and imputed costs
|
|
272
|
+
the reduction weighed, as the exact reduction does.
|
|
273
|
+
- The kill matrices a run writes follow LangAdapter's kill-matrix schema
|
|
274
|
+
version 3. A mutant's status is `killed`, `timeout`, `non_viable` or
|
|
275
|
+
`survived`, a per-suite verdict of a suite that measured nothing is
|
|
276
|
+
`untested`, and a summary counts `detected` and `non_viable`.
|
|
277
|
+
`starter_pool_attribution.json` counts each suite's verdicts under these
|
|
278
|
+
words.
|
|
279
|
+
- The run log tags the starter phase's lines `[starter]`, the mutation runs
|
|
280
|
+
of the starter suites' pool and of the phase ends `[finishing]`, and
|
|
281
|
+
Finishing's lines `[finishing]` or `[metrics]`.
|
|
282
|
+
|
|
283
|
+
Setting up a project:
|
|
284
|
+
|
|
285
|
+
- `testuary setup` and the web interface list the installed languages in
|
|
286
|
+
name order.
|
|
287
|
+
- Setup finds a project's source files by each language's own rules. Test
|
|
288
|
+
files, the files a language names as no sources, such as Python's
|
|
289
|
+
`setup.py`, and empty files are left out. For Python a `tests.py` and a
|
|
290
|
+
test file named in another case count as tests, for Java
|
|
291
|
+
`*TestCase.java` and `*IT.java` do.
|
|
292
|
+
- `testuary check` reports whether each target is in the project
|
|
293
|
+
directory, as the language's check finds it: verified, or missing setup
|
|
294
|
+
with that check's reason. A run checks the targets itself before any
|
|
295
|
+
work.
|
|
296
|
+
- `testuary check --verify` tries the project in a copy as a run does,
|
|
297
|
+
without running a test: for Java it resolves the build without
|
|
298
|
+
downloading, compiles and finds the tests and starts the measurement
|
|
299
|
+
tools a run needs, and for Python it collects the tests and checks where
|
|
300
|
+
they import the project from. The first run tries the build of a project
|
|
301
|
+
with its own build wrapper, which may download its build tool. The report
|
|
302
|
+
names these parts, such as the build of your project and the collection
|
|
303
|
+
of your project's tests.
|
|
304
|
+
- The check report gives each operation of a language its own name, such
|
|
305
|
+
as removing single tests from a test file and counting the tests and
|
|
306
|
+
assertions of a test file.
|
|
307
|
+
- `testuary check` refuses an unknown phase in `--reuse` or
|
|
308
|
+
`--stop-after-phase` in its own report, with exit status 2, as JSON with
|
|
309
|
+
`--json`.
|
|
310
|
+
- A runtime that a project's settings request, such as `python_version`, is
|
|
311
|
+
provisioned by that language's adapter and kept in a directory per
|
|
312
|
+
language in Testuary's cache (`venvs/<language>/<project>`).
|
|
313
|
+
`testuary provision` prepares what every installed language's settings
|
|
314
|
+
request and prints each runtime, or says that none is requested. It
|
|
315
|
+
writes its errors to standard error.
|
|
316
|
+
|
|
317
|
+
Models and their keys:
|
|
318
|
+
|
|
319
|
+
- `testuary init` writes the example aliases in `models.ini` as comments,
|
|
320
|
+
so a new catalogue names no model until you add one.
|
|
321
|
+
- An alias that `testuary setup model` or the Models page writes for
|
|
322
|
+
`ollama`, `openai` or `vllm` holds the address you confirmed, the
|
|
323
|
+
packaged one included. It keeps that address when a global `config.ini`
|
|
324
|
+
sets a gateway in that service's section, so its key goes only to the
|
|
325
|
+
confirmed address. Only `[api] api_url`, which sets the address of every
|
|
326
|
+
model, moves it. To use the gateway, give its address as the alias's
|
|
327
|
+
address.
|
|
328
|
+
- An alias's `chat_model` names a client factory as a Python file or a
|
|
329
|
+
module before its attribute, such as `my_chat.py:factory`. A file is relative to
|
|
330
|
+
the directory of the file that names it and needs no installation, and
|
|
331
|
+
the example `testuary init --examples` copies is one such file.
|
|
332
|
+
- A project's settings file may not set `[api] api_url` or the `api_url` of
|
|
333
|
+
a provider's section, which decide where the model's key goes. A run of
|
|
334
|
+
such a project is refused at its start with exit status 2. Set the
|
|
335
|
+
address in your global `config.ini`, in `models.ini`, in the environment
|
|
336
|
+
or on the command line.
|
|
337
|
+
- A variable ending in `_API_KEY` is always a model service's key and never
|
|
338
|
+
a setting, so a service may be named like a settings section, such as
|
|
339
|
+
`main_x` with `TESTUARY_MAIN_X_API_KEY`.
|
|
340
|
+
- A stored model key goes only to its model's service address, in a Test
|
|
341
|
+
or a run, and no answer of `testuary-web`'s server holds it. The Models
|
|
342
|
+
page says whether a model's key is stored in `models.ini` or comes from
|
|
343
|
+
an environment variable, and Edit changes a stored key.
|
|
344
|
+
- When the record of a failed model request cannot be written, the
|
|
345
|
+
traceback in the run log and on the console leaves out the model
|
|
346
|
+
service's error, which can hold the request and its key.
|
|
347
|
+
- A `max_output_tokens` that is no nonnegative integer is refused wherever
|
|
348
|
+
it is set, in a phase section or a model alias, before any request, as
|
|
349
|
+
the model's own resolution refuses it. The agents' and the starter
|
|
350
|
+
phase's model address, model name and key come from that same
|
|
351
|
+
resolution.
|
|
352
|
+
- Starter suite requests carry no temperature unless you set
|
|
353
|
+
`[starter] temperature`, which is empty by default, so the provider's
|
|
354
|
+
default applies.
|
|
355
|
+
|
|
356
|
+
Extensions:
|
|
357
|
+
|
|
358
|
+
- `[main] extensions` names the extensions a run and `testuary-web` load:
|
|
359
|
+
Python files, such as `.testuary/my_extension.py`, and installed modules.
|
|
360
|
+
An extension adds a language with its top-level `langadapter_language`,
|
|
361
|
+
model hints with `testuary_hints` and `testuary-web` pages with
|
|
362
|
+
`testuary_web`. Each settings file adds its items to those of the files
|
|
363
|
+
before it, and a file item is relative to the directory of the settings
|
|
364
|
+
file that names it. `testuary check` finds each item without running it,
|
|
365
|
+
and the tool check loads it. The Settings page edits each file's own
|
|
366
|
+
items and shows the combined list. `testuary.settings.add_extension` and
|
|
367
|
+
`remove_extension` change the items of the user's `config.ini`, and
|
|
368
|
+
`extension_items` names each item with its source. The generation origin
|
|
369
|
+
records the extensions and their hints, and `testuary tutorial` teaches
|
|
370
|
+
the first extension as one file.
|
|
371
|
+
- A `testuary.web.WebExtension` adds tabs, result views, settings groups,
|
|
372
|
+
jobs and read-only routes to `testuary-web`. It can set the product name
|
|
373
|
+
and the header's Guide link, rename, move and hide Testuary's tabs, and
|
|
374
|
+
put a tab of its own in the place of one of them. Testuary's links to a
|
|
375
|
+
replaced tab open the extension's, its links to a hidden tab are left
|
|
376
|
+
out, and its pages name a renamed or replaced tab as the header does. The
|
|
377
|
+
extension guide lists every part an extension can change. A job's setup
|
|
378
|
+
report in Testuary's readiness format shows on its page as a check's
|
|
379
|
+
report does. `testuary.web.TOOL_CHECK_LABEL`, which `/api/state` carries
|
|
380
|
+
as `tool_check_label`, is the label of Testuary's tool check for a
|
|
381
|
+
replacing Check page.
|
|
382
|
+
- An extension's page shows a Testuary run folded, as Testuary's own pages
|
|
383
|
+
draw it, with `testuaryWeb.nestedRun`: a run's event file as its job's
|
|
384
|
+
page shows it, followed while it is written, or a run directory as
|
|
385
|
+
Results shows it. A job kind names the Testuary runs its jobs start with
|
|
386
|
+
`nested_runs`, each a `testuary.web.NestedRun`, and its job's page folds
|
|
387
|
+
each into its phase.
|
|
388
|
+
- An extension's settings group can name the Testuary settings files its
|
|
389
|
+
runs in a project read instead of the global and the project's settings
|
|
390
|
+
files, its own file last (`SettingsGroup.testuary_settings_files`). The
|
|
391
|
+
Settings page offers that list as a choice of the project, shows every
|
|
392
|
+
setting with the file it comes from as those runs read it, and changes
|
|
393
|
+
the last file through Testuary's validated update. A last file outside
|
|
394
|
+
the project directory is read-only.
|
|
395
|
+
- A settings group can also name the model catalogue files its runs in a
|
|
396
|
+
project read (`testuary_models_files`), its own user directory
|
|
397
|
+
(`extension_user_dir`) and a function that makes its runs read its own
|
|
398
|
+
file alone (`end_models_import`). The Models page shows those models for
|
|
399
|
+
the project, changes the list's last file, and removes a model an
|
|
400
|
+
earlier file defines by ending the import after a question. After the
|
|
401
|
+
Settings page or the Models page saves a file of a group's list, it asks
|
|
402
|
+
the group again, and a group that raises puts the file back.
|
|
403
|
+
|
|
404
|
+
Programs of your own:
|
|
405
|
+
|
|
406
|
+
- `testuary.settings.resolve_settings` takes `models_files`, an ordered
|
|
407
|
+
list of model catalogue files read instead of the user directory's
|
|
408
|
+
`models.ini`, a later file's section overriding the same section of an
|
|
409
|
+
earlier one. A file in the project directory may set neither `api_url`
|
|
410
|
+
nor `api_key_env`, which decide where the model's key goes.
|
|
411
|
+
- `testuary.settings.check_project_settings_file` and
|
|
412
|
+
`testuary.settings.check_project_catalogue_file` check one settings or
|
|
413
|
+
model catalogue file that is not your own, as Testuary checks a
|
|
414
|
+
project's: a settings file may not set an `api_url`, and a catalogue file
|
|
415
|
+
neither an `api_url` nor an `api_key_env`. A program that passes such
|
|
416
|
+
files to its runs calls them for each.
|
|
417
|
+
- `testuary.models.catalogue_chat_model` gives your own program a LangChain
|
|
418
|
+
chat model for an alias of your model catalogue, or with `provider` for a
|
|
419
|
+
model that provider serves, at the provider's address and with its key.
|
|
420
|
+
It resolves the model as a Testuary run resolves `[api] model` and
|
|
421
|
+
`[api] provider`, retries temporary failures as Testuary does, and
|
|
422
|
+
records each request and each model's settings in a directory you name.
|
|
423
|
+
The model, its records and its errors hold no key. Its `config_files`,
|
|
424
|
+
`project_settings` and `models_files` choose the settings files and the
|
|
425
|
+
model catalogue as for `resolve_settings`, so that a program whose runs
|
|
426
|
+
name their own files gets models that read the same files.
|
|
427
|
+
`testuary.models.REQUEST_VALUES` holds the request values it accepts.
|
|
428
|
+
- `testuary.setup.models`, `set_model` and `remove_model` take
|
|
429
|
+
`models_files`, the list of model catalogue files `resolve_settings`
|
|
430
|
+
reads, and change its last file only, writing a model an earlier file
|
|
431
|
+
defines into it whole and no key. Each `ModelEntry` names its `file`. A
|
|
432
|
+
catalogue change is refused when it would leave a model that resolves
|
|
433
|
+
unable to resolve.
|
|
434
|
+
- `testuary.readiness.ReadinessLimit` and
|
|
435
|
+
`testuary.presentation.present_readiness` are on the API reference
|
|
436
|
+
beside `RunReadiness`, so a program builds and prints a setup report as
|
|
437
|
+
`testuary check` does.
|
|
438
|
+
- `testuary.presentation.EventFile` writes the event file `--events`
|
|
439
|
+
writes, for a program of your own, with `EVENT_FILE_SCHEMA`, `RunPlan`
|
|
440
|
+
and `RunCompletion` (the API reference's "Event files").
|
|
441
|
+
`run_pipeline(..., events=PATH)` writes a program's run into an event
|
|
442
|
+
file as `--events` does, with the setup report, the plan and the end of
|
|
443
|
+
the run, so a page that folds the run shows its phases and its summary.
|
|
444
|
+
The event file has `schema_version` 3. Its plan record names Finishing
|
|
445
|
+
`finishing`, Finishing's operation is `step/finishing`, and the step that
|
|
446
|
+
ends each phase is labeled Consolidation.
|
|
447
|
+
- `testuary.generation_evidence.read_suite_slots(manifest, suites_root)`
|
|
448
|
+
and `read_cleaned_source(reference, suites_root)` read the generation
|
|
449
|
+
records.
|
|
450
|
+
- `testuary.results.recorded_source_sha256` reads the SHA-256 of a target
|
|
451
|
+
file's raw bytes that a run recorded, so a program can check that an
|
|
452
|
+
earlier run tested the same file before it reuses that run's suite.
|
|
453
|
+
- `testuary.presentation.read_completion` reads a run's end-of-run
|
|
454
|
+
summary, `results/completion.json`, as its `RunCompletion` and the
|
|
455
|
+
provenance of its values, so a program reads the file as Testuary's
|
|
456
|
+
console and web interface do.
|
|
457
|
+
|
|
458
|
+
### Requirements
|
|
459
|
+
|
|
460
|
+
- Python 3.10 to 3.14 for Testuary, on Windows or Linux. macOS is expected
|
|
461
|
+
to work, but this release has not run on a Mac.
|
|
462
|
+
- For a Python project: pytest, coverage and Cosmic Ray in the project's
|
|
463
|
+
environment. Tested with pytest 8.1.1, coverage 7.6.8 and Cosmic Ray
|
|
464
|
+
8.4.1, on Python 3.14 and 3.15 with pytest 8.4.2 and coverage 7.16.2,
|
|
465
|
+
and project Python 3.10 to 3.15. Python 3.9 is expected to work.
|
|
466
|
+
Testuary itself is not tested on Python 3.15, where tree-sitter, which
|
|
467
|
+
LangAdapter uses, has no wheels, so it installs there only with a C
|
|
468
|
+
compiler.
|
|
469
|
+
- For a Java project: JDK 11 or newer, tested with 11, 17, 21, 25, 26 and
|
|
470
|
+
27, and the project's Maven or Gradle build. LangAdapter downloads the
|
|
471
|
+
JUnit launcher, JaCoCo and PIT on first use.
|
|
472
|
+
- For a C project, on Linux: LangAdapter's extra `langadapter[c]`, the
|
|
473
|
+
project's GCC with its gcov, and Mull with the clang of its LLVM release
|
|
474
|
+
for mutation. Tested with GCC 13.3 on Ubuntu 24.04 under WSL.
|
|
475
|
+
- A model service: OpenAI, an OpenAI-compatible service such as vLLM, or
|
|
476
|
+
Ollama.
|
|
477
|
+
|
|
478
|
+
[Tested environments](https://k-kammler.github.io/testuary/reference/tested-environments/)
|
|
479
|
+
lists the tested combinations in detail.
|
|
480
|
+
|
|
481
|
+
### Known limitations
|
|
482
|
+
|
|
483
|
+
- The model sees the file under test, not the classes it uses. Tests of
|
|
484
|
+
code that depends on other classes of the project can be harder for it
|
|
485
|
+
to write.
|
|
486
|
+
- A test can leave the module under test changed for the tests after it.
|
|
487
|
+
For Python the test run names such a test in a note, and a loop at its
|
|
488
|
+
goal waits for a passing run without one. When the loop's request, token
|
|
489
|
+
or time limit ends it first, the test stays. The measured scores remain
|
|
490
|
+
valid, because they come from running the delivered suite.
|
|
491
|
+
- A test that passes in a normal run can fail while coverage or mutation
|
|
492
|
+
is measured, for example a Java test that counts a class's members by
|
|
493
|
+
reflection, since the measurement tools add hidden members. A tool that
|
|
494
|
+
cannot measure a suite gives it no number. When the failure names tests,
|
|
495
|
+
in the starter phase or at a phase end, those tests are removed and the
|
|
496
|
+
suite is measured again. Otherwise the starter suite is generated again,
|
|
497
|
+
a phase end's suite is no candidate, and in the loops the model is told
|
|
498
|
+
and it counts as a failed test run.
|
|
499
|
+
- With the default settings, every suite a phase can hand on has run under
|
|
500
|
+
both measurement tools. With `phase_end_mutation = false` the candidates
|
|
501
|
+
at the end of the starter and coverage phases are measured under the
|
|
502
|
+
coverage tool only, the end of the mutation phase still measures
|
|
503
|
+
mutation, and with `reduced_suite_mutation = derive` the reduced suite's
|
|
504
|
+
mutation score is derived from the merged suite's run. The results page
|
|
505
|
+
labels that derived score.
|
|
506
|
+
- Adding a language takes a LangAdapter adapter and no Testuary code, as
|
|
507
|
+
[Make a language ready for Testuary](https://k-kammler.github.io/testuary/extending/languages/)
|
|
508
|
+
describes.
|
|
509
|
+
- Some Python source operations read the project's code with the Python
|
|
510
|
+
that runs Testuary, so code written for a newer Python than that one can
|
|
511
|
+
be refused.
|
|
512
|
+
- Gradle projects were not tested in full runs. A Java project with
|
|
513
|
+
both Maven and Gradle build files is refused.
|
|
514
|
+
- Conda, uv, Poetry and Pipenv environments were not tested beyond their
|
|
515
|
+
interpreter.
|
|
516
|
+
- On Windows, a project's commands cannot run in a working directory longer
|
|
517
|
+
than 258 characters.
|
|
518
|
+
|
|
519
|
+
### Documentation
|
|
520
|
+
|
|
521
|
+
The guides are on the
|
|
522
|
+
[documentation site](https://k-kammler.github.io/testuary/),
|
|
523
|
+
starting with
|
|
524
|
+
[Getting started](https://k-kammler.github.io/testuary/getting-started/).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use Testuary, please cite it as below."
|
|
3
|
+
type: software
|
|
4
|
+
title: "Testuary"
|
|
5
|
+
abstract: >-
|
|
6
|
+
Testuary writes unit test suites for Python and Java code, and for C code
|
|
7
|
+
on Linux, with a language model. It measures the generated tests' line and branch coverage and how
|
|
8
|
+
many injected faults (mutants) they detect, asks the model to close the
|
|
9
|
+
gaps, and reduces the suite to the tests that matter.
|
|
10
|
+
authors:
|
|
11
|
+
- family-names: "Kammler"
|
|
12
|
+
given-names: "Kevin"
|
|
13
|
+
license: MIT
|
|
14
|
+
version: "0.2.0"
|
|
15
|
+
keywords:
|
|
16
|
+
- test generation
|
|
17
|
+
- mutation testing
|
|
18
|
+
- large language models
|
|
19
|
+
- automated testing
|
|
20
|
+
repository-code: "https://github.com/k-kammler/testuary"
|
|
21
|
+
url: "https://github.com/k-kammler/testuary"
|
testuary-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kevin Kammler
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|