testuary 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. testuary-0.2.0/.gitignore +14 -0
  2. testuary-0.2.0/CHANGELOG.md +524 -0
  3. testuary-0.2.0/CITATION.cff +21 -0
  4. testuary-0.2.0/LICENSE +21 -0
  5. testuary-0.2.0/PKG-INFO +217 -0
  6. testuary-0.2.0/README.md +147 -0
  7. testuary-0.2.0/THIRD_PARTY_NOTICES.md +51 -0
  8. testuary-0.2.0/docs/contributing.md +65 -0
  9. testuary-0.2.0/docs/explanation/code-organization.md +194 -0
  10. testuary-0.2.0/docs/explanation/measurement-and-recovery.md +245 -0
  11. testuary-0.2.0/docs/extending/hints.md +100 -0
  12. testuary-0.2.0/docs/extending/index.md +39 -0
  13. testuary-0.2.0/docs/extending/languages.md +326 -0
  14. testuary-0.2.0/docs/extending/model-provider.md +290 -0
  15. testuary-0.2.0/docs/extending/prompt-profile.md +138 -0
  16. testuary-0.2.0/docs/extending/web.md +811 -0
  17. testuary-0.2.0/docs/getting-started.md +390 -0
  18. testuary-0.2.0/docs/guide/cluster.md +317 -0
  19. testuary-0.2.0/docs/guide/configuration.md +377 -0
  20. testuary-0.2.0/docs/guide/demonstrations.md +193 -0
  21. testuary-0.2.0/docs/guide/machine.md +96 -0
  22. testuary-0.2.0/docs/guide/output.md +194 -0
  23. testuary-0.2.0/docs/guide/terminal.md +369 -0
  24. testuary-0.2.0/docs/guide/web.md +502 -0
  25. testuary-0.2.0/docs/index.md +125 -0
  26. testuary-0.2.0/docs/models-and-prompts.md +301 -0
  27. testuary-0.2.0/docs/reference/api.md +241 -0
  28. testuary-0.2.0/docs/reference/cli.md +337 -0
  29. testuary-0.2.0/docs/reference/index.md +9 -0
  30. testuary-0.2.0/docs/reference/result-files.md +644 -0
  31. testuary-0.2.0/docs/reference/settings.md +783 -0
  32. testuary-0.2.0/docs/reference/tested-environments.md +108 -0
  33. testuary-0.2.0/examples/quickstart/README.md +112 -0
  34. testuary-0.2.0/examples/quickstart/java-project/.testuary/project.ini +18 -0
  35. testuary-0.2.0/examples/quickstart/java-project/pom.xml +25 -0
  36. testuary-0.2.0/examples/quickstart/java-project/src/main/java/example/Shipping.java +36 -0
  37. testuary-0.2.0/examples/quickstart/python-project/.testuary/project.ini +18 -0
  38. testuary-0.2.0/examples/quickstart/python-project/shipping.py +25 -0
  39. testuary-0.2.0/examples/quickstart/run_demos.ps1 +15 -0
  40. testuary-0.2.0/examples/quickstart/run_demos.py +39 -0
  41. testuary-0.2.0/examples/quickstart/run_demos.sh +12 -0
  42. testuary-0.2.0/examples/quickstart/start_web.py +22 -0
  43. testuary-0.2.0/models.example.ini +8 -0
  44. testuary-0.2.0/pyproject.toml +202 -0
  45. testuary-0.2.0/src/testuary/__init__.py +10 -0
  46. testuary-0.2.0/src/testuary/__main__.py +1075 -0
  47. testuary-0.2.0/src/testuary/_chat_log.py +256 -0
  48. testuary-0.2.0/src/testuary/_chat_models.py +234 -0
  49. testuary-0.2.0/src/testuary/_extensions.py +63 -0
  50. testuary-0.2.0/src/testuary/_model_access.py +129 -0
  51. testuary-0.2.0/src/testuary/_phase_keys.py +13 -0
  52. testuary-0.2.0/src/testuary/_progress.py +614 -0
  53. testuary-0.2.0/src/testuary/_templates/__init__.py +7 -0
  54. testuary-0.2.0/src/testuary/_templates/examples/README.txt +26 -0
  55. testuary-0.2.0/src/testuary/_templates/examples/controls.ini +8 -0
  56. testuary-0.2.0/src/testuary/_templates/examples/hints.json +27 -0
  57. testuary-0.2.0/src/testuary/_templates/examples/language-profile.json +17 -0
  58. testuary-0.2.0/src/testuary/_templates/examples/models.ini +12 -0
  59. testuary-0.2.0/src/testuary/_templates/examples/testuary_example_chat.py +15 -0
  60. testuary-0.2.0/src/testuary/_templates/hints.json +31 -0
  61. testuary-0.2.0/src/testuary/_templates/models.ini +93 -0
  62. testuary-0.2.0/src/testuary/_test_output.py +740 -0
  63. testuary-0.2.0/src/testuary/_words.py +202 -0
  64. testuary-0.2.0/src/testuary/agent/__init__.py +10 -0
  65. testuary-0.2.0/src/testuary/agent/_base.py +2270 -0
  66. testuary-0.2.0/src/testuary/agent/_context.py +417 -0
  67. testuary-0.2.0/src/testuary/agent/_edit_tool.py +733 -0
  68. testuary-0.2.0/src/testuary/agent/_helpers.py +1231 -0
  69. testuary-0.2.0/src/testuary/agent/_langgraph.py +1847 -0
  70. testuary-0.2.0/src/testuary/agent/_limits.py +214 -0
  71. testuary-0.2.0/src/testuary/agent/coverage_agent.py +1896 -0
  72. testuary-0.2.0/src/testuary/agent/mutation_agent.py +2134 -0
  73. testuary-0.2.0/src/testuary/config.ini +747 -0
  74. testuary-0.2.0/src/testuary/config.py +980 -0
  75. testuary-0.2.0/src/testuary/consolidation.py +4737 -0
  76. testuary-0.2.0/src/testuary/demo/__init__.py +61 -0
  77. testuary-0.2.0/src/testuary/demo/backend.py +493 -0
  78. testuary-0.2.0/src/testuary/demo/cli.py +1226 -0
  79. testuary-0.2.0/src/testuary/demo/data/LICENSE-tenacity.txt +212 -0
  80. testuary-0.2.0/src/testuary/demo/data/NOTICE.md +23 -0
  81. testuary-0.2.0/src/testuary/demo/data/java/LICENSE-jsonassert.txt +213 -0
  82. testuary-0.2.0/src/testuary/demo/data/java/NOTICE.md +25 -0
  83. testuary-0.2.0/src/testuary/demo/data/java/manifest.json +206 -0
  84. testuary-0.2.0/src/testuary/demo/data/java/models.ini +4 -0
  85. testuary-0.2.0/src/testuary/demo/data/java/preset.ini +22 -0
  86. testuary-0.2.0/src/testuary/demo/data/java/project.json +6 -0
  87. testuary-0.2.0/src/testuary/demo/data/java/scenario.json.gz +0 -0
  88. testuary-0.2.0/src/testuary/demo/data/manifest.json +215 -0
  89. testuary-0.2.0/src/testuary/demo/data/models.ini +4 -0
  90. testuary-0.2.0/src/testuary/demo/data/preset.ini +23 -0
  91. testuary-0.2.0/src/testuary/demo/data/project.json +7 -0
  92. testuary-0.2.0/src/testuary/demo/data/scenario.json.gz +0 -0
  93. testuary-0.2.0/src/testuary/demo/data/toy/NOTICE.md +18 -0
  94. testuary-0.2.0/src/testuary/demo/data/toy/manifest.json +52 -0
  95. testuary-0.2.0/src/testuary/demo/data/toy/models.ini +4 -0
  96. testuary-0.2.0/src/testuary/demo/data/toy/preset.ini +19 -0
  97. testuary-0.2.0/src/testuary/demo/data/toy/project.json +6 -0
  98. testuary-0.2.0/src/testuary/demo/data/toy/scenario.json.gz +0 -0
  99. testuary-0.2.0/src/testuary/demo/data/tutorial/LICENSE-tenacity.txt +212 -0
  100. testuary-0.2.0/src/testuary/demo/data/tutorial/NOTICE.md +27 -0
  101. testuary-0.2.0/src/testuary/demo/data/tutorial/hints.json +7 -0
  102. testuary-0.2.0/src/testuary/demo/data/tutorial/manifest.json +222 -0
  103. testuary-0.2.0/src/testuary/demo/data/tutorial/models.ini +4 -0
  104. testuary-0.2.0/src/testuary/demo/data/tutorial/preset.ini +24 -0
  105. testuary-0.2.0/src/testuary/demo/data/tutorial/project.json +7 -0
  106. testuary-0.2.0/src/testuary/demo/data/tutorial/prompt_profile.json +17 -0
  107. testuary-0.2.0/src/testuary/demo/data/tutorial/scenario.json.gz +0 -0
  108. testuary-0.2.0/src/testuary/demo/pacing.py +251 -0
  109. testuary-0.2.0/src/testuary/demo/recorded.py +1015 -0
  110. testuary-0.2.0/src/testuary/demo/scenario.py +500 -0
  111. testuary-0.2.0/src/testuary/demo/setup_demo.py +356 -0
  112. testuary-0.2.0/src/testuary/evidence.py +927 -0
  113. testuary-0.2.0/src/testuary/exits.py +107 -0
  114. testuary-0.2.0/src/testuary/generation_evidence.py +669 -0
  115. testuary-0.2.0/src/testuary/hints.py +311 -0
  116. testuary-0.2.0/src/testuary/langadapter_profile.json +86 -0
  117. testuary-0.2.0/src/testuary/languages/README.md +238 -0
  118. testuary-0.2.0/src/testuary/languages/__init__.py +580 -0
  119. testuary-0.2.0/src/testuary/languages/generic.py +1830 -0
  120. testuary-0.2.0/src/testuary/languages/mutation.py +1315 -0
  121. testuary-0.2.0/src/testuary/languages/preparation.py +372 -0
  122. testuary-0.2.0/src/testuary/loadable.py +164 -0
  123. testuary-0.2.0/src/testuary/measurement.py +1789 -0
  124. testuary-0.2.0/src/testuary/model_requests.py +803 -0
  125. testuary-0.2.0/src/testuary/models.py +1034 -0
  126. testuary-0.2.0/src/testuary/normal_run.py +92 -0
  127. testuary-0.2.0/src/testuary/orchestrator.py +2699 -0
  128. testuary-0.2.0/src/testuary/phase_state.py +1290 -0
  129. testuary-0.2.0/src/testuary/presentation/__init__.py +8 -0
  130. testuary-0.2.0/src/testuary/presentation/event_file.py +98 -0
  131. testuary-0.2.0/src/testuary/presentation/events.py +286 -0
  132. testuary-0.2.0/src/testuary/presentation/measurement.py +173 -0
  133. testuary-0.2.0/src/testuary/presentation/readiness.py +462 -0
  134. testuary-0.2.0/src/testuary/presentation/terminal.py +582 -0
  135. testuary-0.2.0/src/testuary/presentation/view.py +873 -0
  136. testuary-0.2.0/src/testuary/project.py +826 -0
  137. testuary-0.2.0/src/testuary/prompts/__init__.py +798 -0
  138. testuary-0.2.0/src/testuary/prompts/test_coverage.json +747 -0
  139. testuary-0.2.0/src/testuary/prompts/test_initial.json +24 -0
  140. testuary-0.2.0/src/testuary/prompts/test_mutation.json +777 -0
  141. testuary-0.2.0/src/testuary/py.typed +0 -0
  142. testuary-0.2.0/src/testuary/readiness.py +1047 -0
  143. testuary-0.2.0/src/testuary/reduction.py +919 -0
  144. testuary-0.2.0/src/testuary/results.py +1247 -0
  145. testuary-0.2.0/src/testuary/settings.py +1807 -0
  146. testuary-0.2.0/src/testuary/setup.py +1698 -0
  147. testuary-0.2.0/src/testuary/starter_salvage.py +621 -0
  148. testuary-0.2.0/src/testuary/starter_selection.py +243 -0
  149. testuary-0.2.0/src/testuary/starters.py +2986 -0
  150. testuary-0.2.0/src/testuary/target_phases.py +3261 -0
  151. testuary-0.2.0/src/testuary/tutorial/__init__.py +480 -0
  152. testuary-0.2.0/src/testuary/web/__init__.py +471 -0
  153. testuary-0.2.0/src/testuary/web/__main__.py +6 -0
  154. testuary-0.2.0/src/testuary/web/_data.py +925 -0
  155. testuary-0.2.0/src/testuary/web/_jobs.py +979 -0
  156. testuary-0.2.0/src/testuary/web/_launch.py +86 -0
  157. testuary-0.2.0/src/testuary/web/_registry.py +344 -0
  158. testuary-0.2.0/src/testuary/web/_results.py +1195 -0
  159. testuary-0.2.0/src/testuary/web/_server.py +1281 -0
  160. testuary-0.2.0/src/testuary/web/extensions.py +174 -0
  161. testuary-0.2.0/src/testuary/web/static/app.js +3545 -0
  162. testuary-0.2.0/src/testuary/web/static/icon.svg +1 -0
  163. testuary-0.2.0/src/testuary/web/static/index.html +34 -0
  164. testuary-0.2.0/src/testuary/web/static/style.css +498 -0
  165. testuary-0.2.0/tools/language_page.py +93 -0
  166. testuary-0.2.0/tools/reference_pages.py +299 -0
@@ -0,0 +1,14 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .pytest_cache/
4
+ .ruff_cache/
5
+ .coverage*
6
+ htmlcov/
7
+ .venv/
8
+ *.egg-info/
9
+ build/
10
+ dist/
11
+ /models.ini
12
+ /projects.ini
13
+ /config.ini
14
+ /results/
@@ -0,0 +1,524 @@
1
+ # Changelog
2
+
3
+ All notable changes are listed here, newest first.
4
+
5
+ ## 0.2.0
6
+
7
+ First public release. Testuary writes unit test suites for Python and Java
8
+ code, and for C code on Linux, with a language model. It runs the generated
9
+ tests, measures their line and branch coverage and their mutation score,
10
+ asks the model to close the gaps within the limits you set, removes tests
11
+ that add no coverage and catch no extra mutant, and delivers the resulting
12
+ suite with its measurements. Language support comes from LangAdapter 0.1.0,
13
+ released with it as a separate package.
14
+
15
+ ### New in this release
16
+
17
+ - Java projects built with Maven or Gradle and tested with JUnit 4, 5 or 6,
18
+ with JaCoCo for coverage and PIT for mutation.
19
+ - C files on Linux, through LangAdapter's C adapter, with gcov for coverage
20
+ and Mull for mutation. Testuary writes their tests in the language the
21
+ adapter names for them: C with utest.h, or Python for a C file built into
22
+ a Python extension. Its prompts and the Results view's place for a
23
+ delivered test follow the tests' language.
24
+ - Language support in LangAdapter, with a way to add a language. Every
25
+ language's runtime is found through its adapter in the same way. Without
26
+ a selected interpreter or `python_version`, a Python project runs with
27
+ its only environment, its `.venv` or the one activated in the shell. An
28
+ environment Testuary runs in outside the project is set aside, so the
29
+ project's `.venv` runs the tests. A `[project]` setting that two
30
+ installed languages declare takes the kind, default and host-path rule of
31
+ the first of them by name.
32
+ - Commands to set up and check a project before a run: `testuary init`,
33
+ `testuary setup`, `testuary check`, `testuary new-language` and
34
+ `testuary tutorial`.
35
+ - Settings in your user directory and in a `.testuary` directory in each
36
+ project, with a reference of every setting.
37
+ - Several repetitions of each file with `--runs`, each repetition's results
38
+ in its own directory, and a summary across them.
39
+ - Progress views in the terminal, and an end-of-run summary saved with the
40
+ results.
41
+ - An offline demonstration that plays back a recorded run without a model:
42
+ `testuary --demo`. It refuses a wrong combination of run options, such as
43
+ `--runs 2 --run 1`, with exit status 2 before the demonstration starts.
44
+ - A quickstart directory, `examples/quickstart` in the repository and the
45
+ source distribution: scripts that play the demonstrations and start the
46
+ web interface, and two small example projects to run with your own model.
47
+ - Prompt profiles and hints files, which change what the model is told. A
48
+ prompt profile may name every placeholder the prompts fill, among them
49
+ `$lang_test_decl_full`, `$lang_test_collection_rule` and
50
+ `$language_level`.
51
+ - A local web interface, started with `testuary-web`. It shows the
52
+ projects, models and settings, tests a model's connection, starts runs
53
+ and checks, follows a run's progress and shows its results and the
54
+ demonstrations. A test file of a delivered suite can be copied from the
55
+ results into its project. A project can be added from its GitHub HTTPS
56
+ address, which git clones into a directory you choose. The `[web]`
57
+ settings choose its port, 8765 by default, and who may use it, every
58
+ program on the computer by default.
59
+
60
+ What the model sees:
61
+
62
+ - The file under test is read as UTF-8 with undecodable bytes replaced and
63
+ CRLF or CR read as LF, in a local and a container run alike, so the model
64
+ and the measurement see the same text in both.
65
+ - Each loop starts by reading the tests, then gets the current measurement,
66
+ then edits. A tool that is not available yet says what makes it
67
+ available.
68
+ - The rules for valid tests are each language's own and name what a test
69
+ may read. Python tests may read a module's `__dict__` and `vars()`.
70
+ - A test that the time limit cut off is named, and the model can fix it
71
+ without losing its other changes.
72
+ - The tool calls of one answer run one after another, in order, so that
73
+ several edits in one answer all take effect. One edit call can carry a
74
+ list of edits.
75
+ - A tool that fails for a temporary reason gets an answer, and the loop
76
+ continues.
77
+ - Text that is cut for length is marked, and every such limit is a setting.
78
+ - A write outside the loop's test output directory is refused, and the
79
+ refusal the model receives names the field it checks, `test_output_dir`.
80
+ - After a passing normal test run, a run under the coverage tool that
81
+ fails, ends without a test result or does not finish is the tool's
82
+ refusal, in both loops. The reply to the loop's measurement names the
83
+ tests that stopped the tool where it can, and it counts as a failed test
84
+ run. Only a normal test run that does not finish is a timeout of the
85
+ suite.
86
+ - A passing test run of a Python file can carry a STANDS ALONE note: the
87
+ tests that left the module under test or a library setting changed,
88
+ and with `report_reverse_order_failures = true` in `[coverage]` or
89
+ `[mutation]` the tests that fail when the file runs as delivered in
90
+ reverse order. After a note that names tests, the model can change the
91
+ tests before it measures, and a loop at its goal ends only after a
92
+ passing run without such a note, within its limits.
93
+
94
+ What is measured:
95
+
96
+ - By default each starter suite gets its own mutation score from a run of
97
+ that suite alone. `[starter] suite_mutation` can derive the starter
98
+ suites' scores from one run of all of them instead, and
99
+ `[main] reduced_suite_mutation` the reduced suite's score at a phase end
100
+ from the merged suite's run.
101
+ - With `per_mutant_timeout_seconds = auto`, the per-mutant time limit for
102
+ C is `per_mutant_timeout_factor` times the mutation tool's own run of the
103
+ passing suite, each test in a process of its own as a mutant's tests
104
+ run. A mutant that no test kills runs all its tests and survives, in a
105
+ large suite too. A fixed value bounds all of a mutant's tests together.
106
+ Logs state a C time limit per mutant.
107
+ - One normal test run decides whether tests pass. For Python it is the
108
+ run as delivered: each test file in its own session, with fixed seeds
109
+ set when the session starts and nothing reset between tests.
110
+ - Tests are removed only where no model is active. The starter suites'
111
+ cleanup removes failing tests. When a suite passes its normal run and a
112
+ coverage or mutation measurement then fails and names tests, in the
113
+ starter phase or at a phase end, Testuary removes the named tests, runs
114
+ the suite again and records each removal, as
115
+ [Tests removed after a failing measurement](https://k-kammler.github.io/testuary/explanation/measurement-and-recovery/#tests-removed-after-a-failing-measurement)
116
+ describes. A loop removes no test from the suite it inherits, and at the
117
+ end of each phase Testuary chooses among the suites that pass with
118
+ complete numbers.
119
+ - A starter suite that the coverage or mutation tool cannot measure first
120
+ loses the tests the failure names. When that does not help, it is
121
+ generated again within its attempts, or left out.
122
+ - Finishing, the fourth phase, takes the delivered suite's coverage from
123
+ the end of the phase that measured it and its mutation results from the
124
+ end of the mutation phase, and measures only what those did not. With
125
+ `[finishing] measure_mutation = false`, Finishing records no mutation
126
+ score and judges no mutation minimum, also when the mutation phase
127
+ measured the delivered suite.
128
+ - Every measurement runs in a private copy of the project.
129
+ - Every mutant has an identifier that stays the same from run to run.
130
+ - A Python test's coverage includes its fixtures' setup and teardown. A
131
+ fixture shared by several tests counts for the first of them, and its
132
+ teardown for the last.
133
+ - Per-test times are precise, and the reduction does not charge a run's
134
+ one-time setup to its first test.
135
+ - Of several suites with equal scores, the end of a phase keeps the
136
+ reduced suite, or else the fastest. When their times do not tell them
137
+ apart, or a test fails in a timing run, the earlier one stays. A tie that
138
+ no candidate's time could break records the input order as its basis,
139
+ and the results page says so.
140
+ - Candidate suites are merged by LangAdapter, which checks the merged file
141
+ with the project's own Python or Java tools. When it refuses a pair,
142
+ nothing is merged and the better single candidate is kept. A merge whose
143
+ tools give no answer is an error, not a refusal. A Python merge keeps one
144
+ copy of a top-level statement both suites hold identically, such as a
145
+ repeated docstring or import guard, where one copy cannot change what a
146
+ test sees, and a definition dropped as shared takes its blank lines along.
147
+ - When the merged suite's per-test evidence covers other lines or branches
148
+ than its own coverage run, as C's tests can when each runs alone, the
149
+ run's coverage stays the suite's coverage, the reduction works on the
150
+ per-test evidence, and the selection audit records both sides
151
+ (`merged_evidence_difference`). A starter suite delivered alone takes the
152
+ coverage of its run under the delivered name
153
+ (`delivered_coverage_difference`). A candidate whose per-test evidence disagrees with its own coverage run is
154
+ scored on that run, without the evidence, with the disagreement as its
155
+ reason.
156
+ - An exact reduction whose time limit runs out before a tier of its
157
+ tie-break records the failure `time_limit_exhausted` with that tier. A
158
+ CBC that cannot run ends the exact reduction with the failure
159
+ `solver_error` and the tier it stopped in, and the phase end writes
160
+ `consolidation_failure.json`. The recorded error names the error type,
161
+ without the solver's message, which can hold paths of your machine.
162
+ - A final mutation measurement that did not finish reports no score.
163
+ - A mutation measurement cut by its time limit counts its untested mutants
164
+ as not killed, so its score is a lower bound. The mutation phase works
165
+ with that score, and a cut final mutation measurement records it beside
166
+ its missing score.
167
+ - When a loop's test run, a mutation measurement or the final coverage run
168
+ reaches its time limit, the run's summary and the end-of-run summary
169
+ carry a warning that names the setting and its value.
170
+
171
+ How runs behave:
172
+
173
+ - A temporary failure of the model service costs no model request.
174
+ - A run ends with exit status 1 when a file failed.
175
+ - A run is refused before any work, as a refused setting is, when its
176
+ required tools are missing or when `--reuse` finds saved work missing for
177
+ a target. It ends with exit status 2, and a program that calls
178
+ `run_pipeline` receives `testuary.settings.ConfigurationError`. For
179
+ missing tools the setup report names what is missing and how
180
+ to fix it, `--events` records that report, and the run's last event reads
181
+ "Setup refused".
182
+ - A target outside the source directory is refused before the run, with or
183
+ without `--resume`, as a file no language handles is. A target whose file
184
+ name holds a comma runs as one target. A target's test file is the one
185
+ its name gives (`test_<stem>.py`, `<Stem>Test.java`), so a reused run
186
+ never takes another target's file from the shared tests directory.
187
+ - A resumed run reuses its saved starter suites, and a resume with changed
188
+ measurement settings is refused. A resume compares the SHA-256 of a model
189
+ provider's file or a language's file and refuses one that changed, naming
190
+ it. A change to one of Finishing's own settings, such as
191
+ `[finishing] measure_lint`, is no refusal: the resume measures Finishing
192
+ again, with no model request.
193
+ - `--resume` judges each repetition's phases against the canonical source
194
+ that repetition's phase recorded when it started: `phase_state.json`
195
+ holds `target_file_path` and `target_source_sha256` in each repetition's
196
+ block of a phase. A repetition made from an earlier source counts as
197
+ unfinished, also after a later repetition ran from the edited source.
198
+ - A resume or reuse knows the model service by its provider and model
199
+ name. Another address, such as a local proxy on another port, is a
200
+ warning.
201
+ - A resumed or reused run that keeps its earlier `tests.csv` row and
202
+ measures mutation in Finishing again writes each of Finishing's metrics
203
+ columns afresh, empty where the new measurement gave no value, and every
204
+ write with a phase result writes `error` afresh. An earlier attempt's
205
+ lower bound, untested count, kill counts or error text does not stay
206
+ beside the new result in `tests.csv` or `summary.json`. A write that
207
+ measured no mutation, such as a timing pass, keeps the earlier scores.
208
+ - A loop that ends at its request, token or time limit names that limit,
209
+ such as TOKEN LIMIT REACHED, an ordinary end, not FAILED.
210
+ - The coverage and the mutation phase give a stop the same status. A
211
+ coverage phase that `max_final_answers_without_measurement` stopped, with
212
+ the stop reason `final_answers_without_measurement`, records that it
213
+ stopped before its goal, as the mutation phase does: its suite is green
214
+ and measured, and its target's row holds the mutation phase's results.
215
+ Repeated test timeouts end either phase before its goal when the suite
216
+ it delivers is measured, else with failing tests. A mutation phase whose
217
+ suite runs longer than its time limit before any edit records
218
+ `skipped_inherited_slow` ("skipped: the suite it received runs longer
219
+ than its time limit"), a failed phase with its reason.
220
+ - A run whose phase named by `--stop-after-phase` is not reached ends with
221
+ its `Stopped:` line, without a traceback.
222
+ - A command refused after its arguments are read leaves an earlier
223
+ `--events` file as it was.
224
+
225
+ What a run records:
226
+
227
+ - A mutation run that the mutation tool's origin check stops, because a
228
+ test fails without a mutant, is logged with its stopping tests and
229
+ recorded in the phase end's `consolidation_record.json` as
230
+ `mutation_origin_stops`.
231
+ - Each phase's `<phase>_duration_s` counts only the work the row keeps: an
232
+ attempt that crashed, or that failed in a phase other than the starter
233
+ phase, counts for nothing there, while `duration_s_target` and the
234
+ `time_*` columns keep a failed attempt's time. `duration_s_target` and
235
+ every `time_*` column are empty when the repetition crashed, or when a
236
+ resume reused a phase or the saved starter suites of a stopped or crashed
237
+ repetition, and `starter_duration_s` too in the second case, since the
238
+ time of that work is not known.
239
+ - `merged_tests` in `consolidation_record.json` and `coverage_merged_tests`
240
+ in `tests.csv` are empty where no merged suite was counted.
241
+ `mutation_merged_tests` and `mutation_delivered_tests` hold 0 for an
242
+ empty count.
243
+ - A phase end's `consolidation_record.json` tells a reduced suite that was
244
+ never built (status `absent`) from one that does not pass (`failing`) or
245
+ was not fully measured (`incomplete`), each with its reason. An absent
246
+ one records no tests, removed tests or source hash, and the results page
247
+ shows it as not built. A merged suite whose run wrote no exact coverage
248
+ matrix is recorded as `incomplete` with that reason.
249
+ - An inherited phase end's `consolidation_record.json` holds its keys in
250
+ the order of a measured phase end's record, followed by `inherited_from`,
251
+ `inherited_artifacts`, `measured` and `measurement_calls`.
252
+ `reduced_coverage_matrix.json` holds the keys of
253
+ `merged_coverage_matrix.json` in their order, followed by `kept_tests`
254
+ and `derive_kept_tests`, and a time map with no measured time is null.
255
+ `selected_coverage_matrix.json` holds per-test times and solver costs
256
+ only for a merged or reduced winner, and a candidate scored on one
257
+ coverage run records the target's statement and branch totals that run
258
+ measured.
259
+ - A phase end's selection audit names the axes the selection compared in
260
+ `selector`, records `runtime_tie_break_candidate_order` as null when no
261
+ timing tie-break ran, and records an input suite that does not pass with
262
+ status `incomplete`. The candidates' times in the audit are their tests'
263
+ measured times summed, or null where a test was not timed.
264
+ - A phase end's `collision_resolution` lists in `renamed_names` the names
265
+ the merge's collision rename gave, when it gave any, and `rename_twins`
266
+ counts the copies of a test that the rename kept beside the original.
267
+ `consolidation_record.json` keeps `collision_resolution` when the merged
268
+ suite fails validation.
269
+ - A phase end's `reduction_solver_metadata` names CBC by its version
270
+ alone, without the solver's path on your machine. With
271
+ `reduction_solver = greedy` it records the measured and imputed costs
272
+ the reduction weighed, as the exact reduction does.
273
+ - The kill matrices a run writes follow LangAdapter's kill-matrix schema
274
+ version 3. A mutant's status is `killed`, `timeout`, `non_viable` or
275
+ `survived`, a per-suite verdict of a suite that measured nothing is
276
+ `untested`, and a summary counts `detected` and `non_viable`.
277
+ `starter_pool_attribution.json` counts each suite's verdicts under these
278
+ words.
279
+ - The run log tags the starter phase's lines `[starter]`, the mutation runs
280
+ of the starter suites' pool and of the phase ends `[finishing]`, and
281
+ Finishing's lines `[finishing]` or `[metrics]`.
282
+
283
+ Setting up a project:
284
+
285
+ - `testuary setup` and the web interface list the installed languages in
286
+ name order.
287
+ - Setup finds a project's source files by each language's own rules. Test
288
+ files, the files a language names as no sources, such as Python's
289
+ `setup.py`, and empty files are left out. For Python a `tests.py` and a
290
+ test file named in another case count as tests, for Java
291
+ `*TestCase.java` and `*IT.java` do.
292
+ - `testuary check` reports whether each target is in the project
293
+ directory, as the language's check finds it: verified, or missing setup
294
+ with that check's reason. A run checks the targets itself before any
295
+ work.
296
+ - `testuary check --verify` tries the project in a copy as a run does,
297
+ without running a test: for Java it resolves the build without
298
+ downloading, compiles and finds the tests and starts the measurement
299
+ tools a run needs, and for Python it collects the tests and checks where
300
+ they import the project from. The first run tries the build of a project
301
+ with its own build wrapper, which may download its build tool. The report
302
+ names these parts, such as the build of your project and the collection
303
+ of your project's tests.
304
+ - The check report gives each operation of a language its own name, such
305
+ as removing single tests from a test file and counting the tests and
306
+ assertions of a test file.
307
+ - `testuary check` refuses an unknown phase in `--reuse` or
308
+ `--stop-after-phase` in its own report, with exit status 2, as JSON with
309
+ `--json`.
310
+ - A runtime that a project's settings request, such as `python_version`, is
311
+ provisioned by that language's adapter and kept in a directory per
312
+ language in Testuary's cache (`venvs/<language>/<project>`).
313
+ `testuary provision` prepares what every installed language's settings
314
+ request and prints each runtime, or says that none is requested. It
315
+ writes its errors to standard error.
316
+
317
+ Models and their keys:
318
+
319
+ - `testuary init` writes the example aliases in `models.ini` as comments,
320
+ so a new catalogue names no model until you add one.
321
+ - An alias that `testuary setup model` or the Models page writes for
322
+ `ollama`, `openai` or `vllm` holds the address you confirmed, the
323
+ packaged one included. It keeps that address when a global `config.ini`
324
+ sets a gateway in that service's section, so its key goes only to the
325
+ confirmed address. Only `[api] api_url`, which sets the address of every
326
+ model, moves it. To use the gateway, give its address as the alias's
327
+ address.
328
+ - An alias's `chat_model` names a client factory as a Python file or a
329
+ module before its attribute, such as `my_chat.py:factory`. A file is relative to
330
+ the directory of the file that names it and needs no installation, and
331
+ the example `testuary init --examples` copies is one such file.
332
+ - A project's settings file may not set `[api] api_url` or the `api_url` of
333
+ a provider's section, which decide where the model's key goes. A run of
334
+ such a project is refused at its start with exit status 2. Set the
335
+ address in your global `config.ini`, in `models.ini`, in the environment
336
+ or on the command line.
337
+ - A variable ending in `_API_KEY` is always a model service's key and never
338
+ a setting, so a service may be named like a settings section, such as
339
+ `main_x` with `TESTUARY_MAIN_X_API_KEY`.
340
+ - A stored model key goes only to its model's service address, in a Test
341
+ or a run, and no answer of `testuary-web`'s server holds it. The Models
342
+ page says whether a model's key is stored in `models.ini` or comes from
343
+ an environment variable, and Edit changes a stored key.
344
+ - When the record of a failed model request cannot be written, the
345
+ traceback in the run log and on the console leaves out the model
346
+ service's error, which can hold the request and its key.
347
+ - A `max_output_tokens` that is no nonnegative integer is refused wherever
348
+ it is set, in a phase section or a model alias, before any request, as
349
+ the model's own resolution refuses it. The agents' and the starter
350
+ phase's model address, model name and key come from that same
351
+ resolution.
352
+ - Starter suite requests carry no temperature unless you set
353
+ `[starter] temperature`, which is empty by default, so the provider's
354
+ default applies.
355
+
356
+ Extensions:
357
+
358
+ - `[main] extensions` names the extensions a run and `testuary-web` load:
359
+ Python files, such as `.testuary/my_extension.py`, and installed modules.
360
+ An extension adds a language with its top-level `langadapter_language`,
361
+ model hints with `testuary_hints` and `testuary-web` pages with
362
+ `testuary_web`. Each settings file adds its items to those of the files
363
+ before it, and a file item is relative to the directory of the settings
364
+ file that names it. `testuary check` finds each item without running it,
365
+ and the tool check loads it. The Settings page edits each file's own
366
+ items and shows the combined list. `testuary.settings.add_extension` and
367
+ `remove_extension` change the items of the user's `config.ini`, and
368
+ `extension_items` names each item with its source. The generation origin
369
+ records the extensions and their hints, and `testuary tutorial` teaches
370
+ the first extension as one file.
371
+ - A `testuary.web.WebExtension` adds tabs, result views, settings groups,
372
+ jobs and read-only routes to `testuary-web`. It can set the product name
373
+ and the header's Guide link, rename, move and hide Testuary's tabs, and
374
+ put a tab of its own in the place of one of them. Testuary's links to a
375
+ replaced tab open the extension's, its links to a hidden tab are left
376
+ out, and its pages name a renamed or replaced tab as the header does. The
377
+ extension guide lists every part an extension can change. A job's setup
378
+ report in Testuary's readiness format shows on its page as a check's
379
+ report does. `testuary.web.TOOL_CHECK_LABEL`, which `/api/state` carries
380
+ as `tool_check_label`, is the label of Testuary's tool check for a
381
+ replacing Check page.
382
+ - An extension's page shows a Testuary run folded, as Testuary's own pages
383
+ draw it, with `testuaryWeb.nestedRun`: a run's event file as its job's
384
+ page shows it, followed while it is written, or a run directory as
385
+ Results shows it. A job kind names the Testuary runs its jobs start with
386
+ `nested_runs`, each a `testuary.web.NestedRun`, and its job's page folds
387
+ each into its phase.
388
+ - An extension's settings group can name the Testuary settings files its
389
+ runs in a project read instead of the global and the project's settings
390
+ files, its own file last (`SettingsGroup.testuary_settings_files`). The
391
+ Settings page offers that list as a choice of the project, shows every
392
+ setting with the file it comes from as those runs read it, and changes
393
+ the last file through Testuary's validated update. A last file outside
394
+ the project directory is read-only.
395
+ - A settings group can also name the model catalogue files its runs in a
396
+ project read (`testuary_models_files`), its own user directory
397
+ (`extension_user_dir`) and a function that makes its runs read its own
398
+ file alone (`end_models_import`). The Models page shows those models for
399
+ the project, changes the list's last file, and removes a model an
400
+ earlier file defines by ending the import after a question. After the
401
+ Settings page or the Models page saves a file of a group's list, it asks
402
+ the group again, and a group that raises puts the file back.
403
+
404
+ Programs of your own:
405
+
406
+ - `testuary.settings.resolve_settings` takes `models_files`, an ordered
407
+ list of model catalogue files read instead of the user directory's
408
+ `models.ini`, a later file's section overriding the same section of an
409
+ earlier one. A file in the project directory may set neither `api_url`
410
+ nor `api_key_env`, which decide where the model's key goes.
411
+ - `testuary.settings.check_project_settings_file` and
412
+ `testuary.settings.check_project_catalogue_file` check one settings or
413
+ model catalogue file that is not your own, as Testuary checks a
414
+ project's: a settings file may not set an `api_url`, and a catalogue file
415
+ neither an `api_url` nor an `api_key_env`. A program that passes such
416
+ files to its runs calls them for each.
417
+ - `testuary.models.catalogue_chat_model` gives your own program a LangChain
418
+ chat model for an alias of your model catalogue, or with `provider` for a
419
+ model that provider serves, at the provider's address and with its key.
420
+ It resolves the model as a Testuary run resolves `[api] model` and
421
+ `[api] provider`, retries temporary failures as Testuary does, and
422
+ records each request and each model's settings in a directory you name.
423
+ The model, its records and its errors hold no key. Its `config_files`,
424
+ `project_settings` and `models_files` choose the settings files and the
425
+ model catalogue as for `resolve_settings`, so that a program whose runs
426
+ name their own files gets models that read the same files.
427
+ `testuary.models.REQUEST_VALUES` holds the request values it accepts.
428
+ - `testuary.setup.models`, `set_model` and `remove_model` take
429
+ `models_files`, the list of model catalogue files `resolve_settings`
430
+ reads, and change its last file only, writing a model an earlier file
431
+ defines into it whole and no key. Each `ModelEntry` names its `file`. A
432
+ catalogue change is refused when it would leave a model that resolves
433
+ unable to resolve.
434
+ - `testuary.readiness.ReadinessLimit` and
435
+ `testuary.presentation.present_readiness` are on the API reference
436
+ beside `RunReadiness`, so a program builds and prints a setup report as
437
+ `testuary check` does.
438
+ - `testuary.presentation.EventFile` writes the event file `--events`
439
+ writes, for a program of your own, with `EVENT_FILE_SCHEMA`, `RunPlan`
440
+ and `RunCompletion` (the API reference's "Event files").
441
+ `run_pipeline(..., events=PATH)` writes a program's run into an event
442
+ file as `--events` does, with the setup report, the plan and the end of
443
+ the run, so a page that folds the run shows its phases and its summary.
444
+ The event file has `schema_version` 3. Its plan record names Finishing
445
+ `finishing`, Finishing's operation is `step/finishing`, and the step that
446
+ ends each phase is labeled Consolidation.
447
+ - `testuary.generation_evidence.read_suite_slots(manifest, suites_root)`
448
+ and `read_cleaned_source(reference, suites_root)` read the generation
449
+ records.
450
+ - `testuary.results.recorded_source_sha256` reads the SHA-256 of a target
451
+ file's raw bytes that a run recorded, so a program can check that an
452
+ earlier run tested the same file before it reuses that run's suite.
453
+ - `testuary.presentation.read_completion` reads a run's end-of-run
454
+ summary, `results/completion.json`, as its `RunCompletion` and the
455
+ provenance of its values, so a program reads the file as Testuary's
456
+ console and web interface do.
457
+
458
+ ### Requirements
459
+
460
+ - Python 3.10 to 3.14 for Testuary, on Windows or Linux. macOS is expected
461
+ to work, but this release has not run on a Mac.
462
+ - For a Python project: pytest, coverage and Cosmic Ray in the project's
463
+ environment. Tested with pytest 8.1.1, coverage 7.6.8 and Cosmic Ray
464
+ 8.4.1, on Python 3.14 and 3.15 with pytest 8.4.2 and coverage 7.16.2,
465
+ and project Python 3.10 to 3.15. Python 3.9 is expected to work.
466
+ Testuary itself is not tested on Python 3.15, where tree-sitter, which
467
+ LangAdapter uses, has no wheels, so it installs there only with a C
468
+ compiler.
469
+ - For a Java project: JDK 11 or newer, tested with 11, 17, 21, 25, 26 and
470
+ 27, and the project's Maven or Gradle build. LangAdapter downloads the
471
+ JUnit launcher, JaCoCo and PIT on first use.
472
+ - For a C project, on Linux: LangAdapter's extra `langadapter[c]`, the
473
+ project's GCC with its gcov, and Mull with the clang of its LLVM release
474
+ for mutation. Tested with GCC 13.3 on Ubuntu 24.04 under WSL.
475
+ - A model service: OpenAI, an OpenAI-compatible service such as vLLM, or
476
+ Ollama.
477
+
478
+ [Tested environments](https://k-kammler.github.io/testuary/reference/tested-environments/)
479
+ lists the tested combinations in detail.
480
+
481
+ ### Known limitations
482
+
483
+ - The model sees the file under test, not the classes it uses. Tests of
484
+ code that depends on other classes of the project can be harder for it
485
+ to write.
486
+ - A test can leave the module under test changed for the tests after it.
487
+ For Python the test run names such a test in a note, and a loop at its
488
+ goal waits for a passing run without one. When the loop's request, token
489
+ or time limit ends it first, the test stays. The measured scores remain
490
+ valid, because they come from running the delivered suite.
491
+ - A test that passes in a normal run can fail while coverage or mutation
492
+ is measured, for example a Java test that counts a class's members by
493
+ reflection, since the measurement tools add hidden members. A tool that
494
+ cannot measure a suite gives it no number. When the failure names tests,
495
+ in the starter phase or at a phase end, those tests are removed and the
496
+ suite is measured again. Otherwise the starter suite is generated again,
497
+ a phase end's suite is no candidate, and in the loops the model is told
498
+ and it counts as a failed test run.
499
+ - With the default settings, every suite a phase can hand on has run under
500
+ both measurement tools. With `phase_end_mutation = false` the candidates
501
+ at the end of the starter and coverage phases are measured under the
502
+ coverage tool only, the end of the mutation phase still measures
503
+ mutation, and with `reduced_suite_mutation = derive` the reduced suite's
504
+ mutation score is derived from the merged suite's run. The results page
505
+ labels that derived score.
506
+ - Adding a language takes a LangAdapter adapter and no Testuary code, as
507
+ [Make a language ready for Testuary](https://k-kammler.github.io/testuary/extending/languages/)
508
+ describes.
509
+ - Some Python source operations read the project's code with the Python
510
+ that runs Testuary, so code written for a newer Python than that one can
511
+ be refused.
512
+ - Gradle projects were not tested in full runs. A Java project with
513
+ both Maven and Gradle build files is refused.
514
+ - Conda, uv, Poetry and Pipenv environments were not tested beyond their
515
+ interpreter.
516
+ - On Windows, a project's commands cannot run in a working directory longer
517
+ than 258 characters.
518
+
519
+ ### Documentation
520
+
521
+ The guides are on the
522
+ [documentation site](https://k-kammler.github.io/testuary/),
523
+ starting with
524
+ [Getting started](https://k-kammler.github.io/testuary/getting-started/).
@@ -0,0 +1,21 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use Testuary, please cite it as below."
3
+ type: software
4
+ title: "Testuary"
5
+ abstract: >-
6
+ Testuary writes unit test suites for Python and Java code, and for C code
7
+ on Linux, with a language model. It measures the generated tests' line and branch coverage and how
8
+ many injected faults (mutants) they detect, asks the model to close the
9
+ gaps, and reduces the suite to the tests that matter.
10
+ authors:
11
+ - family-names: "Kammler"
12
+ given-names: "Kevin"
13
+ license: MIT
14
+ version: "0.2.0"
15
+ keywords:
16
+ - test generation
17
+ - mutation testing
18
+ - large language models
19
+ - automated testing
20
+ repository-code: "https://github.com/k-kammler/testuary"
21
+ url: "https://github.com/k-kammler/testuary"
testuary-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Kevin Kammler
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.