certo 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. {certo-0.2.0 → certo-0.2.2}/.env.example +4 -0
  2. {certo-0.2.0 → certo-0.2.2}/PKG-INFO +2 -2
  3. {certo-0.2.0 → certo-0.2.2}/README.md +1 -1
  4. {certo-0.2.0 → certo-0.2.2}/pyproject.toml +1 -1
  5. {certo-0.2.0 → certo-0.2.2}/src/certo/__init__.py +1 -1
  6. {certo-0.2.0 → certo-0.2.2}/src/certo/admin_cli.py +49 -0
  7. {certo-0.2.0 → certo-0.2.2}/src/certo/bench.py +5 -0
  8. {certo-0.2.0 → certo-0.2.2}/src/certo/bench_config.py +4 -0
  9. {certo-0.2.0 → certo-0.2.2}/src/certo/bench_report.py +8 -0
  10. {certo-0.2.0 → certo-0.2.2}/src/certo/bench_strategies.py +6 -0
  11. {certo-0.2.0 → certo-0.2.2}/src/certo/catalog_check.py +8 -3
  12. {certo-0.2.0 → certo-0.2.2}/src/certo/cli.py +46 -14
  13. {certo-0.2.0 → certo-0.2.2}/src/certo/config.py +5 -0
  14. {certo-0.2.0 → certo-0.2.2}/src/certo/context_window.py +15 -0
  15. {certo-0.2.0 → certo-0.2.2}/src/certo/escalation.py +3 -0
  16. {certo-0.2.0 → certo-0.2.2}/src/certo/llm_check.py +45 -2
  17. {certo-0.2.0 → certo-0.2.2}/src/certo/mcp_server.py +12 -8
  18. {certo-0.2.0 → certo-0.2.2}/src/certo/result.py +2 -0
  19. {certo-0.2.0 → certo-0.2.2}/src/certo/run_history.py +3 -0
  20. certo-0.2.2/src/certo/run_progress.py +130 -0
  21. certo-0.2.2/src/certo/run_report.py +244 -0
  22. {certo-0.2.0 → certo-0.2.2}/src/certo/runner.py +58 -13
  23. {certo-0.2.0 → certo-0.2.2}/src/certo/runs.py +7 -1
  24. {certo-0.2.0 → certo-0.2.2}/src/certo/task_store.py +75 -0
  25. {certo-0.2.0 → certo-0.2.2}/src/certo/text_tool_calls.py +43 -0
  26. {certo-0.2.0 → certo-0.2.2}/tests/conftest.py +4 -2
  27. {certo-0.2.0 → certo-0.2.2}/tests/test_admin_cli.py +67 -0
  28. {certo-0.2.0 → certo-0.2.2}/tests/test_bench.py +14 -0
  29. {certo-0.2.0 → certo-0.2.2}/tests/test_bench_config.py +87 -4
  30. {certo-0.2.0 → certo-0.2.2}/tests/test_bench_report.py +5 -4
  31. {certo-0.2.0 → certo-0.2.2}/tests/test_cli.py +123 -1
  32. {certo-0.2.0 → certo-0.2.2}/tests/test_context_window.py +23 -0
  33. {certo-0.2.0 → certo-0.2.2}/tests/test_escalation.py +7 -0
  34. {certo-0.2.0 → certo-0.2.2}/tests/test_llm_check.py +49 -0
  35. {certo-0.2.0 → certo-0.2.2}/tests/test_result.py +2 -0
  36. {certo-0.2.0 → certo-0.2.2}/tests/test_run_history.py +14 -0
  37. certo-0.2.2/tests/test_run_progress.py +143 -0
  38. certo-0.2.2/tests/test_run_report.py +165 -0
  39. {certo-0.2.0 → certo-0.2.2}/tests/test_runner.py +103 -0
  40. {certo-0.2.0 → certo-0.2.2}/tests/test_runs.py +5 -1
  41. {certo-0.2.0 → certo-0.2.2}/tests/test_task_store.py +61 -0
  42. {certo-0.2.0 → certo-0.2.2}/tests/test_text_tool_calls.py +37 -1
  43. {certo-0.2.0 → certo-0.2.2}/.gitignore +0 -0
  44. {certo-0.2.0 → certo-0.2.2}/LICENSE +0 -0
  45. {certo-0.2.0 → certo-0.2.2}/MANUAL.md +0 -0
  46. {certo-0.2.0 → certo-0.2.2}/docker/agent-server.Dockerfile +0 -0
  47. {certo-0.2.0 → certo-0.2.2}/src/certo/__main__.py +0 -0
  48. {certo-0.2.0 → certo-0.2.2}/src/certo/acceptance.py +0 -0
  49. {certo-0.2.0 → certo-0.2.2}/src/certo/agent.py +0 -0
  50. {certo-0.2.0 → certo-0.2.2}/src/certo/api.py +0 -0
  51. {certo-0.2.0 → certo-0.2.2}/src/certo/artifacts.py +0 -0
  52. {certo-0.2.0 → certo-0.2.2}/src/certo/banner.py +0 -0
  53. {certo-0.2.0 → certo-0.2.2}/src/certo/bench_tasks.py +0 -0
  54. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/VERSION +0 -0
  55. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/duration-parser/hidden_tests/test_hidden_durations.py +0 -0
  56. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/duration-parser/reference/durations.py +0 -0
  57. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/duration-parser/task.md +0 -0
  58. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/hidden_tests/test_hidden_inventory.py +0 -0
  59. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/reference/inventory.py +0 -0
  60. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/reference/test_inventory.py +0 -0
  61. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/starter/inventory.py +0 -0
  62. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/starter/test_inventory.py +0 -0
  63. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/task.md +0 -0
  64. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/hidden_tests/test_hidden_ledger.py +0 -0
  65. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/money.py +0 -0
  66. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/reports.py +0 -0
  67. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/store.py +0 -0
  68. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/transfers.py +0 -0
  69. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/tests/test_money.py +0 -0
  70. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/tests/test_store.py +0 -0
  71. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/tests/test_transfers.py +0 -0
  72. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/README.md +0 -0
  73. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/__init__.py +0 -0
  74. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/__main__.py +0 -0
  75. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/categorize.py +0 -0
  76. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/cli.py +0 -0
  77. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/importers.py +0 -0
  78. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/models.py +0 -0
  79. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/money.py +0 -0
  80. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/reports.py +0 -0
  81. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/store.py +0 -0
  82. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/pyproject.toml +0 -0
  83. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/samples/alpha.csv +0 -0
  84. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/samples/beta.csv +0 -0
  85. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/samples/rules.txt +0 -0
  86. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/__init__.py +0 -0
  87. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/helpers.py +0 -0
  88. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_categorize.py +0 -0
  89. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_cli.py +0 -0
  90. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_importers.py +0 -0
  91. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_money.py +0 -0
  92. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_reports.py +0 -0
  93. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_store.py +0 -0
  94. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/task.md +0 -0
  95. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/hidden_tests/loans_test.go +0 -0
  96. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/reference/api/api.go +0 -0
  97. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/reference/api/api_test.go +0 -0
  98. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/starter/api/api.go +0 -0
  99. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/starter/cmd/server/main.go +0 -0
  100. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/starter/go.mod +0 -0
  101. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/task.md +0 -0
  102. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/page-ranges/hidden_tests/test_hidden_pages.py +0 -0
  103. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/page-ranges/reference/pages.py +0 -0
  104. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/page-ranges/task.md +0 -0
  105. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/pedestrian-crossing/hidden_tests/test_hidden_crossing.py +0 -0
  106. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/pedestrian-crossing/reference/crossing.py +0 -0
  107. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/pedestrian-crossing/task.md +0 -0
  108. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/hidden_tests/room_bookings.test.js +0 -0
  109. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/rooms.js +0 -0
  110. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/server.js +0 -0
  111. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/test/app.test.js +0 -0
  112. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/ui.js +0 -0
  113. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/README.md +0 -0
  114. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/package-lock.json +0 -0
  115. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/package.json +0 -0
  116. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/server.js +0 -0
  117. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/ui.js +0 -0
  118. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/task.md +0 -0
  119. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/hidden_tests/versions.test.js +0 -0
  120. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/reference/package.json +0 -0
  121. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/reference/versions.js +0 -0
  122. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/starter/package.json +0 -0
  123. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/task.md +0 -0
  124. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/hidden_tests/slugify.test.js +0 -0
  125. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/reference/package.json +0 -0
  126. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/reference/slugify.js +0 -0
  127. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/starter/package.json +0 -0
  128. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/task.md +0 -0
  129. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/temperature-table/hidden_tests/test_hidden_temperature.py +0 -0
  130. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/temperature-table/reference/temperature.py +0 -0
  131. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/temperature-table/task.md +0 -0
  132. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/hidden_tests/todo_cli.rs +0 -0
  133. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/reference/src/board.rs +0 -0
  134. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/reference/src/main.rs +0 -0
  135. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/reference/src/store.rs +0 -0
  136. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/starter/Cargo.toml +0 -0
  137. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/starter/src/main.rs +0 -0
  138. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/task.md +0 -0
  139. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/hidden_tests/tokenBucket.test.js +0 -0
  140. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/reference/package.json +0 -0
  141. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/reference/tokenBucket.js +0 -0
  142. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/starter/package.json +0 -0
  143. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/task.md +0 -0
  144. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/wordle-score/hidden_tests/test_hidden_wordle_score.py +0 -0
  145. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/wordle-score/reference/wordle_score.py +0 -0
  146. {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/wordle-score/task.md +0 -0
  147. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/git/commit-conventions.md +0 -0
  148. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/completion-and-release-readiness.md +0 -0
  149. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/debugging-and-failure-repair.md +0 -0
  150. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/documentation-and-operational-readiness.md +0 -0
  151. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/implementation-planning.md +0 -0
  152. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/repository-discovery.md +0 -0
  153. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/requirements-analysis.md +0 -0
  154. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/security-review.md +0 -0
  155. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/testing-and-verification.md +0 -0
  156. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/node/node-project-setup.md +0 -0
  157. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/python/python-environment.md +0 -0
  158. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/python-web/fastapi-conventions.md +0 -0
  159. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/python-web/pin-dependencies.md +0 -0
  160. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/testing/pytest-conventions.md +0 -0
  161. {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/testing/testable-interactive-apps.md +0 -0
  162. {certo-0.2.0 → certo-0.2.2}/src/certo/cancel.py +0 -0
  163. {certo-0.2.0 → certo-0.2.2}/src/certo/catalog_report.py +0 -0
  164. {certo-0.2.0 → certo-0.2.2}/src/certo/check_files.py +0 -0
  165. {certo-0.2.0 → certo-0.2.2}/src/certo/checkpoints.py +0 -0
  166. {certo-0.2.0 → certo-0.2.2}/src/certo/checkpoints_cmd.py +0 -0
  167. {certo-0.2.0 → certo-0.2.2}/src/certo/console.py +0 -0
  168. {certo-0.2.0 → certo-0.2.2}/src/certo/cost_limit.py +0 -0
  169. {certo-0.2.0 → certo-0.2.2}/src/certo/custom_tools/__init__.py +0 -0
  170. {certo-0.2.0 → certo-0.2.2}/src/certo/custom_tools/example_tool.py +0 -0
  171. {certo-0.2.0 → certo-0.2.2}/src/certo/custom_tools/run_tests_tool.py +0 -0
  172. {certo-0.2.0 → certo-0.2.2}/src/certo/deadline.py +0 -0
  173. {certo-0.2.0 → certo-0.2.2}/src/certo/examples/bench.yaml.example +0 -0
  174. {certo-0.2.0 → certo-0.2.2}/src/certo/examples/escalation.yaml.example +0 -0
  175. {certo-0.2.0 → certo-0.2.2}/src/certo/examples/models.yaml.example +0 -0
  176. {certo-0.2.0 → certo-0.2.2}/src/certo/examples/models_list.yaml.example +0 -0
  177. {certo-0.2.0 → certo-0.2.2}/src/certo/examples/task_profiles.yaml.example +0 -0
  178. {certo-0.2.0 → certo-0.2.2}/src/certo/independent_tests.py +0 -0
  179. {certo-0.2.0 → certo-0.2.2}/src/certo/init_cmd.py +0 -0
  180. {certo-0.2.0 → certo-0.2.2}/src/certo/init_models.py +0 -0
  181. {certo-0.2.0 → certo-0.2.2}/src/certo/llm.py +0 -0
  182. {certo-0.2.0 → certo-0.2.2}/src/certo/menus.py +0 -0
  183. {certo-0.2.0 → certo-0.2.2}/src/certo/milestones.py +0 -0
  184. {certo-0.2.0 → certo-0.2.2}/src/certo/model_catalog.py +0 -0
  185. {certo-0.2.0 → certo-0.2.2}/src/certo/model_files.py +0 -0
  186. {certo-0.2.0 → certo-0.2.2}/src/certo/model_pricing.py +0 -0
  187. {certo-0.2.0 → certo-0.2.2}/src/certo/model_selection.py +0 -0
  188. {certo-0.2.0 → certo-0.2.2}/src/certo/models_cmd.py +0 -0
  189. {certo-0.2.0 → certo-0.2.2}/src/certo/plugins.py +0 -0
  190. {certo-0.2.0 → certo-0.2.2}/src/certo/project_env.py +0 -0
  191. {certo-0.2.0 → certo-0.2.2}/src/certo/project_state.py +0 -0
  192. {certo-0.2.0 → certo-0.2.2}/src/certo/run_summary.py +0 -0
  193. {certo-0.2.0 → certo-0.2.2}/src/certo/server.py +0 -0
  194. {certo-0.2.0 → certo-0.2.2}/src/certo/skill_library.py +0 -0
  195. {certo-0.2.0 → certo-0.2.2}/src/certo/skills.py +0 -0
  196. {certo-0.2.0 → certo-0.2.2}/src/certo/skills_cmd.py +0 -0
  197. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/README.md +0 -0
  198. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/gitignore +0 -0
  199. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/providers.toml +0 -0
  200. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/hanoi.md +0 -0
  201. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/invaders.md +0 -0
  202. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/mazeman.md +0 -0
  203. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/space-rocks.md +0 -0
  204. {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/wordle.md +0 -0
  205. {certo-0.2.0 → certo-0.2.2}/src/certo/third_party_skills.py +0 -0
  206. {certo-0.2.0 → certo-0.2.2}/src/certo/tools.py +0 -0
  207. {certo-0.2.0 → certo-0.2.2}/src/certo/update_check.py +0 -0
  208. {certo-0.2.0 → certo-0.2.2}/src/certo/verification.py +0 -0
  209. {certo-0.2.0 → certo-0.2.2}/src/certo/workspace.py +0 -0
  210. {certo-0.2.0 → certo-0.2.2}/src/certo/worktree.py +0 -0
  211. {certo-0.2.0 → certo-0.2.2}/tests/custom_tools/test_run_tests_tool.py +0 -0
  212. {certo-0.2.0 → certo-0.2.2}/tests/model_file_helpers.py +0 -0
  213. {certo-0.2.0 → certo-0.2.2}/tests/test_acceptance.py +0 -0
  214. {certo-0.2.0 → certo-0.2.2}/tests/test_agent.py +0 -0
  215. {certo-0.2.0 → certo-0.2.2}/tests/test_api.py +0 -0
  216. {certo-0.2.0 → certo-0.2.2}/tests/test_artifacts.py +0 -0
  217. {certo-0.2.0 → certo-0.2.2}/tests/test_banner.py +0 -0
  218. {certo-0.2.0 → certo-0.2.2}/tests/test_bench_tasks.py +0 -0
  219. {certo-0.2.0 → certo-0.2.2}/tests/test_catalog_check.py +0 -0
  220. {certo-0.2.0 → certo-0.2.2}/tests/test_check_files.py +0 -0
  221. {certo-0.2.0 → certo-0.2.2}/tests/test_checkpoints.py +0 -0
  222. {certo-0.2.0 → certo-0.2.2}/tests/test_checkpoints_cmd.py +0 -0
  223. {certo-0.2.0 → certo-0.2.2}/tests/test_config.py +0 -0
  224. {certo-0.2.0 → certo-0.2.2}/tests/test_console.py +0 -0
  225. {certo-0.2.0 → certo-0.2.2}/tests/test_cost_limit.py +0 -0
  226. {certo-0.2.0 → certo-0.2.2}/tests/test_deadline.py +0 -0
  227. {certo-0.2.0 → certo-0.2.2}/tests/test_docker_image.py +0 -0
  228. {certo-0.2.0 → certo-0.2.2}/tests/test_docs.py +0 -0
  229. {certo-0.2.0 → certo-0.2.2}/tests/test_escalation_runner.py +0 -0
  230. {certo-0.2.0 → certo-0.2.2}/tests/test_independent_tests.py +0 -0
  231. {certo-0.2.0 → certo-0.2.2}/tests/test_init_cmd.py +0 -0
  232. {certo-0.2.0 → certo-0.2.2}/tests/test_init_models.py +0 -0
  233. {certo-0.2.0 → certo-0.2.2}/tests/test_llm.py +0 -0
  234. {certo-0.2.0 → certo-0.2.2}/tests/test_mcp_server.py +0 -0
  235. {certo-0.2.0 → certo-0.2.2}/tests/test_menus.py +0 -0
  236. {certo-0.2.0 → certo-0.2.2}/tests/test_milestones.py +0 -0
  237. {certo-0.2.0 → certo-0.2.2}/tests/test_model_catalog.py +0 -0
  238. {certo-0.2.0 → certo-0.2.2}/tests/test_model_files.py +0 -0
  239. {certo-0.2.0 → certo-0.2.2}/tests/test_model_selection.py +0 -0
  240. {certo-0.2.0 → certo-0.2.2}/tests/test_models_cmd.py +0 -0
  241. {certo-0.2.0 → certo-0.2.2}/tests/test_packaging.py +0 -0
  242. {certo-0.2.0 → certo-0.2.2}/tests/test_plugins.py +0 -0
  243. {certo-0.2.0 → certo-0.2.2}/tests/test_project_env.py +0 -0
  244. {certo-0.2.0 → certo-0.2.2}/tests/test_project_env_layouts.py +0 -0
  245. {certo-0.2.0 → certo-0.2.2}/tests/test_project_state.py +0 -0
  246. {certo-0.2.0 → certo-0.2.2}/tests/test_readme.py +0 -0
  247. {certo-0.2.0 → certo-0.2.2}/tests/test_run_summary.py +0 -0
  248. {certo-0.2.0 → certo-0.2.2}/tests/test_server.py +0 -0
  249. {certo-0.2.0 → certo-0.2.2}/tests/test_skill_library.py +0 -0
  250. {certo-0.2.0 → certo-0.2.2}/tests/test_skills.py +0 -0
  251. {certo-0.2.0 → certo-0.2.2}/tests/test_skills_cmd.py +0 -0
  252. {certo-0.2.0 → certo-0.2.2}/tests/test_third_party_skills.py +0 -0
  253. {certo-0.2.0 → certo-0.2.2}/tests/test_update_check.py +0 -0
  254. {certo-0.2.0 → certo-0.2.2}/tests/test_verification_wrappers.py +0 -0
  255. {certo-0.2.0 → certo-0.2.2}/tests/test_version.py +0 -0
  256. {certo-0.2.0 → certo-0.2.2}/tests/test_workspace.py +0 -0
  257. {certo-0.2.0 → certo-0.2.2}/tests/test_worktree.py +0 -0
@@ -121,6 +121,10 @@ CERTO_INDEPENDENT_TESTS_STRICT=no
121
121
  # The same as --verbose.
122
122
  CERTO_VERBOSE=no
123
123
 
124
+ # How often a run in the terminal prints its progress line (milestone,
125
+ # fix attempt, last checks, time, cost so far), in seconds. 0 turns it off.
126
+ CERTO_PROGRESS_SECONDS=120
127
+
124
128
  # --- Model selection (see docs/guides/pick-models-automatically.md and
125
129
  # docs/guides/escalate-between-models.md). With manual (the default),
126
130
  # CERTO_LLM_MODEL above is used and no model file is read. With auto or
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: certo
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: A coding-agent harness that isn't tied to one model or provider: it picks a model per task, switches when one fails, and checks the work with your own tests.
5
5
  Project-URL: Homepage, https://github.com/alessioricco/certo
6
6
  Project-URL: Repository, https://github.com/alessioricco/certo
@@ -373,7 +373,7 @@ The full list is in the [documentation](https://github.com/alessioricco/certo/bl
373
373
 
374
374
  ## Contributing
375
375
 
376
- Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). To work on the code:
376
+ Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). Questions and ideas go in [Discussions](https://github.com/alessioricco/certo/discussions). To work on the code:
377
377
 
378
378
  ```bash
379
379
  git clone https://github.com/alessioricco/certo.git
@@ -321,7 +321,7 @@ The full list is in the [documentation](https://github.com/alessioricco/certo/bl
321
321
 
322
322
  ## Contributing
323
323
 
324
- Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). To work on the code:
324
+ Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). Questions and ideas go in [Discussions](https://github.com/alessioricco/certo/discussions). To work on the code:
325
325
 
326
326
  ```bash
327
327
  git clone https://github.com/alessioricco/certo.git
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "certo"
3
- version = "0.2.0"
3
+ version = "0.2.2"
4
4
  description = "A coding-agent harness that isn't tied to one model or provider: it picks a model per task, switches when one fails, and checks the work with your own tests."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12"
@@ -19,4 +19,4 @@ os.environ.setdefault("OPENHANDS_SUPPRESS_BANNER", "1")
19
19
  # which config.py loads without overriding. Certo reads only ./.env.
20
20
  os.environ.setdefault("LITELLM_MODE", "PRODUCTION")
21
21
 
22
- __version__ = "0.2.0"
22
+ __version__ = "0.2.2"
@@ -14,6 +14,7 @@ redis/mysql/postgres store.
14
14
  from __future__ import annotations
15
15
 
16
16
  import argparse
17
+ from datetime import UTC, datetime
17
18
 
18
19
  from . import __version__
19
20
  from .config import ConfigError, load_config, override_llm
@@ -64,6 +65,42 @@ def _cmd_purge(args: argparse.Namespace) -> int:
64
65
  return 0
65
66
 
66
67
 
68
+ # A task's text is cut to this many characters in `list`; `show` has more.
69
+ _LISTED_TASK_CHARS = 50
70
+
71
+
72
+ def _cmd_list(args: argparse.Namespace) -> int:
73
+ if args.limit < 0:
74
+ print("--limit must be 0 (all) or more.")
75
+ return 1
76
+ cfg = load_config()
77
+ store = build_task_store(cfg)
78
+ try:
79
+ summaries = store.list_tasks(args.project)
80
+ finally:
81
+ store.close()
82
+ if not summaries:
83
+ where = f" for project {args.project!r}" if args.project is not None else ""
84
+ print(f"No task records{where}.")
85
+ return 0
86
+ shown = summaries[: args.limit] if args.limit else summaries
87
+ rows = [("ID", "PROJECT", "STATUS", "VERDICT", "CREATED (UTC)", "TASK")]
88
+ for s in shown:
89
+ task = " ".join(s.task.split())
90
+ if len(task) > _LISTED_TASK_CHARS:
91
+ task = task[: _LISTED_TASK_CHARS - 3] + "..."
92
+ created = datetime.fromtimestamp(s.created_at, UTC).strftime("%Y-%m-%d %H:%M")
93
+ rows.append((s.id, s.project or "-", s.status, s.verification_state or "-", created, task))
94
+ widths = [max(len(row[i]) for row in rows) for i in range(len(rows[0]) - 1)]
95
+ for row in rows:
96
+ print(
97
+ " ".join(cell.ljust(w) for cell, w in zip(row, widths, strict=False)) + " " + row[-1]
98
+ )
99
+ if len(shown) < len(summaries):
100
+ print(f"({len(shown)} of {len(summaries)} shown, newest first; --limit 0 shows all.)")
101
+ return 0
102
+
103
+
67
104
  def _cmd_show(args: argparse.Namespace) -> int:
68
105
  cfg = load_config()
69
106
  store = build_task_store(cfg)
@@ -147,6 +184,18 @@ def build_parser() -> argparse.ArgumentParser:
147
184
  )
148
185
  purge.set_defaults(func=_cmd_purge)
149
186
 
187
+ list_cmd = subparsers.add_parser(
188
+ "list", help="List the task records, newest first: id, project, status and verdict."
189
+ )
190
+ list_cmd.add_argument("--project", default=None, help="Only the records of this project.")
191
+ list_cmd.add_argument(
192
+ "--limit",
193
+ type=int,
194
+ default=50,
195
+ help="Show at most this many records (default 50); 0 shows all.",
196
+ )
197
+ list_cmd.set_defaults(func=_cmd_list)
198
+
150
199
  show = subparsers.add_parser("show", help="Print one task record's summary.")
151
200
  show.add_argument("task_id")
152
201
  show.set_defaults(func=_cmd_show)
@@ -210,6 +210,7 @@ def _combined_attempt(attempts: list[AttemptResult]) -> AttemptResult:
210
210
  return AttemptResult(
211
211
  verification_state=last.verification_state,
212
212
  retries_used=sum(a.retries_used for a in attempts),
213
+ tool_call_reminders=sum(a.tool_call_reminders for a in attempts),
213
214
  run_summary=combine_summaries(summaries) if summaries else None,
214
215
  models_used=tuple(dict.fromkeys(m for a in attempts for m in a.models_used)),
215
216
  rungs=tuple(r for a in attempts for r in a.rungs),
@@ -353,6 +354,10 @@ def execute_run(
353
354
  "cost": round(summary.total_cost, 6) if summary else 0.0,
354
355
  "cost_known": summary.cost_complete if summary else True,
355
356
  "retries_used": attempt.retries_used if attempt else 0,
357
+ # How often the agent wrote a tool call as text instead of making it,
358
+ # or ended on a sentence of plans
359
+ # (CERTO_TOOL_CALL_REMINDERS); summed over rungs for an escalation.
360
+ "tool_call_reminders": attempt.tool_call_reminders if attempt else 0,
356
361
  # How often older steps were summarized (the context window filled up).
357
362
  "condensations": summary.condensations if summary else 0,
358
363
  # Escalation only: every rung that ran, and which one gave the verdict.
@@ -306,6 +306,10 @@ def load_bench_config(path: str = DEFAULT_BENCH_FILE) -> BenchConfig:
306
306
  escalation_file=escalation_file,
307
307
  task_profiles_file=task_profiles_file,
308
308
  on_warning=catalog_warnings.append,
309
+ # A file that names its own models.yaml doesn't pick up the
310
+ # ladders and profiles in the current folder: they name other
311
+ # models.
312
+ look_up_optional=not raw.get("models_file"),
309
313
  )
310
314
  catalog = {e.name: e for e in loaded.models}
311
315
  except ModelCatalogError as exc:
@@ -40,6 +40,7 @@ CSV_FIELDS = (
40
40
  "cost",
41
41
  "cost_known",
42
42
  "retries_used",
43
+ "tool_call_reminders",
43
44
  "final_rung",
44
45
  "independent_status",
45
46
  "error",
@@ -256,6 +257,7 @@ def arm_rows(meta: dict[str, Any], records: list[dict[str, Any]]) -> list[dict[s
256
257
  else "-",
257
258
  "Tokens in / out": f"{sum(r['tokens_in'] for r in mine):,} / "
258
259
  f"{sum(r['tokens_out'] for r in mine):,}",
260
+ "Tool-call reminders": str(sum(r.get("tool_call_reminders", 0) for r in mine)),
259
261
  "Cost": _cost(mine) if mine else "-",
260
262
  "Cost per solved task": _cost_per_solved(mine),
261
263
  **(_independent_columns(mine) if independent else {}),
@@ -431,6 +433,12 @@ def render_markdown(meta: dict[str, Any], records: list[dict[str, Any]]) -> str:
431
433
  ),
432
434
  "- **Errors:** the run stopped with an error (for example, the provider failed).",
433
435
  "- **Median time:** wall time of one run, including Certo's checks and retries.",
436
+ (
437
+ "- **Tool-call reminders:** how many times the agent wrote a tool call as text "
438
+ "instead of making it, or ended its turn on a sentence of plans, and Certo sent "
439
+ "it back (`CERTO_TOOL_CALL_REMINDERS`), added up over the arm's runs. An "
440
+ "escalation run counts every rung's own."
441
+ ),
434
442
  "- **Cost:** from LiteLLM's prices; `+ unknown` when a model has no price.",
435
443
  (
436
444
  "- **Effort:** the reasoning effort the model ran at; `default` when none was "
@@ -67,6 +67,9 @@ class AttemptResult:
67
67
  independent_tests: Any = None
68
68
  # Auto only: the task profile the task was classified as.
69
69
  task_profile: str | None = None
70
+ # How many tool-call reminders the run's conversation(s) were sent
71
+ # (TaskOutcome.tool_call_reminders; summed over rungs for an escalation).
72
+ tool_call_reminders: int = 0
70
73
 
71
74
 
72
75
  class Strategy(Protocol):
@@ -193,6 +196,7 @@ class SingleStrategy:
193
196
  run_summary=outcome.run_summary,
194
197
  models_used=(entry.name,),
195
198
  independent_tests=getattr(outcome, "independent_tests", None),
199
+ tool_call_reminders=getattr(outcome, "tool_call_reminders", 0),
196
200
  )
197
201
 
198
202
 
@@ -253,6 +257,7 @@ class EscalateStrategy:
253
257
  rungs=rungs,
254
258
  final_rung=outcome.final_rung,
255
259
  independent_tests=getattr(outcome, "independent_tests", None),
260
+ tool_call_reminders=getattr(outcome, "tool_call_reminders", 0),
256
261
  )
257
262
 
258
263
 
@@ -309,6 +314,7 @@ class AutoStrategy:
309
314
  models_used=used,
310
315
  independent_tests=getattr(outcome, "independent_tests", None),
311
316
  task_profile=decisions[0].task_profile if decisions else None,
317
+ tool_call_reminders=getattr(outcome, "tool_call_reminders", 0),
312
318
  )
313
319
 
314
320
 
@@ -187,13 +187,18 @@ def load_checked_catalog(
187
187
  escalation_file: str = "",
188
188
  on_warning: Callable[[str], None] | None = None,
189
189
  mode: str | None = None,
190
+ look_up_optional: bool = True,
190
191
  ) -> ModelCatalog:
191
192
  """Find, load and check the model files (see model_files.py; "" means
192
193
  look them up; task_profiles.yaml and escalation.yaml are optional).
193
- Raises `ModelCatalogError` listing every error; passes each warning to
194
- `on_warning`. `mode` as in `check_catalog`."""
194
+ With `look_up_optional=False`, "" for one of the optional two means
195
+ none. Raises `ModelCatalogError` listing every error; passes each
196
+ warning to `on_warning`. `mode` as in `check_catalog`."""
195
197
  list_path, models_path = resolve(models_file, models_list_file)
196
- profiles_path, escalation_path = resolve_optional(task_profiles_file, escalation_file)
198
+ if look_up_optional:
199
+ profiles_path, escalation_path = resolve_optional(task_profiles_file, escalation_file)
200
+ else:
201
+ profiles_path, escalation_path = task_profiles_file or None, escalation_file or None
197
202
  catalog = load_model_catalog(
198
203
  list_path,
199
204
  models_path,
@@ -46,6 +46,8 @@ from .project_state import (
46
46
  )
47
47
  from .result import run_result
48
48
  from .run_history import append_run, note_models_in_readme, run_entry
49
+ from .run_progress import RunProgress
50
+ from .run_report import report_relpath, write_report
49
51
  from .worktree import was_created_by_certo, worktree_warning
50
52
 
51
53
  if TYPE_CHECKING:
@@ -716,19 +718,32 @@ def _run_task_command(
716
718
  resume_hint = f"CERTO_PROJECTS_DIR={shlex.quote(cfg.projects_dir)} {resume_hint}"
717
719
  started_at = datetime.now(UTC)
718
720
  history = {"task": resume_from.task if resume_from is not None else (task or "")}
721
+ report: dict[str, str] = {}
719
722
 
720
723
  def record_run(ended: str, outcome=None, summary=None) -> None:
721
724
  if not save_state:
722
725
  return
723
726
  entry = run_entry(cfg, ladder, history["task"], ended, started_at, outcome, summary)
724
- entry = replace(entry, resumed=resume_from is not None)
727
+ entry = replace(entry, resumed=resume_from is not None, report=report_relpath(started_at))
725
728
  try:
726
729
  append_run(cfg.workspace, entry)
727
730
  except OSError as exc:
728
731
  print(f"Warning: couldn't add this run to .certo/RUNS.md ({exc}).", file=sys.stderr)
729
732
 
733
+ # The progress line (run_progress.py): one for the whole run, so its
734
+ # time and cost count every milestone.
735
+ progress = (
736
+ RunProgress(say=_print_progress, every_seconds=cfg.progress_seconds)
737
+ if cfg.progress_seconds > 0
738
+ else None
739
+ )
740
+
730
741
  def run_one(text: str, resume, instruction: str | None, checks):
731
- return run_task(
742
+ if progress is not None and milestone_state is not None:
743
+ total = len(milestone_state.plan.milestones)
744
+ progress.where = f"milestone {milestone_state.current} of {total}"
745
+ progress.checks = ""
746
+ result = run_task(
732
747
  text,
733
748
  cfg=cfg,
734
749
  on_confirm=on_confirm,
@@ -742,13 +757,19 @@ def _run_task_command(
742
757
  ladder=ladder,
743
758
  on_escalate=_confirm_escalation if cfg.interactive else None,
744
759
  on_notice=_print_progress,
760
+ run_progress=progress,
745
761
  )
762
+ summary = result.outcome.run_summary
763
+ if progress is not None and summary is not None:
764
+ progress.cost_before += summary.total_cost
765
+ return result
746
766
 
747
767
  if save_state and milestone_state is None and resume_from is None:
748
768
  clear_state(cfg.workspace) # a new task here isn't the old milestones
749
769
 
750
770
  def keep_result(outcome, error: str | None = None) -> None:
751
- if holder is None:
771
+ # The JSON result for --json, and the run's report (run_report.py).
772
+ if holder is None and not save_state:
752
773
  return
753
774
  milestones = None
754
775
  if milestone_state is not None:
@@ -760,7 +781,7 @@ def _run_task_command(
760
781
  }
761
782
  for m in milestone_state.plan.milestones
762
783
  ]
763
- holder["result"] = run_result(
784
+ result = run_result(
764
785
  outcome,
765
786
  task=history["task"],
766
787
  workspace=os.path.abspath(cfg.workspace),
@@ -768,6 +789,13 @@ def _run_task_command(
768
789
  milestones=milestones,
769
790
  error=error,
770
791
  )
792
+ if holder is not None:
793
+ holder["result"] = result
794
+ if save_state:
795
+ try:
796
+ report["path"] = write_report(cfg.workspace, result, started_at)
797
+ except OSError as exc:
798
+ print(f"Warning: couldn't write the run report ({exc}).", file=sys.stderr)
771
799
 
772
800
  try:
773
801
  if milestone_state is None:
@@ -859,16 +887,13 @@ def _run_task_command(
859
887
  if outcome.rungs and milestone_state is None:
860
888
  for line in format_rung_lines(outcome.rungs, outcome.final_rung):
861
889
  print(f" {line}")
890
+ if report.get("path"):
891
+ print(f"\nReport: {_short_path(report['path'])}")
862
892
  failing = outcome.verification_state in _FAILING_STATES or (
863
893
  args.require_verification and outcome.verification_state in _unproven_states()
864
894
  )
865
895
  if failing and save_state:
866
- status = status_path(cfg.workspace)
867
- try:
868
- status = os.path.relpath(status)
869
- except ValueError:
870
- # Windows cannot make a relative path between different drives.
871
- pass
896
+ status = _short_path(status_path(cfg.workspace))
872
897
  print(f"\nWhere it stopped: {status}. Continue with: {resume_hint}")
873
898
  if save_state and cfg.checkpoints:
874
899
  hint = _checkpoints_hint(cfg.workspace)
@@ -877,6 +902,16 @@ def _run_task_command(
877
902
  return 1 if failing else 0
878
903
 
879
904
 
905
+ def _short_path(path: str) -> str:
906
+ """`path` relative to the current folder, or absolute when that is
907
+ shorter (a project far away would be a long `../..` chain)."""
908
+ path = os.path.abspath(path)
909
+ try:
910
+ return min(path, os.path.relpath(path), key=len)
911
+ except ValueError:
912
+ return path # Windows: no relative path between different drives
913
+
914
+
880
915
  def _checkpoints_hint(workspace: str) -> str | None:
881
916
  """One line naming the project's checkpoints (checkpoints.py)."""
882
917
  from .checkpoints import git_available, list_checkpoints
@@ -887,10 +922,7 @@ def _checkpoints_hint(workspace: str) -> str | None:
887
922
  return None
888
923
  if not count:
889
924
  return None
890
- try:
891
- where = os.path.relpath(workspace)
892
- except ValueError:
893
- where = workspace
925
+ where = _short_path(workspace)
894
926
  path = "" if where == "." else f" {shlex.quote(where)}"
895
927
  return (
896
928
  f"Checkpoints: {count} (list them: certo checkpoints{path}; "
@@ -227,6 +227,9 @@ class Config:
227
227
  # its skills, the SDK's INFO logs. Off by default (console.py shows a
228
228
  # line for each instead).
229
229
  verbose: bool = False
230
+ # CERTO_PROGRESS_SECONDS: how often a terminal run prints its progress
231
+ # line while the agent works (run_progress.py); 0 turns the line off.
232
+ progress_seconds: int = 120
230
233
 
231
234
  # Deterministic auto model selection (see model_catalog.py/
232
235
  # model_selection.py). Default ("manual") is byte-for-byte the
@@ -690,6 +693,7 @@ def load_config(
690
693
  default=False,
691
694
  )
692
695
  verbose = _parse_bool(env, "VERBOSE", default=False)
696
+ progress_seconds = _parse_nonnegative_int(env, "PROGRESS_SECONDS", default=120)
693
697
  independent_tests_strict = _parse_bool(env, "INDEPENDENT_TESTS_STRICT", default=False)
694
698
  independent_tests = (
695
699
  _parse_bool(env, "INDEPENDENT_TESTS", default=False) or independent_tests_strict
@@ -771,6 +775,7 @@ def load_config(
771
775
  context_warn_tokens=context_warn_tokens,
772
776
  tool_call_reminders=tool_call_reminders,
773
777
  verbose=verbose,
778
+ progress_seconds=progress_seconds,
774
779
  model_selection=model_selection,
775
780
  models_file=models_file,
776
781
  models_list_file=models_list_file,
@@ -119,6 +119,21 @@ def ollama_model_digest(model: str, base_url: str | None, timeout: float = 10) -
119
119
  return None
120
120
 
121
121
 
122
+ def ollama_capabilities(model: str, base_url: str | None, timeout: float = 10) -> list[str] | None:
123
+ """What the server says `model` can do (`/api/show`), such as
124
+ `tools` and `thinking`, or None when it can't be learned (a model it
125
+ doesn't have, an old server, no answer). Doesn't load the model."""
126
+ name = model.split("/", 1)[1]
127
+ base = (base_url or _DEFAULT_OLLAMA_URL).rstrip("/")
128
+ try:
129
+ found = _post(f"{base}/api/show", {"model": name}, timeout).get("capabilities")
130
+ except (OSError, ValueError, AttributeError, urllib.error.URLError):
131
+ return None
132
+ if not isinstance(found, list):
133
+ return None
134
+ return [c for c in found if isinstance(c, str)]
135
+
136
+
122
137
  def server_address(base_url: str | None) -> str:
123
138
  """Host and port only: no scheme, path or credentials."""
124
139
  parsed = urllib.parse.urlsplit(base_url or _DEFAULT_OLLAMA_URL)
@@ -332,6 +332,8 @@ class RungResult:
332
332
  changes: FileChanges = FileChanges()
333
333
  failing_checks: tuple[str, ...] = () # "name (`command`)"
334
334
  reasoning_effort: str | None = None # None: no effort set
335
+ # How many tool-call reminders this rung's own conversation was sent.
336
+ tool_call_reminders: int = 0
335
337
 
336
338
  @property
337
339
  def described(self) -> str:
@@ -383,6 +385,7 @@ class RungResult:
383
385
  "verdict": self.verdict,
384
386
  "failure": self.failure,
385
387
  "fix_attempts": self.retries_used,
388
+ "tool_call_reminders": self.tool_call_reminders,
386
389
  "duration_seconds": round(self.duration_seconds, 3),
387
390
  "tokens": summary.total_tokens if summary else 0,
388
391
  "cost": self.cost,
@@ -126,6 +126,7 @@ class LLMCheckResult:
126
126
  # "CERTO_LLM_CONTEXT_TOKENS" or "server" — where context_tokens came from.
127
127
  context_source: str | None = None
128
128
  context_warning: str | None = None # the run's own warning text, if any
129
+ thinking: str | None = None # thinking_line(), when the server says
129
130
 
130
131
 
131
132
  def _scrub(text: str, secret: str | None) -> str:
@@ -339,6 +340,12 @@ def _ollama_details(cfg: Config, *, timeout: int | None = None) -> dict:
339
340
  details["ollama_version"] = context_window.ollama_version(cfg.base_url, t)
340
341
  except Exception: # noqa: BLE001 - best effort
341
342
  pass
343
+ try:
344
+ capabilities = context_window.ollama_capabilities(cfg.model, cfg.base_url, t)
345
+ if capabilities is not None:
346
+ details["thinking"] = thinking_line(cfg.model, capabilities, cfg.reasoning_effort)
347
+ except Exception: # noqa: BLE001
348
+ pass
342
349
  try:
343
350
  tokens = context_window.known_context_tokens(
344
351
  cfg.model, cfg.llm_context_tokens, cfg.base_url, t
@@ -360,11 +367,45 @@ def _ollama_details(cfg: Config, *, timeout: int | None = None) -> dict:
360
367
  return details
361
368
 
362
369
 
370
+ # LiteLLM sends these as `think: true`, and any other effort as `think:
371
+ # false`, except to gpt-oss models, which get the level itself.
372
+ _THINKING_ON = ("low", "medium", "high")
373
+
374
+
375
+ def thinking_line(model: str, capabilities: list[str], effort: str | None) -> str:
376
+ """How an Ollama model will think with this reasoning effort, from what
377
+ the server says it can do."""
378
+ name = model.split("/", 1)[1]
379
+ if "thinking" not in capabilities:
380
+ if effort and effort != "none":
381
+ return (
382
+ f"Thinking: the model has none, but CERTO_LLM_REASONING_EFFORT is {effort}, "
383
+ "so Ollama refuses its requests. Clear the setting or set it to none."
384
+ )
385
+ return "Thinking: the model has none."
386
+ if not effort or effort == "none":
387
+ return (
388
+ "Thinking: the model has it, and CERTO_LLM_REASONING_EFFORT is not set, so it "
389
+ "thinks as Ollama does by default. high turns it on, minimal turns it off."
390
+ )
391
+ if name.startswith("gpt-oss"):
392
+ return f"Thinking: on, at level {effort}."
393
+ if effort in _THINKING_ON:
394
+ return (
395
+ "Thinking: on. Ollama has only on and off for this model, so low, medium "
396
+ "and high think the same."
397
+ )
398
+ return (
399
+ f"Thinking: off. On Ollama, {effort} turns thinking off for this model; only "
400
+ "low, medium and high turn it on."
401
+ )
402
+
403
+
363
404
  def detail_lines(result: LLMCheckResult) -> list[str]:
364
405
  """The extra context lines `certo check` prints for a local (Ollama)
365
406
  model: server version, the context window the run will use and where it
366
- came from, the test request's time, and — separately —
367
- the same small-window warning a run emits. Empty for cloud models, so
407
+ came from, the test request's time, then how the model will think, and
408
+ — separately — the same small-window warning a run emits. Empty for cloud models, so
368
409
  their output is unchanged."""
369
410
  if not _is_ollama(result.model):
370
411
  return []
@@ -381,6 +422,8 @@ def detail_lines(result: LLMCheckResult) -> list[str]:
381
422
  # far below the model's real speed.
382
423
  parts.append(f"{result.latency_seconds:.1f}s")
383
424
  lines = [" · ".join(parts)]
425
+ if result.thinking:
426
+ lines.append(result.thinking)
384
427
  if result.context_warning:
385
428
  lines.append(result.context_warning)
386
429
  return lines
@@ -38,22 +38,26 @@ COMMAND_TOOLS: dict[str, tuple[str, ...]] = {
38
38
  "certo rollback": ("rollback",),
39
39
  }
40
40
 
41
- # The commands with no tool, and why. Those that say "not yet" are still to
42
- # do (TODO.md, "MCP server with every Certo command").
41
+ # The commands with no tool, and why. Those that say "on request" get a
42
+ # tool when someone asks (TODO.md); the others never do (DECISIONS.md).
43
43
  NO_TOOL: dict[str, str] = {
44
44
  "certo init": "asks questions and writes your settings file: run it in a terminal",
45
45
  "certo init models": "asks questions and writes your model files: run it in a terminal",
46
46
  "certo models update": "asks about each change to your models list: run it in a terminal",
47
- "certo skills add": "downloads code, with a license step a person answers: not yet",
48
- "certo skills remove": "changes the skills every run gets: not yet",
49
- "certo skills enable": "turns on downloaded code: not yet",
50
- "certo skills disable": "changes the skills every run gets: not yet",
51
- "certo skills update": "downloads code, with a license step a person answers: not yet",
52
- "certo skills sync": "downloads code, with a license step a person answers: not yet",
47
+ "certo skills add": (
48
+ "from git it downloads code, with a license only a person may accept: run it in a "
49
+ "terminal; a skill file or folder of yours: on request"
50
+ ),
51
+ "certo skills remove": "changes the skills every run gets: on request",
52
+ "certo skills enable": "turns on downloaded code, a person's decision: run it in a terminal",
53
+ "certo skills disable": "changes the skills every run gets: on request",
54
+ "certo skills update": "downloads code, with a license only a person may accept: run it in a terminal",
55
+ "certo skills sync": "downloads code, with a license only a person may accept: run it in a terminal",
53
56
  "certo bench": "runs for hours, and is set up and read in a terminal",
54
57
  "certo mcp": "it is this server",
55
58
  "certo-server": "another server, over HTTP",
56
59
  "certo-admin": "manages certo-server's task store, not this server's runs",
60
+ "certo-admin list": "manages certo-server's task store, not this server's runs",
57
61
  "certo-admin show": "manages certo-server's task store, not this server's runs",
58
62
  "certo-admin delete-task": "manages certo-server's task store, not this server's runs",
59
63
  "certo-admin delete-project": "manages certo-server's task store, not this server's runs",
@@ -126,6 +126,7 @@ def run_result(
126
126
  "checks": [],
127
127
  "acceptance": [],
128
128
  "retries_used": 0,
129
+ "tool_call_reminders": 0,
129
130
  "independent_tests": None,
130
131
  "check_file_changes": [],
131
132
  "files_changed": None,
@@ -146,6 +147,7 @@ def run_result(
146
147
  for r in outcome.acceptance_results
147
148
  ],
148
149
  retries_used=outcome.retries_used,
150
+ tool_call_reminders=outcome.tool_call_reminders,
149
151
  independent_tests=_independent(outcome.independent_tests),
150
152
  check_file_changes=[asdict(c) for c in outcome.check_file_changes],
151
153
  files_changed=(
@@ -41,6 +41,7 @@ class RunEntry:
41
41
  summary: RunSummary | None = None
42
42
  resumed: bool = False
43
43
  note: str = "" # e.g. how the model was chosen
44
+ report: str = "" # the run's report inside .certo/ (run_report.py), if any
44
45
 
45
46
 
46
47
  def runs_path(workspace: str) -> str:
@@ -70,6 +71,8 @@ def render_entry(entry: RunEntry) -> str:
70
71
  f"- Time: {format_duration(s.duration_seconds)} · "
71
72
  f"Tokens: {s.total_tokens:,} · Cost: {cost}"
72
73
  )
74
+ if entry.report:
75
+ lines.append(f"- Report: [{entry.report}]({entry.report})")
73
76
  return "\n".join(lines) + "\n"
74
77
 
75
78