certo 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {certo-0.2.0 → certo-0.2.2}/.env.example +4 -0
- {certo-0.2.0 → certo-0.2.2}/PKG-INFO +2 -2
- {certo-0.2.0 → certo-0.2.2}/README.md +1 -1
- {certo-0.2.0 → certo-0.2.2}/pyproject.toml +1 -1
- {certo-0.2.0 → certo-0.2.2}/src/certo/__init__.py +1 -1
- {certo-0.2.0 → certo-0.2.2}/src/certo/admin_cli.py +49 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/bench.py +5 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/bench_config.py +4 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/bench_report.py +8 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/bench_strategies.py +6 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/catalog_check.py +8 -3
- {certo-0.2.0 → certo-0.2.2}/src/certo/cli.py +46 -14
- {certo-0.2.0 → certo-0.2.2}/src/certo/config.py +5 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/context_window.py +15 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/escalation.py +3 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/llm_check.py +45 -2
- {certo-0.2.0 → certo-0.2.2}/src/certo/mcp_server.py +12 -8
- {certo-0.2.0 → certo-0.2.2}/src/certo/result.py +2 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/run_history.py +3 -0
- certo-0.2.2/src/certo/run_progress.py +130 -0
- certo-0.2.2/src/certo/run_report.py +244 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/runner.py +58 -13
- {certo-0.2.0 → certo-0.2.2}/src/certo/runs.py +7 -1
- {certo-0.2.0 → certo-0.2.2}/src/certo/task_store.py +75 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/text_tool_calls.py +43 -0
- {certo-0.2.0 → certo-0.2.2}/tests/conftest.py +4 -2
- {certo-0.2.0 → certo-0.2.2}/tests/test_admin_cli.py +67 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_bench.py +14 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_bench_config.py +87 -4
- {certo-0.2.0 → certo-0.2.2}/tests/test_bench_report.py +5 -4
- {certo-0.2.0 → certo-0.2.2}/tests/test_cli.py +123 -1
- {certo-0.2.0 → certo-0.2.2}/tests/test_context_window.py +23 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_escalation.py +7 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_llm_check.py +49 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_result.py +2 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_run_history.py +14 -0
- certo-0.2.2/tests/test_run_progress.py +143 -0
- certo-0.2.2/tests/test_run_report.py +165 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_runner.py +103 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_runs.py +5 -1
- {certo-0.2.0 → certo-0.2.2}/tests/test_task_store.py +61 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_text_tool_calls.py +37 -1
- {certo-0.2.0 → certo-0.2.2}/.gitignore +0 -0
- {certo-0.2.0 → certo-0.2.2}/LICENSE +0 -0
- {certo-0.2.0 → certo-0.2.2}/MANUAL.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/docker/agent-server.Dockerfile +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/__main__.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/acceptance.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/agent.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/api.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/artifacts.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/banner.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/bench_tasks.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/VERSION +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/duration-parser/hidden_tests/test_hidden_durations.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/duration-parser/reference/durations.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/duration-parser/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/hidden_tests/test_hidden_inventory.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/reference/inventory.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/reference/test_inventory.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/starter/inventory.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/starter/test_inventory.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/inventory-fix/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/hidden_tests/test_hidden_ledger.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/money.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/reports.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/store.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/ledger/transfers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/tests/test_money.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/tests/test_store.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/reference/tests/test_transfers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/README.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/__init__.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/__main__.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/categorize.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/cli.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/importers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/models.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/money.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/reports.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/ledger/store.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/pyproject.toml +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/samples/alpha.csv +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/samples/beta.csv +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/samples/rules.txt +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/__init__.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/helpers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_categorize.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_cli.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_importers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_money.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_reports.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/starter/tests/test_store.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/ledger-transfers/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/hidden_tests/loans_test.go +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/reference/api/api.go +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/reference/api/api_test.go +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/starter/api/api.go +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/starter/cmd/server/main.go +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/starter/go.mod +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/library-loans/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/page-ranges/hidden_tests/test_hidden_pages.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/page-ranges/reference/pages.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/page-ranges/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/pedestrian-crossing/hidden_tests/test_hidden_crossing.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/pedestrian-crossing/reference/crossing.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/pedestrian-crossing/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/hidden_tests/room_bookings.test.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/rooms.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/server.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/test/app.test.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/reference/ui.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/README.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/package-lock.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/server.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/starter/ui.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/room-bookings/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/hidden_tests/versions.test.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/reference/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/reference/versions.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/starter/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/semver-order/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/hidden_tests/slugify.test.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/reference/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/reference/slugify.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/starter/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/slugify/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/temperature-table/hidden_tests/test_hidden_temperature.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/temperature-table/reference/temperature.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/temperature-table/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/hidden_tests/todo_cli.rs +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/reference/src/board.rs +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/reference/src/main.rs +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/reference/src/store.rs +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/starter/Cargo.toml +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/starter/src/main.rs +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/todo-deps/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/hidden_tests/tokenBucket.test.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/reference/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/reference/tokenBucket.js +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/starter/package.json +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/token-bucket/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/wordle-score/hidden_tests/test_hidden_wordle_score.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/wordle-score/reference/wordle_score.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/benchmark_tasks/wordle-score/task.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/git/commit-conventions.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/completion-and-release-readiness.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/debugging-and-failure-repair.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/documentation-and-operational-readiness.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/implementation-planning.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/repository-discovery.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/requirements-analysis.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/security-review.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/lifecycle/testing-and-verification.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/node/node-project-setup.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/python/python-environment.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/python-web/fastapi-conventions.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/python-web/pin-dependencies.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/testing/pytest-conventions.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/builtin_skills/testing/testable-interactive-apps.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/cancel.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/catalog_report.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/check_files.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/checkpoints.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/checkpoints_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/console.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/cost_limit.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/custom_tools/__init__.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/custom_tools/example_tool.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/custom_tools/run_tests_tool.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/deadline.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/examples/bench.yaml.example +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/examples/escalation.yaml.example +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/examples/models.yaml.example +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/examples/models_list.yaml.example +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/examples/task_profiles.yaml.example +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/independent_tests.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/init_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/init_models.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/llm.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/menus.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/milestones.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/model_catalog.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/model_files.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/model_pricing.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/model_selection.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/models_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/plugins.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/project_env.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/project_state.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/run_summary.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/server.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/skill_library.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/skills.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/skills_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/README.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/gitignore +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/providers.toml +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/hanoi.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/invaders.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/mazeman.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/space-rocks.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/starter/tasks/wordle.md +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/third_party_skills.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/tools.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/update_check.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/verification.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/workspace.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/src/certo/worktree.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/custom_tools/test_run_tests_tool.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/model_file_helpers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_acceptance.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_agent.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_api.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_artifacts.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_banner.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_bench_tasks.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_catalog_check.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_check_files.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_checkpoints.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_checkpoints_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_config.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_console.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_cost_limit.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_deadline.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_docker_image.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_docs.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_escalation_runner.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_independent_tests.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_init_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_init_models.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_llm.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_mcp_server.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_menus.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_milestones.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_model_catalog.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_model_files.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_model_selection.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_models_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_packaging.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_plugins.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_project_env.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_project_env_layouts.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_project_state.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_readme.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_run_summary.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_server.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_skill_library.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_skills.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_skills_cmd.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_third_party_skills.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_update_check.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_verification_wrappers.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_version.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_workspace.py +0 -0
- {certo-0.2.0 → certo-0.2.2}/tests/test_worktree.py +0 -0
|
@@ -121,6 +121,10 @@ CERTO_INDEPENDENT_TESTS_STRICT=no
|
|
|
121
121
|
# The same as --verbose.
|
|
122
122
|
CERTO_VERBOSE=no
|
|
123
123
|
|
|
124
|
+
# How often a run in the terminal prints its progress line (milestone,
|
|
125
|
+
# fix attempt, last checks, time, cost so far), in seconds. 0 turns it off.
|
|
126
|
+
CERTO_PROGRESS_SECONDS=120
|
|
127
|
+
|
|
124
128
|
# --- Model selection (see docs/guides/pick-models-automatically.md and
|
|
125
129
|
# docs/guides/escalate-between-models.md). With manual (the default),
|
|
126
130
|
# CERTO_LLM_MODEL above is used and no model file is read. With auto or
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: certo
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: A coding-agent harness that isn't tied to one model or provider: it picks a model per task, switches when one fails, and checks the work with your own tests.
|
|
5
5
|
Project-URL: Homepage, https://github.com/alessioricco/certo
|
|
6
6
|
Project-URL: Repository, https://github.com/alessioricco/certo
|
|
@@ -373,7 +373,7 @@ The full list is in the [documentation](https://github.com/alessioricco/certo/bl
|
|
|
373
373
|
|
|
374
374
|
## Contributing
|
|
375
375
|
|
|
376
|
-
Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). To work on the code:
|
|
376
|
+
Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). Questions and ideas go in [Discussions](https://github.com/alessioricco/certo/discussions). To work on the code:
|
|
377
377
|
|
|
378
378
|
```bash
|
|
379
379
|
git clone https://github.com/alessioricco/certo.git
|
|
@@ -321,7 +321,7 @@ The full list is in the [documentation](https://github.com/alessioricco/certo/bl
|
|
|
321
321
|
|
|
322
322
|
## Contributing
|
|
323
323
|
|
|
324
|
-
Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). To work on the code:
|
|
324
|
+
Issues and pull requests are welcome. Issues labelled [good first issue](https://github.com/alessioricco/certo/labels/good%20first%20issue) are a good place to start, and so is [adding a benchmark task](https://github.com/alessioricco/certo/blob/main/docs/guides/contribute-a-benchmark-task.md). Questions and ideas go in [Discussions](https://github.com/alessioricco/certo/discussions). To work on the code:
|
|
325
325
|
|
|
326
326
|
```bash
|
|
327
327
|
git clone https://github.com/alessioricco/certo.git
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "certo"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "A coding-agent harness that isn't tied to one model or provider: it picks a model per task, switches when one fails, and checks the work with your own tests."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -14,6 +14,7 @@ redis/mysql/postgres store.
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import argparse
|
|
17
|
+
from datetime import UTC, datetime
|
|
17
18
|
|
|
18
19
|
from . import __version__
|
|
19
20
|
from .config import ConfigError, load_config, override_llm
|
|
@@ -64,6 +65,42 @@ def _cmd_purge(args: argparse.Namespace) -> int:
|
|
|
64
65
|
return 0
|
|
65
66
|
|
|
66
67
|
|
|
68
|
+
# A task's text is cut to this many characters in `list`; `show` has more.
|
|
69
|
+
_LISTED_TASK_CHARS = 50
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _cmd_list(args: argparse.Namespace) -> int:
|
|
73
|
+
if args.limit < 0:
|
|
74
|
+
print("--limit must be 0 (all) or more.")
|
|
75
|
+
return 1
|
|
76
|
+
cfg = load_config()
|
|
77
|
+
store = build_task_store(cfg)
|
|
78
|
+
try:
|
|
79
|
+
summaries = store.list_tasks(args.project)
|
|
80
|
+
finally:
|
|
81
|
+
store.close()
|
|
82
|
+
if not summaries:
|
|
83
|
+
where = f" for project {args.project!r}" if args.project is not None else ""
|
|
84
|
+
print(f"No task records{where}.")
|
|
85
|
+
return 0
|
|
86
|
+
shown = summaries[: args.limit] if args.limit else summaries
|
|
87
|
+
rows = [("ID", "PROJECT", "STATUS", "VERDICT", "CREATED (UTC)", "TASK")]
|
|
88
|
+
for s in shown:
|
|
89
|
+
task = " ".join(s.task.split())
|
|
90
|
+
if len(task) > _LISTED_TASK_CHARS:
|
|
91
|
+
task = task[: _LISTED_TASK_CHARS - 3] + "..."
|
|
92
|
+
created = datetime.fromtimestamp(s.created_at, UTC).strftime("%Y-%m-%d %H:%M")
|
|
93
|
+
rows.append((s.id, s.project or "-", s.status, s.verification_state or "-", created, task))
|
|
94
|
+
widths = [max(len(row[i]) for row in rows) for i in range(len(rows[0]) - 1)]
|
|
95
|
+
for row in rows:
|
|
96
|
+
print(
|
|
97
|
+
" ".join(cell.ljust(w) for cell, w in zip(row, widths, strict=False)) + " " + row[-1]
|
|
98
|
+
)
|
|
99
|
+
if len(shown) < len(summaries):
|
|
100
|
+
print(f"({len(shown)} of {len(summaries)} shown, newest first; --limit 0 shows all.)")
|
|
101
|
+
return 0
|
|
102
|
+
|
|
103
|
+
|
|
67
104
|
def _cmd_show(args: argparse.Namespace) -> int:
|
|
68
105
|
cfg = load_config()
|
|
69
106
|
store = build_task_store(cfg)
|
|
@@ -147,6 +184,18 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
147
184
|
)
|
|
148
185
|
purge.set_defaults(func=_cmd_purge)
|
|
149
186
|
|
|
187
|
+
list_cmd = subparsers.add_parser(
|
|
188
|
+
"list", help="List the task records, newest first: id, project, status and verdict."
|
|
189
|
+
)
|
|
190
|
+
list_cmd.add_argument("--project", default=None, help="Only the records of this project.")
|
|
191
|
+
list_cmd.add_argument(
|
|
192
|
+
"--limit",
|
|
193
|
+
type=int,
|
|
194
|
+
default=50,
|
|
195
|
+
help="Show at most this many records (default 50); 0 shows all.",
|
|
196
|
+
)
|
|
197
|
+
list_cmd.set_defaults(func=_cmd_list)
|
|
198
|
+
|
|
150
199
|
show = subparsers.add_parser("show", help="Print one task record's summary.")
|
|
151
200
|
show.add_argument("task_id")
|
|
152
201
|
show.set_defaults(func=_cmd_show)
|
|
@@ -210,6 +210,7 @@ def _combined_attempt(attempts: list[AttemptResult]) -> AttemptResult:
|
|
|
210
210
|
return AttemptResult(
|
|
211
211
|
verification_state=last.verification_state,
|
|
212
212
|
retries_used=sum(a.retries_used for a in attempts),
|
|
213
|
+
tool_call_reminders=sum(a.tool_call_reminders for a in attempts),
|
|
213
214
|
run_summary=combine_summaries(summaries) if summaries else None,
|
|
214
215
|
models_used=tuple(dict.fromkeys(m for a in attempts for m in a.models_used)),
|
|
215
216
|
rungs=tuple(r for a in attempts for r in a.rungs),
|
|
@@ -353,6 +354,10 @@ def execute_run(
|
|
|
353
354
|
"cost": round(summary.total_cost, 6) if summary else 0.0,
|
|
354
355
|
"cost_known": summary.cost_complete if summary else True,
|
|
355
356
|
"retries_used": attempt.retries_used if attempt else 0,
|
|
357
|
+
# How often the agent wrote a tool call as text instead of making it,
|
|
358
|
+
# or ended on a sentence of plans
|
|
359
|
+
# (CERTO_TOOL_CALL_REMINDERS); summed over rungs for an escalation.
|
|
360
|
+
"tool_call_reminders": attempt.tool_call_reminders if attempt else 0,
|
|
356
361
|
# How often older steps were summarized (the context window filled up).
|
|
357
362
|
"condensations": summary.condensations if summary else 0,
|
|
358
363
|
# Escalation only: every rung that ran, and which one gave the verdict.
|
|
@@ -306,6 +306,10 @@ def load_bench_config(path: str = DEFAULT_BENCH_FILE) -> BenchConfig:
|
|
|
306
306
|
escalation_file=escalation_file,
|
|
307
307
|
task_profiles_file=task_profiles_file,
|
|
308
308
|
on_warning=catalog_warnings.append,
|
|
309
|
+
# A file that names its own models.yaml doesn't pick up the
|
|
310
|
+
# ladders and profiles in the current folder: they name other
|
|
311
|
+
# models.
|
|
312
|
+
look_up_optional=not raw.get("models_file"),
|
|
309
313
|
)
|
|
310
314
|
catalog = {e.name: e for e in loaded.models}
|
|
311
315
|
except ModelCatalogError as exc:
|
|
@@ -40,6 +40,7 @@ CSV_FIELDS = (
|
|
|
40
40
|
"cost",
|
|
41
41
|
"cost_known",
|
|
42
42
|
"retries_used",
|
|
43
|
+
"tool_call_reminders",
|
|
43
44
|
"final_rung",
|
|
44
45
|
"independent_status",
|
|
45
46
|
"error",
|
|
@@ -256,6 +257,7 @@ def arm_rows(meta: dict[str, Any], records: list[dict[str, Any]]) -> list[dict[s
|
|
|
256
257
|
else "-",
|
|
257
258
|
"Tokens in / out": f"{sum(r['tokens_in'] for r in mine):,} / "
|
|
258
259
|
f"{sum(r['tokens_out'] for r in mine):,}",
|
|
260
|
+
"Tool-call reminders": str(sum(r.get("tool_call_reminders", 0) for r in mine)),
|
|
259
261
|
"Cost": _cost(mine) if mine else "-",
|
|
260
262
|
"Cost per solved task": _cost_per_solved(mine),
|
|
261
263
|
**(_independent_columns(mine) if independent else {}),
|
|
@@ -431,6 +433,12 @@ def render_markdown(meta: dict[str, Any], records: list[dict[str, Any]]) -> str:
|
|
|
431
433
|
),
|
|
432
434
|
"- **Errors:** the run stopped with an error (for example, the provider failed).",
|
|
433
435
|
"- **Median time:** wall time of one run, including Certo's checks and retries.",
|
|
436
|
+
(
|
|
437
|
+
"- **Tool-call reminders:** how many times the agent wrote a tool call as text "
|
|
438
|
+
"instead of making it, or ended its turn on a sentence of plans, and Certo sent "
|
|
439
|
+
"it back (`CERTO_TOOL_CALL_REMINDERS`), added up over the arm's runs. An "
|
|
440
|
+
"escalation run counts every rung's own."
|
|
441
|
+
),
|
|
434
442
|
"- **Cost:** from LiteLLM's prices; `+ unknown` when a model has no price.",
|
|
435
443
|
(
|
|
436
444
|
"- **Effort:** the reasoning effort the model ran at; `default` when none was "
|
|
@@ -67,6 +67,9 @@ class AttemptResult:
|
|
|
67
67
|
independent_tests: Any = None
|
|
68
68
|
# Auto only: the task profile the task was classified as.
|
|
69
69
|
task_profile: str | None = None
|
|
70
|
+
# How many tool-call reminders the run's conversation(s) were sent
|
|
71
|
+
# (TaskOutcome.tool_call_reminders; summed over rungs for an escalation).
|
|
72
|
+
tool_call_reminders: int = 0
|
|
70
73
|
|
|
71
74
|
|
|
72
75
|
class Strategy(Protocol):
|
|
@@ -193,6 +196,7 @@ class SingleStrategy:
|
|
|
193
196
|
run_summary=outcome.run_summary,
|
|
194
197
|
models_used=(entry.name,),
|
|
195
198
|
independent_tests=getattr(outcome, "independent_tests", None),
|
|
199
|
+
tool_call_reminders=getattr(outcome, "tool_call_reminders", 0),
|
|
196
200
|
)
|
|
197
201
|
|
|
198
202
|
|
|
@@ -253,6 +257,7 @@ class EscalateStrategy:
|
|
|
253
257
|
rungs=rungs,
|
|
254
258
|
final_rung=outcome.final_rung,
|
|
255
259
|
independent_tests=getattr(outcome, "independent_tests", None),
|
|
260
|
+
tool_call_reminders=getattr(outcome, "tool_call_reminders", 0),
|
|
256
261
|
)
|
|
257
262
|
|
|
258
263
|
|
|
@@ -309,6 +314,7 @@ class AutoStrategy:
|
|
|
309
314
|
models_used=used,
|
|
310
315
|
independent_tests=getattr(outcome, "independent_tests", None),
|
|
311
316
|
task_profile=decisions[0].task_profile if decisions else None,
|
|
317
|
+
tool_call_reminders=getattr(outcome, "tool_call_reminders", 0),
|
|
312
318
|
)
|
|
313
319
|
|
|
314
320
|
|
|
@@ -187,13 +187,18 @@ def load_checked_catalog(
|
|
|
187
187
|
escalation_file: str = "",
|
|
188
188
|
on_warning: Callable[[str], None] | None = None,
|
|
189
189
|
mode: str | None = None,
|
|
190
|
+
look_up_optional: bool = True,
|
|
190
191
|
) -> ModelCatalog:
|
|
191
192
|
"""Find, load and check the model files (see model_files.py; "" means
|
|
192
193
|
look them up; task_profiles.yaml and escalation.yaml are optional).
|
|
193
|
-
|
|
194
|
-
|
|
194
|
+
With `look_up_optional=False`, "" for one of the optional two means
|
|
195
|
+
none. Raises `ModelCatalogError` listing every error; passes each
|
|
196
|
+
warning to `on_warning`. `mode` as in `check_catalog`."""
|
|
195
197
|
list_path, models_path = resolve(models_file, models_list_file)
|
|
196
|
-
|
|
198
|
+
if look_up_optional:
|
|
199
|
+
profiles_path, escalation_path = resolve_optional(task_profiles_file, escalation_file)
|
|
200
|
+
else:
|
|
201
|
+
profiles_path, escalation_path = task_profiles_file or None, escalation_file or None
|
|
197
202
|
catalog = load_model_catalog(
|
|
198
203
|
list_path,
|
|
199
204
|
models_path,
|
|
@@ -46,6 +46,8 @@ from .project_state import (
|
|
|
46
46
|
)
|
|
47
47
|
from .result import run_result
|
|
48
48
|
from .run_history import append_run, note_models_in_readme, run_entry
|
|
49
|
+
from .run_progress import RunProgress
|
|
50
|
+
from .run_report import report_relpath, write_report
|
|
49
51
|
from .worktree import was_created_by_certo, worktree_warning
|
|
50
52
|
|
|
51
53
|
if TYPE_CHECKING:
|
|
@@ -716,19 +718,32 @@ def _run_task_command(
|
|
|
716
718
|
resume_hint = f"CERTO_PROJECTS_DIR={shlex.quote(cfg.projects_dir)} {resume_hint}"
|
|
717
719
|
started_at = datetime.now(UTC)
|
|
718
720
|
history = {"task": resume_from.task if resume_from is not None else (task or "")}
|
|
721
|
+
report: dict[str, str] = {}
|
|
719
722
|
|
|
720
723
|
def record_run(ended: str, outcome=None, summary=None) -> None:
|
|
721
724
|
if not save_state:
|
|
722
725
|
return
|
|
723
726
|
entry = run_entry(cfg, ladder, history["task"], ended, started_at, outcome, summary)
|
|
724
|
-
entry = replace(entry, resumed=resume_from is not None)
|
|
727
|
+
entry = replace(entry, resumed=resume_from is not None, report=report_relpath(started_at))
|
|
725
728
|
try:
|
|
726
729
|
append_run(cfg.workspace, entry)
|
|
727
730
|
except OSError as exc:
|
|
728
731
|
print(f"Warning: couldn't add this run to .certo/RUNS.md ({exc}).", file=sys.stderr)
|
|
729
732
|
|
|
733
|
+
# The progress line (run_progress.py): one for the whole run, so its
|
|
734
|
+
# time and cost count every milestone.
|
|
735
|
+
progress = (
|
|
736
|
+
RunProgress(say=_print_progress, every_seconds=cfg.progress_seconds)
|
|
737
|
+
if cfg.progress_seconds > 0
|
|
738
|
+
else None
|
|
739
|
+
)
|
|
740
|
+
|
|
730
741
|
def run_one(text: str, resume, instruction: str | None, checks):
|
|
731
|
-
|
|
742
|
+
if progress is not None and milestone_state is not None:
|
|
743
|
+
total = len(milestone_state.plan.milestones)
|
|
744
|
+
progress.where = f"milestone {milestone_state.current} of {total}"
|
|
745
|
+
progress.checks = ""
|
|
746
|
+
result = run_task(
|
|
732
747
|
text,
|
|
733
748
|
cfg=cfg,
|
|
734
749
|
on_confirm=on_confirm,
|
|
@@ -742,13 +757,19 @@ def _run_task_command(
|
|
|
742
757
|
ladder=ladder,
|
|
743
758
|
on_escalate=_confirm_escalation if cfg.interactive else None,
|
|
744
759
|
on_notice=_print_progress,
|
|
760
|
+
run_progress=progress,
|
|
745
761
|
)
|
|
762
|
+
summary = result.outcome.run_summary
|
|
763
|
+
if progress is not None and summary is not None:
|
|
764
|
+
progress.cost_before += summary.total_cost
|
|
765
|
+
return result
|
|
746
766
|
|
|
747
767
|
if save_state and milestone_state is None and resume_from is None:
|
|
748
768
|
clear_state(cfg.workspace) # a new task here isn't the old milestones
|
|
749
769
|
|
|
750
770
|
def keep_result(outcome, error: str | None = None) -> None:
|
|
751
|
-
|
|
771
|
+
# The JSON result for --json, and the run's report (run_report.py).
|
|
772
|
+
if holder is None and not save_state:
|
|
752
773
|
return
|
|
753
774
|
milestones = None
|
|
754
775
|
if milestone_state is not None:
|
|
@@ -760,7 +781,7 @@ def _run_task_command(
|
|
|
760
781
|
}
|
|
761
782
|
for m in milestone_state.plan.milestones
|
|
762
783
|
]
|
|
763
|
-
|
|
784
|
+
result = run_result(
|
|
764
785
|
outcome,
|
|
765
786
|
task=history["task"],
|
|
766
787
|
workspace=os.path.abspath(cfg.workspace),
|
|
@@ -768,6 +789,13 @@ def _run_task_command(
|
|
|
768
789
|
milestones=milestones,
|
|
769
790
|
error=error,
|
|
770
791
|
)
|
|
792
|
+
if holder is not None:
|
|
793
|
+
holder["result"] = result
|
|
794
|
+
if save_state:
|
|
795
|
+
try:
|
|
796
|
+
report["path"] = write_report(cfg.workspace, result, started_at)
|
|
797
|
+
except OSError as exc:
|
|
798
|
+
print(f"Warning: couldn't write the run report ({exc}).", file=sys.stderr)
|
|
771
799
|
|
|
772
800
|
try:
|
|
773
801
|
if milestone_state is None:
|
|
@@ -859,16 +887,13 @@ def _run_task_command(
|
|
|
859
887
|
if outcome.rungs and milestone_state is None:
|
|
860
888
|
for line in format_rung_lines(outcome.rungs, outcome.final_rung):
|
|
861
889
|
print(f" {line}")
|
|
890
|
+
if report.get("path"):
|
|
891
|
+
print(f"\nReport: {_short_path(report['path'])}")
|
|
862
892
|
failing = outcome.verification_state in _FAILING_STATES or (
|
|
863
893
|
args.require_verification and outcome.verification_state in _unproven_states()
|
|
864
894
|
)
|
|
865
895
|
if failing and save_state:
|
|
866
|
-
status = status_path(cfg.workspace)
|
|
867
|
-
try:
|
|
868
|
-
status = os.path.relpath(status)
|
|
869
|
-
except ValueError:
|
|
870
|
-
# Windows cannot make a relative path between different drives.
|
|
871
|
-
pass
|
|
896
|
+
status = _short_path(status_path(cfg.workspace))
|
|
872
897
|
print(f"\nWhere it stopped: {status}. Continue with: {resume_hint}")
|
|
873
898
|
if save_state and cfg.checkpoints:
|
|
874
899
|
hint = _checkpoints_hint(cfg.workspace)
|
|
@@ -877,6 +902,16 @@ def _run_task_command(
|
|
|
877
902
|
return 1 if failing else 0
|
|
878
903
|
|
|
879
904
|
|
|
905
|
+
def _short_path(path: str) -> str:
|
|
906
|
+
"""`path` relative to the current folder, or absolute when that is
|
|
907
|
+
shorter (a project far away would be a long `../..` chain)."""
|
|
908
|
+
path = os.path.abspath(path)
|
|
909
|
+
try:
|
|
910
|
+
return min(path, os.path.relpath(path), key=len)
|
|
911
|
+
except ValueError:
|
|
912
|
+
return path # Windows: no relative path between different drives
|
|
913
|
+
|
|
914
|
+
|
|
880
915
|
def _checkpoints_hint(workspace: str) -> str | None:
|
|
881
916
|
"""One line naming the project's checkpoints (checkpoints.py)."""
|
|
882
917
|
from .checkpoints import git_available, list_checkpoints
|
|
@@ -887,10 +922,7 @@ def _checkpoints_hint(workspace: str) -> str | None:
|
|
|
887
922
|
return None
|
|
888
923
|
if not count:
|
|
889
924
|
return None
|
|
890
|
-
|
|
891
|
-
where = os.path.relpath(workspace)
|
|
892
|
-
except ValueError:
|
|
893
|
-
where = workspace
|
|
925
|
+
where = _short_path(workspace)
|
|
894
926
|
path = "" if where == "." else f" {shlex.quote(where)}"
|
|
895
927
|
return (
|
|
896
928
|
f"Checkpoints: {count} (list them: certo checkpoints{path}; "
|
|
@@ -227,6 +227,9 @@ class Config:
|
|
|
227
227
|
# its skills, the SDK's INFO logs. Off by default (console.py shows a
|
|
228
228
|
# line for each instead).
|
|
229
229
|
verbose: bool = False
|
|
230
|
+
# CERTO_PROGRESS_SECONDS: how often a terminal run prints its progress
|
|
231
|
+
# line while the agent works (run_progress.py); 0 turns the line off.
|
|
232
|
+
progress_seconds: int = 120
|
|
230
233
|
|
|
231
234
|
# Deterministic auto model selection (see model_catalog.py/
|
|
232
235
|
# model_selection.py). Default ("manual") is byte-for-byte the
|
|
@@ -690,6 +693,7 @@ def load_config(
|
|
|
690
693
|
default=False,
|
|
691
694
|
)
|
|
692
695
|
verbose = _parse_bool(env, "VERBOSE", default=False)
|
|
696
|
+
progress_seconds = _parse_nonnegative_int(env, "PROGRESS_SECONDS", default=120)
|
|
693
697
|
independent_tests_strict = _parse_bool(env, "INDEPENDENT_TESTS_STRICT", default=False)
|
|
694
698
|
independent_tests = (
|
|
695
699
|
_parse_bool(env, "INDEPENDENT_TESTS", default=False) or independent_tests_strict
|
|
@@ -771,6 +775,7 @@ def load_config(
|
|
|
771
775
|
context_warn_tokens=context_warn_tokens,
|
|
772
776
|
tool_call_reminders=tool_call_reminders,
|
|
773
777
|
verbose=verbose,
|
|
778
|
+
progress_seconds=progress_seconds,
|
|
774
779
|
model_selection=model_selection,
|
|
775
780
|
models_file=models_file,
|
|
776
781
|
models_list_file=models_list_file,
|
|
@@ -119,6 +119,21 @@ def ollama_model_digest(model: str, base_url: str | None, timeout: float = 10) -
|
|
|
119
119
|
return None
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
def ollama_capabilities(model: str, base_url: str | None, timeout: float = 10) -> list[str] | None:
|
|
123
|
+
"""What the server says `model` can do (`/api/show`), such as
|
|
124
|
+
`tools` and `thinking`, or None when it can't be learned (a model it
|
|
125
|
+
doesn't have, an old server, no answer). Doesn't load the model."""
|
|
126
|
+
name = model.split("/", 1)[1]
|
|
127
|
+
base = (base_url or _DEFAULT_OLLAMA_URL).rstrip("/")
|
|
128
|
+
try:
|
|
129
|
+
found = _post(f"{base}/api/show", {"model": name}, timeout).get("capabilities")
|
|
130
|
+
except (OSError, ValueError, AttributeError, urllib.error.URLError):
|
|
131
|
+
return None
|
|
132
|
+
if not isinstance(found, list):
|
|
133
|
+
return None
|
|
134
|
+
return [c for c in found if isinstance(c, str)]
|
|
135
|
+
|
|
136
|
+
|
|
122
137
|
def server_address(base_url: str | None) -> str:
|
|
123
138
|
"""Host and port only: no scheme, path or credentials."""
|
|
124
139
|
parsed = urllib.parse.urlsplit(base_url or _DEFAULT_OLLAMA_URL)
|
|
@@ -332,6 +332,8 @@ class RungResult:
|
|
|
332
332
|
changes: FileChanges = FileChanges()
|
|
333
333
|
failing_checks: tuple[str, ...] = () # "name (`command`)"
|
|
334
334
|
reasoning_effort: str | None = None # None: no effort set
|
|
335
|
+
# How many tool-call reminders this rung's own conversation was sent.
|
|
336
|
+
tool_call_reminders: int = 0
|
|
335
337
|
|
|
336
338
|
@property
|
|
337
339
|
def described(self) -> str:
|
|
@@ -383,6 +385,7 @@ class RungResult:
|
|
|
383
385
|
"verdict": self.verdict,
|
|
384
386
|
"failure": self.failure,
|
|
385
387
|
"fix_attempts": self.retries_used,
|
|
388
|
+
"tool_call_reminders": self.tool_call_reminders,
|
|
386
389
|
"duration_seconds": round(self.duration_seconds, 3),
|
|
387
390
|
"tokens": summary.total_tokens if summary else 0,
|
|
388
391
|
"cost": self.cost,
|
|
@@ -126,6 +126,7 @@ class LLMCheckResult:
|
|
|
126
126
|
# "CERTO_LLM_CONTEXT_TOKENS" or "server" — where context_tokens came from.
|
|
127
127
|
context_source: str | None = None
|
|
128
128
|
context_warning: str | None = None # the run's own warning text, if any
|
|
129
|
+
thinking: str | None = None # thinking_line(), when the server says
|
|
129
130
|
|
|
130
131
|
|
|
131
132
|
def _scrub(text: str, secret: str | None) -> str:
|
|
@@ -339,6 +340,12 @@ def _ollama_details(cfg: Config, *, timeout: int | None = None) -> dict:
|
|
|
339
340
|
details["ollama_version"] = context_window.ollama_version(cfg.base_url, t)
|
|
340
341
|
except Exception: # noqa: BLE001 - best effort
|
|
341
342
|
pass
|
|
343
|
+
try:
|
|
344
|
+
capabilities = context_window.ollama_capabilities(cfg.model, cfg.base_url, t)
|
|
345
|
+
if capabilities is not None:
|
|
346
|
+
details["thinking"] = thinking_line(cfg.model, capabilities, cfg.reasoning_effort)
|
|
347
|
+
except Exception: # noqa: BLE001
|
|
348
|
+
pass
|
|
342
349
|
try:
|
|
343
350
|
tokens = context_window.known_context_tokens(
|
|
344
351
|
cfg.model, cfg.llm_context_tokens, cfg.base_url, t
|
|
@@ -360,11 +367,45 @@ def _ollama_details(cfg: Config, *, timeout: int | None = None) -> dict:
|
|
|
360
367
|
return details
|
|
361
368
|
|
|
362
369
|
|
|
370
|
+
# LiteLLM sends these as `think: true`, and any other effort as `think:
|
|
371
|
+
# false`, except to gpt-oss models, which get the level itself.
|
|
372
|
+
_THINKING_ON = ("low", "medium", "high")
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def thinking_line(model: str, capabilities: list[str], effort: str | None) -> str:
|
|
376
|
+
"""How an Ollama model will think with this reasoning effort, from what
|
|
377
|
+
the server says it can do."""
|
|
378
|
+
name = model.split("/", 1)[1]
|
|
379
|
+
if "thinking" not in capabilities:
|
|
380
|
+
if effort and effort != "none":
|
|
381
|
+
return (
|
|
382
|
+
f"Thinking: the model has none, but CERTO_LLM_REASONING_EFFORT is {effort}, "
|
|
383
|
+
"so Ollama refuses its requests. Clear the setting or set it to none."
|
|
384
|
+
)
|
|
385
|
+
return "Thinking: the model has none."
|
|
386
|
+
if not effort or effort == "none":
|
|
387
|
+
return (
|
|
388
|
+
"Thinking: the model has it, and CERTO_LLM_REASONING_EFFORT is not set, so it "
|
|
389
|
+
"thinks as Ollama does by default. high turns it on, minimal turns it off."
|
|
390
|
+
)
|
|
391
|
+
if name.startswith("gpt-oss"):
|
|
392
|
+
return f"Thinking: on, at level {effort}."
|
|
393
|
+
if effort in _THINKING_ON:
|
|
394
|
+
return (
|
|
395
|
+
"Thinking: on. Ollama has only on and off for this model, so low, medium "
|
|
396
|
+
"and high think the same."
|
|
397
|
+
)
|
|
398
|
+
return (
|
|
399
|
+
f"Thinking: off. On Ollama, {effort} turns thinking off for this model; only "
|
|
400
|
+
"low, medium and high turn it on."
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
|
|
363
404
|
def detail_lines(result: LLMCheckResult) -> list[str]:
|
|
364
405
|
"""The extra context lines `certo check` prints for a local (Ollama)
|
|
365
406
|
model: server version, the context window the run will use and where it
|
|
366
|
-
came from, the test request's time,
|
|
367
|
-
the same small-window warning a run emits. Empty for cloud models, so
|
|
407
|
+
came from, the test request's time, then how the model will think, and
|
|
408
|
+
— separately — the same small-window warning a run emits. Empty for cloud models, so
|
|
368
409
|
their output is unchanged."""
|
|
369
410
|
if not _is_ollama(result.model):
|
|
370
411
|
return []
|
|
@@ -381,6 +422,8 @@ def detail_lines(result: LLMCheckResult) -> list[str]:
|
|
|
381
422
|
# far below the model's real speed.
|
|
382
423
|
parts.append(f"{result.latency_seconds:.1f}s")
|
|
383
424
|
lines = [" · ".join(parts)]
|
|
425
|
+
if result.thinking:
|
|
426
|
+
lines.append(result.thinking)
|
|
384
427
|
if result.context_warning:
|
|
385
428
|
lines.append(result.context_warning)
|
|
386
429
|
return lines
|
|
@@ -38,22 +38,26 @@ COMMAND_TOOLS: dict[str, tuple[str, ...]] = {
|
|
|
38
38
|
"certo rollback": ("rollback",),
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
# The commands with no tool, and why. Those that say "
|
|
42
|
-
#
|
|
41
|
+
# The commands with no tool, and why. Those that say "on request" get a
|
|
42
|
+
# tool when someone asks (TODO.md); the others never do (DECISIONS.md).
|
|
43
43
|
NO_TOOL: dict[str, str] = {
|
|
44
44
|
"certo init": "asks questions and writes your settings file: run it in a terminal",
|
|
45
45
|
"certo init models": "asks questions and writes your model files: run it in a terminal",
|
|
46
46
|
"certo models update": "asks about each change to your models list: run it in a terminal",
|
|
47
|
-
"certo skills add":
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
"certo skills
|
|
52
|
-
"certo skills
|
|
47
|
+
"certo skills add": (
|
|
48
|
+
"from git it downloads code, with a license only a person may accept: run it in a "
|
|
49
|
+
"terminal; a skill file or folder of yours: on request"
|
|
50
|
+
),
|
|
51
|
+
"certo skills remove": "changes the skills every run gets: on request",
|
|
52
|
+
"certo skills enable": "turns on downloaded code, a person's decision: run it in a terminal",
|
|
53
|
+
"certo skills disable": "changes the skills every run gets: on request",
|
|
54
|
+
"certo skills update": "downloads code, with a license only a person may accept: run it in a terminal",
|
|
55
|
+
"certo skills sync": "downloads code, with a license only a person may accept: run it in a terminal",
|
|
53
56
|
"certo bench": "runs for hours, and is set up and read in a terminal",
|
|
54
57
|
"certo mcp": "it is this server",
|
|
55
58
|
"certo-server": "another server, over HTTP",
|
|
56
59
|
"certo-admin": "manages certo-server's task store, not this server's runs",
|
|
60
|
+
"certo-admin list": "manages certo-server's task store, not this server's runs",
|
|
57
61
|
"certo-admin show": "manages certo-server's task store, not this server's runs",
|
|
58
62
|
"certo-admin delete-task": "manages certo-server's task store, not this server's runs",
|
|
59
63
|
"certo-admin delete-project": "manages certo-server's task store, not this server's runs",
|
|
@@ -126,6 +126,7 @@ def run_result(
|
|
|
126
126
|
"checks": [],
|
|
127
127
|
"acceptance": [],
|
|
128
128
|
"retries_used": 0,
|
|
129
|
+
"tool_call_reminders": 0,
|
|
129
130
|
"independent_tests": None,
|
|
130
131
|
"check_file_changes": [],
|
|
131
132
|
"files_changed": None,
|
|
@@ -146,6 +147,7 @@ def run_result(
|
|
|
146
147
|
for r in outcome.acceptance_results
|
|
147
148
|
],
|
|
148
149
|
retries_used=outcome.retries_used,
|
|
150
|
+
tool_call_reminders=outcome.tool_call_reminders,
|
|
149
151
|
independent_tests=_independent(outcome.independent_tests),
|
|
150
152
|
check_file_changes=[asdict(c) for c in outcome.check_file_changes],
|
|
151
153
|
files_changed=(
|
|
@@ -41,6 +41,7 @@ class RunEntry:
|
|
|
41
41
|
summary: RunSummary | None = None
|
|
42
42
|
resumed: bool = False
|
|
43
43
|
note: str = "" # e.g. how the model was chosen
|
|
44
|
+
report: str = "" # the run's report inside .certo/ (run_report.py), if any
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
def runs_path(workspace: str) -> str:
|
|
@@ -70,6 +71,8 @@ def render_entry(entry: RunEntry) -> str:
|
|
|
70
71
|
f"- Time: {format_duration(s.duration_seconds)} · "
|
|
71
72
|
f"Tokens: {s.total_tokens:,} · Cost: {cost}"
|
|
72
73
|
)
|
|
74
|
+
if entry.report:
|
|
75
|
+
lines.append(f"- Report: [{entry.report}]({entry.report})")
|
|
73
76
|
return "\n".join(lines) + "\n"
|
|
74
77
|
|
|
75
78
|
|