coder-eval 0.8.2__tar.gz → 0.8.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (540) hide show
  1. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/release.yml +82 -75
  2. {coder_eval-0.8.2 → coder_eval-0.8.3}/CHANGELOG.md +36 -0
  3. {coder_eval-0.8.2 → coder_eval-0.8.3}/PKG-INFO +1 -1
  4. coder_eval-0.8.3/evalboard/app/_components/harness-badge.tsx +37 -0
  5. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/page.tsx +14 -1
  6. coder_eval-0.8.3/evalboard/public/harness/antigravity.png +0 -0
  7. coder_eval-0.8.3/evalboard/public/harness/claude-code.png +0 -0
  8. coder_eval-0.8.3/evalboard/public/harness/codex.png +0 -0
  9. {coder_eval-0.8.2 → coder_eval-0.8.3}/pyproject.toml +1 -1
  10. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/__init__.py +1 -1
  11. {coder_eval-0.8.2 → coder_eval-0.8.3}/uv.lock +1 -1
  12. coder_eval-0.8.2/.github/workflows/publish-dev.yml +0 -155
  13. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-code-review-full.md +0 -0
  14. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-code-review-wf.md +0 -0
  15. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-code-review.md +0 -0
  16. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-create-plan.md +0 -0
  17. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-implement-plan.md +0 -0
  18. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-review.md +0 -0
  19. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-run-analysis.md +0 -0
  20. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/commands/coder-eval-task-create.md +0 -0
  21. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/harness-candidates.md +0 -0
  22. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/shared/axes.md +0 -0
  23. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/shared/multi-model-review.md +0 -0
  24. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/shared/review-rubric.md +0 -0
  25. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/shared/run-layout.md +0 -0
  26. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/workflows/cr-axis.js +0 -0
  27. {coder_eval-0.8.2 → coder_eval-0.8.3}/.claude/workflows/cr-parent.js +0 -0
  28. {coder_eval-0.8.2 → coder_eval-0.8.3}/.env.example +0 -0
  29. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/CODEOWNERS +0 -0
  30. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  31. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  32. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/code_review.md +0 -0
  33. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/dependabot.yml +0 -0
  34. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/claude-pr-review.yml +0 -0
  35. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/codeql.yml +0 -0
  36. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/conventional-commits.yml +0 -0
  37. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/docker-publish.yml +0 -0
  38. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/pr-checks.yml +0 -0
  39. {coder_eval-0.8.2 → coder_eval-0.8.3}/.github/workflows/publish-testpypi.yml +0 -0
  40. {coder_eval-0.8.2 → coder_eval-0.8.3}/.gitignore +0 -0
  41. {coder_eval-0.8.2 → coder_eval-0.8.3}/.pre-commit-config.yaml +0 -0
  42. {coder_eval-0.8.2 → coder_eval-0.8.3}/.python-version +0 -0
  43. {coder_eval-0.8.2 → coder_eval-0.8.3}/CLAUDE.md +0 -0
  44. {coder_eval-0.8.2 → coder_eval-0.8.3}/CODE_OF_CONDUCT.md +0 -0
  45. {coder_eval-0.8.2 → coder_eval-0.8.3}/CONTRIBUTING.md +0 -0
  46. {coder_eval-0.8.2 → coder_eval-0.8.3}/LICENSE +0 -0
  47. {coder_eval-0.8.2 → coder_eval-0.8.3}/Makefile +0 -0
  48. {coder_eval-0.8.2 → coder_eval-0.8.3}/NOTICE +0 -0
  49. {coder_eval-0.8.2 → coder_eval-0.8.3}/README.md +0 -0
  50. {coder_eval-0.8.2 → coder_eval-0.8.3}/SECURITY.md +0 -0
  51. {coder_eval-0.8.2 → coder_eval-0.8.3}/docker/Dockerfile +0 -0
  52. {coder_eval-0.8.2 → coder_eval-0.8.3}/docker/Dockerfile.runtime +0 -0
  53. {coder_eval-0.8.2 → coder_eval-0.8.3}/docker/coder_eval_entrypoint.sh +0 -0
  54. {coder_eval-0.8.2 → coder_eval-0.8.3}/docker/coder_eval_runtime_entrypoint.sh +0 -0
  55. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/AB_EXPERIMENTS.md +0 -0
  56. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/BYOD.md +0 -0
  57. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/CODEX_AGENT_GUIDE.md +0 -0
  58. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/DOCKER_ISOLATION.md +0 -0
  59. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/IDEAS.md +0 -0
  60. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/TASK_DEFINITION_GUIDE.md +0 -0
  61. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/USER_GUIDE.md +0 -0
  62. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/assets/hero.gif +0 -0
  63. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/01-first-evaluation.md +0 -0
  64. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/02-ci-pipeline.md +0 -0
  65. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/03-evalboard-local.md +0 -0
  66. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/04-writing-a-task.md +0 -0
  67. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/05-comparing-models.md +0 -0
  68. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/06-use-docker-isolation.md +0 -0
  69. {coder_eval-0.8.2 → coder_eval-0.8.3}/docs/tutorials/README.md +0 -0
  70. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/.gitignore +0 -0
  71. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/README.md +0 -0
  72. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/col-help.tsx +0 -0
  73. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/collapsible-rail.tsx +0 -0
  74. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/scroll-table.tsx +0 -0
  75. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/search-box.tsx +0 -0
  76. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/unit-toggle.tsx +0 -0
  77. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/version-list.tsx +0 -0
  78. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_components/window-selector.tsx +0 -0
  79. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_overview/daily-chart.tsx +0 -0
  80. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_overview/tag-rail.tsx +0 -0
  81. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_overview/turn-budget-chart.tsx +0 -0
  82. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/_overview/window-summary.tsx +0 -0
  83. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/api/download/route.ts +0 -0
  84. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/api/file/route.ts +0 -0
  85. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/api/refresh/__tests__/route.test.ts +0 -0
  86. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/api/refresh/route.ts +0 -0
  87. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/error.tsx +0 -0
  88. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/globals.css +0 -0
  89. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/icon.png +0 -0
  90. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/layout.tsx +0 -0
  91. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/path-to-ga/page.tsx +0 -0
  92. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/trends/__tests__/trends-view.test.tsx +0 -0
  93. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/trends/actions.ts +0 -0
  94. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/trends/page.tsx +0 -0
  95. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/trends/trends-view.tsx +0 -0
  96. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx +0 -0
  97. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/watchlist/page.tsx +0 -0
  98. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/app/watchlist/watchlist-view.tsx +0 -0
  99. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/next-env.d.ts +0 -0
  100. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/next.config.mjs +0 -0
  101. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/package.json +0 -0
  102. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/pnpm-lock.yaml +0 -0
  103. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/postcss.config.mjs +0 -0
  104. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/public/uipath.png +0 -0
  105. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/tailwind.config.ts +0 -0
  106. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/tsconfig.json +0 -0
  107. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/vitest.config.ts +0 -0
  108. {coder_eval-0.8.2 → coder_eval-0.8.3}/evalboard/vitest.setup.ts +0 -0
  109. {coder_eval-0.8.2 → coder_eval-0.8.3}/experiments/default.yaml +0 -0
  110. {coder_eval-0.8.2 → coder_eval-0.8.3}/experiments/model-comparison.yaml +0 -0
  111. {coder_eval-0.8.2 → coder_eval-0.8.3}/experiments/permissions-smoke.yaml +0 -0
  112. {coder_eval-0.8.2 → coder_eval-0.8.3}/experiments/plugin-comparison.yaml +0 -0
  113. {coder_eval-0.8.2 → coder_eval-0.8.3}/experiments/prompt-mutations-example.yaml +0 -0
  114. {coder_eval-0.8.2 → coder_eval-0.8.3}/experiments/smoke_variants.yaml +0 -0
  115. {coder_eval-0.8.2 → coder_eval-0.8.3}/osv-scanner.toml +0 -0
  116. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/.gitattributes +0 -0
  117. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agent.py +0 -0
  118. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/__init__.py +0 -0
  119. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/_logging.py +0 -0
  120. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/antigravity_agent.py +0 -0
  121. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/claude_code_agent.py +0 -0
  122. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/codex_agent.py +0 -0
  123. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/noop_agent.py +0 -0
  124. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/registry.py +0 -0
  125. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/agents/watchdog.py +0 -0
  126. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/analysis.py +0 -0
  127. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/__init__.py +0 -0
  128. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/aggregate_command.py +0 -0
  129. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/console.py +0 -0
  130. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/evaluate_command.py +0 -0
  131. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/plan_command.py +0 -0
  132. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/report_command.py +0 -0
  133. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/run_command.py +0 -0
  134. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/run_helpers.py +0 -0
  135. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/run_task_internal_command.py +0 -0
  136. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/cli/utils.py +0 -0
  137. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/config.py +0 -0
  138. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/__init__.py +0 -0
  139. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/_classification_aggregate.py +0 -0
  140. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/agent_judge.py +0 -0
  141. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/base.py +0 -0
  142. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/classification_match.py +0 -0
  143. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/command_executed.py +0 -0
  144. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/commands_efficiency.py +0 -0
  145. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/file_check.py +0 -0
  146. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/file_contains.py +0 -0
  147. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/file_exists.py +0 -0
  148. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/file_matches_regex.py +0 -0
  149. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/json_check.py +0 -0
  150. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/llm_judge.py +0 -0
  151. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/reference_comparison.py +0 -0
  152. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/run_command.py +0 -0
  153. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/skill_triggered.py +0 -0
  154. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/criteria/uipath_eval.py +0 -0
  155. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/__init__.py +0 -0
  156. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/agent.py +0 -0
  157. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/budget.py +0 -0
  158. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/categories.py +0 -0
  159. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/categorization.py +0 -0
  160. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/executor.py +0 -0
  161. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/judge.py +0 -0
  162. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/retry.py +0 -0
  163. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/errors/timeout.py +0 -0
  164. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/__init__.py +0 -0
  165. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/checker.py +0 -0
  166. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/judge_anthropic.py +0 -0
  167. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/judge_bedrock.py +0 -0
  168. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/judge_context.py +0 -0
  169. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/judge_models.py +0 -0
  170. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/judge_persistence.py +0 -0
  171. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/judge_usage.py +0 -0
  172. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/sub_agent.py +0 -0
  173. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/summaries.py +0 -0
  174. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/evaluation/verdict_tool.py +0 -0
  175. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/formatting.py +0 -0
  176. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/isolation/__init__.py +0 -0
  177. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/isolation/docker_runner.py +0 -0
  178. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/logging_config.py +0 -0
  179. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/__init__.py +0 -0
  180. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/agent_config.py +0 -0
  181. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/container_paths.py +0 -0
  182. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/criteria.py +0 -0
  183. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/enums.py +0 -0
  184. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/experiment.py +0 -0
  185. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/judge.py +0 -0
  186. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/judge_defaults.py +0 -0
  187. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/limits.py +0 -0
  188. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/merge_strategy.py +0 -0
  189. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/mutations.py +0 -0
  190. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/results.py +0 -0
  191. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/routing.py +0 -0
  192. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/sandbox.py +0 -0
  193. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/tasks.py +0 -0
  194. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/telemetry.py +0 -0
  195. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/models/templates.py +0 -0
  196. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/__init__.py +0 -0
  197. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/batch.py +0 -0
  198. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/config.py +0 -0
  199. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/config_merge.py +0 -0
  200. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/evaluation.py +0 -0
  201. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/experiment.py +0 -0
  202. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/overrides.py +0 -0
  203. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestration/task_loader.py +0 -0
  204. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/orchestrator.py +0 -0
  205. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/path_utils.py +0 -0
  206. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/plugins.py +0 -0
  207. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/pricing.py +0 -0
  208. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/py.typed +0 -0
  209. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/reports.py +0 -0
  210. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/reports_experiment.py +0 -0
  211. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/reports_html.py +0 -0
  212. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/reports_stats.py +0 -0
  213. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/resources/__init__.py +0 -0
  214. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/resources/default_ignore_patterns.yaml +0 -0
  215. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/resources/tags.yaml +0 -0
  216. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/sandbox.py +0 -0
  217. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/__init__.py +0 -0
  218. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/ast_similarity.py +0 -0
  219. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/complexity.py +0 -0
  220. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/quality.py +0 -0
  221. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/signature_similarity.py +0 -0
  222. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/similarity.py +0 -0
  223. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/scoring/token_similarity.py +0 -0
  224. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/simulation/__init__.py +0 -0
  225. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/simulation/termination.py +0 -0
  226. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/simulation/user_simulator.py +0 -0
  227. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/streaming/__init__.py +0 -0
  228. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/streaming/callbacks.py +0 -0
  229. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/streaming/collector.py +0 -0
  230. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/streaming/events.py +0 -0
  231. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/streaming/renderers.py +0 -0
  232. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/streaming/wire.py +0 -0
  233. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/telemetry.py +0 -0
  234. {coder_eval-0.8.2 → coder_eval-0.8.3}/src/coder_eval/utils.py +0 -0
  235. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/README.md +0 -0
  236. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agentless_smoke_test.yaml +0 -0
  237. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/antigravity_hello_world.yaml +0 -0
  238. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/antigravity_hello_world_docker.yaml +0 -0
  239. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/claude_hello_world.yaml +0 -0
  240. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/claude_hello_world_docker.yaml +0 -0
  241. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/claude_parallel_single_gen.yaml +0 -0
  242. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/claude_subagent_test.yaml +0 -0
  243. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_disallowed_tools_test.yaml +0 -0
  244. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_hello_world.yaml +0 -0
  245. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_parallel_commands.yaml +0 -0
  246. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_parallel_single_gen.yaml +0 -0
  247. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_skills_test.yaml +0 -0
  248. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_string_utils.yaml +0 -0
  249. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/codex_subagent_test.yaml +0 -0
  250. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/subagent_bash_long_input.yaml +0 -0
  251. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/agents/subagent_merge_sort.yaml +0 -0
  252. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/byod_smoke_test.yaml +0 -0
  253. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dataset_example.yaml +0 -0
  254. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/datasets/sentiment.jsonl +0 -0
  255. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dockerfile_build_example/dockerfile_build_example.yaml +0 -0
  256. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dockerfile_build_example/environment/Dockerfile +0 -0
  257. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dockerfile_build_example/environment/Dockerfile.workdir +0 -0
  258. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dockerfile_build_example/environment/input.txt +0 -0
  259. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dockerfile_build_example/working_dir_auto_example.yaml +0 -0
  260. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/dockerfile_build_example/working_dir_concrete_example.yaml +0 -0
  261. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/fibonacci_with_template.yaml +0 -0
  262. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/hello_date.yaml +0 -0
  263. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/inline_starter_example.yaml +0 -0
  264. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/internal/session_resumption.yaml +0 -0
  265. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/mock_path_dirs_smoke.yaml +0 -0
  266. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/mock_path_dirs_template_dir/mock-cli-bins/README.md +0 -0
  267. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/mock_path_dirs_template_dir/mock-cli-bins/mocks/echo_args +0 -0
  268. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/mock_path_dirs_template_dir/mock-cli-bins/mocks/fixtures/config.json +0 -0
  269. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/mock_path_dirs_template_dir/mock-cli-bins/mocks/say_hello +0 -0
  270. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/mock_path_dirs_template_dir/task.yaml +0 -0
  271. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/python_cli_simulated_judged/echo_simulated_judged.yaml +0 -0
  272. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/3d-scan-calc/3d-scan-calc.yaml +0 -0
  273. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/3d-scan-calc/environment/Dockerfile +0 -0
  274. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/3d-scan-calc/environment/material_density_table.md +0 -0
  275. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/3d-scan-calc/environment/scan_data.stl +0 -0
  276. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/3d-scan-calc/verifier/test.sh +0 -0
  277. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/3d-scan-calc/verifier/test_outputs.py +0 -0
  278. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/README.md +0 -0
  279. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml +0 -0
  280. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile +0 -0
  281. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/court-form-filling/environment/sc100-blank.pdf +0 -0
  282. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/court-form-filling/verifier/test.sh +0 -0
  283. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +0 -0
  284. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/dialogue-parser/dialogue-parser.yaml +0 -0
  285. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/dialogue-parser/environment/Dockerfile +0 -0
  286. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/dialogue-parser/environment/script.txt +0 -0
  287. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/dialogue-parser/verifier/test.sh +0 -0
  288. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/samples/skillsbench/dialogue-parser/verifier/test_outputs.py +0 -0
  289. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/sentiment_classification.yaml +0 -0
  290. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_agent_judge.yaml +0 -0
  291. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_budget_exceeded.yaml +0 -0
  292. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_cost_budget_exceeded.yaml +0 -0
  293. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_llm_judge.yaml +0 -0
  294. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_negative_path.yaml +0 -0
  295. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_task_timeout.yaml +0 -0
  296. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/smoke_variants.yaml +0 -0
  297. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/test_sandbox.yaml +0 -0
  298. {coder_eval-0.8.2 → coder_eval-0.8.3}/tasks/token_check.yaml +0 -0
  299. {coder_eval-0.8.2 → coder_eval-0.8.3}/templates/byod_smoke_test/Dockerfile +0 -0
  300. {coder_eval-0.8.2 → coder_eval-0.8.3}/templates/fibonacci-starter/README.md +0 -0
  301. {coder_eval-0.8.2 → coder_eval-0.8.3}/templates/fibonacci-starter/src/main.py +0 -0
  302. {coder_eval-0.8.2 → coder_eval-0.8.3}/templates/fibonacci-starter/tests/test_main.py +0 -0
  303. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/__init__.py +0 -0
  304. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/__init__.py +0 -0
  305. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/__init__.py +0 -0
  306. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/_scrub.py +0 -0
  307. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/claude_fixtures.py +0 -0
  308. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/codex_fixtures.py +0 -0
  309. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json +0 -0
  310. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json +0 -0
  311. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json +0 -0
  312. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json +0 -0
  313. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json +0 -0
  314. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json +0 -0
  315. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json +0 -0
  316. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json +0 -0
  317. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json +0 -0
  318. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json +0 -0
  319. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json +0 -0
  320. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json +0 -0
  321. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json +0 -0
  322. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json +0 -0
  323. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json +0 -0
  324. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json +0 -0
  325. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json +0 -0
  326. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json +0 -0
  327. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/__init__.py +0 -0
  328. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/_snapshot.py +0 -0
  329. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/experiment_2variant.md +0 -0
  330. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/experiment_3variant.md +0 -0
  331. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/experiment_replicates.md +0 -0
  332. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/run_full.md +0 -0
  333. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_fixtures/report_snapshots/run_minimal.md +0 -0
  334. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/_path_helpers.py +0 -0
  335. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/conftest.py +0 -0
  336. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/__init__.py +0 -0
  337. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/byoa_demo_plugin/byoa_demo.py +0 -0
  338. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/byoa_demo_plugin/pyproject.toml +0 -0
  339. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/mock_agent.py +0 -0
  340. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/tasks/test_task_multiple_criteria.yaml +0 -0
  341. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/tasks/test_task_pass.yaml +0 -0
  342. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/fixtures/text_stub_agent.py +0 -0
  343. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/__init__.py +0 -0
  344. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/__init__.py +0 -0
  345. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/base.py +0 -0
  346. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce014_merge_strategy_declared.py +0 -0
  347. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce015_create_subprocess_limit.py +0 -0
  348. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce016_no_computed_tokenusage_kwargs.py +0 -0
  349. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce017_models_lazy_agent_imports.py +0 -0
  350. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce018_no_final_status_name_denylist.py +0 -0
  351. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce019_telemetry_non_fatal.py +0 -0
  352. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce020_no_sdk_typed_base_agent_fields.py +0 -0
  353. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce021_guarded_evaluationresult_parse.py +0 -0
  354. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce022_dialog_loop_statement_cap.py +0 -0
  355. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce023_no_proxy_shim_import.py +0 -0
  356. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/ce024_discriminated_unions.py +0 -0
  357. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_agent_timing_access.py +0 -0
  358. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_blocking_io_in_async.py +0 -0
  359. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_cli_imports_in_core.py +0 -0
  360. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_silent_except.py +0 -0
  361. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_submodule_model_imports.py +0 -0
  362. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_top_level_run_limits_access.py +0 -0
  363. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_transcript_regex_in_eval.py +0 -0
  364. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/no_type_name_string_dispatch.py +0 -0
  365. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/open_explicit_encoding.py +0 -0
  366. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/read_text_explicit_encoding.py +0 -0
  367. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/register_criterion_required.py +0 -0
  368. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/subprocess_run_explicit_encoding.py +0 -0
  369. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/rules/yaml_models_forbid_extras.py +0 -0
  370. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/runner.py +0 -0
  371. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/lint/violation.py +0 -0
  372. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent.py +0 -0
  373. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_config_no_timing_fields.py +0 -0
  374. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_config_optional_type.py +0 -0
  375. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_config_registry_dispatch.py +0 -0
  376. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_config_sdk_decoupling.py +0 -0
  377. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_golden_master.py +0 -0
  378. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_judge_criterion.py +0 -0
  379. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_telemetry.py +0 -0
  380. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_telemetry_advanced.py +0 -0
  381. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agent_timeout.py +0 -0
  382. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_agentless.py +0 -0
  383. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_aggregate.py +0 -0
  384. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_antigravity_agent.py +0 -0
  385. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_byoa_plugin.py +0 -0
  386. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_byoa_plugin_live.py +0 -0
  387. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_byod_feature.py +0 -0
  388. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_checker_logging.py +0 -0
  389. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_classification_match.py +0 -0
  390. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_claude_settings_enforcement_live.py +0 -0
  391. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cleanup_preservation_guard.py +0 -0
  392. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cli_backend_flag.py +0 -0
  393. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cli_empty_glob.py +0 -0
  394. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cli_sdk_options.py +0 -0
  395. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cli_set_overrides.py +0 -0
  396. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cli_telemetry.py +0 -0
  397. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_cli_type_flag.py +0 -0
  398. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_code_review_bugs.py +0 -0
  399. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_codex_agent.py +0 -0
  400. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_codex_agent_live.py +0 -0
  401. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_codex_agent_unit.py +0 -0
  402. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_codex_token_mapping.py +0 -0
  403. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_command_executed.py +0 -0
  404. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_command_statistics.py +0 -0
  405. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_command_telemetry_result_data.py +0 -0
  406. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_commands_efficiency.py +0 -0
  407. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_config_lineage.py +0 -0
  408. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_config_merge_engine.py +0 -0
  409. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_config_precedence.py +0 -0
  410. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_continuous_scoring.py +0 -0
  411. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_criterion_result_round_trip.py +0 -0
  412. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_custom_lint.py +0 -0
  413. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_dataset_expansion.py +0 -0
  414. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_debug_logging.py +0 -0
  415. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_docker_build_failure.py +0 -0
  416. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_docker_runner_container_death.py +0 -0
  417. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_docker_runner_mounts.py +0 -0
  418. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_docker_runner_stream_limit.py +0 -0
  419. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_docker_wildcard_env.py +0 -0
  420. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_docker_workdir_live.py +0 -0
  421. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_driver_resolver.py +0 -0
  422. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_error_handling.py +0 -0
  423. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_evaluate_command.py +0 -0
  424. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_evaluator.py +0 -0
  425. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_event_collector.py +0 -0
  426. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_experiment_cli.py +0 -0
  427. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_experiment_loader.py +0 -0
  428. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_experiment_models.py +0 -0
  429. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_experiment_reports.py +0 -0
  430. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_experiment_resolver.py +0 -0
  431. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_experiment_runner.py +0 -0
  432. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_file_check.py +0 -0
  433. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_file_contains_scoring.py +0 -0
  434. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_formatting.py +0 -0
  435. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_git_clone_failure.py +0 -0
  436. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_heartbeat_watchdog.py +0 -0
  437. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_ignore_patterns_negation.py +0 -0
  438. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_image_from_dockerfiles.py +0 -0
  439. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_integration.py +0 -0
  440. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_json_check.py +0 -0
  441. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_judge_anthropic.py +0 -0
  442. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_judge_bedrock.py +0 -0
  443. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_judge_burn_in_live.py +0 -0
  444. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_judge_context_builder.py +0 -0
  445. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_judge_models.py +0 -0
  446. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_judge_persistence.py +0 -0
  447. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_lint_no_top_level_run_limits.py +0 -0
  448. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_lint_runner.py +0 -0
  449. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_llm_judge_criterion.py +0 -0
  450. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_log_tail_buffer.py +0 -0
  451. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_logging.py +0 -0
  452. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_logging_isolation.py +0 -0
  453. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_merge_characterization.py +0 -0
  454. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_merge_strategy_annotations.py +0 -0
  455. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_merge_unification.py +0 -0
  456. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_models.py +0 -0
  457. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_mutations.py +0 -0
  458. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_new_criteria.py +0 -0
  459. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_node_env_config.py +0 -0
  460. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_optional_dependencies.py +0 -0
  461. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_orchestrator.py +0 -0
  462. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_orchestrator_error_log_tail.py +0 -0
  463. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_orchestrator_telemetry.py +0 -0
  464. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_overrides_engine.py +0 -0
  465. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_parallel.py +0 -0
  466. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_path_utils.py +0 -0
  467. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_plan_command.py +0 -0
  468. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_plugin_processing.py +0 -0
  469. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_plugins.py +0 -0
  470. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_post_run.py +0 -0
  471. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_pr_review_workflow.py +0 -0
  472. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_pre_run.py +0 -0
  473. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_preservation_mode.py +0 -0
  474. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_pricing_registry.py +0 -0
  475. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reference_comparison_scoring.py +0 -0
  476. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reference_evaluator.py +0 -0
  477. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reference_missing_file.py +0 -0
  478. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reference_models.py +0 -0
  479. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reference_orchestrator.py +0 -0
  480. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_registry.py +0 -0
  481. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_replicate_stats.py +0 -0
  482. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_report_command.py +0 -0
  483. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reports.py +0 -0
  484. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reports_experiment.py +0 -0
  485. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reports_html.py +0 -0
  486. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_reports_stats.py +0 -0
  487. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_resolve_task_files.py +0 -0
  488. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_resume.py +0 -0
  489. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_retry_logic_comprehensive.py +0 -0
  490. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_routing.py +0 -0
  491. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_run_command_stdout.py +0 -0
  492. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_run_limits_models.py +0 -0
  493. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_run_limits_orchestrator.py +0 -0
  494. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_run_limits_resolver.py +0 -0
  495. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_runtime_tool_versions.py +0 -0
  496. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sandbox.py +0 -0
  497. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sandbox_layer_builder.py +0 -0
  498. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sandbox_optional.py +0 -0
  499. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sandbox_security.py +0 -0
  500. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sandbox_symlink_preservation.py +0 -0
  501. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sandbox_templates.py +0 -0
  502. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_scorers.py +0 -0
  503. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_scoring_quality.py +0 -0
  504. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sdk_option_classification.py +0 -0
  505. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_simulation_config.py +0 -0
  506. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_simulation_integration.py +0 -0
  507. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_simulation_termination.py +0 -0
  508. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_simulation_trials.py +0 -0
  509. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_skill_triggered.py +0 -0
  510. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_agent_integration.py +0 -0
  511. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_batch.py +0 -0
  512. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_callbacks.py +0 -0
  513. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_cli.py +0 -0
  514. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_events.py +0 -0
  515. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_orchestrator.py +0 -0
  516. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_renderers.py +0 -0
  517. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_streaming_wire.py +0 -0
  518. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_sub_agent_runner.py +0 -0
  519. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_success_criterion_union.py +0 -0
  520. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_suite_rollup.py +0 -0
  521. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_summaries.py +0 -0
  522. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_tags.py +0 -0
  523. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_task_definition_unknown_fields.py +0 -0
  524. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_telemetry.py +0 -0
  525. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_template_env_expansion.py +0 -0
  526. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_threshold_enforcement.py +0 -0
  527. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_timeout_batch.py +0 -0
  528. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_timeout_categorization.py +0 -0
  529. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_timeout_exceptions.py +0 -0
  530. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_timeout_models.py +0 -0
  531. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_timeout_orchestrator.py +0 -0
  532. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_token_usage.py +0 -0
  533. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_uipath_eval.py +0 -0
  534. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_user_simulator.py +0 -0
  535. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_utils.py +0 -0
  536. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_utterance_extraction.py +0 -0
  537. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_variant_prompt_file.py +0 -0
  538. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_verdict_tool.py +0 -0
  539. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_watchdog.py +0 -0
  540. {coder_eval-0.8.2 → coder_eval-0.8.3}/tests/test_yaml_migration.py +0 -0
@@ -1,39 +1,49 @@
1
1
  name: Release
2
2
 
3
- # Manually-triggered release: build the CURRENT version (from pyproject.toml)
4
- # and publish it to public PyPI + the UiPath `coder_eval` Azure Artifacts feed
5
- # (wheel) and GHCR (agent image), then push the `v<version>` git tag.
3
+ # Manually-triggered release: bump the version, tag, build, and publish to
4
+ # public PyPI (wheel) and GHCR (agent image). Merges to main do NOT
5
+ # auto-release -- run this workflow from the Actions tab when you want to cut a
6
+ # release.
6
7
  #
7
- # Version management: `main` is protected by a ruleset (changes via PR only),
8
- # so this workflow does NOT commit a version bump to main. Bump `version` in
9
- # pyproject.toml (+ src/coder_eval/__init__.py) through a normal PR; once that
10
- # merges, run this workflow to release whatever version main currently
11
- # declares. It refuses to run if the `v<version>` tag already exists (i.e. you
12
- # forgot to bump). Only the git TAG is pushed -- tags are outside the branch
13
- # ruleset -- never a commit to main.
8
+ # The bump level is CHOSEN at dispatch, not derived from commit messages:
9
+ # `patch` (default), `minor`, or `major`. A dispatch always cuts a release.
10
+ # semantic-release only does the mechanics -- write the new version to
11
+ # pyproject.toml + __init__.py, tag `v<version>`, regenerate the changelog from
12
+ # the commits since the last tag (notes only -- they don't affect the version),
13
+ # and push.
14
14
  # (Continuous :latest / :sha- agent images still publish on every main push via
15
15
  # docker-publish.yml -- only the versioned release artifacts gate on this run.)
16
16
 
17
17
  on:
18
18
  workflow_dispatch:
19
+ inputs:
20
+ bump:
21
+ description: 'Version bump to release'
22
+ type: choice
23
+ default: patch
24
+ options:
25
+ - patch
26
+ - minor
27
+ - major
19
28
 
20
29
  concurrency:
21
30
  group: release-${{ github.ref }}
22
31
  cancel-in-progress: false
23
32
 
24
- # The release pushes only the `v<version>` git TAG (tags are not covered by
25
- # the branch ruleset protecting main), so the built-in GITHUB_TOKEN with
26
- # contents: write suffices -- no GitHub App / ruleset bypass required.
33
+ # The version-bump commit + tag are pushed with a GitHub App installation token
34
+ # (not GITHUB_TOKEN): main is protected by a ruleset (changes via PR only), and
35
+ # only the release app has a ruleset bypass. GITHUB_TOKEN is only used for the
36
+ # checkout's read access and the GHCR login.
27
37
  permissions:
28
- contents: write # push the v<version> tag
38
+ contents: read
29
39
  packages: write # push the versioned agent image to ghcr.io on release
30
40
 
31
41
  jobs:
32
42
  release:
33
43
  name: Bump version and publish
34
44
  # GitHub-hosted so cutting a release does not depend on the self-hosted
35
- # `uipath-ubuntu-latest` pool. uv, twine, and the docker buildx -> GHCR
36
- # push all run fine here.
45
+ # `uipath-ubuntu-latest` pool. semantic-release, uv, twine, and the docker
46
+ # buildx -> GHCR push all run fine here.
37
47
  runs-on: ubuntu-latest
38
48
  timeout-minutes: 15
39
49
  outputs:
@@ -48,8 +58,20 @@ jobs:
48
58
  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex"
49
59
 
50
60
  steps:
61
+ - name: Mint release app token
62
+ id: app-token
63
+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
64
+ with:
65
+ app-id: ${{ secrets.RELEASE_APP_ID }}
66
+ private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
67
+
51
68
  - name: Checkout code
52
69
  uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
70
+ with:
71
+ fetch-depth: 0 # semantic-release needs full history for tags + changelog
72
+ # Persisted in .git config so semantic-release's push to main is
73
+ # authenticated as the app (which bypasses the branch ruleset).
74
+ token: ${{ steps.app-token.outputs.token }}
53
75
 
54
76
  - name: Set up Python 3.13
55
77
  uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
@@ -61,21 +83,44 @@ jobs:
61
83
  with:
62
84
  enable-cache: true
63
85
 
64
- - name: Resolve version + guard against re-release
86
+ - name: Install build + release tools
87
+ run: uv tool install python-semantic-release && uv tool install twine
88
+
89
+ - name: Run semantic-release (bump + tag, no push yet)
65
90
  id: release
91
+ env:
92
+ GH_TOKEN: ${{ steps.app-token.outputs.token }}
93
+ # Passed via env (not interpolated into the script) per GitHub's
94
+ # injection guidance; it's a constrained choice input regardless.
95
+ BUMP: ${{ inputs.bump }}
66
96
  run: |
67
97
  set -euo pipefail
68
- # Version is whatever main currently declares. Bumps land via a PR
69
- # (main is ruleset-protected); this workflow never rewrites it.
70
- VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
71
- echo "Releasing coder-eval ${VERSION}"
72
- # Refuse to re-release an existing version: a PyPI version is
73
- # immutable, so this is almost always a forgotten version bump.
74
- if git ls-remote --exit-code --tags origin "refs/tags/v${VERSION}" >/dev/null 2>&1; then
75
- echo "::error::Tag v${VERSION} already exists. Bump 'version' in pyproject.toml (+ src/coder_eval/__init__.py) via a PR before releasing."
76
- exit 1
98
+ # The bump level comes from the dispatch input, not commit messages:
99
+ # force a patch/minor/major bump outright. `version` writes the new
100
+ # version to pyproject.toml + __init__.py, tags it, and regenerates the
101
+ # changelog, but does not push yet (--no-push) so we can regenerate
102
+ # uv.lock and amend before sending.
103
+ PSR="uv tool run --from python-semantic-release semantic-release"
104
+ $PSR version "--$BUMP" --no-vcs-release --no-push --changelog
105
+ # A dispatch always cuts a release; report the just-published version.
106
+ echo "version=$($PSR version --print)" >> "$GITHUB_OUTPUT"
107
+
108
+ - name: Regenerate uv.lock and amend release commit
109
+ if: steps.release.outputs.version != ''
110
+ run: |
111
+ uv lock
112
+ if ! git diff --quiet uv.lock; then
113
+ git config user.email "github-actions[bot]@users.noreply.github.com"
114
+ git config user.name "github-actions[bot]"
115
+ git add uv.lock
116
+ git commit --amend --no-edit
117
+ # Amend replaced the commit the tag points at; re-point it before pushing.
118
+ git tag -f "v${{ steps.release.outputs.version }}"
77
119
  fi
78
- echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
120
+
121
+ - name: Push release commit and tags
122
+ if: steps.release.outputs.version != ''
123
+ run: git push origin main "v${{ steps.release.outputs.version }}"
79
124
 
80
125
  - name: Build wheel + sdist
81
126
  if: steps.release.outputs.version != ''
@@ -92,24 +137,12 @@ jobs:
92
137
  path: dist/
93
138
  if-no-files-found: error
94
139
 
95
- - name: Publish to coder_eval feed
96
- # Best-effort: the private Azure feed is a UiPath-internal target; its
97
- # failure (e.g. an expired PAT) must NOT block the public PyPI publish.
98
- continue-on-error: true
99
- if: steps.release.outputs.version != ''
100
- env:
101
- # Azure Artifacts accepts any non-empty username when paired with a
102
- # PAT, so we hardcode it. The PAT (Packaging Read & write scope)
103
- # comes from UV_INDEX_UIPATH_PASSWORD, reused from pr-checks.yml.
104
- TWINE_USERNAME: "azure-artifacts"
105
- TWINE_PASSWORD: ${{ secrets.UV_INDEX_UIPATH_PASSWORD }}
106
- TWINE_REPOSITORY_URL: https://pkgs.dev.azure.com/uipath/ML%20Platform/_packaging/coder_eval/pypi/upload/
107
- run: uv tool run --from twine twine upload --non-interactive --verbose dist/*
108
-
109
- # Build + push the agent image HERE, in the release job, tagged with the
110
- # version main declares. docker-publish.yml only publishes :latest / :sha-
111
- # on every main push, so this is the authoritative versioned image; it
112
- # also repoints `:latest` to the exact release artifact.
140
+ # Build + push the agent image HERE, in the same job that cut the release,
141
+ # so the `:<version>` tag is built from the BUMPED pyproject (the version
142
+ # only exists after the semantic-release step above). docker-publish.yml
143
+ # runs on the triggering commit, BEFORE the bump, so it can never tag the
144
+ # release version -- this is the authoritative versioned image. It also
145
+ # repoints `:latest` to the exact release artifact.
113
146
  - name: Lowercase owner for GHCR
114
147
  if: steps.release.outputs.version != ''
115
148
  id: img
@@ -149,11 +182,10 @@ jobs:
149
182
  # back from here, to avoid two concurrent writers racing the cache tag.
150
183
  cache-from: type=registry,ref=ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:buildcache
151
184
 
152
- # Publish the SAME wheel+sdist to public PyPI (pkgs live alongside the private
153
- # Azure Artifacts feed, which the release job above still populates). This runs
154
- # as its own job so OIDC Trusted Publishing is scoped to a dedicated,
155
- # environment-gated context on GitHub-hosted runners -- no PyPI token/secret is
156
- # stored. Gated on the release job having actually cut a version.
185
+ # Publish the wheel+sdist to public PyPI. This runs as its own job so OIDC
186
+ # Trusted Publishing is scoped to a dedicated, environment-gated context on
187
+ # GitHub-hosted runners -- no PyPI token/secret is stored. Gated on the
188
+ # release job having actually cut a version.
157
189
  publish-pypi:
158
190
  name: Publish to PyPI
159
191
  needs: release
@@ -179,28 +211,3 @@ jobs:
179
211
  # Trusted Publisher is configured on pypi.org for this repo +
180
212
  # workflow (release.yml) + environment (pypi); no password needed.
181
213
  packages-dir: dist/
182
-
183
- # Tag the release ONLY after PyPI has published, so a `v<version>` tag marks a
184
- # completed public release and the re-release guard never blocks retrying a
185
- # failed publish. Tags are outside the branch ruleset, so GITHUB_TOKEN suffices.
186
- tag-release:
187
- name: Push release tag
188
- needs: [release, publish-pypi]
189
- if: needs.release.outputs.version != ''
190
- runs-on: ubuntu-latest
191
- timeout-minutes: 5
192
- permissions:
193
- contents: write # push the v<version> tag
194
- steps:
195
- - name: Checkout code
196
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
197
-
198
- - name: Push v<version> tag
199
- env:
200
- VERSION: ${{ needs.release.outputs.version }}
201
- run: |
202
- set -euo pipefail
203
- git config user.email "github-actions[bot]@users.noreply.github.com"
204
- git config user.name "github-actions[bot]"
205
- git tag -a "v${VERSION}" -m "Release v${VERSION}"
206
- git push origin "v${VERSION}"
@@ -2,6 +2,42 @@
2
2
 
3
3
  <!-- version list -->
4
4
 
5
+ ## v0.8.3 (2026-07-09)
6
+
7
+ ### Bug Fixes
8
+
9
+ - **evalboard**: Fall back to agent_config type when run_config omits harness
10
+ ([#10](https://github.com/UiPath/coder_eval/pull/10),
11
+ [`59a240c`](https://github.com/UiPath/coder_eval/commit/59a240c3d87d3ec4aa7f43a77673dd0561a0e006))
12
+
13
+ ### Continuous Integration
14
+
15
+ - Remove Azure Artifacts publishing ([#11](https://github.com/UiPath/coder_eval/pull/11),
16
+ [`2a5124f`](https://github.com/UiPath/coder_eval/commit/2a5124f2c2c8b4151b83a2a9ff2c0bc464683a34))
17
+
18
+ - Restore auto-bump release (semantic-release + app token)
19
+ ([#12](https://github.com/UiPath/coder_eval/pull/12),
20
+ [`0b9d378`](https://github.com/UiPath/coder_eval/commit/0b9d378e5a99a878a211046a9a9a9ab7f40faaa4))
21
+
22
+ ### Features
23
+
24
+ - **evalboard**: Add a Harness (RunConfig) column to the runs tables
25
+ ([#10](https://github.com/UiPath/coder_eval/pull/10),
26
+ [`59a240c`](https://github.com/UiPath/coder_eval/commit/59a240c3d87d3ec4aa7f43a77673dd0561a0e006))
27
+
28
+ - **evalboard**: Add Gemini rates to the frontend pricing table
29
+ ([#10](https://github.com/UiPath/coder_eval/pull/10),
30
+ [`59a240c`](https://github.com/UiPath/coder_eval/commit/59a240c3d87d3ec4aa7f43a77673dd0561a0e006))
31
+
32
+ - **evalboard**: Show harness as a vendor logo (internal, main table only)
33
+ ([#10](https://github.com/UiPath/coder_eval/pull/10),
34
+ [`59a240c`](https://github.com/UiPath/coder_eval/commit/59a240c3d87d3ec4aa7f43a77673dd0561a0e006))
35
+
36
+ - **evalboard**: Show the harness (RunConfig) column on the runs tables
37
+ ([#10](https://github.com/UiPath/coder_eval/pull/10),
38
+ [`59a240c`](https://github.com/UiPath/coder_eval/commit/59a240c3d87d3ec4aa7f43a77673dd0561a0e006))
39
+
40
+
5
41
  ## v0.8.2 (2026-07-07)
6
42
 
7
43
  ### Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: coder-eval
3
- Version: 0.8.2
3
+ Version: 0.8.3
4
4
  Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
5
5
  Project-URL: Homepage, https://github.com/UiPath/coder_eval
6
6
  Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -0,0 +1,37 @@
1
+ import Image from "next/image";
2
+
3
+ // Vendor logo for a run's harness (RunConfig). Renders the recognizable
4
+ // vendor mark instead of raw "claude-code"/"codex"/"antigravity" text,
5
+ // mirroring the Slack rollup's vendor emoji. A missing harness defaults to
6
+ // claude-code (the nightly). This is an internal-only column — the caller
7
+ // gates it behind isInternal (see lib/edition.ts).
8
+ const HARNESS_LOGO: Record<string, { src: string; label: string }> = {
9
+ "claude-code": {
10
+ src: "/harness/claude-code.png",
11
+ label: "Claude Code · Anthropic",
12
+ },
13
+ codex: { src: "/harness/codex.png", label: "Codex · OpenAI" },
14
+ antigravity: {
15
+ src: "/harness/antigravity.png",
16
+ label: "Antigravity · Google Gemini",
17
+ },
18
+ };
19
+
20
+ export function HarnessBadge({ harness }: { harness?: string | null }) {
21
+ const key = harness ?? "claude-code";
22
+ const logo = HARNESS_LOGO[key];
23
+ // Unknown harness: show the raw id rather than a misleading logo.
24
+ if (!logo) {
25
+ return <span className="text-xs text-gray-700">{key}</span>;
26
+ }
27
+ return (
28
+ <Image
29
+ src={logo.src}
30
+ alt={logo.label}
31
+ title={logo.label}
32
+ width={20}
33
+ height={20}
34
+ className="rounded-sm"
35
+ />
36
+ );
37
+ }
@@ -15,6 +15,7 @@ import { ChipLegend, MergedTagRail } from "./_overview/tag-rail";
15
15
  import { TableScroll } from "./_components/scroll-table";
16
16
  import { CollapsibleRail } from "./_components/collapsible-rail";
17
17
  import { isInternal } from "@/lib/edition";
18
+ import { HarnessBadge } from "@/app/_components/harness-badge";
18
19
 
19
20
  export const dynamic = "force-dynamic";
20
21
 
@@ -330,6 +331,11 @@ export default async function Page({
330
331
  <thead>
331
332
  <tr className="bg-gray-50 border-b border-gray-200 text-left text-gray-600">
332
333
  <th className="py-3 px-4 font-medium">Run</th>
334
+ {isInternal && (
335
+ <th className="py-3 px-4 font-medium">
336
+ Harness
337
+ </th>
338
+ )}
333
339
  <th className="py-3 px-4 font-medium">
334
340
  Pass rate
335
341
  </th>
@@ -363,6 +369,13 @@ export default async function Page({
363
369
  {fmtRunTime(r.id)}
364
370
  </Link>
365
371
  </td>
372
+ {isInternal && (
373
+ <td className="py-3 px-4">
374
+ <HarnessBadge
375
+ harness={r.harness}
376
+ />
377
+ </td>
378
+ )}
366
379
  <td className="py-3 px-4 tabular-nums">
367
380
  <span
368
381
  className={`font-medium ${passClass(
@@ -393,7 +406,7 @@ export default async function Page({
393
406
  {listing.rows.length === 0 && (
394
407
  <tr>
395
408
  <td
396
- colSpan={5}
409
+ colSpan={isInternal ? 6 : 5}
397
410
  className="py-6 px-4 text-center text-sm text-gray-500"
398
411
  >
399
412
  {isFiltered
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "coder-eval"
3
- version = "0.8.2"
3
+ version = "0.8.3"
4
4
  description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites."
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -1,3 +1,3 @@
1
1
  """coder_eval - A framework for evaluating AI coding agents."""
2
2
 
3
- __version__ = "0.8.2"
3
+ __version__ = "0.8.3"
@@ -443,7 +443,7 @@ wheels = [
443
443
 
444
444
  [[package]]
445
445
  name = "coder-eval"
446
- version = "0.8.2"
446
+ version = "0.8.3"
447
447
  source = { editable = "." }
448
448
  dependencies = [
449
449
  { name = "anthropic" },
@@ -1,155 +0,0 @@
1
- name: Publish Dev Build
2
-
3
- # Per-PR dev wheel publishing for end-to-end testing of the release pipeline
4
- # without bumping the canonical version on main.
5
- #
6
- # How to use:
7
- # 1. Open a PR.
8
- # 2. Add the `build:dev` label. Re-add (or push new commits) to refresh.
9
- # 3. Workflow rewrites pyproject.toml version to:
10
- # <current>.dev1<PR#:05d><run#:04d>
11
- # builds wheel+sdist, publishes to the coder_eval feed, and edits the
12
- # PR description with a copy-pasteable install snippet.
13
- #
14
- # Inspired by uipath-core-python/.github/workflows/publish-dev.yml.
15
-
16
- on:
17
- pull_request:
18
- types: [opened, synchronize, reopened, labeled]
19
-
20
- concurrency:
21
- group: publish-dev-${{ github.event.pull_request.number }}
22
- cancel-in-progress: true
23
-
24
- permissions:
25
- contents: read
26
- pull-requests: write # for editing PR description with install snippet
27
-
28
- jobs:
29
- publish-dev:
30
- name: Publish dev wheel for PR
31
- runs-on: ubuntu-latest
32
- if: contains(github.event.pull_request.labels.*.name, 'build:dev')
33
- timeout-minutes: 15
34
- env:
35
- # uipath-ubuntu-latest runners enforce a minimum package-age safe-chain
36
- # check on uv installs; match the exclusions used by pr-checks.yml.
37
- SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex"
38
-
39
- steps:
40
- - name: Checkout code
41
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
42
-
43
- - name: Set up Python 3.13
44
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
45
- with:
46
- python-version: "3.13"
47
-
48
- - name: Install uv
49
- uses: astral-sh/setup-uv@v4
50
- with:
51
- enable-cache: true
52
-
53
- - name: Compute and apply dev version
54
- id: ver
55
- env:
56
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
57
- PR_NUMBER: ${{ github.event.pull_request.number }}
58
- RUN_NUMBER: ${{ github.run_number }}
59
- REPO: ${{ github.repository }}
60
- run: |
61
- set -euo pipefail
62
- PROJECT_NAME=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['name'])")
63
- CURRENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
64
- PADDED_PR=$(printf '%05d' "$PR_NUMBER")
65
- PADDED_RUN=$(printf '%04d' "$RUN_NUMBER")
66
- PADDED_NEXT_PR=$(printf '%05d' "$((PR_NUMBER + 1))")
67
- # PEP 440 dev version: .dev1<PR><RUN> sorts cleanly inside [MIN, MAX).
68
- DEV_VERSION="${CURRENT_VERSION}.dev1${PADDED_PR}${PADDED_RUN}"
69
- MIN_VERSION="${CURRENT_VERSION}.dev1${PADDED_PR}0000"
70
- MAX_VERSION="${CURRENT_VERSION}.dev1${PADDED_NEXT_PR}0000"
71
-
72
- python3 - <<PY
73
- import re, pathlib
74
- p = pathlib.Path("pyproject.toml")
75
- s = p.read_text()
76
- s = re.sub(r'(?m)^version\s*=\s*".+?"$', 'version = "${DEV_VERSION}"', s, count=1)
77
- p.write_text(s)
78
- PY
79
-
80
- echo "version=${DEV_VERSION}" >> "$GITHUB_OUTPUT"
81
- echo "min=${MIN_VERSION}" >> "$GITHUB_OUTPUT"
82
- echo "max=${MAX_VERSION}" >> "$GITHUB_OUTPUT"
83
- echo "name=${PROJECT_NAME}" >> "$GITHUB_OUTPUT"
84
- echo "Set version to ${DEV_VERSION}"
85
-
86
- - name: Build wheel + sdist
87
- run: uv build
88
-
89
- - name: Publish to coder_eval feed
90
- env:
91
- # Azure Artifacts accepts any non-empty username when paired with a
92
- # PAT, so we hardcode it; the PAT (read+write Packaging scope) lives
93
- # in UV_INDEX_UIPATH_PASSWORD, reused from pr-checks.yml.
94
- TWINE_USERNAME: "azure-artifacts"
95
- TWINE_PASSWORD: ${{ secrets.UV_INDEX_UIPATH_PASSWORD }}
96
- TWINE_REPOSITORY_URL: https://pkgs.dev.azure.com/uipath/ML%20Platform/_packaging/coder_eval/pypi/upload/
97
- run: uv tool run --from twine twine upload --non-interactive --verbose dist/*
98
-
99
- - name: Update PR description with install snippet
100
- env:
101
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
102
- PR_NUMBER: ${{ github.event.pull_request.number }}
103
- REPO: ${{ github.repository }}
104
- PKG: ${{ steps.ver.outputs.name }}
105
- DEV_VERSION: ${{ steps.ver.outputs.version }}
106
- MIN_VERSION: ${{ steps.ver.outputs.min }}
107
- MAX_VERSION: ${{ steps.ver.outputs.max }}
108
- run: |
109
- set -euo pipefail
110
- START="<!-- DEV_PACKAGE_START -->"
111
- END="<!-- DEV_PACKAGE_END -->"
112
- read -r -d '' SNIPPET <<EOF || true
113
- $START
114
- ## Development Package (\`${DEV_VERSION}\`)
115
-
116
- Install the exact build from this PR via the coder_eval feed:
117
-
118
- \`\`\`toml
119
- [project]
120
- dependencies = [
121
- # Exact version:
122
- "${PKG}==${DEV_VERSION}",
123
- # Or any version from this PR:
124
- "${PKG}>=${MIN_VERSION},<${MAX_VERSION}",
125
- ]
126
-
127
- [[tool.uv.index]]
128
- name = "coder_eval"
129
- url = "https://pkgs.dev.azure.com/uipath/ML%20Platform/_packaging/coder_eval/pypi/simple/"
130
- publish-url = "https://pkgs.dev.azure.com/uipath/ML%20Platform/_packaging/coder_eval/pypi/upload/"
131
- explicit = true
132
-
133
- [tool.uv.sources]
134
- ${PKG} = { index = "coder_eval" }
135
- \`\`\`
136
- $END
137
- EOF
138
-
139
- export CURRENT SNIPPET START END
140
- CURRENT=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq .body)
141
- if echo "$CURRENT" | grep -q "$START"; then
142
- NEW=$(CURRENT="$CURRENT" SNIPPET="$SNIPPET" START="$START" END="$END" python3 - <<'PY'
143
- import os, re
144
- body = os.environ["CURRENT"]
145
- snippet = os.environ["SNIPPET"]
146
- pat = re.compile(re.escape(os.environ["START"]) + r".*?" + re.escape(os.environ["END"]), re.DOTALL)
147
- print(pat.sub(snippet, body))
148
- PY
149
- )
150
- else
151
- NEW=$(printf '%s\n\n%s' "$CURRENT" "$SNIPPET")
152
- fi
153
-
154
- gh api -X PATCH "repos/${REPO}/pulls/${PR_NUMBER}" -f body="$NEW" >/dev/null
155
- echo "Updated PR description"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes