coder-eval 0.8.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. coder_eval/.gitattributes +2 -0
  2. coder_eval/__init__.py +3 -0
  3. coder_eval/agent.py +302 -0
  4. coder_eval/agents/__init__.py +41 -0
  5. coder_eval/agents/_logging.py +89 -0
  6. coder_eval/agents/antigravity_agent.py +811 -0
  7. coder_eval/agents/claude_code_agent.py +1701 -0
  8. coder_eval/agents/codex_agent.py +2055 -0
  9. coder_eval/agents/noop_agent.py +100 -0
  10. coder_eval/agents/registry.py +163 -0
  11. coder_eval/agents/watchdog.py +116 -0
  12. coder_eval/analysis.py +88 -0
  13. coder_eval/cli/__init__.py +87 -0
  14. coder_eval/cli/aggregate_command.py +153 -0
  15. coder_eval/cli/console.py +7 -0
  16. coder_eval/cli/evaluate_command.py +164 -0
  17. coder_eval/cli/plan_command.py +152 -0
  18. coder_eval/cli/report_command.py +121 -0
  19. coder_eval/cli/run_command.py +686 -0
  20. coder_eval/cli/run_helpers.py +137 -0
  21. coder_eval/cli/run_task_internal_command.py +210 -0
  22. coder_eval/cli/utils.py +37 -0
  23. coder_eval/config.py +212 -0
  24. coder_eval/criteria/__init__.py +163 -0
  25. coder_eval/criteria/_classification_aggregate.py +106 -0
  26. coder_eval/criteria/agent_judge.py +425 -0
  27. coder_eval/criteria/base.py +248 -0
  28. coder_eval/criteria/classification_match.py +107 -0
  29. coder_eval/criteria/command_executed.py +181 -0
  30. coder_eval/criteria/commands_efficiency.py +73 -0
  31. coder_eval/criteria/file_check.py +119 -0
  32. coder_eval/criteria/file_contains.py +85 -0
  33. coder_eval/criteria/file_exists.py +45 -0
  34. coder_eval/criteria/file_matches_regex.py +86 -0
  35. coder_eval/criteria/json_check.py +184 -0
  36. coder_eval/criteria/llm_judge.py +260 -0
  37. coder_eval/criteria/reference_comparison.py +130 -0
  38. coder_eval/criteria/run_command.py +220 -0
  39. coder_eval/criteria/skill_triggered.py +111 -0
  40. coder_eval/criteria/uipath_eval.py +223 -0
  41. coder_eval/errors/__init__.py +26 -0
  42. coder_eval/errors/agent.py +26 -0
  43. coder_eval/errors/budget.py +28 -0
  44. coder_eval/errors/categories.py +205 -0
  45. coder_eval/errors/categorization.py +240 -0
  46. coder_eval/errors/executor.py +124 -0
  47. coder_eval/errors/judge.py +12 -0
  48. coder_eval/errors/retry.py +211 -0
  49. coder_eval/errors/timeout.py +63 -0
  50. coder_eval/evaluation/__init__.py +10 -0
  51. coder_eval/evaluation/checker.py +260 -0
  52. coder_eval/evaluation/judge_anthropic.py +55 -0
  53. coder_eval/evaluation/judge_bedrock.py +114 -0
  54. coder_eval/evaluation/judge_context.py +497 -0
  55. coder_eval/evaluation/judge_models.py +53 -0
  56. coder_eval/evaluation/judge_persistence.py +291 -0
  57. coder_eval/evaluation/judge_usage.py +50 -0
  58. coder_eval/evaluation/sub_agent.py +231 -0
  59. coder_eval/evaluation/summaries.py +48 -0
  60. coder_eval/evaluation/verdict_tool.py +213 -0
  61. coder_eval/formatting.py +191 -0
  62. coder_eval/isolation/__init__.py +15 -0
  63. coder_eval/isolation/docker_runner.py +1253 -0
  64. coder_eval/logging_config.py +399 -0
  65. coder_eval/models/__init__.py +337 -0
  66. coder_eval/models/agent_config.py +373 -0
  67. coder_eval/models/container_paths.py +26 -0
  68. coder_eval/models/criteria.py +942 -0
  69. coder_eval/models/enums.py +121 -0
  70. coder_eval/models/experiment.py +314 -0
  71. coder_eval/models/judge.py +90 -0
  72. coder_eval/models/judge_defaults.py +11 -0
  73. coder_eval/models/limits.py +89 -0
  74. coder_eval/models/merge_strategy.py +132 -0
  75. coder_eval/models/mutations.py +95 -0
  76. coder_eval/models/results.py +787 -0
  77. coder_eval/models/routing.py +160 -0
  78. coder_eval/models/sandbox.py +371 -0
  79. coder_eval/models/tasks.py +631 -0
  80. coder_eval/models/telemetry.py +454 -0
  81. coder_eval/models/templates.py +108 -0
  82. coder_eval/orchestration/__init__.py +13 -0
  83. coder_eval/orchestration/batch.py +690 -0
  84. coder_eval/orchestration/config.py +111 -0
  85. coder_eval/orchestration/config_merge.py +431 -0
  86. coder_eval/orchestration/evaluation.py +94 -0
  87. coder_eval/orchestration/experiment.py +837 -0
  88. coder_eval/orchestration/overrides.py +206 -0
  89. coder_eval/orchestration/task_loader.py +437 -0
  90. coder_eval/orchestrator.py +2078 -0
  91. coder_eval/path_utils.py +79 -0
  92. coder_eval/plugins.py +81 -0
  93. coder_eval/pricing.py +169 -0
  94. coder_eval/py.typed +0 -0
  95. coder_eval/reports.py +1066 -0
  96. coder_eval/reports_experiment.py +761 -0
  97. coder_eval/reports_html.py +1659 -0
  98. coder_eval/reports_stats.py +250 -0
  99. coder_eval/resources/__init__.py +137 -0
  100. coder_eval/resources/default_experiment.yaml +85 -0
  101. coder_eval/resources/default_ignore_patterns.yaml +60 -0
  102. coder_eval/resources/tags.yaml +55 -0
  103. coder_eval/sandbox.py +1127 -0
  104. coder_eval/scoring/__init__.py +11 -0
  105. coder_eval/scoring/ast_similarity.py +38 -0
  106. coder_eval/scoring/complexity.py +115 -0
  107. coder_eval/scoring/quality.py +154 -0
  108. coder_eval/scoring/signature_similarity.py +57 -0
  109. coder_eval/scoring/similarity.py +103 -0
  110. coder_eval/scoring/token_similarity.py +44 -0
  111. coder_eval/simulation/__init__.py +27 -0
  112. coder_eval/simulation/termination.py +88 -0
  113. coder_eval/simulation/user_simulator.py +324 -0
  114. coder_eval/streaming/__init__.py +44 -0
  115. coder_eval/streaming/callbacks.py +57 -0
  116. coder_eval/streaming/collector.py +193 -0
  117. coder_eval/streaming/events.py +198 -0
  118. coder_eval/streaming/renderers.py +248 -0
  119. coder_eval/streaming/wire.py +115 -0
  120. coder_eval/telemetry.py +407 -0
  121. coder_eval/utils.py +517 -0
  122. coder_eval-0.8.2.dist-info/METADATA +242 -0
  123. coder_eval-0.8.2.dist-info/RECORD +127 -0
  124. coder_eval-0.8.2.dist-info/WHEEL +4 -0
  125. coder_eval-0.8.2.dist-info/entry_points.txt +5 -0
  126. coder_eval-0.8.2.dist-info/licenses/LICENSE +201 -0
  127. coder_eval-0.8.2.dist-info/licenses/NOTICE +18 -0
coder_eval/models/container_paths.py ADDED
@@ -0,0 +1,26 @@
1
+ """Canonical container-side paths (single source of truth).
2
+
3
+ These absolute paths are framework-owned inside a ``driver: docker`` container.
4
+ They are defined here -- a dependency-free leaf module -- so both the
5
+ import-light ``models`` layer (``SandboxConfig`` validation) and the heavier
6
+ ``isolation.docker_runner`` can share one definition instead of each carrying a
7
+ copy. ``docker_runner`` re-exports the ``CONTAINER_*`` names, so existing
8
+ importers (and tests) that read them from ``docker_runner`` are unaffected.
9
+
10
+ Also mirrored as a comment in ``docker/coder_eval_entrypoint.sh``.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+
16
+ CONTAINER_WORK_DIR = "/work"
17
+ CONTAINER_INPUT_DIR = "/work/input"
18
+ CONTAINER_OUTPUT_DIR = "/work/output"
19
+ CONTAINER_TASK_DIR = "/work/task_dir"
20
+
21
+ # Paths a task's WORKDIR must never collide with: the container root and every
22
+ # framework-owned mount under /work. Consumed by SandboxConfig's working_dir
23
+ # validator (models/sandbox.py) and re-asserted host-side in docker_runner.
24
+ RESERVED_CONTAINER_DIRS = frozenset(
25
+ {"/", CONTAINER_WORK_DIR, CONTAINER_INPUT_DIR, CONTAINER_OUTPUT_DIR, CONTAINER_TASK_DIR}
26
+ )