evalrx 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (339) hide show
  1. evalrx/__init__.py +139 -0
  2. evalrx/agent_assets/__init__.py +2 -0
  3. evalrx/agent_assets/skills/README.md +28 -0
  4. evalrx/agent_assets/skills/eval-chart-style/SKILL.md +172 -0
  5. evalrx/agent_assets/skills/evalrx-report-ui/SKILL.md +116 -0
  6. evalrx/agent_assets/skills/nature-figure/LICENSE +201 -0
  7. evalrx/agent_assets/skills/nature-figure/README.md +412 -0
  8. evalrx/agent_assets/skills/nature-figure/SKILL.md +60 -0
  9. evalrx/agent_assets/skills/nature-figure/manifest.yaml +59 -0
  10. evalrx/agent_assets/skills/nature-figure/references/api.md +436 -0
  11. evalrx/agent_assets/skills/nature-figure/references/backend-selection.md +100 -0
  12. evalrx/agent_assets/skills/nature-figure/references/chart-types.md +281 -0
  13. evalrx/agent_assets/skills/nature-figure/references/common-patterns.md +350 -0
  14. evalrx/agent_assets/skills/nature-figure/references/demos.md +65 -0
  15. evalrx/agent_assets/skills/nature-figure/references/design-theory.md +439 -0
  16. evalrx/agent_assets/skills/nature-figure/references/figure-contract.md +93 -0
  17. evalrx/agent_assets/skills/nature-figure/references/figure-legend-conventions.md +71 -0
  18. evalrx/agent_assets/skills/nature-figure/references/nature-2026-observations.md +112 -0
  19. evalrx/agent_assets/skills/nature-figure/references/qa-contract.md +119 -0
  20. evalrx/agent_assets/skills/nature-figure/references/r-template-index.md +66 -0
  21. evalrx/agent_assets/skills/nature-figure/references/r-workflow.md +161 -0
  22. evalrx/agent_assets/skills/nature-figure/references/tutorials.md +251 -0
  23. evalrx/agent_assets/skills/nature-figure/static/core/contract.md +29 -0
  24. evalrx/agent_assets/skills/nature-figure/static/core/stance.md +37 -0
  25. evalrx/agent_assets/skills/nature-figure/static/fragments/backend/python.md +37 -0
  26. evalrx/agent_assets/skills/nature-figure/static/fragments/backend/r.md +44 -0
  27. evalrx/agent_assets/skills/outcome-driver-analysis/SKILL.md +213 -0
  28. evalrx/agent_assets/skills/outcome-driver-analysis/assets/analysis_report_template.md +53 -0
  29. evalrx/agent_assets/skills/outcome-driver-analysis/references/model_selection.md +72 -0
  30. evalrx/agent_assets/skills/outcome-driver-analysis/scripts/explanatory_var_eda.R +130 -0
  31. evalrx/agent_assets/skills/outcome-driver-analysis/scripts/explanatory_var_eda.py +150 -0
  32. evalrx/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.R +181 -0
  33. evalrx/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.py +186 -0
  34. evalrx/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.R +149 -0
  35. evalrx/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.py +177 -0
  36. evalrx/agent_assets/skills.py +27 -0
  37. evalrx/agent_runtime/__init__.py +78 -0
  38. evalrx/agent_runtime/_docker_runner.py +89 -0
  39. evalrx/agent_runtime/cli_runtime.py +103 -0
  40. evalrx/agent_runtime/cli_transcript.py +138 -0
  41. evalrx/agent_runtime/cli_types.py +68 -0
  42. evalrx/agent_runtime/codegen/__init__.py +5 -0
  43. evalrx/agent_runtime/codegen/runner.py +94 -0
  44. evalrx/agent_runtime/experiment_harness.py +117 -0
  45. evalrx/agent_runtime/factory.py +102 -0
  46. evalrx/agent_runtime/json_shape.py +44 -0
  47. evalrx/agent_runtime/judges/__init__.py +28 -0
  48. evalrx/agent_runtime/judges/agy.py +179 -0
  49. evalrx/agent_runtime/judges/autodetect.py +135 -0
  50. evalrx/agent_runtime/judges/claude.py +159 -0
  51. evalrx/agent_runtime/judges/codex.py +120 -0
  52. evalrx/agent_runtime/providers/__init__.py +21 -0
  53. evalrx/agent_runtime/providers/antigravity.py +31 -0
  54. evalrx/agent_runtime/providers/base.py +145 -0
  55. evalrx/agent_runtime/providers/claude_code.py +49 -0
  56. evalrx/agent_runtime/providers/codex.py +37 -0
  57. evalrx/agent_runtime/providers/gemini_cli.py +26 -0
  58. evalrx/agent_runtime/providers/kimi_cli.py +27 -0
  59. evalrx/agent_runtime/providers/opencode.py +27 -0
  60. evalrx/agent_runtime/providers/registry.py +58 -0
  61. evalrx/agent_runtime/sandbox.py +517 -0
  62. evalrx/agent_runtime/skill_audit.py +143 -0
  63. evalrx/agent_runtime/skills/__init__.py +19 -0
  64. evalrx/agent_runtime/skills/installer.py +68 -0
  65. evalrx/agent_runtime/skills/prompt_policy.py +86 -0
  66. evalrx/agent_runtime/skills/resolver.py +19 -0
  67. evalrx/analysis/__init__.py +132 -0
  68. evalrx/analysis/adjudicate.py +154 -0
  69. evalrx/analysis/analysis_module.py +361 -0
  70. evalrx/analysis/api.py +171 -0
  71. evalrx/analysis/case_studio.py +651 -0
  72. evalrx/analysis/cli.py +114 -0
  73. evalrx/analysis/dashboard.py +350 -0
  74. evalrx/analysis/eval_case_matrix.py +118 -0
  75. evalrx/analysis/eval_viz_theme.py +833 -0
  76. evalrx/analysis/explore_run.py +333 -0
  77. evalrx/analysis/explorer.py +1276 -0
  78. evalrx/analysis/failure_modes.py +607 -0
  79. evalrx/analysis/fused_pipeline.py +489 -0
  80. evalrx/analysis/holdout.py +300 -0
  81. evalrx/analysis/hypothesis_agent.py +230 -0
  82. evalrx/analysis/narration.py +177 -0
  83. evalrx/analysis/operationalize.py +442 -0
  84. evalrx/analysis/plain_language.py +42 -0
  85. evalrx/analysis/planner.py +283 -0
  86. evalrx/analysis/probe_search.py +203 -0
  87. evalrx/analysis/profile.py +268 -0
  88. evalrx/analysis/prompts/__init__.py +0 -0
  89. evalrx/analysis/prompts/explorer.py +417 -0
  90. evalrx/analysis/prompts/failure_modes.py +33 -0
  91. evalrx/analysis/prompts/holdout.py +27 -0
  92. evalrx/analysis/prompts/hypothesis_agent.py +78 -0
  93. evalrx/analysis/prompts/run_codebase.py +47 -0
  94. evalrx/analysis/prompts/stats_agent.py +72 -0
  95. evalrx/analysis/prompts/stats_tool_generator.py +43 -0
  96. evalrx/analysis/result_marker.py +47 -0
  97. evalrx/analysis/run_codebase.py +242 -0
  98. evalrx/analysis/run_view.py +205 -0
  99. evalrx/analysis/stage_views.py +93 -0
  100. evalrx/analysis/stats_agent.py +944 -0
  101. evalrx/analysis/stats_tool_agent.py +261 -0
  102. evalrx/analysis/stats_tool_generator.py +415 -0
  103. evalrx/analysis/stats_tools.py +1153 -0
  104. evalrx/analysis/trajectory_records.py +193 -0
  105. evalrx/analysis/workbench.py +431 -0
  106. evalrx/analyzers/__init__.py +42 -0
  107. evalrx/analyzers/agent/__init__.py +25 -0
  108. evalrx/analyzers/agent/counterfactual.py +84 -0
  109. evalrx/analyzers/agent/first_error_judge.py +96 -0
  110. evalrx/analyzers/agent/ignored_obs.py +81 -0
  111. evalrx/analyzers/agent/loop_detect.py +79 -0
  112. evalrx/analyzers/agent/reliability.py +165 -0
  113. evalrx/analyzers/agent/tool_shap.py +225 -0
  114. evalrx/analyzers/agent/trajectory_rubric.py +168 -0
  115. evalrx/analyzers/attention/__init__.py +19 -0
  116. evalrx/analyzers/attention/relative_attn.py +610 -0
  117. evalrx/analyzers/attention/rollout.py +73 -0
  118. evalrx/analyzers/attention/sink.py +56 -0
  119. evalrx/analyzers/attention/summary.py +190 -0
  120. evalrx/analyzers/attribution/__init__.py +6 -0
  121. evalrx/analyzers/attribution/generic_attn.py +31 -0
  122. evalrx/analyzers/attribution/gradcam.py +30 -0
  123. evalrx/analyzers/base.py +12 -0
  124. evalrx/analyzers/geometry/__init__.py +6 -0
  125. evalrx/analyzers/geometry/cka.py +70 -0
  126. evalrx/analyzers/geometry/linear_probe.py +157 -0
  127. evalrx/analyzers/hallucination/__init__.py +9 -0
  128. evalrx/analyzers/hallucination/chair.py +78 -0
  129. evalrx/analyzers/hallucination/opera.py +29 -0
  130. evalrx/analyzers/hallucination/pope.py +119 -0
  131. evalrx/analyzers/hallucination/selfcheck.py +155 -0
  132. evalrx/analyzers/hallucination/vcd.py +29 -0
  133. evalrx/analyzers/lens/__init__.py +7 -0
  134. evalrx/analyzers/lens/layer_contrast.py +133 -0
  135. evalrx/analyzers/lens/logit_lens.py +138 -0
  136. evalrx/analyzers/lens/tuned_lens.py +30 -0
  137. evalrx/analyzers/patching/__init__.py +5 -0
  138. evalrx/analyzers/patching/causal_trace.py +30 -0
  139. evalrx/analyzers/perturbation/__init__.py +23 -0
  140. evalrx/analyzers/perturbation/_shapley.py +54 -0
  141. evalrx/analyzers/perturbation/context_shap.py +174 -0
  142. evalrx/analyzers/perturbation/cot_faithfulness.py +239 -0
  143. evalrx/analyzers/perturbation/format_sensitivity.py +237 -0
  144. evalrx/analyzers/perturbation/mm_shap.py +146 -0
  145. evalrx/analyzers/perturbation/modality_ablation.py +196 -0
  146. evalrx/analyzers/perturbation/perturbation_battery.py +274 -0
  147. evalrx/analyzers/perturbation/prompt_contrast.py +265 -0
  148. evalrx/analyzers/perturbation/rise.py +94 -0
  149. evalrx/analyzers/perturbation/vl_shap.py +102 -0
  150. evalrx/analyzers/reasoning/__init__.py +33 -0
  151. evalrx/analyzers/reasoning/_text.py +328 -0
  152. evalrx/analyzers/reasoning/answer_extraction_audit.py +327 -0
  153. evalrx/analyzers/reasoning/arith_audit.py +226 -0
  154. evalrx/analyzers/reasoning/contamination.py +214 -0
  155. evalrx/analyzers/reasoning/knowledge_split.py +253 -0
  156. evalrx/analyzers/reasoning/self_repair.py +246 -0
  157. evalrx/analyzers/reasoning/step_rollout_value.py +216 -0
  158. evalrx/analyzers/reasoning/termination_audit.py +258 -0
  159. evalrx/analyzers/uncertainty/__init__.py +18 -0
  160. evalrx/analyzers/uncertainty/calibration.py +174 -0
  161. evalrx/analyzers/uncertainty/coverage_gap.py +199 -0
  162. evalrx/analyzers/uncertainty/entropy.py +90 -0
  163. evalrx/analyzers/uncertainty/logprob_entropy.py +69 -0
  164. evalrx/analyzers/uncertainty/self_consistency.py +204 -0
  165. evalrx/analyzers/uncertainty/verbalized_conf.py +64 -0
  166. evalrx/cli.py +411 -0
  167. evalrx/config.py +77 -0
  168. evalrx/contract/__init__.py +179 -0
  169. evalrx/contract/common.py +452 -0
  170. evalrx/contract/emit.py +948 -0
  171. evalrx/contract/export.py +237 -0
  172. evalrx/contract/m1.py +325 -0
  173. evalrx/contract/m2.py +317 -0
  174. evalrx/contract/m3.py +165 -0
  175. evalrx/contract/m4.py +130 -0
  176. evalrx/contract/m5.py +292 -0
  177. evalrx/contract/methodology.py +76 -0
  178. evalrx/contract/pre_m1.py +58 -0
  179. evalrx/contract/typescript.py +140 -0
  180. evalrx/core/__init__.py +85 -0
  181. evalrx/core/analyzer.py +174 -0
  182. evalrx/core/capability.py +54 -0
  183. evalrx/core/case.py +443 -0
  184. evalrx/core/experiment.py +106 -0
  185. evalrx/core/model.py +198 -0
  186. evalrx/core/pipeline.py +42 -0
  187. evalrx/core/registry.py +142 -0
  188. evalrx/core/result.py +64 -0
  189. evalrx/core/spec.py +173 -0
  190. evalrx/core/tokentype.py +165 -0
  191. evalrx/core/tool.py +92 -0
  192. evalrx/datasets/__init__.py +41 -0
  193. evalrx/datasets/base.py +68 -0
  194. evalrx/datasets/gui_os.py +52 -0
  195. evalrx/datasets/llm_qa.py +57 -0
  196. evalrx/datasets/pure_qa.py +12 -0
  197. evalrx/datasets/vlm_qa.py +695 -0
  198. evalrx/datasets/web_search_qa.py +52 -0
  199. evalrx/eval_agent/__init__.py +341 -0
  200. evalrx/eval_agent/_tools.py +81 -0
  201. evalrx/eval_agent/ab_runner.py +50 -0
  202. evalrx/eval_agent/agentic/__init__.py +43 -0
  203. evalrx/eval_agent/agentic/actions.py +216 -0
  204. evalrx/eval_agent/agentic/board.py +107 -0
  205. evalrx/eval_agent/agentic/loop.py +190 -0
  206. evalrx/eval_agent/agentic/tools.py +538 -0
  207. evalrx/eval_agent/checkpoint.py +57 -0
  208. evalrx/eval_agent/cli_agent.py +59 -0
  209. evalrx/eval_agent/cli_skills.py +5 -0
  210. evalrx/eval_agent/evolution.py +396 -0
  211. evalrx/eval_agent/git_manager.py +215 -0
  212. evalrx/eval_agent/hypothesis.py +172 -0
  213. evalrx/eval_agent/label_quarantine.py +209 -0
  214. evalrx/eval_agent/legacy.py +530 -0
  215. evalrx/eval_agent/log_schema.py +497 -0
  216. evalrx/eval_agent/loop.py +2159 -0
  217. evalrx/eval_agent/loop_reports.py +116 -0
  218. evalrx/eval_agent/model_instrumentation.py +282 -0
  219. evalrx/eval_agent/narration.py +193 -0
  220. evalrx/eval_agent/nl_runner.py +460 -0
  221. evalrx/eval_agent/orchestrator.py +61 -0
  222. evalrx/eval_agent/preregister.py +93 -0
  223. evalrx/eval_agent/prompts/__init__.py +1 -0
  224. evalrx/eval_agent/prompts/agentic.py +46 -0
  225. evalrx/eval_agent/prompts/case_discovery.py +25 -0
  226. evalrx/eval_agent/prompts/diagnosis.py +125 -0
  227. evalrx/eval_agent/prompts/experiment_writer.py +265 -0
  228. evalrx/eval_agent/prompts/explore_step.py +37 -0
  229. evalrx/eval_agent/prompts/fix_agent.py +257 -0
  230. evalrx/eval_agent/prompts/hypothesis_tester.py +15 -0
  231. evalrx/eval_agent/prompts/nl_runner.py +38 -0
  232. evalrx/eval_agent/prompts/probe_agent.py +25 -0
  233. evalrx/eval_agent/prompts/probe_candidate_generator.py +14 -0
  234. evalrx/eval_agent/prompts/probe_generator.py +35 -0
  235. evalrx/eval_agent/prompts/whitebox_probe_generator.py +38 -0
  236. evalrx/eval_agent/report.py +58 -0
  237. evalrx/eval_agent/run_context.py +354 -0
  238. evalrx/eval_agent/run_log.schema.json +1215 -0
  239. evalrx/eval_agent/run_logger_v2.py +1764 -0
  240. evalrx/eval_agent/run_metadata.py +208 -0
  241. evalrx/eval_agent/stages/__init__.py +56 -0
  242. evalrx/eval_agent/stages/case_discovery.py +293 -0
  243. evalrx/eval_agent/stages/diagnosis.py +1017 -0
  244. evalrx/eval_agent/stages/experiment_writer.py +1634 -0
  245. evalrx/eval_agent/stages/fix_agent.py +3916 -0
  246. evalrx/eval_agent/stages/fix_internals.py +499 -0
  247. evalrx/eval_agent/stages/fix_pipeline.py +725 -0
  248. evalrx/eval_agent/stages/fix_tiers.py +187 -0
  249. evalrx/eval_agent/stages/fix_tools.py +1034 -0
  250. evalrx/eval_agent/stages/hypothesis_tester.py +1014 -0
  251. evalrx/eval_agent/stages/probe.py +439 -0
  252. evalrx/eval_agent/stages/probe_agent.py +1079 -0
  253. evalrx/eval_agent/stages/probe_candidate_generator.py +128 -0
  254. evalrx/eval_agent/stages/probe_generator.py +326 -0
  255. evalrx/eval_agent/stages/probe_search_agent.py +106 -0
  256. evalrx/eval_agent/stages/protocol.py +112 -0
  257. evalrx/eval_agent/stages/repair_catalog.py +273 -0
  258. evalrx/eval_agent/stages/surgery.py +524 -0
  259. evalrx/eval_agent/stages/whitebox_probe_generator.py +351 -0
  260. evalrx/eval_agent/store.py +231 -0
  261. evalrx/logging_utils.py +112 -0
  262. evalrx/models/__init__.py +161 -0
  263. evalrx/models/_discover.py +101 -0
  264. evalrx/models/agent.py +380 -0
  265. evalrx/models/backends/__init__.py +58 -0
  266. evalrx/models/backends/api.py +169 -0
  267. evalrx/models/backends/base.py +57 -0
  268. evalrx/models/backends/gemini_compat.py +579 -0
  269. evalrx/models/backends/hf_local.py +2074 -0
  270. evalrx/models/backends/openai_compat.py +301 -0
  271. evalrx/models/backends/vllm_offline.py +116 -0
  272. evalrx/models/base.py +24 -0
  273. evalrx/models/blackbox/__init__.py +4 -0
  274. evalrx/models/blackbox/agent.py +31 -0
  275. evalrx/models/blackbox/base.py +29 -0
  276. evalrx/models/blackbox/gemini.py +279 -0
  277. evalrx/models/blackbox/llm_api.py +17 -0
  278. evalrx/models/blackbox/vlm_api.py +17 -0
  279. evalrx/models/compose.py +66 -0
  280. evalrx/models/inference.py +88 -0
  281. evalrx/models/paper_methods/__init__.py +8 -0
  282. evalrx/models/paper_methods/aad.py +53 -0
  283. evalrx/models/paper_methods/ifcd.py +204 -0
  284. evalrx/models/paper_methods/pai.py +164 -0
  285. evalrx/models/paper_methods/tcd.py +202 -0
  286. evalrx/models/paper_methods/vcd.py +45 -0
  287. evalrx/models/paper_methods/vicrop.py +137 -0
  288. evalrx/models/toolcodec.py +143 -0
  289. evalrx/models/tools/__init__.py +20 -0
  290. evalrx/models/tools/perception.py +300 -0
  291. evalrx/models/tools/visual.py +174 -0
  292. evalrx/models/whitebox/__init__.py +26 -0
  293. evalrx/models/whitebox/agent.py +31 -0
  294. evalrx/models/whitebox/base.py +24 -0
  295. evalrx/models/whitebox/qwen.py +61 -0
  296. evalrx/models/whitebox/qwen2_5_omni.py +29 -0
  297. evalrx/models/whitebox/qwen2_audio.py +25 -0
  298. evalrx/models/whitebox/qwen_omni.py +53 -0
  299. evalrx/models/whitebox/qwen_vl.py +62 -0
  300. evalrx/observability/__init__.py +21 -0
  301. evalrx/observability/envelope.py +122 -0
  302. evalrx/observability/outbox.py +111 -0
  303. evalrx/observability/tracer.py +882 -0
  304. evalrx/reporting/__init__.py +28 -0
  305. evalrx/reporting/case_study.py +947 -0
  306. evalrx/reporting/compiler.py +587 -0
  307. evalrx/reporting/dynamic.py +1882 -0
  308. evalrx/reporting/html_report.py +2225 -0
  309. evalrx/reporting/langfuse_exporter.py +38 -0
  310. evalrx/reporting/langfuse_source.py +155 -0
  311. evalrx/reporting/model.py +151 -0
  312. evalrx/reporting/run_events.py +184 -0
  313. evalrx/reporting/server.py +557 -0
  314. evalrx/reporting/stages.py +58 -0
  315. evalrx/reporting/static_export.py +142 -0
  316. evalrx/reporting/web_dist/index.html +146 -0
  317. evalrx/specs.py +727 -0
  318. evalrx/stats/__init__.py +47 -0
  319. evalrx/stats/api.py +192 -0
  320. evalrx/stats/bootstrap.py +86 -0
  321. evalrx/stats/ebh.py +27 -0
  322. evalrx/stats/evalue.py +98 -0
  323. evalrx/stats/friedman.py +138 -0
  324. evalrx/stats/mcnemar.py +40 -0
  325. evalrx/stats/multiplicity.py +159 -0
  326. evalrx/stats/subset_sampling.py +55 -0
  327. evalrx/term_links.py +43 -0
  328. evalrx/viz/__init__.py +7 -0
  329. evalrx/viz/labels.py +77 -0
  330. evalrx/viz/prompts.py +39 -0
  331. evalrx/viz/renderer.py +590 -0
  332. evalrx/viz/schema.py +36 -0
  333. evalrx/viz/style.py +134 -0
  334. evalrx-0.1.2.dist-info/METADATA +532 -0
  335. evalrx-0.1.2.dist-info/RECORD +339 -0
  336. evalrx-0.1.2.dist-info/WHEEL +5 -0
  337. evalrx-0.1.2.dist-info/entry_points.txt +3 -0
  338. evalrx-0.1.2.dist-info/licenses/LICENSE +121 -0
  339. evalrx-0.1.2.dist-info/top_level.txt +1 -0
evalrx/reporting/__init__.py ADDED
@@ -0,0 +1,28 @@
1
+ """Claim/evidence diagnostic reporting."""
2
+
3
+ from evalrx.reporting.compiler import compile_diagnostic_report
4
+ from evalrx.reporting.dynamic import (
5
+ PublishedReport,
6
+ ReportAgent,
7
+ build_report_data,
8
+ publish_report,
9
+ )
10
+ from evalrx.reporting.html_report import build_html_report
11
+ from evalrx.reporting.model import Claim, DiagnosticReport, Evidence, ReportStep
12
+ from evalrx.reporting.stages import STAGE_SPECS, StageSpec, stage_specs_as_dicts
13
+
14
+ __all__ = [
15
+ "Claim",
16
+ "DiagnosticReport",
17
+ "Evidence",
18
+ "ReportStep",
19
+ "STAGE_SPECS",
20
+ "StageSpec",
21
+ "build_html_report",
22
+ "build_report_data",
23
+ "compile_diagnostic_report",
24
+ "publish_report",
25
+ "PublishedReport",
26
+ "ReportAgent",
27
+ "stage_specs_as_dicts",
28
+ ]