evalrx 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalrx/__init__.py +139 -0
- evalrx/agent_assets/__init__.py +2 -0
- evalrx/agent_assets/skills/README.md +28 -0
- evalrx/agent_assets/skills/eval-chart-style/SKILL.md +172 -0
- evalrx/agent_assets/skills/evalrx-report-ui/SKILL.md +116 -0
- evalrx/agent_assets/skills/nature-figure/LICENSE +201 -0
- evalrx/agent_assets/skills/nature-figure/README.md +412 -0
- evalrx/agent_assets/skills/nature-figure/SKILL.md +60 -0
- evalrx/agent_assets/skills/nature-figure/manifest.yaml +59 -0
- evalrx/agent_assets/skills/nature-figure/references/api.md +436 -0
- evalrx/agent_assets/skills/nature-figure/references/backend-selection.md +100 -0
- evalrx/agent_assets/skills/nature-figure/references/chart-types.md +281 -0
- evalrx/agent_assets/skills/nature-figure/references/common-patterns.md +350 -0
- evalrx/agent_assets/skills/nature-figure/references/demos.md +65 -0
- evalrx/agent_assets/skills/nature-figure/references/design-theory.md +439 -0
- evalrx/agent_assets/skills/nature-figure/references/figure-contract.md +93 -0
- evalrx/agent_assets/skills/nature-figure/references/figure-legend-conventions.md +71 -0
- evalrx/agent_assets/skills/nature-figure/references/nature-2026-observations.md +112 -0
- evalrx/agent_assets/skills/nature-figure/references/qa-contract.md +119 -0
- evalrx/agent_assets/skills/nature-figure/references/r-template-index.md +66 -0
- evalrx/agent_assets/skills/nature-figure/references/r-workflow.md +161 -0
- evalrx/agent_assets/skills/nature-figure/references/tutorials.md +251 -0
- evalrx/agent_assets/skills/nature-figure/static/core/contract.md +29 -0
- evalrx/agent_assets/skills/nature-figure/static/core/stance.md +37 -0
- evalrx/agent_assets/skills/nature-figure/static/fragments/backend/python.md +37 -0
- evalrx/agent_assets/skills/nature-figure/static/fragments/backend/r.md +44 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/SKILL.md +213 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/assets/analysis_report_template.md +53 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/references/model_selection.md +72 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/scripts/explanatory_var_eda.R +130 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/scripts/explanatory_var_eda.py +150 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.R +181 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.py +186 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.R +149 -0
- evalrx/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.py +177 -0
- evalrx/agent_assets/skills.py +27 -0
- evalrx/agent_runtime/__init__.py +78 -0
- evalrx/agent_runtime/_docker_runner.py +89 -0
- evalrx/agent_runtime/cli_runtime.py +103 -0
- evalrx/agent_runtime/cli_transcript.py +138 -0
- evalrx/agent_runtime/cli_types.py +68 -0
- evalrx/agent_runtime/codegen/__init__.py +5 -0
- evalrx/agent_runtime/codegen/runner.py +94 -0
- evalrx/agent_runtime/experiment_harness.py +117 -0
- evalrx/agent_runtime/factory.py +102 -0
- evalrx/agent_runtime/json_shape.py +44 -0
- evalrx/agent_runtime/judges/__init__.py +28 -0
- evalrx/agent_runtime/judges/agy.py +179 -0
- evalrx/agent_runtime/judges/autodetect.py +135 -0
- evalrx/agent_runtime/judges/claude.py +159 -0
- evalrx/agent_runtime/judges/codex.py +120 -0
- evalrx/agent_runtime/providers/__init__.py +21 -0
- evalrx/agent_runtime/providers/antigravity.py +31 -0
- evalrx/agent_runtime/providers/base.py +145 -0
- evalrx/agent_runtime/providers/claude_code.py +49 -0
- evalrx/agent_runtime/providers/codex.py +37 -0
- evalrx/agent_runtime/providers/gemini_cli.py +26 -0
- evalrx/agent_runtime/providers/kimi_cli.py +27 -0
- evalrx/agent_runtime/providers/opencode.py +27 -0
- evalrx/agent_runtime/providers/registry.py +58 -0
- evalrx/agent_runtime/sandbox.py +517 -0
- evalrx/agent_runtime/skill_audit.py +143 -0
- evalrx/agent_runtime/skills/__init__.py +19 -0
- evalrx/agent_runtime/skills/installer.py +68 -0
- evalrx/agent_runtime/skills/prompt_policy.py +86 -0
- evalrx/agent_runtime/skills/resolver.py +19 -0
- evalrx/analysis/__init__.py +132 -0
- evalrx/analysis/adjudicate.py +154 -0
- evalrx/analysis/analysis_module.py +361 -0
- evalrx/analysis/api.py +171 -0
- evalrx/analysis/case_studio.py +651 -0
- evalrx/analysis/cli.py +114 -0
- evalrx/analysis/dashboard.py +350 -0
- evalrx/analysis/eval_case_matrix.py +118 -0
- evalrx/analysis/eval_viz_theme.py +833 -0
- evalrx/analysis/explore_run.py +333 -0
- evalrx/analysis/explorer.py +1276 -0
- evalrx/analysis/failure_modes.py +607 -0
- evalrx/analysis/fused_pipeline.py +489 -0
- evalrx/analysis/holdout.py +300 -0
- evalrx/analysis/hypothesis_agent.py +230 -0
- evalrx/analysis/narration.py +177 -0
- evalrx/analysis/operationalize.py +442 -0
- evalrx/analysis/plain_language.py +42 -0
- evalrx/analysis/planner.py +283 -0
- evalrx/analysis/probe_search.py +203 -0
- evalrx/analysis/profile.py +268 -0
- evalrx/analysis/prompts/__init__.py +0 -0
- evalrx/analysis/prompts/explorer.py +417 -0
- evalrx/analysis/prompts/failure_modes.py +33 -0
- evalrx/analysis/prompts/holdout.py +27 -0
- evalrx/analysis/prompts/hypothesis_agent.py +78 -0
- evalrx/analysis/prompts/run_codebase.py +47 -0
- evalrx/analysis/prompts/stats_agent.py +72 -0
- evalrx/analysis/prompts/stats_tool_generator.py +43 -0
- evalrx/analysis/result_marker.py +47 -0
- evalrx/analysis/run_codebase.py +242 -0
- evalrx/analysis/run_view.py +205 -0
- evalrx/analysis/stage_views.py +93 -0
- evalrx/analysis/stats_agent.py +944 -0
- evalrx/analysis/stats_tool_agent.py +261 -0
- evalrx/analysis/stats_tool_generator.py +415 -0
- evalrx/analysis/stats_tools.py +1153 -0
- evalrx/analysis/trajectory_records.py +193 -0
- evalrx/analysis/workbench.py +431 -0
- evalrx/analyzers/__init__.py +42 -0
- evalrx/analyzers/agent/__init__.py +25 -0
- evalrx/analyzers/agent/counterfactual.py +84 -0
- evalrx/analyzers/agent/first_error_judge.py +96 -0
- evalrx/analyzers/agent/ignored_obs.py +81 -0
- evalrx/analyzers/agent/loop_detect.py +79 -0
- evalrx/analyzers/agent/reliability.py +165 -0
- evalrx/analyzers/agent/tool_shap.py +225 -0
- evalrx/analyzers/agent/trajectory_rubric.py +168 -0
- evalrx/analyzers/attention/__init__.py +19 -0
- evalrx/analyzers/attention/relative_attn.py +610 -0
- evalrx/analyzers/attention/rollout.py +73 -0
- evalrx/analyzers/attention/sink.py +56 -0
- evalrx/analyzers/attention/summary.py +190 -0
- evalrx/analyzers/attribution/__init__.py +6 -0
- evalrx/analyzers/attribution/generic_attn.py +31 -0
- evalrx/analyzers/attribution/gradcam.py +30 -0
- evalrx/analyzers/base.py +12 -0
- evalrx/analyzers/geometry/__init__.py +6 -0
- evalrx/analyzers/geometry/cka.py +70 -0
- evalrx/analyzers/geometry/linear_probe.py +157 -0
- evalrx/analyzers/hallucination/__init__.py +9 -0
- evalrx/analyzers/hallucination/chair.py +78 -0
- evalrx/analyzers/hallucination/opera.py +29 -0
- evalrx/analyzers/hallucination/pope.py +119 -0
- evalrx/analyzers/hallucination/selfcheck.py +155 -0
- evalrx/analyzers/hallucination/vcd.py +29 -0
- evalrx/analyzers/lens/__init__.py +7 -0
- evalrx/analyzers/lens/layer_contrast.py +133 -0
- evalrx/analyzers/lens/logit_lens.py +138 -0
- evalrx/analyzers/lens/tuned_lens.py +30 -0
- evalrx/analyzers/patching/__init__.py +5 -0
- evalrx/analyzers/patching/causal_trace.py +30 -0
- evalrx/analyzers/perturbation/__init__.py +23 -0
- evalrx/analyzers/perturbation/_shapley.py +54 -0
- evalrx/analyzers/perturbation/context_shap.py +174 -0
- evalrx/analyzers/perturbation/cot_faithfulness.py +239 -0
- evalrx/analyzers/perturbation/format_sensitivity.py +237 -0
- evalrx/analyzers/perturbation/mm_shap.py +146 -0
- evalrx/analyzers/perturbation/modality_ablation.py +196 -0
- evalrx/analyzers/perturbation/perturbation_battery.py +274 -0
- evalrx/analyzers/perturbation/prompt_contrast.py +265 -0
- evalrx/analyzers/perturbation/rise.py +94 -0
- evalrx/analyzers/perturbation/vl_shap.py +102 -0
- evalrx/analyzers/reasoning/__init__.py +33 -0
- evalrx/analyzers/reasoning/_text.py +328 -0
- evalrx/analyzers/reasoning/answer_extraction_audit.py +327 -0
- evalrx/analyzers/reasoning/arith_audit.py +226 -0
- evalrx/analyzers/reasoning/contamination.py +214 -0
- evalrx/analyzers/reasoning/knowledge_split.py +253 -0
- evalrx/analyzers/reasoning/self_repair.py +246 -0
- evalrx/analyzers/reasoning/step_rollout_value.py +216 -0
- evalrx/analyzers/reasoning/termination_audit.py +258 -0
- evalrx/analyzers/uncertainty/__init__.py +18 -0
- evalrx/analyzers/uncertainty/calibration.py +174 -0
- evalrx/analyzers/uncertainty/coverage_gap.py +199 -0
- evalrx/analyzers/uncertainty/entropy.py +90 -0
- evalrx/analyzers/uncertainty/logprob_entropy.py +69 -0
- evalrx/analyzers/uncertainty/self_consistency.py +204 -0
- evalrx/analyzers/uncertainty/verbalized_conf.py +64 -0
- evalrx/cli.py +411 -0
- evalrx/config.py +77 -0
- evalrx/contract/__init__.py +179 -0
- evalrx/contract/common.py +452 -0
- evalrx/contract/emit.py +948 -0
- evalrx/contract/export.py +237 -0
- evalrx/contract/m1.py +325 -0
- evalrx/contract/m2.py +317 -0
- evalrx/contract/m3.py +165 -0
- evalrx/contract/m4.py +130 -0
- evalrx/contract/m5.py +292 -0
- evalrx/contract/methodology.py +76 -0
- evalrx/contract/pre_m1.py +58 -0
- evalrx/contract/typescript.py +140 -0
- evalrx/core/__init__.py +85 -0
- evalrx/core/analyzer.py +174 -0
- evalrx/core/capability.py +54 -0
- evalrx/core/case.py +443 -0
- evalrx/core/experiment.py +106 -0
- evalrx/core/model.py +198 -0
- evalrx/core/pipeline.py +42 -0
- evalrx/core/registry.py +142 -0
- evalrx/core/result.py +64 -0
- evalrx/core/spec.py +173 -0
- evalrx/core/tokentype.py +165 -0
- evalrx/core/tool.py +92 -0
- evalrx/datasets/__init__.py +41 -0
- evalrx/datasets/base.py +68 -0
- evalrx/datasets/gui_os.py +52 -0
- evalrx/datasets/llm_qa.py +57 -0
- evalrx/datasets/pure_qa.py +12 -0
- evalrx/datasets/vlm_qa.py +695 -0
- evalrx/datasets/web_search_qa.py +52 -0
- evalrx/eval_agent/__init__.py +341 -0
- evalrx/eval_agent/_tools.py +81 -0
- evalrx/eval_agent/ab_runner.py +50 -0
- evalrx/eval_agent/agentic/__init__.py +43 -0
- evalrx/eval_agent/agentic/actions.py +216 -0
- evalrx/eval_agent/agentic/board.py +107 -0
- evalrx/eval_agent/agentic/loop.py +190 -0
- evalrx/eval_agent/agentic/tools.py +538 -0
- evalrx/eval_agent/checkpoint.py +57 -0
- evalrx/eval_agent/cli_agent.py +59 -0
- evalrx/eval_agent/cli_skills.py +5 -0
- evalrx/eval_agent/evolution.py +396 -0
- evalrx/eval_agent/git_manager.py +215 -0
- evalrx/eval_agent/hypothesis.py +172 -0
- evalrx/eval_agent/label_quarantine.py +209 -0
- evalrx/eval_agent/legacy.py +530 -0
- evalrx/eval_agent/log_schema.py +497 -0
- evalrx/eval_agent/loop.py +2159 -0
- evalrx/eval_agent/loop_reports.py +116 -0
- evalrx/eval_agent/model_instrumentation.py +282 -0
- evalrx/eval_agent/narration.py +193 -0
- evalrx/eval_agent/nl_runner.py +460 -0
- evalrx/eval_agent/orchestrator.py +61 -0
- evalrx/eval_agent/preregister.py +93 -0
- evalrx/eval_agent/prompts/__init__.py +1 -0
- evalrx/eval_agent/prompts/agentic.py +46 -0
- evalrx/eval_agent/prompts/case_discovery.py +25 -0
- evalrx/eval_agent/prompts/diagnosis.py +125 -0
- evalrx/eval_agent/prompts/experiment_writer.py +265 -0
- evalrx/eval_agent/prompts/explore_step.py +37 -0
- evalrx/eval_agent/prompts/fix_agent.py +257 -0
- evalrx/eval_agent/prompts/hypothesis_tester.py +15 -0
- evalrx/eval_agent/prompts/nl_runner.py +38 -0
- evalrx/eval_agent/prompts/probe_agent.py +25 -0
- evalrx/eval_agent/prompts/probe_candidate_generator.py +14 -0
- evalrx/eval_agent/prompts/probe_generator.py +35 -0
- evalrx/eval_agent/prompts/whitebox_probe_generator.py +38 -0
- evalrx/eval_agent/report.py +58 -0
- evalrx/eval_agent/run_context.py +354 -0
- evalrx/eval_agent/run_log.schema.json +1215 -0
- evalrx/eval_agent/run_logger_v2.py +1764 -0
- evalrx/eval_agent/run_metadata.py +208 -0
- evalrx/eval_agent/stages/__init__.py +56 -0
- evalrx/eval_agent/stages/case_discovery.py +293 -0
- evalrx/eval_agent/stages/diagnosis.py +1017 -0
- evalrx/eval_agent/stages/experiment_writer.py +1634 -0
- evalrx/eval_agent/stages/fix_agent.py +3916 -0
- evalrx/eval_agent/stages/fix_internals.py +499 -0
- evalrx/eval_agent/stages/fix_pipeline.py +725 -0
- evalrx/eval_agent/stages/fix_tiers.py +187 -0
- evalrx/eval_agent/stages/fix_tools.py +1034 -0
- evalrx/eval_agent/stages/hypothesis_tester.py +1014 -0
- evalrx/eval_agent/stages/probe.py +439 -0
- evalrx/eval_agent/stages/probe_agent.py +1079 -0
- evalrx/eval_agent/stages/probe_candidate_generator.py +128 -0
- evalrx/eval_agent/stages/probe_generator.py +326 -0
- evalrx/eval_agent/stages/probe_search_agent.py +106 -0
- evalrx/eval_agent/stages/protocol.py +112 -0
- evalrx/eval_agent/stages/repair_catalog.py +273 -0
- evalrx/eval_agent/stages/surgery.py +524 -0
- evalrx/eval_agent/stages/whitebox_probe_generator.py +351 -0
- evalrx/eval_agent/store.py +231 -0
- evalrx/logging_utils.py +112 -0
- evalrx/models/__init__.py +161 -0
- evalrx/models/_discover.py +101 -0
- evalrx/models/agent.py +380 -0
- evalrx/models/backends/__init__.py +58 -0
- evalrx/models/backends/api.py +169 -0
- evalrx/models/backends/base.py +57 -0
- evalrx/models/backends/gemini_compat.py +579 -0
- evalrx/models/backends/hf_local.py +2074 -0
- evalrx/models/backends/openai_compat.py +301 -0
- evalrx/models/backends/vllm_offline.py +116 -0
- evalrx/models/base.py +24 -0
- evalrx/models/blackbox/__init__.py +4 -0
- evalrx/models/blackbox/agent.py +31 -0
- evalrx/models/blackbox/base.py +29 -0
- evalrx/models/blackbox/gemini.py +279 -0
- evalrx/models/blackbox/llm_api.py +17 -0
- evalrx/models/blackbox/vlm_api.py +17 -0
- evalrx/models/compose.py +66 -0
- evalrx/models/inference.py +88 -0
- evalrx/models/paper_methods/__init__.py +8 -0
- evalrx/models/paper_methods/aad.py +53 -0
- evalrx/models/paper_methods/ifcd.py +204 -0
- evalrx/models/paper_methods/pai.py +164 -0
- evalrx/models/paper_methods/tcd.py +202 -0
- evalrx/models/paper_methods/vcd.py +45 -0
- evalrx/models/paper_methods/vicrop.py +137 -0
- evalrx/models/toolcodec.py +143 -0
- evalrx/models/tools/__init__.py +20 -0
- evalrx/models/tools/perception.py +300 -0
- evalrx/models/tools/visual.py +174 -0
- evalrx/models/whitebox/__init__.py +26 -0
- evalrx/models/whitebox/agent.py +31 -0
- evalrx/models/whitebox/base.py +24 -0
- evalrx/models/whitebox/qwen.py +61 -0
- evalrx/models/whitebox/qwen2_5_omni.py +29 -0
- evalrx/models/whitebox/qwen2_audio.py +25 -0
- evalrx/models/whitebox/qwen_omni.py +53 -0
- evalrx/models/whitebox/qwen_vl.py +62 -0
- evalrx/observability/__init__.py +21 -0
- evalrx/observability/envelope.py +122 -0
- evalrx/observability/outbox.py +111 -0
- evalrx/observability/tracer.py +882 -0
- evalrx/reporting/__init__.py +28 -0
- evalrx/reporting/case_study.py +947 -0
- evalrx/reporting/compiler.py +587 -0
- evalrx/reporting/dynamic.py +1882 -0
- evalrx/reporting/html_report.py +2225 -0
- evalrx/reporting/langfuse_exporter.py +38 -0
- evalrx/reporting/langfuse_source.py +155 -0
- evalrx/reporting/model.py +151 -0
- evalrx/reporting/run_events.py +184 -0
- evalrx/reporting/server.py +557 -0
- evalrx/reporting/stages.py +58 -0
- evalrx/reporting/static_export.py +142 -0
- evalrx/reporting/web_dist/index.html +146 -0
- evalrx/specs.py +727 -0
- evalrx/stats/__init__.py +47 -0
- evalrx/stats/api.py +192 -0
- evalrx/stats/bootstrap.py +86 -0
- evalrx/stats/ebh.py +27 -0
- evalrx/stats/evalue.py +98 -0
- evalrx/stats/friedman.py +138 -0
- evalrx/stats/mcnemar.py +40 -0
- evalrx/stats/multiplicity.py +159 -0
- evalrx/stats/subset_sampling.py +55 -0
- evalrx/term_links.py +43 -0
- evalrx/viz/__init__.py +7 -0
- evalrx/viz/labels.py +77 -0
- evalrx/viz/prompts.py +39 -0
- evalrx/viz/renderer.py +590 -0
- evalrx/viz/schema.py +36 -0
- evalrx/viz/style.py +134 -0
- evalrx-0.1.2.dist-info/METADATA +532 -0
- evalrx-0.1.2.dist-info/RECORD +339 -0
- evalrx-0.1.2.dist-info/WHEEL +5 -0
- evalrx-0.1.2.dist-info/entry_points.txt +3 -0
- evalrx-0.1.2.dist-info/licenses/LICENSE +121 -0
- evalrx-0.1.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Visual perception tools — the T0 tier of the default visual-reasoning kit.
|
|
2
|
+
|
|
3
|
+
``zoom_in`` is the single highest-value visual tool (the o3 / DeepEyes /
|
|
4
|
+
Qwen3-VL-demo mechanism): crop a region, upscale it, and hand it back as a new
|
|
5
|
+
image so the model can inspect fine detail. Torch-free; PIL imported lazily.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import itertools
|
|
11
|
+
import os
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from evalrx.core.tool import Tool, ToolResult
|
|
15
|
+
|
|
16
|
+
_MIN_SIDE_FRAC = 0.05 # never zoom into a sliver thinner than 5% of a side
|
|
17
|
+
_TARGET_SHORT_SIDE = 672 # upscale the crop so detail is actually visible
|
|
18
|
+
_MAX_UPSCALE = 4.0 # ... but never blow pixels up more than 4x
|
|
19
|
+
|
|
20
|
+
ZOOM_IN_DESCRIPTION = (
|
|
21
|
+
"Zoom into a rectangular region of the CURRENT image to inspect fine detail. "
|
|
22
|
+
"Pass bbox=[x0, y0, x1, y1] as FRACTIONS of the image width/height, each in "
|
|
23
|
+
"[0, 1], with (x0, y0) the top-left and (x1, y1) the bottom-right corner "
|
|
24
|
+
"(e.g. the right half is [0.5, 0.0, 1.0, 1.0]). The cropped, enlarged view "
|
|
25
|
+
"comes back as a new image. Zoom before answering questions about small "
|
|
26
|
+
"objects, text, or fine attributes."
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
ZOOM_IN_PARAMETERS = {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"properties": {
|
|
32
|
+
"bbox": {
|
|
33
|
+
"type": "array",
|
|
34
|
+
"items": {"type": "number"},
|
|
35
|
+
"minItems": 4,
|
|
36
|
+
"maxItems": 4,
|
|
37
|
+
"description": "[x0, y0, x1, y1], fractions of width/height in [0, 1].",
|
|
38
|
+
},
|
|
39
|
+
"target": {
|
|
40
|
+
"type": "string",
|
|
41
|
+
"description": "What you are trying to see in this region (short phrase).",
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
"required": ["bbox"],
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _to_pil(image: Any):
|
|
49
|
+
"""Return *image* as a PIL image (paths are opened and converted to RGB)."""
|
|
50
|
+
if hasattr(image, "size") and hasattr(image, "mode"):
|
|
51
|
+
return image
|
|
52
|
+
from PIL import Image
|
|
53
|
+
|
|
54
|
+
return Image.open(image).convert("RGB")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _sanitize_bbox(bbox: Any, width: int, height: int) -> "tuple[list[float], str]":
|
|
58
|
+
"""Normalize *bbox* to fractional [x0, y0, x1, y1] and report the coord mode.
|
|
59
|
+
|
|
60
|
+
Models that ground in pixel coordinates are auto-detected (any value > 1.5)
|
|
61
|
+
and converted — the mode is recorded so the trajectory keeps evidence of
|
|
62
|
+
which convention the model actually used.
|
|
63
|
+
"""
|
|
64
|
+
try:
|
|
65
|
+
vals = [float(v) for v in bbox]
|
|
66
|
+
except (TypeError, ValueError):
|
|
67
|
+
raise ValueError(f"bbox must be four numbers [x0, y0, x1, y1], got {bbox!r}")
|
|
68
|
+
if len(vals) != 4:
|
|
69
|
+
raise ValueError(f"bbox must have exactly 4 values [x0, y0, x1, y1], got {len(vals)}")
|
|
70
|
+
|
|
71
|
+
mode = "normalized"
|
|
72
|
+
if max(vals) > 1.5: # not fractions: pixel coordinates or a Qwen-style 0-1000 grid
|
|
73
|
+
fits_pixels = (
|
|
74
|
+
max(vals[0], vals[2]) <= width * 1.05
|
|
75
|
+
and max(vals[1], vals[3]) <= height * 1.05
|
|
76
|
+
)
|
|
77
|
+
if fits_pixels:
|
|
78
|
+
mode = "pixel"
|
|
79
|
+
scale = (width, height, width, height)
|
|
80
|
+
else: # out of pixel bounds -> 0-1000 normalized grid
|
|
81
|
+
mode = "grid_1000"
|
|
82
|
+
scale = (1000.0, 1000.0, 1000.0, 1000.0)
|
|
83
|
+
vals = [v / s for v, s in zip(vals, scale)]
|
|
84
|
+
|
|
85
|
+
x0, y0, x1, y1 = vals
|
|
86
|
+
if x1 < x0:
|
|
87
|
+
x0, x1 = x1, x0
|
|
88
|
+
if y1 < y0:
|
|
89
|
+
y0, y1 = y1, y0
|
|
90
|
+
x0, y0 = max(0.0, x0), max(0.0, y0)
|
|
91
|
+
x1, y1 = min(1.0, x1), min(1.0, y1)
|
|
92
|
+
|
|
93
|
+
# expand degenerate/sliver boxes around their center to a usable minimum
|
|
94
|
+
for lo, hi in ((0, 2), (1, 3)):
|
|
95
|
+
box = [x0, y0, x1, y1]
|
|
96
|
+
if box[hi] - box[lo] < _MIN_SIDE_FRAC:
|
|
97
|
+
center = (box[lo] + box[hi]) / 2
|
|
98
|
+
lo_v = min(max(0.0, center - _MIN_SIDE_FRAC / 2), 1.0 - _MIN_SIDE_FRAC)
|
|
99
|
+
if lo == 0:
|
|
100
|
+
x0, x1 = lo_v, lo_v + _MIN_SIDE_FRAC
|
|
101
|
+
else:
|
|
102
|
+
y0, y1 = lo_v, lo_v + _MIN_SIDE_FRAC
|
|
103
|
+
return [x0, y0, x1, y1], mode
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def zoom_in_tool(
|
|
107
|
+
image: Any,
|
|
108
|
+
*,
|
|
109
|
+
save_dir: "str | None" = None,
|
|
110
|
+
target_short_side: int = _TARGET_SHORT_SIDE,
|
|
111
|
+
max_upscale: float = _MAX_UPSCALE,
|
|
112
|
+
) -> Tool:
|
|
113
|
+
"""Build the ``image_zoom_in`` tool bound to *image* (PIL or path).
|
|
114
|
+
|
|
115
|
+
The returned crop is upscaled (LANCZOS) so its short side reaches
|
|
116
|
+
*target_short_side* (capped at *max_upscale*), and re-injected into the
|
|
117
|
+
conversation by the agent loop. When *save_dir* is given every crop is
|
|
118
|
+
also written there (``zoom_00.png``, ...) and the path recorded in
|
|
119
|
+
``ToolResult.meta`` — trajectories reference media rather than embed it.
|
|
120
|
+
"""
|
|
121
|
+
base = _to_pil(image)
|
|
122
|
+
width, height = base.size
|
|
123
|
+
counter = itertools.count()
|
|
124
|
+
|
|
125
|
+
def _zoom(bbox, target: str = "") -> ToolResult:
|
|
126
|
+
frac, mode = _sanitize_bbox(bbox, width, height)
|
|
127
|
+
px = (
|
|
128
|
+
int(round(frac[0] * width)),
|
|
129
|
+
int(round(frac[1] * height)),
|
|
130
|
+
int(round(frac[2] * width)),
|
|
131
|
+
int(round(frac[3] * height)),
|
|
132
|
+
)
|
|
133
|
+
crop = base.crop(px)
|
|
134
|
+
short = min(crop.size)
|
|
135
|
+
if short > 0 and short < target_short_side:
|
|
136
|
+
factor = min(target_short_side / short, max_upscale)
|
|
137
|
+
if factor > 1.0:
|
|
138
|
+
from PIL import Image
|
|
139
|
+
|
|
140
|
+
crop = crop.resize(
|
|
141
|
+
(int(round(crop.size[0] * factor)), int(round(crop.size[1] * factor))),
|
|
142
|
+
Image.LANCZOS,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
meta: dict = {
|
|
146
|
+
"bbox_frac": [round(v, 4) for v in frac],
|
|
147
|
+
"crop_px": list(px),
|
|
148
|
+
"coord_mode": mode,
|
|
149
|
+
}
|
|
150
|
+
if target:
|
|
151
|
+
meta["target"] = target
|
|
152
|
+
if save_dir:
|
|
153
|
+
os.makedirs(save_dir, exist_ok=True)
|
|
154
|
+
path = os.path.join(save_dir, f"zoom_{next(counter):02d}.png")
|
|
155
|
+
crop.save(path)
|
|
156
|
+
meta["saved_path"] = path
|
|
157
|
+
|
|
158
|
+
note = "" if mode == "normalized" else f" (coordinates interpreted as {mode})"
|
|
159
|
+
return ToolResult(
|
|
160
|
+
text=(
|
|
161
|
+
f"Zoomed into region x:[{frac[0]:.2f}, {frac[2]:.2f}] "
|
|
162
|
+
f"y:[{frac[1]:.2f}, {frac[3]:.2f}] of the {width}x{height} image{note}. "
|
|
163
|
+
f"The enlarged {crop.size[0]}x{crop.size[1]} view is attached as a new image."
|
|
164
|
+
),
|
|
165
|
+
images=[crop],
|
|
166
|
+
meta=meta,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
return Tool(
|
|
170
|
+
name="image_zoom_in",
|
|
171
|
+
description=ZOOM_IN_DESCRIPTION,
|
|
172
|
+
parameters=ZOOM_IN_PARAMETERS,
|
|
173
|
+
fn=_zoom,
|
|
174
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""White-box (local) model entry points — per-version convenience factories.
|
|
2
|
+
|
|
3
|
+
Identity lives in ``evalrx.specs``; these are thin wrappers over
|
|
4
|
+
``compose(spec, "hf_local")``. Import a specific version factory, e.g.::
|
|
5
|
+
|
|
6
|
+
from evalrx.models.whitebox import qwen3_8b, qwen3_vl_8b_instruct, qwen3_omni_30b_a3b_instruct
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from evalrx.models.whitebox import qwen as _qwen
|
|
10
|
+
from evalrx.models.whitebox import qwen2_5_omni as _qwen2_5_omni
|
|
11
|
+
from evalrx.models.whitebox import qwen2_audio as _qwen2_audio
|
|
12
|
+
from evalrx.models.whitebox import qwen_omni as _qwen_omni
|
|
13
|
+
from evalrx.models.whitebox import qwen_vl as _qwen_vl
|
|
14
|
+
from evalrx.models.whitebox.qwen import * # noqa: F401,F403 (QwenLLM + qwen text factories)
|
|
15
|
+
from evalrx.models.whitebox.qwen2_5_omni import * # noqa: F401,F403 (qwen2.5-omni factory)
|
|
16
|
+
from evalrx.models.whitebox.qwen2_audio import * # noqa: F401,F403 (qwen2-audio factory)
|
|
17
|
+
from evalrx.models.whitebox.qwen_omni import * # noqa: F401,F403 (qwen3-omni factories)
|
|
18
|
+
from evalrx.models.whitebox.qwen_vl import * # noqa: F401,F403 (QwenVL + qwen-vl factories)
|
|
19
|
+
|
|
20
|
+
__all__ = sorted(
|
|
21
|
+
set(_qwen.__all__)
|
|
22
|
+
| set(_qwen_vl.__all__)
|
|
23
|
+
| set(_qwen_omni.__all__)
|
|
24
|
+
| set(_qwen2_5_omni.__all__)
|
|
25
|
+
| set(_qwen2_audio.__all__)
|
|
26
|
+
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Local agent wrapper with tool calling (white-box).
|
|
2
|
+
|
|
3
|
+
Planned for Stage 2.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from evalrx.core.capability import Capability
|
|
11
|
+
from evalrx.core.model import Trace
|
|
12
|
+
from evalrx.models.base import BaseAgent
|
|
13
|
+
from evalrx.models.whitebox.base import WhiteboxModel
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class WhiteboxAgent(WhiteboxModel, BaseAgent):
|
|
17
|
+
"""Local model that can call external tools over multiple steps."""
|
|
18
|
+
|
|
19
|
+
capabilities = frozenset({Capability.GENERATE})
|
|
20
|
+
|
|
21
|
+
def load(self) -> None:
|
|
22
|
+
raise NotImplementedError("WhiteboxAgent is planned for Stage 2.")
|
|
23
|
+
|
|
24
|
+
def generate(self, inputs: Any, **kwargs) -> str:
|
|
25
|
+
raise NotImplementedError("WhiteboxAgent is planned for Stage 2.")
|
|
26
|
+
|
|
27
|
+
def forward(self, inputs: Any, capture: set[Capability]) -> Trace:
|
|
28
|
+
raise NotImplementedError("WhiteboxAgent is planned for Stage 2.")
|
|
29
|
+
|
|
30
|
+
def step(self, observation: str, **kwargs) -> str:
|
|
31
|
+
raise NotImplementedError("WhiteboxAgent is planned for Stage 2.")
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Base class for locally-deployed (white-box) models.
|
|
2
|
+
|
|
3
|
+
White-box models have full parameter access, so they can provide rich
|
|
4
|
+
capabilities (attention, hidden states, gradients). Concrete subclasses
|
|
5
|
+
declare their ``capabilities`` and implement ``generate`` + ``forward``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from abc import abstractmethod
|
|
11
|
+
|
|
12
|
+
from evalrx.core.model import Model
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class WhiteboxModel(Model):
|
|
16
|
+
"""Base for open-source models loaded with full parameter access.
|
|
17
|
+
|
|
18
|
+
Subclasses must implement :meth:`load`, :meth:`generate`, and
|
|
19
|
+
:meth:`forward`, and set the ``capabilities`` class attribute.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def load(self) -> None:
|
|
24
|
+
"""Load model weights into memory (called lazily on first use)."""
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Qwen text LLMs (white-box, local) — per-version convenience factories.
|
|
2
|
+
|
|
3
|
+
Model identity lives in :mod:`evalrx.specs`; these factories are **thin
|
|
4
|
+
wrappers** over ``compose(spec, "hf_local")`` (via :func:`evalrx.load`), so
|
|
5
|
+
there is NO per-version ``load``/``forward`` duplication — the single generic
|
|
6
|
+
``hf_local`` backend does the work. Each ``qwen…(**runtime)`` returns an
|
|
7
|
+
hf_local model (caps: GENERATE, LOGITS, LOGPROBS, HIDDEN_STATES, ATTENTION; plus
|
|
8
|
+
TOOL_CALLS when the model's chat template supports tools)::
|
|
9
|
+
|
|
10
|
+
from evalrx.models.whitebox.qwen import qwen3_8b
|
|
11
|
+
model = qwen3_8b(device="cuda", dtype="bfloat16")
|
|
12
|
+
# equivalent to: evalrx.load("qwen3-8b", backend="hf_local", ...)
|
|
13
|
+
|
|
14
|
+
``QwenLLM(...)`` remains as a deprecated back-compat shim.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import warnings
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
# Spec keys exposed as factories (each must exist in evalrx.specs).
|
|
23
|
+
_TEXT_KEYS = [
|
|
24
|
+
"qwen2.5-7b-instruct", "qwen2.5-14b-instruct", "qwen2.5-32b-instruct", "qwen2.5-72b-instruct",
|
|
25
|
+
"qwen3-4b", "qwen3-8b", "qwen3-14b", "qwen3-32b", "qwen3-30b-a3b", "qwen3-235b-a22b",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _version_factory(key: str):
|
|
30
|
+
"""A thin ``(**runtime) -> hf_local model`` builder for a spec *key*."""
|
|
31
|
+
def build(**runtime: Any):
|
|
32
|
+
from evalrx.models import load
|
|
33
|
+
return load(key, backend="hf_local", **runtime)
|
|
34
|
+
|
|
35
|
+
name = key.replace("-", "_").replace(".", "_")
|
|
36
|
+
build.__name__ = build.__qualname__ = name
|
|
37
|
+
build.__doc__ = (
|
|
38
|
+
f"Build {key!r} on the hf_local (white-box) backend. "
|
|
39
|
+
f"Thin wrapper over compose('{key}', 'hf_local')."
|
|
40
|
+
)
|
|
41
|
+
return build
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
for _k in _TEXT_KEYS:
|
|
45
|
+
globals()[_k.replace("-", "_").replace(".", "_")] = _version_factory(_k)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def QwenLLM(checkpoint: str | None = None, device: str = "auto", dtype: str = "bfloat16", **runtime: Any):
|
|
49
|
+
"""Deprecated. Use a version factory (e.g. ``qwen2_5_7b_instruct()``) or
|
|
50
|
+
``evalrx.load('qwen2.5-7b-instruct')``."""
|
|
51
|
+
warnings.warn(
|
|
52
|
+
"QwenLLM is deprecated; use a version factory (e.g. qwen2_5_7b_instruct()) "
|
|
53
|
+
"or evalrx.load('qwen2.5-7b-instruct').",
|
|
54
|
+
DeprecationWarning, stacklevel=2,
|
|
55
|
+
)
|
|
56
|
+
from evalrx.models import load
|
|
57
|
+
return load("qwen2.5-7b-instruct", backend="hf_local",
|
|
58
|
+
checkpoint=checkpoint, device=device, dtype=dtype, **runtime)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
__all__ = ["QwenLLM"] + [k.replace("-", "_").replace(".", "_") for k in _TEXT_KEYS]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Qwen2.5-Omni (white-box, local) — convenience factory.
|
|
2
|
+
|
|
3
|
+
Same pattern as :mod:`evalrx.models.whitebox.qwen_omni`: a thin wrapper over
|
|
4
|
+
``compose(spec, "hf_local")`` (via :func:`evalrx.load`); identity lives in
|
|
5
|
+
:mod:`evalrx.specs`. Registered separately from the Qwen3-Omni family
|
|
6
|
+
because it is the checkpoint published paper methods (e.g. audio contrastive
|
|
7
|
+
decoding) tend to report hyperparameters against — see the spec's caveats for
|
|
8
|
+
its 300s audio-window truncation guard::
|
|
9
|
+
|
|
10
|
+
from evalrx.models.whitebox.qwen2_5_omni import qwen2_5_omni_7b
|
|
11
|
+
model = qwen2_5_omni_7b(device="cuda", dtype="bfloat16")
|
|
12
|
+
model.modalities # frozenset({'text', 'image', 'audio', 'video'})
|
|
13
|
+
|
|
14
|
+
Reference: https://github.com/QwenLM/Qwen2.5-Omni
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def qwen2_5_omni_7b(**runtime: Any):
|
|
23
|
+
"""Build 'qwen2.5-omni-7b' on the hf_local (white-box) backend."""
|
|
24
|
+
from evalrx.models import load
|
|
25
|
+
|
|
26
|
+
return load("qwen2.5-omni-7b", backend="hf_local", **runtime)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
__all__ = ["qwen2_5_omni_7b"]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Qwen2-Audio-Instruct (white-box, local) — convenience factory.
|
|
2
|
+
|
|
3
|
+
Audio-in / text-out only (no vision tower) — the simplest audio-capable spec
|
|
4
|
+
in the registry. Same thin-wrapper pattern as the other whitebox factories::
|
|
5
|
+
|
|
6
|
+
from evalrx.models.whitebox.qwen2_audio import qwen2_audio_7b_instruct
|
|
7
|
+
model = qwen2_audio_7b_instruct(device="cuda", dtype="bfloat16")
|
|
8
|
+
model.modalities # frozenset({'text', 'audio'})
|
|
9
|
+
|
|
10
|
+
Reference: https://github.com/QwenLM/Qwen2-Audio
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def qwen2_audio_7b_instruct(**runtime: Any):
|
|
19
|
+
"""Build 'qwen2-audio-7b-instruct' on the hf_local (white-box) backend."""
|
|
20
|
+
from evalrx.models import load
|
|
21
|
+
|
|
22
|
+
return load("qwen2-audio-7b-instruct", backend="hf_local", **runtime)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__all__ = ["qwen2_audio_7b_instruct"]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Qwen3-Omni models (white-box, local) — per-version convenience factories.
|
|
2
|
+
|
|
3
|
+
Same pattern as :mod:`evalrx.models.whitebox.qwen` /
|
|
4
|
+
:mod:`~evalrx.models.whitebox.qwen_vl`: thin wrappers over
|
|
5
|
+
``compose(spec, "hf_local")`` (via :func:`evalrx.load`); identity lives in
|
|
6
|
+
:mod:`evalrx.specs`. An "omni" model is **not** a new class — it is a spec
|
|
7
|
+
that carries vision *and* audio (+ video), so its modality set is
|
|
8
|
+
``{"text", "image", "audio", "video"}`` and analyzers match on it::
|
|
9
|
+
|
|
10
|
+
from evalrx.models.whitebox.qwen_omni import qwen3_omni_30b_a3b_instruct
|
|
11
|
+
model = qwen3_omni_30b_a3b_instruct(device="cuda", dtype="bfloat16")
|
|
12
|
+
model.modalities # frozenset({'text', 'image', 'audio', 'video'})
|
|
13
|
+
|
|
14
|
+
NOTE: text generation works via the thinker; full multimodal generate
|
|
15
|
+
(image/audio/video in) and white-box forward-capture over the vision/audio towers
|
|
16
|
+
are Stage-2 (needs ``transformers>=5.2.0`` and ``qwen_omni_utils.process_mm_info``;
|
|
17
|
+
pass ``use_audio_in_video`` consistently). The Captioner is audio-in / text-out
|
|
18
|
+
(modalities ``{"text", "audio"}``).
|
|
19
|
+
|
|
20
|
+
Reference: https://github.com/QwenLM/Qwen3-Omni
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
# Spec keys exposed as factories (each must exist in evalrx.specs).
|
|
28
|
+
_OMNI_KEYS = [
|
|
29
|
+
"qwen3-omni-30b-a3b-instruct",
|
|
30
|
+
"qwen3-omni-30b-a3b-thinking",
|
|
31
|
+
"qwen3-omni-30b-a3b-captioner",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _version_factory(key: str):
|
|
36
|
+
def build(**runtime: Any):
|
|
37
|
+
from evalrx.models import load
|
|
38
|
+
return load(key, backend="hf_local", **runtime)
|
|
39
|
+
|
|
40
|
+
name = key.replace("-", "_").replace(".", "_")
|
|
41
|
+
build.__name__ = build.__qualname__ = name
|
|
42
|
+
build.__doc__ = (
|
|
43
|
+
f"Build {key!r} on the hf_local (white-box) backend. "
|
|
44
|
+
f"Thin wrapper over compose('{key}', 'hf_local')."
|
|
45
|
+
)
|
|
46
|
+
return build
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
for _k in _OMNI_KEYS:
|
|
50
|
+
globals()[_k.replace("-", "_").replace(".", "_")] = _version_factory(_k)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
__all__ = [k.replace("-", "_").replace(".", "_") for k in _OMNI_KEYS]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Qwen vision-language models (white-box, local) — per-version convenience factories.
|
|
2
|
+
|
|
3
|
+
Same pattern as :mod:`evalrx.models.whitebox.qwen`: thin wrappers over
|
|
4
|
+
``compose(spec, "hf_local")`` (via :func:`evalrx.load`); identity lives in
|
|
5
|
+
:mod:`evalrx.specs`. Each ``qwen…vl…(**runtime)`` returns an hf_local VLM
|
|
6
|
+
(modalities ``{"text", "image"}``)::
|
|
7
|
+
|
|
8
|
+
from evalrx.models.whitebox.qwen_vl import qwen3_vl_8b_instruct
|
|
9
|
+
model = qwen3_vl_8b_instruct(device="cuda", dtype="bfloat16")
|
|
10
|
+
|
|
11
|
+
NOTE: text generation + logprobs work today; white-box forward-capture over image
|
|
12
|
+
tokens (attention/hidden) uses the hf_local VLM path + ``core.tokentype.TokenTypeMap``.
|
|
13
|
+
``QwenVL(...)`` remains as a deprecated back-compat shim.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import warnings
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
# Spec keys exposed as factories (each must exist in evalrx.specs).
|
|
22
|
+
_VL_KEYS = [
|
|
23
|
+
"qwen2-vl-7b-instruct",
|
|
24
|
+
"qwen2.5-vl-3b-instruct", "qwen2.5-vl-7b-instruct",
|
|
25
|
+
"qwen2.5-vl-32b-instruct", "qwen2.5-vl-72b-instruct",
|
|
26
|
+
"qwen3-vl-2b-instruct", "qwen3-vl-4b-instruct", "qwen3-vl-8b-instruct",
|
|
27
|
+
"qwen3-vl-30b-a3b-instruct", "qwen3-vl-235b-a22b-instruct",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _version_factory(key: str):
|
|
32
|
+
def build(**runtime: Any):
|
|
33
|
+
from evalrx.models import load
|
|
34
|
+
return load(key, backend="hf_local", **runtime)
|
|
35
|
+
|
|
36
|
+
name = key.replace("-", "_").replace(".", "_")
|
|
37
|
+
build.__name__ = build.__qualname__ = name
|
|
38
|
+
build.__doc__ = (
|
|
39
|
+
f"Build {key!r} on the hf_local (white-box) backend. "
|
|
40
|
+
f"Thin wrapper over compose('{key}', 'hf_local')."
|
|
41
|
+
)
|
|
42
|
+
return build
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
for _k in _VL_KEYS:
|
|
46
|
+
globals()[_k.replace("-", "_").replace(".", "_")] = _version_factory(_k)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def QwenVL(checkpoint: str | None = None, device: str = "auto", dtype: str = "bfloat16", **runtime: Any):
|
|
50
|
+
"""Deprecated. Use a version factory (e.g. ``qwen3_vl_8b_instruct()``) or
|
|
51
|
+
``evalrx.load('qwen2.5-vl-7b-instruct')``."""
|
|
52
|
+
warnings.warn(
|
|
53
|
+
"QwenVL is deprecated; use a version factory (e.g. qwen3_vl_8b_instruct()) "
|
|
54
|
+
"or evalrx.load('qwen2.5-vl-7b-instruct').",
|
|
55
|
+
DeprecationWarning, stacklevel=2,
|
|
56
|
+
)
|
|
57
|
+
from evalrx.models import load
|
|
58
|
+
return load("qwen2.5-vl-7b-instruct", backend="hf_local",
|
|
59
|
+
checkpoint=checkpoint, device=device, dtype=dtype, **runtime)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
__all__ = ["QwenVL"] + [k.replace("-", "_").replace(".", "_") for k in _VL_KEYS]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""EvalRX Observability Engine.
|
|
2
|
+
|
|
3
|
+
Full OpenTelemetry and Langfuse integration for agent auditing, execution tracing,
|
|
4
|
+
multi-dimensional probe measurement, and model I/O tracking.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from evalrx.observability.tracer import (
|
|
10
|
+
DiagnosticTracer,
|
|
11
|
+
backfill_run_to_langfuse,
|
|
12
|
+
export_to_langfuse_bundle,
|
|
13
|
+
sync_to_langfuse_live,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"DiagnosticTracer",
|
|
18
|
+
"backfill_run_to_langfuse",
|
|
19
|
+
"export_to_langfuse_bundle",
|
|
20
|
+
"sync_to_langfuse_live",
|
|
21
|
+
]
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Stable event and artifact envelopes shared by writers and future readers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import mimetypes
|
|
8
|
+
import uuid
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any # noqa: I001 - kept with Path for Python 3.10 compatibility notes
|
|
11
|
+
|
|
12
|
+
OBSERVABILITY_SCHEMA_VERSION = 1
|
|
13
|
+
_EVENT_NAMESPACE = uuid.UUID("3a09f2ed-28b1-4dd4-914f-b8da031edab2")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def make_event_envelope(
|
|
17
|
+
event: dict[str, Any], *, trace_id: str, event_seq: int,
|
|
18
|
+
artifact_refs: list[dict[str, Any]] | None = None,
|
|
19
|
+
) -> dict[str, Any]:
|
|
20
|
+
"""Normalize a run-log record into an idempotent observability event."""
|
|
21
|
+
event_type = str(event.get("event") or "event")
|
|
22
|
+
event_id = str(uuid.uuid5(_EVENT_NAMESPACE, f"{trace_id}:{event_seq}:{event_type}"))
|
|
23
|
+
return {
|
|
24
|
+
"schema_version": OBSERVABILITY_SCHEMA_VERSION,
|
|
25
|
+
"event_id": event_id,
|
|
26
|
+
"trace_id": trace_id,
|
|
27
|
+
"event_seq": event_seq,
|
|
28
|
+
"timestamp": event.get("ts"),
|
|
29
|
+
"event_type": event_type,
|
|
30
|
+
"stage": _stage_for(event),
|
|
31
|
+
"cycle": event.get("cycle"),
|
|
32
|
+
"parent_id": event.get("span_id"),
|
|
33
|
+
"status": _status_for(event),
|
|
34
|
+
"payload": event,
|
|
35
|
+
"artifact_refs": artifact_refs or [],
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def artifact_manifest(path: str | Path, *, run_dir: str | Path, role: str = "artifact") -> dict[str, Any]:
|
|
40
|
+
"""Describe an artifact without reading it into the event payload."""
|
|
41
|
+
artifact_path = Path(path)
|
|
42
|
+
root = Path(run_dir).resolve()
|
|
43
|
+
digest = hashlib.sha256()
|
|
44
|
+
with artifact_path.open("rb") as handle:
|
|
45
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
46
|
+
digest.update(chunk)
|
|
47
|
+
try:
|
|
48
|
+
relative = str(artifact_path.relative_to(root))
|
|
49
|
+
except ValueError:
|
|
50
|
+
relative = artifact_path.name
|
|
51
|
+
return {
|
|
52
|
+
"artifact_id": f"sha256:{digest.hexdigest()}",
|
|
53
|
+
"role": role,
|
|
54
|
+
"path": relative,
|
|
55
|
+
"filename": artifact_path.name,
|
|
56
|
+
"mime_type": mimetypes.guess_type(artifact_path.name)[0] or "application/octet-stream",
|
|
57
|
+
"size_bytes": artifact_path.stat().st_size,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def artifact_manifests_for_event(event: dict[str, Any], *, run_dir: str | Path) -> list[dict[str, Any]]:
|
|
62
|
+
"""Find file references in a run event and turn them into content manifests.
|
|
63
|
+
|
|
64
|
+
Event fields intentionally contain relative paths, never inline binary
|
|
65
|
+
content. This small adapter gives the Langfuse writer a complete upload
|
|
66
|
+
manifest without changing the established JSONL event shapes.
|
|
67
|
+
"""
|
|
68
|
+
root = Path(run_dir).resolve()
|
|
69
|
+
candidates: list[tuple[str, str]] = []
|
|
70
|
+
|
|
71
|
+
def visit(value: Any, key: str = "artifact", path_like: bool = False) -> None:
|
|
72
|
+
if isinstance(value, dict):
|
|
73
|
+
for child_key, child in value.items():
|
|
74
|
+
child_key = str(child_key)
|
|
75
|
+
visit(child, child_key, path_like or child_key.endswith("path") or child_key.endswith("_paths") or child_key == "record")
|
|
76
|
+
elif isinstance(value, list):
|
|
77
|
+
for child in value:
|
|
78
|
+
visit(child, key, path_like)
|
|
79
|
+
elif isinstance(value, str) and path_like:
|
|
80
|
+
candidates.append((key, value))
|
|
81
|
+
|
|
82
|
+
visit(event)
|
|
83
|
+
manifests: list[dict[str, Any]] = []
|
|
84
|
+
seen: set[Path] = set()
|
|
85
|
+
for role, relative in candidates:
|
|
86
|
+
path = (root / relative).resolve()
|
|
87
|
+
if root not in path.parents:
|
|
88
|
+
continue
|
|
89
|
+
if not path.is_file() or path in seen:
|
|
90
|
+
continue
|
|
91
|
+
seen.add(path)
|
|
92
|
+
try:
|
|
93
|
+
manifests.append(artifact_manifest(path, run_dir=root, role=role))
|
|
94
|
+
except OSError:
|
|
95
|
+
continue
|
|
96
|
+
return manifests
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def canonical_json(value: Any) -> str:
|
|
100
|
+
"""Canonical form used by parity checks and tests."""
|
|
101
|
+
return json.dumps(value, ensure_ascii=False, default=str, sort_keys=True, separators=(",", ":"))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _stage_for(event: dict[str, Any]) -> str:
|
|
105
|
+
# Lifecycle events carry their target stage in the payload; preserving it
|
|
106
|
+
# keeps a deliberately skipped M5 beside the rest of its evidence in
|
|
107
|
+
# Langfuse instead of burying it under an undifferentiated RUN row.
|
|
108
|
+
event_type = str(event.get("event") or "event")
|
|
109
|
+
if event_type == "stage_skipped":
|
|
110
|
+
return str(event.get("stage") or "RUN")
|
|
111
|
+
return {
|
|
112
|
+
"probe_search": "PRE_M1", "probe": "M1", "analysis": "M2",
|
|
113
|
+
"explore": "M2", "diagnosis": "M3", "surgery": "M4",
|
|
114
|
+
"fix": "M5", "experiment": "M5",
|
|
115
|
+
"case_record": "DATA", "report_published": "REPORT",
|
|
116
|
+
}.get(event_type, "RUN")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _status_for(event: dict[str, Any]) -> str:
|
|
120
|
+
if event.get("ok") is False or event.get("status") in {"failed", "error"}:
|
|
121
|
+
return "failed"
|
|
122
|
+
return "completed"
|