benchflow 0.7.6.dev2072__tar.gz → 0.7.6.dev2102__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/CHANGELOG.md +41 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/PKG-INFO +2 -1
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/pyproject.toml +7 -1
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/eval_artifacts.py +44 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/main.py +38 -3
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/eval_artifacts.py +57 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/eval_plan.py +12 -2
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/huggingface.py +108 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/__init__.py +77 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/__main__.py +13 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/boot.js +135 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/catalog.js +331 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/detail.js +710 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/fonts/OFL-google-sans-code.txt +93 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/fonts/google-sans-code-400-latin.woff2 +0 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/fonts/google-sans-code-600-latin.woff2 +0 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/template.html +36 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/theme.css +129 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/viewer.css +331 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/assets/viewer.js +67 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/catalog.py +90 -0
- benchflow-0.7.6.dev2072/src/benchflow/trajectories/viewer.py → benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/legacy.py +142 -254
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/models.py +401 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/payload.py +593 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/render.py +93 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/server.py +439 -0
- benchflow-0.7.6.dev2102/src/benchflow/trajectories/viewer/sources.py +177 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_arg_validation.py +23 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_artifact_cli.py +248 -0
- benchflow-0.7.6.dev2102/tests/test_publish_huggingface.py +147 -0
- benchflow-0.7.6.dev2102/tests/test_viewer_confirm.py +679 -0
- benchflow-0.7.6.dev2102/tests/trajectories/test_viewer_acp_renderer.py +918 -0
- benchflow-0.7.6.dev2102/tests/trajectories/test_viewer_browser.py +847 -0
- benchflow-0.7.6.dev2072/tests/test_viewer_confirm.py +0 -279
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/.gitignore +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/LICENSE +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/README.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_dotenv.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_paths.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_run.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_types.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/benchmark_repos.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/build_context_stage.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/config_override.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/content_address.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/dataset_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/env_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/evaluation_results.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/hf_datasets.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/json_safe.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/learner_memory.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/live_activity.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/result_metadata.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/reward_events.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/scoring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/source_provenance.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/task_authoring/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/task_authoring/_evidence_paths.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/task_authoring/acceptance_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/task_authoring/scaffolding.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/task_authoring/structural_checks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/text.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/_utils/yaml_loader.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/client.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/container_transport.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/selection.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/session.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/transport.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/acp/types.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/_toolathlon.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/_toolathlon_container.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/_toolathlon_fake_mail.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/_toolathlon_poste_users.tsv +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/_toolathlon_services.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/harbor.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/inbound.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/inspect_ai.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/ors.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/resources/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/resources/mcp_atlas_bridge.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/resources/mcp_atlas_judge.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/adapters/source.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agent_router.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agent_router_guide.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agent_router_parity.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agent_router_scaffold.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/codex_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/credentials.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/deepagents_acp_shim.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/errors.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/harvey_lab_acp_shim.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/install.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/openclaw_acp_shim.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/pi_acp_launcher.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/providers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/agents/remote_manifests.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/branch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_failure_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_hosted_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_live_progress.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_options.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_shared.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_traj_tui.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/_traj_upload_ui.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/adopt.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/agent.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/continue_cmd.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/environment.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/eval_lift.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/hub.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/monitor.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/review.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/sandbox.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/skills.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/tasks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/trace_import.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/train.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/cli/traj.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/continue_run/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/continue_run/batch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/continue_run/orchestrator.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/continue_run/replay_proxy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/continue_run/run_folder.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/continue_run/sandbox_proxy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/contracts/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/contracts/planes.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/contracts/user.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/demo_task/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/demo_task/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/demo_task/task.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/demo_task/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/demo_task/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/diagnostics.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/_registry/env0@outage.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/_registry/env0@prod.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/manifest_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/environment/readiness.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/eval_lift.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/eval_sharding.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/eval_worker.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/evaluation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/experimental/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/experimental/mcp/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/experimental/mcp/hooks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/experimental/mcp/reviewer_server.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/hosted_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/hub/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/hub/harbor_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/integrations/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/integrations/trl/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/integrations/trl/spec.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/learner_skills.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/learner_store.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/loop_strategies.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/loop_sweep.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/metrics.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/models.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/monitor.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/litellm_bedrock_patch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/litellm_bedrock_preflight.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/litellm_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/litellm_logging.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/litellm_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/providers/runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/_progress.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/azure_blob.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/broker.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/redact.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/traj_capture.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/publish/traj_report.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/py.typed +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/default-rubric.json +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/prompts.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/runner.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/scoring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/review/wrapper.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/README.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/builtins.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/events.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/file_readers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/llm.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/memory_scorer.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/node.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/rubric.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/rubric_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rewards/validation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_deadline.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_results.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_setup.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_skills.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_usage.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/_user_loop.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/session_factory_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout/task_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout_branch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/rollout_planes.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_base.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_cache_reclaim.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_compose.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_compose_files/docker-compose-base.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_compose_files/docker-compose-build.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_compose_files/docker-compose-no-network.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_compose_files/docker-compose-prebuilt.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/_sdk_ops.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/agentcore.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/agentcore_builder.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/agentcore_image.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/agentcore_provisioning.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/agentcore_reaper.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/agentcore_sealed.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/apple_container.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/daytona.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/daytona_dind.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/daytona_pty.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/daytona_reaper.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/daytona_strategies.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/docker.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/lockdown.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/metadata.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/modal_impl.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/process/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/process/_base.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/process/agentcore.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/process/apple.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/process/daytona.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/process/docker.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/providers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/services.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/setup.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/snapshot.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sandbox/user.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/scenes.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/sdk.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/self_gen.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/skill_eval/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/skill_eval/_core.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/skill_eval/gepa_export.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/skill_eval/schema.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/skill_policy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/skills.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/_document_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/_document_normalize.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/_document_parse.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/_document_profiles.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/acceptance_live.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/acceptance_live_model.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/acceptance_live_report.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/acceptance_live_validation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/discovery.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/document.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/export.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/imports.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/output_format.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/package.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/paths.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/prompts.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/runtime_capabilities.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/runtime_view.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/task.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_core.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_document.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_errors.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_judge_inputs.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_ors_episode.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_reward_kit.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_scan.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/task/verifier_script_strategy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/templates/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/templates/judge.py.tmpl +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/templates/test.sh.tmpl +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/traces/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/traces/huggingface.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/traces/local.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/traces/models.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/traces/parsers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/traces/task_gen.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/training/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/training/backends/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/training/backends/prime_rl.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/training/run_manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/_capture.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/_export_common.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/_llm_capture.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/call_purpose.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/export.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/export_adp.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/export_atif.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/export_prime_sft.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/export_trl_sft.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/metrics.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/results.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/sessions.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/tree.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/trl_sft_tokenization.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/trajectories/types.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/src/benchflow/usage_tracking.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/acceptance_live_harness.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_manifest_dirscan.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_manifest_loader.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_manifest_parity.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_manifest_wiring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_namespace_shorthand.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_plugin_entry_points.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/agents/test_remote_manifest_autoload.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/README.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/environment/docker-compose.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/environment/skills/conformance-writer/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/task.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/acp_smoke/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/conformance-results.json +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/proof_multi_agent.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/proof_snapshot.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/run_conformance.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conformance/self_gen_smoke_skills/skill-creator/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/conftest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/continue_run/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/continue_run/_helpers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/continue_run/test_batch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/continue_run/test_orchestrator.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/continue_run/test_replay_proxy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/continue_run/test_run_folder.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/test_chibench_manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/test_clawsbench_manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/test_manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/test_manifest_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/test_protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/environment/test_readiness.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/hello-world-task/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/hello-world-task/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/hello-world-task/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/hello-world-task/task.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/hello-world-task/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/hello-world-task/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/test_claude.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/test_codex.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/test_codex_custom_provider.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/test_gemini.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/test_openclaw.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/traces/minimal-claude.jsonl +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/examples/traces/minimal-opentraces.jsonl +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/README.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/alpha-task/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/alpha-task/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/alpha-task/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/alpha-task/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/beta-task/data/empty.bin +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/beta-task/data/unicode.txt +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/beta-task/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/beta-task/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/digest/not-a-task/notes.txt +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/mock_acp_agent.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/mock_acp_agent_interleaved.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/mock_acp_agent_multi_turn.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/mock_openai_responses_server.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/data/michigan_stations.txt +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/flood-detection/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/nws-flood-thresholds/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/usgs-data-download/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/flood-risk-analysis/tests/test_outputs.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/climate.csv +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/hydrology.csv +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/land_cover.csv +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/water_temperature.csv +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/contribution-analysis/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/meteorology-driver-classification/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/pca-decomposition/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/trend-analysis/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/lake-warming-attribution/tests/test_outputs.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/manifest.json +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/r2r_simulator.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/finite-horizon-lqr/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/integral-action-design/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/mpc-horizon-tuning/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/state-space-linearization/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/system_config.json +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/r2r-mpc-control/tests/test_outputs.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/generate_training_pcaps.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/local.rules +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/pcap-triage-tshark/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/pcap-triage-tshark/scripts/summarize_http_requests.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-offline-evejson/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-offline-evejson/scripts/run_suricata_offline.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-rules-basics/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/suricata.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/tests/test_outputs.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/Dockerfile +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syz-extract-constants/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syzkaller-build-loop/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syzlang-ioctl-basics/SKILL.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/instruction.md +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/solution/solve.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/task.toml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/tests/test.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/tests/test_outputs.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/_consistency.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/agent_judge.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/check_adapter_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/check_hosted_env_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/check_results.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/check_skillsbench_harbor_parity.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/check_trace_to_task_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/claude-agent-acp.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/codex-acp.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/gemini.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/harvey-lab-harness.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/mimo.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/openclaw.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/opencode.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/openhands.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/configs/pi-acp.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/deepagents_harness.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/rubric_checks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/run.sh +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/run_suite.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/scenarios.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/integration/suites/release.yaml +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acceptance_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acceptance_live.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acceptance_live_execution.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acp.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acp_capability_advertising.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acp_mcp_servers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acp_model_config_dispatch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acp_pinned_protocol_guard.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_acp_setup_failure_propagation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_adapter_scripts.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_adapters.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_env_resolution.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_gemini_defaults.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_idle_timeout_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_model_decouple.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_router.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_router_cli_e2e.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_router_scaffold.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_setup.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agent_spec.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agentcore_builder.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agentcore_parallel.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agentcore_reaper.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agentcore_sandbox.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_agentcore_transport.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_api_error_capture.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_apple_container_sandbox.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_bare_model_provider.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_base_install_imports.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_bedrock_thinking.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_benchflow_experiment_review_validator.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_benchmark_repos_resolve_repo_path.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_branch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_build_context_stage.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_build_review_pack.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_capture_trajectory.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_check_skillsbench_harbor_parity.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_clawsbench_slice.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_adopt_aliases.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_daytona.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_docs_drift.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_edge_case_hardening.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_hub_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_live_progress.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_cli_misc.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_codex_review.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_codex_self_write_auth.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_config_override.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_config_redaction.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_connect_as_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_console_error_truncation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_console_progress.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_continuallearningbench_adapter.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_dataset_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_daytona_command_polling.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_daytona_dind_compose_up.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_daytona_download.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_daytona_litellm_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_daytona_poll_deadline.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_daytona_reap.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_deepagents_agent.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_diagnostics.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_docker_prune_scoping.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_docker_uploads.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_docs_examples.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eng50_capabilities.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_env_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_env_setup.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_environment_manifest_controls.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_filters_applied.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_lift.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_sharding.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_single_task_summary.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_source_provenance.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_worker_retry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_eval_zero_task_guard.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_evaluation_environment_manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_exclude_tasks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_filter_credentialed_cells.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_harvey_lab_shim.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_hilbench_adapter.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_hosted_env.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_hosted_env_rollout_contract.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_hub_harbor_registry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_inbound_adapter_manifest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_inbound_adapters.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_agent_judge.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_check_results.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_consistency.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_matrix.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_provider_selection.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_run_suite.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_integration_suite.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_internet_policy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_job.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_job_sequential_shared.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_job_sequential_shared_resume.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_judge_robustness.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_judge_symlink_ingestion.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_learner_skills.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_learner_skills_traversal.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_learner_store.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_learner_store_persistence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_hardening.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_logging.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_model_via_env_generic.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_reasoning_passthrough.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_litellm_smoke.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_live_llm_trajectory.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_llm_judge.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_llm_judge_event_tags.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_llm_judge_verifier.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_loop_strategies.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_loop_summary.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_loop_sweep.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_memory_scorer.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_metrics.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_mle_bench_adapter.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_mock_openai_responses_server.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_monitor_scaffold.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_native_acp_usage.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_no_cross_provider_fallback.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_notification_order_real.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_opencode_family_proxy_tracking.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_oracle.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_oracle_chokepoint.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_paths_safe.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_paths_symlink_helpers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_pi_acp_launcher.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_pi_acp_proxy_routing.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_process.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_project_skills_layout.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_provider_auth_detection.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_providers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_publish_azure_blob.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_reexport.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_registry_invariants.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_release_hardening_r5.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_release_version.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_release_workflow_wiring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_resolve_env_helpers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_review_boundaries.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_review_rubric.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_review_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_reward_lenient.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_reward_node.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_reward_range.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_reward_unified_contract.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rewards.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rewards_jsonl.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_architecture.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_branch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_config_path_coercion.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_environment.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_hard_deadline.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_import_no_side_effects.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_on_ask_user_wiring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_planes_contract.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_probe_sandbox_health.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_session_factory_dispatch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_setup_commands.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rollout_upload.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rubric_checks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_rubric_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_runtime_capabilities.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_runtime_config_wired.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_runtime_live_sandbox.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_exec_secret_handling.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_hardening.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_isolation_copy_traversal.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_live_process.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_multi_service.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_prebuilt_validation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_protocol.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_provider_registry_drift.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_setup.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_snapshot_contract.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_upload_symlink.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sandbox_verifier_workspace.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_scene.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_scene_outbox_trial.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_scene_parallel_group.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_scene_result_aggregation.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_scoring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sdk_internals.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sdk_lockdown.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sdk_run_alias.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_sealed_channel_container.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_self_gen_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_self_gen_export_error_channel.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_self_gen_export_failures.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_self_gen_orchestration.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_session_factory_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_session_request_permission_dispatch.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_eval.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_eval_dryrun.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_eval_integration.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_eval_sweep.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_eval_traversal.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_invocation_artifacts.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skill_policy.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skills.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skills_dir_agent_home_link.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skillsbench_conversion_conformance.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_skillsbench_harbor_run_suite.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_smoke.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_smoke_wiring.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_source_adapters.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_subscription_auth.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_check_eval_consistency.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_digest.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_document.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_download.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_export.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_output_format.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_package.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_task_runtime_primitive.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_tasks.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_token_usage_normalization.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trace_import_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trace_task_gen_traversal.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trace_to_task_evidence.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traces_huggingface.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traces_parsers.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traces_task_gen.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_train_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_train_mode_artifact_emission.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_train_trl_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traj_report.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traj_staging_phases.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traj_tui.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traj_upload_cli.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traj_upload_skill.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_traj_workspace.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trajectory_integration.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trajectory_streaming.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trajectory_upload_layout.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trajectory_upload_service.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trial_agent_timeout_verify.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trial_install_agent_timeout.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trial_litellm_runtime.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_trl_integration.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_usage_litellm.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_usage_required.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_usage_tracking.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_user.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_v06_guard_ports.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verifier_document.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verifier_multi_container.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verifier_output.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verifier_output_freshness.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verifier_strategies.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verifier_wedge_retry.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_verify.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_workflow_action_pinning.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/test_yaml_config.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/__init__.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_export.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_export_adp.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_export_atif.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_export_common.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_export_nan_handling.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_export_prime_sft.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_redaction.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_step_granularity.py +0 -0
- {benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2102}/tests/trajectories/test_tree.py +0 -0
|
@@ -2,6 +2,47 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
### Added
|
|
6
|
+
- **ACP rollout directories render as an interactive reviewer page.**
|
|
7
|
+
`bench eval view <rollout>` now assembles a JSON payload (normalized
|
|
8
|
+
events + result/timing/verifier metadata) and renders it client-side in a
|
|
9
|
+
self-contained template (no build step, zero network requests): full
|
|
10
|
+
event stream with collapse instead of truncation, harness/model/skills
|
|
11
|
+
identity row, reward badge and `result.json` failure-diagnostic banners,
|
|
12
|
+
Verifier and Metrics tabs, Focus/Full modes, per-kind filters and hues,
|
|
13
|
+
text search, and per-event `#e42` anchors. Trajectory content is treated
|
|
14
|
+
as untrusted end to end (data-only embedding, `textContent` rendering).
|
|
15
|
+
Raw session JSONL files and legacy `turn*.txt` runs remain supported, and
|
|
16
|
+
the `--confirm` approve/reject contract remains compatible.
|
|
17
|
+
- **Directories of rollouts serve a run catalog.** Pointing
|
|
18
|
+
`bench eval view` at a job directory (or a whole `jobs/` tree) serves a
|
|
19
|
+
browsable index: corpus counts, grouping by task or model + harness with
|
|
20
|
+
per-group pass/fail aggregates and pass rates, sorting by
|
|
21
|
+
name/reward/duration/cost, text filtering, collapsible groups with
|
|
22
|
+
incremental pagination, and URL-preserved state — selecting a run opens
|
|
23
|
+
the detail page and the back control restores the exact catalog view.
|
|
24
|
+
Traces load dynamically from `/api/rollout?id=…` (ids resolve only by
|
|
25
|
+
exact membership in a fresh directory scan, so crafted ids cannot reach
|
|
26
|
+
the filesystem) and `?run=<id>` deep-links a run. `--confirm` on a
|
|
27
|
+
multi-run directory errors out: a confirmation needs exactly one
|
|
28
|
+
trajectory.
|
|
29
|
+
- **`hf://` sources browse HuggingFace trajectory datasets directly.**
|
|
30
|
+
`bench eval view hf://<org>/<dataset>[@revision][/subpath]` fetches the
|
|
31
|
+
viewer-relevant slice of a trajectory dataset into the shared
|
|
32
|
+
`huggingface_hub` cache and serves it through browse mode, making the
|
|
33
|
+
community ground-truth uploads reviewable with one command. The download
|
|
34
|
+
allowlist is exact — trajectories, result/timing/prompts, and the four
|
|
35
|
+
verifier sidecars the viewer renders — with no wildcards, so large run
|
|
36
|
+
artifacts and `llm_trajectory`/`trainer` exports are never fetched. The
|
|
37
|
+
CLI passes the spec as a string into a typed
|
|
38
|
+
`LocalPathSource | HfDatasetSource` parser (never through `pathlib.Path`,
|
|
39
|
+
whose normalization would mangle `hf://` into `hf:/`), and dataset
|
|
40
|
+
subpaths are validated.
|
|
41
|
+
- **The viewer renders per-event timelines the moment captures provide
|
|
42
|
+
timestamps.** Steps gain a `+m:ss` offset chip and tool calls a duration
|
|
43
|
+
chip whenever events carry `ts` / `started_at` / `finished_at` fields;
|
|
44
|
+
with today's captures (which carry none) nothing changes visually.
|
|
45
|
+
|
|
5
46
|
## 0.7.5 — 2026-08-19
|
|
6
47
|
|
|
7
48
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: benchflow
|
|
3
|
-
Version: 0.7.6.
|
|
3
|
+
Version: 0.7.6.dev2102
|
|
4
4
|
Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
|
|
5
5
|
Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
|
|
6
6
|
Project-URL: Repository, https://github.com/benchflow-ai/benchflow
|
|
@@ -45,6 +45,7 @@ Requires-Dist: deepagents>=0.6; extra == 'deepagents'
|
|
|
45
45
|
Requires-Dist: langchain-openai>=0.2; extra == 'deepagents'
|
|
46
46
|
Provides-Extra: dev
|
|
47
47
|
Requires-Dist: pathspec>=0.12; extra == 'dev'
|
|
48
|
+
Requires-Dist: playwright>=1.49; extra == 'dev'
|
|
48
49
|
Requires-Dist: pre-commit>=3.7; extra == 'dev'
|
|
49
50
|
Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
|
|
50
51
|
Requires-Dist: pytest>=9.0.3; extra == 'dev'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "benchflow"
|
|
3
|
-
version = "0.7.6.
|
|
3
|
+
version = "0.7.6.dev2102"
|
|
4
4
|
description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -61,6 +61,11 @@ dev = [
|
|
|
61
61
|
"pathspec>=0.12",
|
|
62
62
|
"pytest>=9.0.3",
|
|
63
63
|
"pytest-asyncio>=0.24.0",
|
|
64
|
+
# Browser-level viewer tests (tests/trajectories/test_viewer_browser.py):
|
|
65
|
+
# drives the interactive template in real Chromium. Tests self-skip when
|
|
66
|
+
# playwright or its browser binary is absent, so plain `pytest tests/`
|
|
67
|
+
# stays runnable without `playwright install chromium`.
|
|
68
|
+
"playwright>=1.49",
|
|
64
69
|
"ruff>=0.7.0",
|
|
65
70
|
"ty>=0.0.1a1",
|
|
66
71
|
]
|
|
@@ -154,6 +159,7 @@ testpaths = ["tests"]
|
|
|
154
159
|
markers = [
|
|
155
160
|
"live: requires real Anthropic API and Docker daemon (run with -m live)",
|
|
156
161
|
"integration: full integration tests — requires GEMINI_API_KEY + DAYTONA_API_KEY (run with -m integration)",
|
|
162
|
+
"browser: drives the viewer template in real Chromium via Playwright (self-skips without playwright/chromium; deselect with -m 'not browser')",
|
|
157
163
|
]
|
|
158
164
|
|
|
159
165
|
[tool.ruff]
|
|
@@ -66,7 +66,9 @@ def postprocess_eval_artifacts(
|
|
|
66
66
|
eval_config = _apply_source_sidecar(eval_config, resolved_tasks_dir)
|
|
67
67
|
from benchflow.eval_artifacts import (
|
|
68
68
|
TaskManifestOptions,
|
|
69
|
+
canonical_task_rows,
|
|
69
70
|
materialize_canonical_job,
|
|
71
|
+
render_run_summary_markdown,
|
|
70
72
|
write_canonical_selection,
|
|
71
73
|
write_health_summary,
|
|
72
74
|
write_task_manifest,
|
|
@@ -138,6 +140,12 @@ def postprocess_eval_artifacts(
|
|
|
138
140
|
console.print(
|
|
139
141
|
f"[green]Canonical jobs:[/green] {escape(str(req.canonical_jobs_dir))}"
|
|
140
142
|
)
|
|
143
|
+
if req.publish_hf is not None or req.publish_bucket is not None:
|
|
144
|
+
(job_dir / "README.md").write_text(
|
|
145
|
+
render_run_summary_markdown(
|
|
146
|
+
job_dir, agent=eval_config.agent, model=eval_config.model
|
|
147
|
+
)
|
|
148
|
+
)
|
|
141
149
|
if req.publish_hf is not None:
|
|
142
150
|
from benchflow.publish.huggingface import publish_folder_to_hf
|
|
143
151
|
|
|
@@ -154,6 +162,42 @@ def postprocess_eval_artifacts(
|
|
|
154
162
|
print_error(str(exc))
|
|
155
163
|
raise typer.Exit(1) from None
|
|
156
164
|
console.print(f"[green]Published HF artifacts:[/green] {escape(published.url)}")
|
|
165
|
+
if req.publish_bucket is not None:
|
|
166
|
+
from benchflow.publish.huggingface import publish_folder_to_bucket
|
|
167
|
+
|
|
168
|
+
prefix = req.hf_prefix or job_dir.name
|
|
169
|
+
try:
|
|
170
|
+
published = publish_folder_to_bucket(
|
|
171
|
+
job_dir, bucket_id=req.publish_bucket, path_in_repo=prefix
|
|
172
|
+
)
|
|
173
|
+
except ValueError as exc:
|
|
174
|
+
print_error(str(exc))
|
|
175
|
+
raise typer.Exit(1) from None
|
|
176
|
+
console.print(f"[green]Published HF bucket:[/green] {escape(published.url)}")
|
|
177
|
+
if req.eval_results_model is not None:
|
|
178
|
+
from benchflow.publish.huggingface import open_eval_results_pr
|
|
179
|
+
|
|
180
|
+
rewards = [
|
|
181
|
+
row["reward"] for row in canonical_task_rows(job_dir) if row["scored"]
|
|
182
|
+
]
|
|
183
|
+
if rewards:
|
|
184
|
+
value = round(100.0 * sum(rewards) / len(rewards), 2)
|
|
185
|
+
try:
|
|
186
|
+
pr_url = open_eval_results_pr(
|
|
187
|
+
model_repo=req.eval_results_model,
|
|
188
|
+
dataset_id=cast(str, req.eval_results_dataset),
|
|
189
|
+
task_id=cast(str, req.eval_results_task),
|
|
190
|
+
value=value,
|
|
191
|
+
)
|
|
192
|
+
except ValueError as exc:
|
|
193
|
+
print_error(str(exc))
|
|
194
|
+
raise typer.Exit(1) from None
|
|
195
|
+
console.print(f"[green]Eval-results PR:[/green] {escape(pr_url or '')}")
|
|
196
|
+
else:
|
|
197
|
+
console.print(
|
|
198
|
+
"[yellow]--eval-results-model set but no scored rollouts were "
|
|
199
|
+
"found; skipping eval-results PR[/yellow]"
|
|
200
|
+
)
|
|
157
201
|
|
|
158
202
|
|
|
159
203
|
def _load_eval_matrix(path: Path) -> dict[str, dict]:
|
|
@@ -581,6 +581,31 @@ def eval_run(
|
|
|
581
581
|
"--hf-public-read-check", help="Verify public HF reads after upload"
|
|
582
582
|
),
|
|
583
583
|
] = False,
|
|
584
|
+
publish_bucket: Annotated[
|
|
585
|
+
str | None,
|
|
586
|
+
typer.Option(
|
|
587
|
+
"--publish-bucket", help="Upload final eval artifacts to a HF bucket"
|
|
588
|
+
),
|
|
589
|
+
] = None,
|
|
590
|
+
eval_results_model: Annotated[
|
|
591
|
+
str | None,
|
|
592
|
+
typer.Option(
|
|
593
|
+
"--eval-results-model",
|
|
594
|
+
help="HF model repo to open a community eval-results PR on",
|
|
595
|
+
),
|
|
596
|
+
] = None,
|
|
597
|
+
eval_results_dataset: Annotated[
|
|
598
|
+
str | None,
|
|
599
|
+
typer.Option(
|
|
600
|
+
"--eval-results-dataset", help="HF benchmark dataset id, e.g. org/benchmark"
|
|
601
|
+
),
|
|
602
|
+
] = None,
|
|
603
|
+
eval_results_task: Annotated[
|
|
604
|
+
str | None,
|
|
605
|
+
typer.Option(
|
|
606
|
+
"--eval-results-task", help="Benchmark task_id from the dataset's eval.yaml"
|
|
607
|
+
),
|
|
608
|
+
] = None,
|
|
584
609
|
matrix: Annotated[
|
|
585
610
|
Path | None,
|
|
586
611
|
typer.Option("--matrix", help="YAML model matrix for repeated evals"),
|
|
@@ -653,6 +678,10 @@ def eval_run(
|
|
|
653
678
|
publish_hf=publish_hf,
|
|
654
679
|
hf_prefix=hf_prefix,
|
|
655
680
|
hf_public_read_check=hf_public_read_check,
|
|
681
|
+
publish_bucket=publish_bucket,
|
|
682
|
+
eval_results_model=eval_results_model,
|
|
683
|
+
eval_results_dataset=eval_results_dataset,
|
|
684
|
+
eval_results_task=eval_results_task,
|
|
656
685
|
matrix=matrix,
|
|
657
686
|
trials=trials,
|
|
658
687
|
)
|
|
@@ -1222,9 +1251,15 @@ def eval_metrics(
|
|
|
1222
1251
|
@eval_app.command("view")
|
|
1223
1252
|
def eval_view(
|
|
1224
1253
|
rollout_dir: Annotated[
|
|
1225
|
-
Path
|
|
1254
|
+
# A str, not a Path: hf:// specs must arrive verbatim — Path
|
|
1255
|
+
# normalization would collapse the double slash into hf:/.
|
|
1256
|
+
str,
|
|
1226
1257
|
typer.Argument(
|
|
1227
|
-
help=
|
|
1258
|
+
help=(
|
|
1259
|
+
"Rollout directory, job directory, trajectory JSONL file, or "
|
|
1260
|
+
"an hf://<org>/<dataset> trajectory dataset "
|
|
1261
|
+
"(optionally with /subpath)"
|
|
1262
|
+
)
|
|
1228
1263
|
),
|
|
1229
1264
|
],
|
|
1230
1265
|
port: Annotated[int, typer.Option(help="Server port")] = 8888,
|
|
@@ -1256,7 +1291,7 @@ def eval_view(
|
|
|
1256
1291
|
from benchflow.trajectories import viewer
|
|
1257
1292
|
|
|
1258
1293
|
decision = viewer.serve(
|
|
1259
|
-
|
|
1294
|
+
rollout_dir, port, confirm=confirm, redaction_summary=redaction_summary
|
|
1260
1295
|
)
|
|
1261
1296
|
# Exit-code contract for --confirm: 0 approved, 3 rejected. 3 is chosen
|
|
1262
1297
|
# so a rejection never collides with the CLI's existing error exits
|
|
@@ -9,6 +9,7 @@ from pathlib import Path
|
|
|
9
9
|
from typing import Any, Literal
|
|
10
10
|
|
|
11
11
|
from benchflow._utils.task_authoring import task_digest
|
|
12
|
+
from benchflow._utils.text import truncate_end
|
|
12
13
|
from benchflow.task.discovery import is_task_dir, resolve_task_collection_root
|
|
13
14
|
from benchflow.trajectories.export_prime_sft import (
|
|
14
15
|
PrimeSftTrajectoryJsonlError,
|
|
@@ -268,6 +269,62 @@ def write_health_summary(path: Path, job_dir: Path) -> dict[str, Any]:
|
|
|
268
269
|
return summary
|
|
269
270
|
|
|
270
271
|
|
|
272
|
+
def _escape_md_cell(text: str) -> str:
|
|
273
|
+
return text.replace("|", "\\|").replace("\n", " ").strip()
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
_ISSUE_CELL_LIMIT = 120
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def canonical_task_rows(job_dir: Path) -> list[dict[str, Any]]:
|
|
280
|
+
"""One row per task_id — the best-scored/highest-reward rollout.
|
|
281
|
+
|
|
282
|
+
Reused by the README summary and ``--eval-results-model`` so retry
|
|
283
|
+
attempts (multiple rollout dirs for the same task) are never counted
|
|
284
|
+
or averaged as if they were distinct tasks.
|
|
285
|
+
"""
|
|
286
|
+
return build_canonical_selection(job_dir, policy="one-healthy-per-task")["selected"]
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def render_run_summary_markdown(
|
|
290
|
+
job_dir: Path, *, agent: str | None = None, model: str | None = None
|
|
291
|
+
) -> str:
|
|
292
|
+
"""Render a Markdown run summary (results + issues) for publish folders."""
|
|
293
|
+
rows = canonical_task_rows(job_dir)
|
|
294
|
+
rewards = [row["reward"] for row in rows if row["scored"]]
|
|
295
|
+
mean_reward = sum(rewards) / len(rewards) if rewards else None
|
|
296
|
+
scored_count = sum(1 for row in rows if row["scored"])
|
|
297
|
+
tool_call_count = sum(1 for row in rows if row["tool_calls"] > 0)
|
|
298
|
+
|
|
299
|
+
lines = ["# BenchFlow run summary", ""]
|
|
300
|
+
if agent:
|
|
301
|
+
lines.append(f"- Agent: `{agent}`")
|
|
302
|
+
if model:
|
|
303
|
+
lines.append(f"- Model: `{model}`")
|
|
304
|
+
lines += [
|
|
305
|
+
f"- Tasks: {len(rows)}",
|
|
306
|
+
f"- Scored: {scored_count} / Unscored: {len(rows) - scored_count}",
|
|
307
|
+
f"- Mean reward: {mean_reward:.3f}"
|
|
308
|
+
if mean_reward is not None
|
|
309
|
+
else "- Mean reward: n/a",
|
|
310
|
+
f"- Tasks with tool calls: {tool_call_count}",
|
|
311
|
+
"",
|
|
312
|
+
"## Tasks",
|
|
313
|
+
"",
|
|
314
|
+
"| Task | Reward | Tool calls | Issue |",
|
|
315
|
+
"| --- | --- | --- | --- |",
|
|
316
|
+
]
|
|
317
|
+
for row in rows:
|
|
318
|
+
reward = f"{row['reward']:.3f}" if isinstance(row["reward"], float) else "—"
|
|
319
|
+
issue = row.get("error") or row.get("verifier_error") or ""
|
|
320
|
+
issue = truncate_end(_escape_md_cell(str(issue)), _ISSUE_CELL_LIMIT)
|
|
321
|
+
lines.append(
|
|
322
|
+
f"| {_escape_md_cell(str(row['task_id']))} | {reward} | "
|
|
323
|
+
f"{row['tool_calls']} | {issue} |"
|
|
324
|
+
)
|
|
325
|
+
return "\n".join(lines) + "\n"
|
|
326
|
+
|
|
327
|
+
|
|
271
328
|
def _canonical_score(row: dict[str, Any]) -> tuple[int, float, int]:
|
|
272
329
|
scored = 1 if row["scored"] else 0
|
|
273
330
|
reward = row["reward"] if isinstance(row["reward"], float) else float("-inf")
|
|
@@ -126,6 +126,10 @@ class EvalCreateRequest:
|
|
|
126
126
|
publish_hf: str | None = None
|
|
127
127
|
hf_prefix: str | None = None
|
|
128
128
|
hf_public_read_check: bool = False
|
|
129
|
+
publish_bucket: str | None = None
|
|
130
|
+
eval_results_model: str | None = None
|
|
131
|
+
eval_results_dataset: str | None = None
|
|
132
|
+
eval_results_task: str | None = None
|
|
129
133
|
matrix: Path | None = None
|
|
130
134
|
trials: int = 1
|
|
131
135
|
|
|
@@ -274,8 +278,14 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan:
|
|
|
274
278
|
raise EvalPlanError("--retry-attempts must be >= 0")
|
|
275
279
|
if request.retry_concurrency is not None and request.retry_concurrency < 1:
|
|
276
280
|
raise EvalPlanError("--retry-concurrency must be >= 1")
|
|
277
|
-
if request.hf_prefix and not request.publish_hf:
|
|
278
|
-
raise EvalPlanError("--hf-prefix requires --publish-hf")
|
|
281
|
+
if request.hf_prefix and not (request.publish_hf or request.publish_bucket):
|
|
282
|
+
raise EvalPlanError("--hf-prefix requires --publish-hf or --publish-bucket")
|
|
283
|
+
if request.eval_results_model and not (
|
|
284
|
+
request.eval_results_dataset and request.eval_results_task
|
|
285
|
+
):
|
|
286
|
+
raise EvalPlanError(
|
|
287
|
+
"--eval-results-model requires --eval-results-dataset and --eval-results-task"
|
|
288
|
+
)
|
|
279
289
|
if request.tasks_dir and not Path(request.tasks_dir).exists():
|
|
280
290
|
raise EvalPlanError(f"--tasks-dir not found: {request.tasks_dir}")
|
|
281
291
|
if request.matrix is not None and not Path(request.matrix).is_file():
|
|
@@ -38,6 +38,14 @@ def _resolve_url(repo_id: str, repo_type: str, path_in_repo: str) -> str:
|
|
|
38
38
|
return f"https://huggingface.co/{kind}{repo_id}/resolve/main/{path_in_repo}"
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
def _bucket_tree_url(bucket_id: str, path_in_repo: str) -> str:
|
|
42
|
+
# Buckets are non-versioned (no "main" branch), so the browsable listing
|
|
43
|
+
# is /tree/<path> directly, not /resolve/<path> (which 404s) or
|
|
44
|
+
# /tree/main/<path> (the git-repo convention _tree_url uses).
|
|
45
|
+
suffix = f"/tree/{path_in_repo.strip('/')}" if path_in_repo else ""
|
|
46
|
+
return f"https://huggingface.co/buckets/{bucket_id}{suffix}"
|
|
47
|
+
|
|
48
|
+
|
|
41
49
|
def _check_public_files(repo_id: str, repo_type: str, path_in_repo: str) -> None:
|
|
42
50
|
index_url = _tree_url(repo_id, repo_type, path_in_repo)
|
|
43
51
|
response = httpx.get(index_url, follow_redirects=True, timeout=20)
|
|
@@ -79,6 +87,106 @@ def publish_folder_to_hf(
|
|
|
79
87
|
)
|
|
80
88
|
|
|
81
89
|
|
|
90
|
+
def publish_folder_to_bucket(
|
|
91
|
+
folder: Path,
|
|
92
|
+
*,
|
|
93
|
+
bucket_id: str,
|
|
94
|
+
path_in_repo: str = "",
|
|
95
|
+
private: bool = False,
|
|
96
|
+
) -> HfPublishResult:
|
|
97
|
+
if not folder.is_dir():
|
|
98
|
+
raise ValueError(f"publish source folder not found: {folder}")
|
|
99
|
+
try:
|
|
100
|
+
from huggingface_hub import create_bucket, sync_bucket
|
|
101
|
+
from huggingface_hub.errors import HfHubHTTPError
|
|
102
|
+
except ImportError as exc: # pragma: no cover - depends on env
|
|
103
|
+
raise ValueError(
|
|
104
|
+
"huggingface_hub with bucket support (create_bucket/sync_bucket) is "
|
|
105
|
+
"required for --publish-bucket; upgrade huggingface_hub"
|
|
106
|
+
) from exc
|
|
107
|
+
try:
|
|
108
|
+
create_bucket(bucket_id, private=private)
|
|
109
|
+
except HfHubHTTPError as exc:
|
|
110
|
+
if exc.response is None or exc.response.status_code != 409:
|
|
111
|
+
raise
|
|
112
|
+
prefix = path_in_repo.strip("/")
|
|
113
|
+
remote = (
|
|
114
|
+
f"hf://buckets/{bucket_id}/{prefix}" if prefix else f"hf://buckets/{bucket_id}"
|
|
115
|
+
)
|
|
116
|
+
sync_bucket(str(folder), remote)
|
|
117
|
+
return HfPublishResult(
|
|
118
|
+
repo_id=bucket_id,
|
|
119
|
+
repo_type="bucket",
|
|
120
|
+
path_in_repo=prefix,
|
|
121
|
+
url=_bucket_tree_url(bucket_id, prefix),
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
_EVAL_RESULTS_PATH = ".eval_results/benchflow.yaml"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _existing_eval_results(model_repo: str) -> list[dict]:
|
|
129
|
+
from huggingface_hub import hf_hub_download
|
|
130
|
+
from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError
|
|
131
|
+
|
|
132
|
+
try:
|
|
133
|
+
downloaded = hf_hub_download(
|
|
134
|
+
repo_id=model_repo, filename=_EVAL_RESULTS_PATH, repo_type="model"
|
|
135
|
+
)
|
|
136
|
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
|
137
|
+
return []
|
|
138
|
+
import yaml
|
|
139
|
+
|
|
140
|
+
data = yaml.safe_load(Path(downloaded).read_text())
|
|
141
|
+
return data if isinstance(data, list) else []
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _upsert_eval_result(existing: list[dict], entry: dict) -> list[dict]:
|
|
145
|
+
key = (entry["dataset"]["id"], entry["dataset"]["task_id"])
|
|
146
|
+
kept = [
|
|
147
|
+
row
|
|
148
|
+
for row in existing
|
|
149
|
+
if (row.get("dataset", {}).get("id"), row.get("dataset", {}).get("task_id"))
|
|
150
|
+
!= key
|
|
151
|
+
]
|
|
152
|
+
return [*kept, entry]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def open_eval_results_pr(
|
|
156
|
+
*,
|
|
157
|
+
model_repo: str,
|
|
158
|
+
dataset_id: str,
|
|
159
|
+
task_id: str,
|
|
160
|
+
value: float,
|
|
161
|
+
source_url: str | None = None,
|
|
162
|
+
notes: str | None = None,
|
|
163
|
+
) -> str | None:
|
|
164
|
+
try:
|
|
165
|
+
import yaml
|
|
166
|
+
from huggingface_hub import CommitOperationAdd, HfApi
|
|
167
|
+
except ImportError as exc: # pragma: no cover - depends on env
|
|
168
|
+
raise ValueError(
|
|
169
|
+
"huggingface_hub is required for --eval-results-model"
|
|
170
|
+
) from exc
|
|
171
|
+
entry: dict = {"dataset": {"id": dataset_id, "task_id": task_id}, "value": value}
|
|
172
|
+
if source_url:
|
|
173
|
+
entry["source"] = {"url": source_url}
|
|
174
|
+
if notes:
|
|
175
|
+
entry["notes"] = notes
|
|
176
|
+
merged = _upsert_eval_result(_existing_eval_results(model_repo), entry)
|
|
177
|
+
content = yaml.safe_dump(merged, sort_keys=False).encode("utf-8")
|
|
178
|
+
commit = HfApi().create_commit(
|
|
179
|
+
repo_id=model_repo,
|
|
180
|
+
repo_type="model",
|
|
181
|
+
operations=[
|
|
182
|
+
CommitOperationAdd(path_in_repo=_EVAL_RESULTS_PATH, path_or_fileobj=content)
|
|
183
|
+
],
|
|
184
|
+
commit_message=f"Add {dataset_id} eval results",
|
|
185
|
+
create_pr=True,
|
|
186
|
+
)
|
|
187
|
+
return commit.pr_url
|
|
188
|
+
|
|
189
|
+
|
|
82
190
|
def publish_file_to_hf(
|
|
83
191
|
file_path: Path,
|
|
84
192
|
*,
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Trajectory viewer package.
|
|
2
|
+
|
|
3
|
+
Import surface preserved from the former single-module viewer: everything
|
|
4
|
+
tests and the CLI reached via ``benchflow.trajectories.viewer`` re-exports
|
|
5
|
+
here. See the sibling modules for the actual responsibilities.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .catalog import (
|
|
9
|
+
_discover_rollouts,
|
|
10
|
+
_resolve_browse_rollout,
|
|
11
|
+
_rollout_summary,
|
|
12
|
+
_runs_cap,
|
|
13
|
+
)
|
|
14
|
+
from .legacy import (
|
|
15
|
+
_NO_TRAJECTORIES_HTML,
|
|
16
|
+
_VIEWER_CSS,
|
|
17
|
+
_confirm_bar_html,
|
|
18
|
+
_inject_confirm_bar,
|
|
19
|
+
_tool_accent_class,
|
|
20
|
+
render_jsonl_file,
|
|
21
|
+
render_rollout,
|
|
22
|
+
render_turn,
|
|
23
|
+
)
|
|
24
|
+
from .payload import (
|
|
25
|
+
_DIAGNOSTIC_KEYS_FALLBACK,
|
|
26
|
+
_build_acp_payload,
|
|
27
|
+
_diagnostic_keys,
|
|
28
|
+
_is_acp_rollout_dir,
|
|
29
|
+
_load_prompts,
|
|
30
|
+
_parse_jsonl,
|
|
31
|
+
_safe_json,
|
|
32
|
+
_tool_content_texts,
|
|
33
|
+
)
|
|
34
|
+
from .render import _render_acp_trajectory, _render_shell
|
|
35
|
+
from .server import serve
|
|
36
|
+
from .sources import (
|
|
37
|
+
_HF_VIEWER_FILES,
|
|
38
|
+
HfDatasetSource,
|
|
39
|
+
LocalPathSource,
|
|
40
|
+
ViewerSourceError,
|
|
41
|
+
parse_source,
|
|
42
|
+
resolve_hf_dataset,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
__all__ = [
|
|
46
|
+
# public surface
|
|
47
|
+
"render_rollout",
|
|
48
|
+
"render_turn",
|
|
49
|
+
"render_jsonl_file",
|
|
50
|
+
"serve",
|
|
51
|
+
"parse_source",
|
|
52
|
+
"resolve_hf_dataset",
|
|
53
|
+
"LocalPathSource",
|
|
54
|
+
"HfDatasetSource",
|
|
55
|
+
"ViewerSourceError",
|
|
56
|
+
# internals reached by tests and siblings through the historical module
|
|
57
|
+
"_DIAGNOSTIC_KEYS_FALLBACK",
|
|
58
|
+
"_HF_VIEWER_FILES",
|
|
59
|
+
"_NO_TRAJECTORIES_HTML",
|
|
60
|
+
"_VIEWER_CSS",
|
|
61
|
+
"_build_acp_payload",
|
|
62
|
+
"_confirm_bar_html",
|
|
63
|
+
"_diagnostic_keys",
|
|
64
|
+
"_discover_rollouts",
|
|
65
|
+
"_inject_confirm_bar",
|
|
66
|
+
"_is_acp_rollout_dir",
|
|
67
|
+
"_load_prompts",
|
|
68
|
+
"_parse_jsonl",
|
|
69
|
+
"_render_acp_trajectory",
|
|
70
|
+
"_render_shell",
|
|
71
|
+
"_resolve_browse_rollout",
|
|
72
|
+
"_rollout_summary",
|
|
73
|
+
"_runs_cap",
|
|
74
|
+
"_safe_json",
|
|
75
|
+
"_tool_accent_class",
|
|
76
|
+
"_tool_content_texts",
|
|
77
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""``python -m benchflow.trajectories.viewer <rollout_dir_or_jsonl> [port]``"""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from .server import serve
|
|
6
|
+
|
|
7
|
+
if len(sys.argv) < 2:
|
|
8
|
+
print(
|
|
9
|
+
"Usage: python -m benchflow.trajectories.viewer <rollout_dir_or_jsonl> [port]"
|
|
10
|
+
)
|
|
11
|
+
sys.exit(1)
|
|
12
|
+
port = int(sys.argv[2]) if len(sys.argv) > 2 else 8888
|
|
13
|
+
serve(sys.argv[1], port)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
BF.navigation = (() => {
|
|
2
|
+
let generation = 0;
|
|
3
|
+
let request = null;
|
|
4
|
+
let loadedRun = null;
|
|
5
|
+
let selectedRun = null;
|
|
6
|
+
|
|
7
|
+
function beginTransition() {
|
|
8
|
+
generation += 1;
|
|
9
|
+
if (request) request.abort();
|
|
10
|
+
request = null;
|
|
11
|
+
BF.detail.cancel();
|
|
12
|
+
return generation;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function showCatalog(push) {
|
|
16
|
+
beginTransition();
|
|
17
|
+
loadedRun = null;
|
|
18
|
+
if (push) BF.catalog.writeURL(null, true);
|
|
19
|
+
BF.catalog.show({ focusRun: selectedRun });
|
|
20
|
+
document.title = "runs - benchflow trajectory";
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function openRun(runId, push, sourceButton = null) {
|
|
24
|
+
const index = document.getElementById("view-index");
|
|
25
|
+
if (!index.classList.contains("hidden")) BF.catalog.rememberScroll();
|
|
26
|
+
selectedRun = runId;
|
|
27
|
+
const transition = beginTransition();
|
|
28
|
+
loadedRun = null;
|
|
29
|
+
if (push) BF.catalog.writeURL(runId, true);
|
|
30
|
+
BF.core.showDetailShell(true);
|
|
31
|
+
BF.detail.showLoading(runId);
|
|
32
|
+
|
|
33
|
+
const controller = new AbortController();
|
|
34
|
+
request = controller;
|
|
35
|
+
try {
|
|
36
|
+
const response = await fetch("/api/rollout?id=" + encodeURIComponent(runId), {
|
|
37
|
+
signal: controller.signal,
|
|
38
|
+
});
|
|
39
|
+
if (!response.ok) throw new Error("HTTP " + response.status + " loading run " + runId);
|
|
40
|
+
const body = await response.text();
|
|
41
|
+
let payload;
|
|
42
|
+
try {
|
|
43
|
+
payload = JSON.parse(body);
|
|
44
|
+
} catch (error) {
|
|
45
|
+
throw new TypeError("malformed JSON from the rollout API: " + error.message);
|
|
46
|
+
}
|
|
47
|
+
BF.core.requirePayload(payload);
|
|
48
|
+
if (transition !== generation) return;
|
|
49
|
+
BF.detail.loadPayload(payload, { focusHeading: Boolean(sourceButton) });
|
|
50
|
+
loadedRun = runId;
|
|
51
|
+
window.scrollTo(0, 0);
|
|
52
|
+
} catch (error) {
|
|
53
|
+
if (controller.signal.aborted || transition !== generation) return;
|
|
54
|
+
BF.detail.showError("Failed to load run " + runId + ": " + error.message);
|
|
55
|
+
} finally {
|
|
56
|
+
if (transition === generation) request = null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function showUnknownRun(runId) {
|
|
61
|
+
selectedRun = runId;
|
|
62
|
+
beginTransition();
|
|
63
|
+
loadedRun = null;
|
|
64
|
+
BF.core.showDetailShell(true);
|
|
65
|
+
BF.detail.showError(BF.catalog.unknownRunMessage(runId));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function applyLocation() {
|
|
69
|
+
const runId = BF.catalog.readURL();
|
|
70
|
+
if (runId && runId === loadedRun && !document.getElementById("content").classList.contains("hidden")) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (runId && BF.catalog.hasRun(runId)) openRun(runId, false);
|
|
74
|
+
else if (runId) showUnknownRun(runId);
|
|
75
|
+
else showCatalog(false);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function startBrowse(boot) {
|
|
79
|
+
BF.catalog.init(boot, (runId, sourceButton) => openRun(runId, true, sourceButton));
|
|
80
|
+
const back = document.getElementById("backbtn");
|
|
81
|
+
back.addEventListener("click", () => showCatalog(true));
|
|
82
|
+
window.addEventListener("popstate", applyLocation);
|
|
83
|
+
applyLocation();
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function startSingle(payload) {
|
|
87
|
+
beginTransition();
|
|
88
|
+
BF.core.showDetailShell(false);
|
|
89
|
+
try {
|
|
90
|
+
BF.detail.loadPayload(payload);
|
|
91
|
+
} catch (error) {
|
|
92
|
+
BF.detail.showError("The embedded trajectory payload is malformed: " + error.message, "viewer data error");
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return { startBrowse, startSingle };
|
|
97
|
+
})();
|
|
98
|
+
|
|
99
|
+
(() => {
|
|
100
|
+
function bootError(message) {
|
|
101
|
+
BF.core.showDetailShell(false);
|
|
102
|
+
BF.detail.showError(message, "viewer data error");
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const node = document.getElementById("bf-payload");
|
|
106
|
+
let boot;
|
|
107
|
+
try {
|
|
108
|
+
boot = JSON.parse(node ? node.textContent : "");
|
|
109
|
+
} catch (error) {
|
|
110
|
+
bootError("The embedded viewer data is not valid JSON: " + error.message);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
if (!BF.core.isRecord(boot)) {
|
|
114
|
+
bootError("The embedded viewer data must be a JSON object.");
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (boot.mode === "single") {
|
|
118
|
+
BF.navigation.startSingle(boot.payload);
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
if (boot.mode === "browse") {
|
|
122
|
+
if (!Array.isArray(boot.rollouts)) {
|
|
123
|
+
bootError("Browse-mode viewer data must contain a rollouts array.");
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
const invalid = boot.rollouts.find((run) => !BF.core.isRecord(run) || typeof run.id !== "string" || !run.id);
|
|
127
|
+
if (invalid) {
|
|
128
|
+
bootError("A catalog entry is malformed: every rollout must be an object with a non-empty string id.");
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
BF.navigation.startBrowse(boot);
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
bootError('Unknown viewer mode: expected "single" or "browse".');
|
|
135
|
+
})();
|