benchflow 0.6.9.dev1917__tar.gz → 0.6.9.dev1920__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/CHANGELOG.md +7 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/PKG-INFO +1 -1
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/pyproject.toml +1 -1
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/traj.py +10 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/publish/broker.py +2 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/publish/traj_capture.py +62 -2
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_publish_azure_blob.py +47 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_traj_upload_cli.py +84 -12
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trajectory_upload_service.py +126 -10
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/.gitignore +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/LICENSE +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/README.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_dotenv.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_paths.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_run.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_types.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/benchmark_repos.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/build_context_stage.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/config_override.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/content_address.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/dataset_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/env_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/evaluation_results.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/hf_datasets.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/json_safe.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/learner_memory.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/live_activity.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/result_metadata.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/reward_events.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/scoring.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/source_provenance.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/task_authoring/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/task_authoring/_evidence_paths.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/task_authoring/acceptance_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/task_authoring/scaffolding.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/task_authoring/structural_checks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/text.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/_utils/yaml_loader.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/client.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/container_transport.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/selection.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/session.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/transport.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/acp/types.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/_toolathlon.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/_toolathlon_container.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/_toolathlon_fake_mail.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/_toolathlon_poste_users.tsv +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/_toolathlon_services.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/harbor.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/inbound.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/inspect_ai.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/ors.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/resources/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/resources/mcp_atlas_bridge.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/resources/mcp_atlas_judge.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/adapters/source.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agent_router.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agent_router_guide.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agent_router_parity.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agent_router_scaffold.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/codex_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/credentials.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/deepagents_acp_shim.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/errors.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/harvey_lab_acp_shim.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/install.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/openclaw_acp_shim.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/pi_acp_launcher.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/providers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/agents/remote_manifests.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/branch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/_failure_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/_hosted_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/_live_progress.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/_options.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/_shared.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/adopt.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/agent.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/continue_cmd.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/environment.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/eval_artifacts.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/eval_lift.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/hub.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/main.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/monitor.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/review.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/sandbox.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/skills.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/tasks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/trace_import.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/cli/train.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/continue_run/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/continue_run/batch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/continue_run/orchestrator.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/continue_run/replay_proxy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/continue_run/run_folder.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/continue_run/sandbox_proxy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/contracts/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/contracts/planes.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/contracts/user.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/demo_task/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/demo_task/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/demo_task/task.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/demo_task/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/demo_task/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/diagnostics.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/_registry/env0@outage.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/_registry/env0@prod.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/manifest_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/environment/readiness.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/eval_artifacts.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/eval_lift.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/eval_plan.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/eval_sharding.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/eval_worker.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/evaluation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/experimental/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/experimental/mcp/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/experimental/mcp/hooks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/experimental/mcp/reviewer_server.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/hosted_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/hub/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/hub/harbor_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/integrations/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/integrations/trl/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/integrations/trl/spec.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/learner_skills.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/learner_store.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/loop_strategies.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/loop_sweep.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/metrics.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/models.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/monitor.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/litellm_bedrock_patch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/litellm_bedrock_preflight.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/litellm_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/litellm_logging.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/litellm_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/providers/runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/publish/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/publish/azure_blob.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/publish/huggingface.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/publish/redact.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/py.typed +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/review/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/review/config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/review/default-rubric.json +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/review/prompts.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/review/runner.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/review/wrapper.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/README.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/builtins.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/events.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/file_readers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/llm.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/memory_scorer.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/node.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/rubric.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/rubric_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rewards/validation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_deadline.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_results.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_setup.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_skills.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_usage.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/_user_loop.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/session_factory_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout/task_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout_branch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/rollout_planes.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_base.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_cache_reclaim.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_compose.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_compose_files/docker-compose-base.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_compose_files/docker-compose-build.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_compose_files/docker-compose-no-network.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_compose_files/docker-compose-prebuilt.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/_sdk_ops.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/agentcore.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/agentcore_builder.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/agentcore_image.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/agentcore_provisioning.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/agentcore_reaper.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/agentcore_sealed.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/apple_container.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/daytona.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/daytona_dind.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/daytona_pty.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/daytona_reaper.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/daytona_strategies.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/docker.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/lockdown.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/metadata.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/modal_impl.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/_base.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/agentcore.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/apple.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/daytona.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/docker.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/providers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/services.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/setup.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/snapshot.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/user.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/scenes.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/sdk.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/self_gen.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/skill_eval/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/skill_eval/_core.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/skill_eval/gepa_export.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/skill_eval/schema.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/skill_policy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/skills.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/_document_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/_document_normalize.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/_document_parse.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/_document_profiles.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/acceptance_live.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/acceptance_live_model.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/acceptance_live_report.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/acceptance_live_validation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/discovery.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/document.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/export.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/imports.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/output_format.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/package.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/paths.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/prompts.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/runtime_capabilities.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/runtime_view.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/task.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_core.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_document.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_errors.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_judge_inputs.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_ors_episode.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_reward_kit.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_scan.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/task/verifier_script_strategy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/templates/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/templates/judge.py.tmpl +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/templates/test.sh.tmpl +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/traces/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/traces/huggingface.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/traces/local.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/traces/models.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/traces/parsers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/traces/task_gen.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/training/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/training/backends/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/training/backends/prime_rl.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/training/run_manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/_capture.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/_export_common.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/_llm_capture.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/call_purpose.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/export.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/export_adp.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/export_atif.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/export_prime_sft.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/export_trl_sft.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/metrics.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/results.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/tree.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/trl_sft_tokenization.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/types.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/trajectories/viewer.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/src/benchflow/usage_tracking.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/acceptance_live_harness.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_manifest_dirscan.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_manifest_loader.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_manifest_parity.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_manifest_wiring.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_namespace_shorthand.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_plugin_entry_points.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/agents/test_remote_manifest_autoload.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/README.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/environment/docker-compose.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/environment/skills/conformance-writer/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/task.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/acp_smoke/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/conformance-results.json +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/proof_multi_agent.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/proof_snapshot.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/run_conformance.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conformance/self_gen_smoke_skills/skill-creator/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/conftest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/continue_run/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/continue_run/_helpers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/continue_run/test_batch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/continue_run/test_orchestrator.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/continue_run/test_replay_proxy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/continue_run/test_run_folder.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/test_chibench_manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/test_clawsbench_manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/test_manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/test_manifest_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/test_protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/environment/test_readiness.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/hello-world-task/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/hello-world-task/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/hello-world-task/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/hello-world-task/task.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/hello-world-task/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/hello-world-task/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/test_claude.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/test_codex.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/test_codex_custom_provider.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/test_gemini.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/test_openclaw.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/traces/minimal-claude.jsonl +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/examples/traces/minimal-opentraces.jsonl +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/README.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/alpha-task/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/alpha-task/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/alpha-task/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/alpha-task/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/beta-task/data/empty.bin +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/beta-task/data/unicode.txt +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/beta-task/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/beta-task/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/digest/not-a-task/notes.txt +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/mock_acp_agent.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/mock_acp_agent_interleaved.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/mock_acp_agent_multi_turn.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/mock_openai_responses_server.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/data/michigan_stations.txt +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/flood-detection/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/nws-flood-thresholds/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/usgs-data-download/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/flood-risk-analysis/tests/test_outputs.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/climate.csv +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/hydrology.csv +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/land_cover.csv +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/water_temperature.csv +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/contribution-analysis/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/meteorology-driver-classification/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/pca-decomposition/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/trend-analysis/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/lake-warming-attribution/tests/test_outputs.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/manifest.json +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/r2r_simulator.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/finite-horizon-lqr/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/integral-action-design/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/mpc-horizon-tuning/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/state-space-linearization/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/system_config.json +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/r2r-mpc-control/tests/test_outputs.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/generate_training_pcaps.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/local.rules +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/pcap-triage-tshark/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/pcap-triage-tshark/scripts/summarize_http_requests.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-offline-evejson/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-offline-evejson/scripts/run_suricata_offline.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-rules-basics/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/suricata.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/tests/test_outputs.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/Dockerfile +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syz-extract-constants/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syzkaller-build-loop/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syzlang-ioctl-basics/SKILL.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/instruction.md +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/solution/solve.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/task.toml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/tests/test.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/tests/test_outputs.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/_consistency.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/agent_judge.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/check_adapter_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/check_hosted_env_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/check_results.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/check_skillsbench_harbor_parity.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/check_trace_to_task_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/claude-agent-acp.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/codex-acp.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/gemini.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/harvey-lab-harness.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/mimo.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/openclaw.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/opencode.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/openhands.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/configs/pi-acp.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/deepagents_harness.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/rubric_checks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/run.sh +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/run_suite.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/scenarios.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/integration/suites/release.yaml +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acceptance_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acceptance_live.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acceptance_live_execution.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acp.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acp_capability_advertising.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acp_mcp_servers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acp_model_config_dispatch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acp_pinned_protocol_guard.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_acp_setup_failure_propagation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_adapter_scripts.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_adapters.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_env_resolution.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_gemini_defaults.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_idle_timeout_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_model_decouple.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_router.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_router_cli_e2e.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_router_scaffold.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_setup.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agent_spec.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agentcore_builder.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agentcore_parallel.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agentcore_reaper.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agentcore_sandbox.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_agentcore_transport.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_api_error_capture.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_apple_container_sandbox.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_bare_model_provider.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_base_install_imports.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_bedrock_thinking.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_benchflow_experiment_review_validator.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_benchmark_repos_resolve_repo_path.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_branch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_build_context_stage.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_build_review_pack.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_capture_trajectory.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_check_skillsbench_harbor_parity.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_clawsbench_slice.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_adopt_aliases.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_arg_validation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_daytona.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_docs_drift.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_edge_case_hardening.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_hub_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_live_progress.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_cli_misc.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_codex_review.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_codex_self_write_auth.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_config_override.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_config_redaction.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_connect_as_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_console_error_truncation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_console_progress.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_continuallearningbench_adapter.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_dataset_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_daytona_command_polling.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_daytona_dind_compose_up.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_daytona_download.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_daytona_litellm_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_daytona_poll_deadline.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_daytona_reap.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_deepagents_agent.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_diagnostics.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_docker_prune_scoping.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_docker_uploads.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_docs_examples.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eng50_capabilities.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_env_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_env_setup.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_environment_manifest_controls.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_artifact_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_filters_applied.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_lift.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_sharding.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_single_task_summary.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_source_provenance.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_worker_retry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_eval_zero_task_guard.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_evaluation_environment_manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_exclude_tasks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_filter_credentialed_cells.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_harvey_lab_shim.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_hilbench_adapter.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_hosted_env.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_hosted_env_rollout_contract.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_hub_harbor_registry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_inbound_adapter_manifest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_inbound_adapters.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_agent_judge.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_check_results.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_consistency.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_matrix.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_provider_selection.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_run_suite.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_integration_suite.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_internet_policy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_job.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_job_sequential_shared.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_job_sequential_shared_resume.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_judge_robustness.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_judge_symlink_ingestion.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_learner_skills.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_learner_skills_traversal.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_learner_store.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_learner_store_persistence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_hardening.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_logging.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_model_via_env_generic.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_reasoning_passthrough.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_litellm_smoke.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_live_llm_trajectory.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_llm_judge.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_llm_judge_event_tags.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_llm_judge_verifier.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_loop_strategies.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_loop_summary.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_loop_sweep.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_memory_scorer.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_metrics.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_mle_bench_adapter.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_mock_openai_responses_server.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_monitor_scaffold.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_native_acp_usage.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_no_cross_provider_fallback.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_notification_order_real.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_opencode_family_proxy_tracking.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_oracle.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_oracle_chokepoint.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_paths_safe.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_paths_symlink_helpers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_pi_acp_launcher.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_pi_acp_proxy_routing.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_process.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_project_skills_layout.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_provider_auth_detection.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_providers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_reexport.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_registry_invariants.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_release_hardening_r5.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_release_version.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_release_workflow_wiring.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_resolve_env_helpers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_review_boundaries.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_review_rubric.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_review_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_reward_lenient.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_reward_node.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_reward_range.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_reward_unified_contract.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rewards.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rewards_jsonl.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_architecture.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_branch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_config_path_coercion.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_environment.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_hard_deadline.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_import_no_side_effects.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_on_ask_user_wiring.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_planes_contract.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_probe_sandbox_health.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_session_factory_dispatch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_setup_commands.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rollout_upload.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rubric_checks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_rubric_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_runtime_capabilities.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_runtime_config_wired.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_runtime_live_sandbox.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_exec_secret_handling.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_hardening.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_isolation_copy_traversal.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_live_process.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_multi_service.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_prebuilt_validation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_protocol.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_provider_registry_drift.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_setup.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_snapshot_contract.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_upload_symlink.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sandbox_verifier_workspace.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_scene.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_scene_outbox_trial.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_scene_parallel_group.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_scene_result_aggregation.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_scoring.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sdk_internals.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sdk_lockdown.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sdk_run_alias.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_sealed_channel_container.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_self_gen_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_self_gen_export_error_channel.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_self_gen_export_failures.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_self_gen_orchestration.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_session_factory_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_session_request_permission_dispatch.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_eval.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_eval_dryrun.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_eval_integration.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_eval_sweep.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_eval_traversal.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_invocation_artifacts.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skill_policy.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skills.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skills_dir_agent_home_link.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skillsbench_conversion_conformance.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_skillsbench_harbor_run_suite.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_smoke.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_smoke_wiring.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_source_adapters.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_subscription_auth.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_check_eval_consistency.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_digest.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_document.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_download.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_export.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_output_format.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_package.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_task_runtime_primitive.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_tasks.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_token_usage_normalization.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trace_import_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trace_task_gen_traversal.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trace_to_task_evidence.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_traces_huggingface.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_traces_parsers.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_traces_task_gen.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_train_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_train_mode_artifact_emission.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_train_trl_cli.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trajectory_integration.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trajectory_streaming.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trial_agent_timeout_verify.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trial_install_agent_timeout.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trial_litellm_runtime.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_trl_integration.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_usage_litellm.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_usage_required.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_usage_tracking.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_user.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_v06_guard_ports.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verifier_document.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verifier_multi_container.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verifier_output.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verifier_output_freshness.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verifier_strategies.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verifier_wedge_retry.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_verify.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_workflow_action_pinning.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/test_yaml_config.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/__init__.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_export.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_export_adp.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_export_atif.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_export_common.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_export_nan_handling.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_export_prime_sft.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_redaction.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_step_granularity.py +0 -0
- {benchflow-0.6.9.dev1917 → benchflow-0.6.9.dev1920}/tests/trajectories/test_tree.py +0 -0
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
### Changed
|
|
6
|
+
- **Trajectory uploads require contributor provenance.** The single
|
|
7
|
+
`bench traj upload` command now requires `--github-id` and `--email`; both
|
|
8
|
+
values are validated locally and server-side and stored in `manifest.json`
|
|
9
|
+
under the structured `contributor` field. Existing 0.6.8 manifests remain
|
|
10
|
+
readable by the validator during the client transition.
|
|
11
|
+
|
|
5
12
|
## 0.6.8 — 2026-08-15
|
|
6
13
|
|
|
7
14
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: benchflow
|
|
3
|
-
Version: 0.6.9.
|
|
3
|
+
Version: 0.6.9.dev1920
|
|
4
4
|
Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
|
|
5
5
|
Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
|
|
6
6
|
Project-URL: Repository, https://github.com/benchflow-ai/benchflow
|
|
@@ -29,6 +29,14 @@ def register_traj(app: typer.Typer) -> None:
|
|
|
29
29
|
Path,
|
|
30
30
|
typer.Argument(help="Trajectory JSONL file, directory, or trial directory"),
|
|
31
31
|
],
|
|
32
|
+
github_id: Annotated[
|
|
33
|
+
str,
|
|
34
|
+
typer.Option("--github-id", help="Contributor GitHub username"),
|
|
35
|
+
],
|
|
36
|
+
email: Annotated[
|
|
37
|
+
str,
|
|
38
|
+
typer.Option("--email", help="Contributor email stored in the manifest"),
|
|
39
|
+
],
|
|
32
40
|
source_id: Annotated[
|
|
33
41
|
str | None,
|
|
34
42
|
typer.Option("--source-id", help="Stable contributor source identifier"),
|
|
@@ -84,6 +92,8 @@ def register_traj(app: typer.Typer) -> None:
|
|
|
84
92
|
path,
|
|
85
93
|
source_id=selected_source_id,
|
|
86
94
|
uploaded_by=os.environ.get("BENCHFLOW_TRAJ_UPLOADED_BY"),
|
|
95
|
+
github_id=github_id,
|
|
96
|
+
email=email,
|
|
87
97
|
) as staged:
|
|
88
98
|
if dry_run:
|
|
89
99
|
_print_dry_run(staged, destination=destination, direct=direct)
|
|
@@ -42,6 +42,8 @@ def upload_capture_via_broker(
|
|
|
42
42
|
"uploaded_by": staged.manifest["uploaded_by"],
|
|
43
43
|
"artifacts": artifacts,
|
|
44
44
|
}
|
|
45
|
+
if contributor := staged.manifest.get("contributor"):
|
|
46
|
+
request_body["contributor"] = contributor
|
|
45
47
|
manager = nullcontext(http_client) if http_client is not None else httpx.Client()
|
|
46
48
|
try:
|
|
47
49
|
with _quiet_httpx_request_logging(), manager as client:
|
|
@@ -23,11 +23,18 @@ MAX_CAPTURE_BYTES = 2 * MAX_FILE_BYTES
|
|
|
23
23
|
MAX_MANIFEST_BYTES = 1024**2
|
|
24
24
|
MAX_RUN_METADATA_BYTES = 1024**2
|
|
25
25
|
MAX_UPLOADED_BY_LENGTH = 256
|
|
26
|
+
MAX_GITHUB_ID_LENGTH = 39
|
|
27
|
+
MAX_EMAIL_LENGTH = 254
|
|
26
28
|
MAX_JSONL_RECORD_BYTES = 8 * 1024**2
|
|
27
29
|
MAX_JSON_NESTING = 100
|
|
28
30
|
MAX_JSON_NODES = 100_000
|
|
29
31
|
SOURCE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$")
|
|
30
32
|
ARTIFACT_NAME_PATTERN = re.compile(r"^trajectory/[A-Za-z0-9._-]{1,128}\.jsonl$")
|
|
33
|
+
GITHUB_ID_PATTERN = re.compile(
|
|
34
|
+
rf"^[A-Za-z0-9](?:[A-Za-z0-9-]{{0,{MAX_GITHUB_ID_LENGTH - 2}}}[A-Za-z0-9])?$"
|
|
35
|
+
)
|
|
36
|
+
EMAIL_LOCAL_PATTERN = re.compile(r"^[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+$")
|
|
37
|
+
EMAIL_DOMAIN_LABEL_PATTERN = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$")
|
|
31
38
|
|
|
32
39
|
|
|
33
40
|
def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
@@ -102,6 +109,42 @@ def validate_source_id(source_id: str) -> str:
|
|
|
102
109
|
return normalized
|
|
103
110
|
|
|
104
111
|
|
|
112
|
+
def validate_github_id(github_id: str) -> str:
|
|
113
|
+
"""Validate a self-asserted GitHub username stored as contributor provenance."""
|
|
114
|
+
normalized = github_id.strip()
|
|
115
|
+
if not GITHUB_ID_PATTERN.fullmatch(normalized) or "--" in normalized:
|
|
116
|
+
raise ValueError(
|
|
117
|
+
"invalid GitHub ID; use the 1-39 character username without '@'"
|
|
118
|
+
)
|
|
119
|
+
return normalized
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def validate_email(email: str) -> str:
|
|
123
|
+
"""Validate a bounded ASCII contributor email address."""
|
|
124
|
+
normalized = email.strip()
|
|
125
|
+
if len(normalized) > MAX_EMAIL_LENGTH or normalized.count("@") != 1:
|
|
126
|
+
raise ValueError("invalid contributor email address")
|
|
127
|
+
local, domain = normalized.rsplit("@", 1)
|
|
128
|
+
labels = domain.split(".")
|
|
129
|
+
if (
|
|
130
|
+
not local
|
|
131
|
+
or len(local) > 64
|
|
132
|
+
or local.startswith(".")
|
|
133
|
+
or local.endswith(".")
|
|
134
|
+
or ".." in local
|
|
135
|
+
or not EMAIL_LOCAL_PATTERN.fullmatch(local)
|
|
136
|
+
or len(labels) < 2
|
|
137
|
+
or any(
|
|
138
|
+
not label
|
|
139
|
+
or len(label) > 63
|
|
140
|
+
or not EMAIL_DOMAIN_LABEL_PATTERN.fullmatch(label)
|
|
141
|
+
for label in labels
|
|
142
|
+
)
|
|
143
|
+
):
|
|
144
|
+
raise ValueError("invalid contributor email address")
|
|
145
|
+
return normalized
|
|
146
|
+
|
|
147
|
+
|
|
105
148
|
def validate_artifact_name(name: str) -> str:
|
|
106
149
|
"""Require the canonical public trajectory object namespace."""
|
|
107
150
|
if not ARTIFACT_NAME_PATTERN.fullmatch(name):
|
|
@@ -119,6 +162,8 @@ def stage_trajectory_capture(
|
|
|
119
162
|
source_id: str,
|
|
120
163
|
redact: bool = True,
|
|
121
164
|
uploaded_by: str | None = None,
|
|
165
|
+
github_id: str | None = None,
|
|
166
|
+
email: str | None = None,
|
|
122
167
|
) -> Iterator[StagedCapture]:
|
|
123
168
|
"""Validate and stage a trajectory capture without mutating its source."""
|
|
124
169
|
source_id = validate_source_id(source_id)
|
|
@@ -126,6 +171,14 @@ def stage_trajectory_capture(
|
|
|
126
171
|
raise ValueError(
|
|
127
172
|
f"trajectory contributor label exceeds {MAX_UPLOADED_BY_LENGTH} characters"
|
|
128
173
|
)
|
|
174
|
+
contributor: dict[str, str] | None = None
|
|
175
|
+
if github_id is not None or email is not None:
|
|
176
|
+
if github_id is None or email is None:
|
|
177
|
+
raise ValueError("GitHub ID and email must be provided together")
|
|
178
|
+
contributor = {
|
|
179
|
+
"github_id": validate_github_id(github_id),
|
|
180
|
+
"email": validate_email(email),
|
|
181
|
+
}
|
|
129
182
|
resolved = _resolve_input(path)
|
|
130
183
|
if len(resolved.files) > MAX_ARTIFACTS:
|
|
131
184
|
raise ValueError(
|
|
@@ -176,6 +229,7 @@ def stage_trajectory_capture(
|
|
|
176
229
|
payloads=payloads,
|
|
177
230
|
metadata_dir=resolved.metadata_dir,
|
|
178
231
|
uploaded_by=uploaded_by,
|
|
232
|
+
contributor=contributor,
|
|
179
233
|
redact=redact,
|
|
180
234
|
replacement_count=replacement_count,
|
|
181
235
|
)
|
|
@@ -183,6 +237,8 @@ def stage_trajectory_capture(
|
|
|
183
237
|
redacted_manifest, manifest_replacements = redact_value(manifest)
|
|
184
238
|
if redacted_manifest["source_id"] != manifest["source_id"]:
|
|
185
239
|
raise ValueError("source id contains a secret-like value")
|
|
240
|
+
if redacted_manifest.get("contributor") != manifest.get("contributor"):
|
|
241
|
+
raise ValueError("contributor metadata resembles a secret-like value")
|
|
186
242
|
replacement_count += manifest_replacements
|
|
187
243
|
redacted_manifest["redaction"]["replacements"] = replacement_count
|
|
188
244
|
manifest = redacted_manifest
|
|
@@ -384,11 +440,12 @@ def _build_manifest(
|
|
|
384
440
|
payloads: list[StagedFile],
|
|
385
441
|
metadata_dir: Path | None,
|
|
386
442
|
uploaded_by: str | None,
|
|
443
|
+
contributor: dict[str, str] | None,
|
|
387
444
|
redact: bool,
|
|
388
445
|
replacement_count: int,
|
|
389
446
|
) -> dict[str, Any]:
|
|
390
|
-
|
|
391
|
-
"schema_version": "1.0.0",
|
|
447
|
+
manifest = {
|
|
448
|
+
"schema_version": "1.1.0" if contributor is not None else "1.0.0",
|
|
392
449
|
"kind": "bronze.trajectory",
|
|
393
450
|
"created_at": datetime.now(UTC)
|
|
394
451
|
.replace(microsecond=0)
|
|
@@ -405,6 +462,9 @@ def _build_manifest(
|
|
|
405
462
|
],
|
|
406
463
|
"redaction": {"applied": redact, "replacements": replacement_count},
|
|
407
464
|
}
|
|
465
|
+
if contributor is not None:
|
|
466
|
+
manifest["contributor"] = contributor
|
|
467
|
+
return manifest
|
|
408
468
|
|
|
409
469
|
|
|
410
470
|
def _load_run_metadata(metadata_dir: Path | None) -> dict[str, Any]:
|
|
@@ -271,6 +271,53 @@ def test_staging_enforces_contributor_label_bound(tmp_path: Path) -> None:
|
|
|
271
271
|
).__enter__()
|
|
272
272
|
|
|
273
273
|
|
|
274
|
+
def test_manifest_contains_validated_contributor_provenance(tmp_path: Path) -> None:
|
|
275
|
+
"""CLI contributor parameters survive in the manifest uploaded last."""
|
|
276
|
+
trial = _trial(tmp_path)
|
|
277
|
+
contributor = {
|
|
278
|
+
"github_id": "benchflow-ai",
|
|
279
|
+
"email": "contributor+demo@benchflow.ai",
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
with stage_trajectory_capture(
|
|
283
|
+
trial,
|
|
284
|
+
source_id="demo",
|
|
285
|
+
github_id=contributor["github_id"],
|
|
286
|
+
email=contributor["email"],
|
|
287
|
+
) as staged:
|
|
288
|
+
assert staged.manifest["schema_version"] == "1.1.0"
|
|
289
|
+
assert staged.manifest["contributor"] == contributor
|
|
290
|
+
persisted = json.loads(staged.files[-1].local_path.read_text(encoding="utf-8"))
|
|
291
|
+
assert persisted["contributor"] == contributor
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
@pytest.mark.parametrize(
|
|
295
|
+
("github_id", "email", "message"),
|
|
296
|
+
[
|
|
297
|
+
("@benchflow-ai", "user@benchflow.ai", "GitHub ID"),
|
|
298
|
+
("benchflow--ai", "user@benchflow.ai", "GitHub ID"),
|
|
299
|
+
("benchflow-ai", "not-an-email", "email"),
|
|
300
|
+
("benchflow-ai", "user@localhost", "email"),
|
|
301
|
+
(None, "user@benchflow.ai", "provided together"),
|
|
302
|
+
("benchflow-ai", None, "provided together"),
|
|
303
|
+
],
|
|
304
|
+
)
|
|
305
|
+
def test_staging_rejects_invalid_or_partial_contributor_provenance(
|
|
306
|
+
tmp_path: Path,
|
|
307
|
+
github_id: str | None,
|
|
308
|
+
email: str | None,
|
|
309
|
+
message: str,
|
|
310
|
+
) -> None:
|
|
311
|
+
"""Malformed contributor metadata never reaches a transport."""
|
|
312
|
+
with pytest.raises(ValueError, match=message):
|
|
313
|
+
stage_trajectory_capture(
|
|
314
|
+
_trial(tmp_path),
|
|
315
|
+
source_id="demo",
|
|
316
|
+
github_id=github_id,
|
|
317
|
+
email=email,
|
|
318
|
+
).__enter__()
|
|
319
|
+
|
|
320
|
+
|
|
274
321
|
@pytest.mark.parametrize(
|
|
275
322
|
"filename", ["capture with space.jsonl", "capture.JSONL", "x" * 129 + ".jsonl"]
|
|
276
323
|
)
|
|
@@ -7,6 +7,7 @@ import logging
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from types import SimpleNamespace
|
|
9
9
|
|
|
10
|
+
import click
|
|
10
11
|
import httpx
|
|
11
12
|
import pytest
|
|
12
13
|
from typer.testing import CliRunner
|
|
@@ -16,6 +17,8 @@ from benchflow.publish.broker import upload_capture_via_broker
|
|
|
16
17
|
from benchflow.publish.traj_capture import stage_trajectory_capture
|
|
17
18
|
|
|
18
19
|
runner = CliRunner()
|
|
20
|
+
GITHUB_ID = "benchflow-user"
|
|
21
|
+
EMAIL = "user@example.com"
|
|
19
22
|
|
|
20
23
|
|
|
21
24
|
def _trial(tmp_path: Path) -> Path:
|
|
@@ -28,6 +31,19 @@ def _trial(tmp_path: Path) -> Path:
|
|
|
28
31
|
return trial
|
|
29
32
|
|
|
30
33
|
|
|
34
|
+
def _upload_command(path: Path, *args: str) -> list[str]:
|
|
35
|
+
return [
|
|
36
|
+
"traj",
|
|
37
|
+
"upload",
|
|
38
|
+
str(path),
|
|
39
|
+
"--github-id",
|
|
40
|
+
GITHUB_ID,
|
|
41
|
+
"--email",
|
|
42
|
+
EMAIL,
|
|
43
|
+
*args,
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
|
|
31
47
|
def test_stock_cli_has_the_verified_public_broker() -> None:
|
|
32
48
|
"""A wheel install can contribute without private endpoint configuration."""
|
|
33
49
|
from benchflow.cli.traj import DEFAULT_TRAJ_BROKER_URL
|
|
@@ -72,12 +88,13 @@ def test_dry_run_stages_without_constructing_a_transport(
|
|
|
72
88
|
raise AssertionError("network client constructed during --dry-run")
|
|
73
89
|
|
|
74
90
|
monkeypatch.setattr(httpx, "Client", fail_client)
|
|
75
|
-
result = runner.invoke(app,
|
|
91
|
+
result = runner.invoke(app, _upload_command(trial, "--dry-run"))
|
|
76
92
|
|
|
77
93
|
assert result.exit_code == 0, result.output
|
|
78
94
|
assert "sha256:" in result.output
|
|
79
95
|
assert "trajectory/acp_trajectory.jsonl" in result.output
|
|
80
96
|
assert "manifest.json" in result.output
|
|
97
|
+
assert EMAIL not in result.output
|
|
81
98
|
|
|
82
99
|
|
|
83
100
|
def test_direct_mode_reports_azure_destination(
|
|
@@ -87,6 +104,10 @@ def test_direct_mode_reports_azure_destination(
|
|
|
87
104
|
trial = _trial(tmp_path)
|
|
88
105
|
|
|
89
106
|
def fake_upload(staged, *, container_url):
|
|
107
|
+
assert staged.manifest["contributor"] == {
|
|
108
|
+
"github_id": GITHUB_ID,
|
|
109
|
+
"email": EMAIL,
|
|
110
|
+
}
|
|
90
111
|
return SimpleNamespace(
|
|
91
112
|
url=f"{container_url}/sources/demo/{staged.traj_digest}/",
|
|
92
113
|
uploaded=("payload", "manifest"),
|
|
@@ -98,14 +119,12 @@ def test_direct_mode_reports_azure_destination(
|
|
|
98
119
|
)
|
|
99
120
|
result = runner.invoke(
|
|
100
121
|
app,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
"upload",
|
|
104
|
-
str(trial),
|
|
122
|
+
_upload_command(
|
|
123
|
+
trial,
|
|
105
124
|
"--direct",
|
|
106
125
|
"--container-url",
|
|
107
126
|
"https://tasksminerdata.blob.core.windows.net/bronze",
|
|
108
|
-
|
|
127
|
+
),
|
|
109
128
|
)
|
|
110
129
|
|
|
111
130
|
assert result.exit_code == 0, result.output
|
|
@@ -130,9 +149,20 @@ def test_broker_mode_uses_exact_manifest_and_server_order(
|
|
|
130
149
|
"source_id",
|
|
131
150
|
"traj_digest",
|
|
132
151
|
"uploaded_by",
|
|
152
|
+
"contributor",
|
|
133
153
|
"artifacts",
|
|
134
154
|
}
|
|
155
|
+
assert body["contributor"] == {
|
|
156
|
+
"github_id": GITHUB_ID,
|
|
157
|
+
"email": EMAIL,
|
|
158
|
+
}
|
|
159
|
+
assert body["schema_version"] == "1.1.0"
|
|
135
160
|
return httpx.Response(200, json=_broker_payload(request))
|
|
161
|
+
if request.url.path.endswith("manifest.json"):
|
|
162
|
+
assert json.loads(request.content)["contributor"] == {
|
|
163
|
+
"github_id": GITHUB_ID,
|
|
164
|
+
"email": EMAIL,
|
|
165
|
+
}
|
|
136
166
|
assert request.headers["x-ms-blob-type"] == "BlockBlob"
|
|
137
167
|
assert request.headers["if-none-match"] == "*"
|
|
138
168
|
return httpx.Response(201)
|
|
@@ -140,7 +170,7 @@ def test_broker_mode_uses_exact_manifest_and_server_order(
|
|
|
140
170
|
client = httpx.Client(transport=httpx.MockTransport(handler))
|
|
141
171
|
monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: client)
|
|
142
172
|
monkeypatch.setenv("BENCHFLOW_TRAJ_BROKER_URL", "https://broker.test")
|
|
143
|
-
result = runner.invoke(app,
|
|
173
|
+
result = runner.invoke(app, _upload_command(trial))
|
|
144
174
|
|
|
145
175
|
assert result.exit_code == 0, result.output
|
|
146
176
|
assert [request.method for request in requests] == ["POST", "PUT", "PUT"]
|
|
@@ -198,12 +228,12 @@ def test_broker_conflict_is_success_and_rate_limit_is_actionable(
|
|
|
198
228
|
)
|
|
199
229
|
)
|
|
200
230
|
monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: conflict)
|
|
201
|
-
result = runner.invoke(app,
|
|
231
|
+
result = runner.invoke(app, _upload_command(trial))
|
|
202
232
|
assert result.exit_code == 0, result.output
|
|
203
233
|
assert "Already uploaded" in result.output
|
|
204
234
|
|
|
205
235
|
monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: limited)
|
|
206
|
-
result = runner.invoke(app,
|
|
236
|
+
result = runner.invoke(app, _upload_command(trial))
|
|
207
237
|
assert result.exit_code == 1
|
|
208
238
|
assert "retry after 60" in result.output
|
|
209
239
|
|
|
@@ -215,7 +245,7 @@ def test_missing_broker_names_both_available_modes(
|
|
|
215
245
|
trial = _trial(tmp_path)
|
|
216
246
|
monkeypatch.delenv("BENCHFLOW_TRAJ_BROKER_URL", raising=False)
|
|
217
247
|
monkeypatch.setattr("benchflow.cli.traj.DEFAULT_TRAJ_BROKER_URL", None)
|
|
218
|
-
result = runner.invoke(app,
|
|
248
|
+
result = runner.invoke(app, _upload_command(trial))
|
|
219
249
|
|
|
220
250
|
assert result.exit_code == 1
|
|
221
251
|
assert "BENCHFLOW_TRAJ_BROKER_URL" in result.output
|
|
@@ -231,7 +261,7 @@ def test_validation_failure_names_the_bad_file(
|
|
|
231
261
|
path = trial / "trajectory" / "acp_trajectory.jsonl"
|
|
232
262
|
path.write_text("{bad\n", encoding="utf-8")
|
|
233
263
|
monkeypatch.setenv("BENCHFLOW_TRAJ_BROKER_URL", "https://broker.test")
|
|
234
|
-
result = runner.invoke(app,
|
|
264
|
+
result = runner.invoke(app, _upload_command(trial))
|
|
235
265
|
|
|
236
266
|
assert result.exit_code == 1
|
|
237
267
|
assert "acp_trajectory.jsonl" in result.output.replace("\n", "")
|
|
@@ -290,7 +320,7 @@ def test_broker_put_conflicts_are_cloud_neutral_skips(
|
|
|
290
320
|
|
|
291
321
|
|
|
292
322
|
def test_help_exposes_only_the_planned_upload_command() -> None:
|
|
293
|
-
"""
|
|
323
|
+
"""Guards PR #992 while ignoring Rich's environment-specific ANSI styling."""
|
|
294
324
|
traj_group = next(group for group in app.registered_groups if group.name == "traj")
|
|
295
325
|
assert {
|
|
296
326
|
command.name for command in traj_group.typer_instance.registered_commands
|
|
@@ -299,3 +329,45 @@ def test_help_exposes_only_the_planned_upload_command() -> None:
|
|
|
299
329
|
result = runner.invoke(app, ["traj", "--help"])
|
|
300
330
|
assert result.exit_code == 0
|
|
301
331
|
assert "upload" in result.output
|
|
332
|
+
|
|
333
|
+
upload_help = runner.invoke(app, ["traj", "upload", "--help"])
|
|
334
|
+
assert upload_help.exit_code == 0
|
|
335
|
+
upload_help_output = click.unstyle(upload_help.output)
|
|
336
|
+
assert "--github-id" in upload_help_output
|
|
337
|
+
assert "--email" in upload_help_output
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def test_upload_requires_github_id_and_email_parameters(tmp_path: Path) -> None:
|
|
341
|
+
"""Guards PR #992: every CLI upload must name its contributor."""
|
|
342
|
+
trial = _trial(tmp_path)
|
|
343
|
+
result = runner.invoke(app, ["traj", "upload", str(trial)])
|
|
344
|
+
|
|
345
|
+
assert result.exit_code == 2
|
|
346
|
+
assert "--github-id" in click.unstyle(result.output)
|
|
347
|
+
|
|
348
|
+
result = runner.invoke(
|
|
349
|
+
app,
|
|
350
|
+
["traj", "upload", str(trial), "--github-id", GITHUB_ID],
|
|
351
|
+
)
|
|
352
|
+
assert result.exit_code == 2
|
|
353
|
+
assert "--email" in click.unstyle(result.output)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
@pytest.mark.parametrize(
|
|
357
|
+
("args", "message"),
|
|
358
|
+
[
|
|
359
|
+
(("--github-id", "@not-a-github-id", "--email", EMAIL), "GitHub ID"),
|
|
360
|
+
(("--github-id", GITHUB_ID, "--email", "not-an-email"), "email"),
|
|
361
|
+
],
|
|
362
|
+
)
|
|
363
|
+
def test_upload_validates_contributor_parameters_locally(
|
|
364
|
+
tmp_path: Path, args: tuple[str, ...], message: str
|
|
365
|
+
) -> None:
|
|
366
|
+
"""Malformed contributor provenance fails before the upload handshake."""
|
|
367
|
+
result = runner.invoke(
|
|
368
|
+
app,
|
|
369
|
+
["traj", "upload", str(_trial(tmp_path)), *args, "--dry-run"],
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
assert result.exit_code == 1
|
|
373
|
+
assert message in result.output
|
|
@@ -53,7 +53,7 @@ def _trial(tmp_path: Path, text: str = "safe") -> Path:
|
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
def _request_from_manifest(manifest: dict) -> dict:
|
|
56
|
-
|
|
56
|
+
request = {
|
|
57
57
|
key: manifest[key]
|
|
58
58
|
for key in (
|
|
59
59
|
"schema_version",
|
|
@@ -64,6 +64,9 @@ def _request_from_manifest(manifest: dict) -> dict:
|
|
|
64
64
|
"artifacts",
|
|
65
65
|
)
|
|
66
66
|
}
|
|
67
|
+
if "contributor" in manifest:
|
|
68
|
+
request["contributor"] = manifest["contributor"]
|
|
69
|
+
return request
|
|
67
70
|
|
|
68
71
|
|
|
69
72
|
class FakeBroker:
|
|
@@ -158,7 +161,12 @@ def test_upload_contract_recomputes_digest_and_rejects_object_injection(
|
|
|
158
161
|
) -> None:
|
|
159
162
|
"""The broker accepts only content-addressed trajectory JSONL object names."""
|
|
160
163
|
trial = _trial(tmp_path)
|
|
161
|
-
with stage_trajectory_capture(
|
|
164
|
+
with stage_trajectory_capture(
|
|
165
|
+
trial,
|
|
166
|
+
source_id="demo",
|
|
167
|
+
github_id="benchflow-ai",
|
|
168
|
+
email="contributor@benchflow.ai",
|
|
169
|
+
) as staged:
|
|
162
170
|
body = _request_from_manifest(staged.manifest)
|
|
163
171
|
request = UploadRequest.model_validate(body)
|
|
164
172
|
assert request.traj_digest == staged.manifest["traj_digest"]
|
|
@@ -167,19 +175,90 @@ def test_upload_contract_recomputes_digest_and_rejects_object_injection(
|
|
|
167
175
|
with pytest.raises(ValidationError, match="outside trajectory"):
|
|
168
176
|
UploadRequest.model_validate(body)
|
|
169
177
|
|
|
170
|
-
with stage_trajectory_capture(
|
|
178
|
+
with stage_trajectory_capture(
|
|
179
|
+
trial,
|
|
180
|
+
source_id="demo",
|
|
181
|
+
github_id="benchflow-ai",
|
|
182
|
+
email="contributor@benchflow.ai",
|
|
183
|
+
) as staged:
|
|
171
184
|
body = _request_from_manifest(staged.manifest)
|
|
172
185
|
body["traj_digest"] = "sha256:" + "0" * 64
|
|
173
186
|
with pytest.raises(ValidationError, match="does not match"):
|
|
174
187
|
UploadRequest.model_validate(body)
|
|
175
188
|
|
|
176
189
|
|
|
190
|
+
def test_contributor_contract_is_validated_at_broker_and_manifest_boundaries(
|
|
191
|
+
tmp_path: Path,
|
|
192
|
+
) -> None:
|
|
193
|
+
"""Self-asserted GitHub and email provenance remains typed server-side."""
|
|
194
|
+
trial = _trial(tmp_path)
|
|
195
|
+
with stage_trajectory_capture(
|
|
196
|
+
trial,
|
|
197
|
+
source_id="demo",
|
|
198
|
+
github_id="benchflow-ai",
|
|
199
|
+
email="contributor@benchflow.ai",
|
|
200
|
+
) as staged:
|
|
201
|
+
request = UploadRequest.model_validate(_request_from_manifest(staged.manifest))
|
|
202
|
+
manifest = validate_manifest_bytes(staged.files[-1].local_path.read_bytes())
|
|
203
|
+
|
|
204
|
+
assert request.contributor is not None
|
|
205
|
+
assert request.schema_version == "1.1.0"
|
|
206
|
+
assert request.contributor.github_id == "benchflow-ai"
|
|
207
|
+
assert request.contributor.email == "contributor@benchflow.ai"
|
|
208
|
+
assert manifest.contributor == request.contributor
|
|
209
|
+
|
|
210
|
+
invalid = _request_from_manifest(staged.manifest)
|
|
211
|
+
invalid["contributor"] = {
|
|
212
|
+
"github_id": "@invalid",
|
|
213
|
+
"email": "not-an-email",
|
|
214
|
+
}
|
|
215
|
+
with pytest.raises(ValidationError):
|
|
216
|
+
UploadRequest.model_validate(invalid)
|
|
217
|
+
|
|
218
|
+
missing = _request_from_manifest(staged.manifest)
|
|
219
|
+
missing.pop("contributor")
|
|
220
|
+
with pytest.raises(ValidationError, match="Field required"):
|
|
221
|
+
UploadRequest.model_validate(missing)
|
|
222
|
+
|
|
223
|
+
legacy_request = dict(missing)
|
|
224
|
+
legacy_request["schema_version"] = "1.0.0"
|
|
225
|
+
with pytest.raises(ValidationError, match="literal_error"):
|
|
226
|
+
UploadRequest.model_validate(legacy_request)
|
|
227
|
+
|
|
228
|
+
with stage_trajectory_capture(trial, source_id="legacy") as legacy:
|
|
229
|
+
legacy_manifest = validate_manifest_bytes(
|
|
230
|
+
legacy.files[-1].local_path.read_bytes()
|
|
231
|
+
)
|
|
232
|
+
assert legacy_manifest.schema_version == "1.0.0"
|
|
233
|
+
assert legacy_manifest.contributor is None
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def test_broker_rejects_new_legacy_uploads_without_contributor(
|
|
237
|
+
tmp_path: Path,
|
|
238
|
+
) -> None:
|
|
239
|
+
"""Guards PR #992 against granting contributor-free schema 1.0 uploads."""
|
|
240
|
+
with stage_trajectory_capture(_trial(tmp_path), source_id="legacy") as staged:
|
|
241
|
+
body = _request_from_manifest(staged.manifest)
|
|
242
|
+
|
|
243
|
+
response = TestClient(create_app(FakeBroker(AssertionError()))).post(
|
|
244
|
+
"/v1/uploads", json=body
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
assert response.status_code == 400
|
|
248
|
+
assert response.json()["detail"][0]["type"] == "literal_error"
|
|
249
|
+
|
|
250
|
+
|
|
177
251
|
def test_broker_http_surface_returns_scoped_grants_and_protocol_statuses(
|
|
178
252
|
tmp_path: Path,
|
|
179
253
|
) -> None:
|
|
180
254
|
"""The public endpoint emits v1 grants, conflict, and Retry-After responses."""
|
|
181
255
|
trial = _trial(tmp_path)
|
|
182
|
-
with stage_trajectory_capture(
|
|
256
|
+
with stage_trajectory_capture(
|
|
257
|
+
trial,
|
|
258
|
+
source_id="demo",
|
|
259
|
+
github_id="benchflow-ai",
|
|
260
|
+
email="contributor@benchflow.ai",
|
|
261
|
+
) as staged:
|
|
183
262
|
body = _request_from_manifest(staged.manifest)
|
|
184
263
|
grant = UploadGrant(
|
|
185
264
|
upload_id="u_demo",
|
|
@@ -240,7 +319,12 @@ def test_broker_validation_errors_are_fail_closed_and_json_safe(
|
|
|
240
319
|
) -> None:
|
|
241
320
|
"""Guards the live Azure fix after malformed handshakes returned HTTP 500."""
|
|
242
321
|
trial = _trial(tmp_path)
|
|
243
|
-
with stage_trajectory_capture(
|
|
322
|
+
with stage_trajectory_capture(
|
|
323
|
+
trial,
|
|
324
|
+
source_id="demo",
|
|
325
|
+
github_id="benchflow-ai",
|
|
326
|
+
email="contributor@benchflow.ai",
|
|
327
|
+
) as staged:
|
|
244
328
|
body = _request_from_manifest(staged.manifest)
|
|
245
329
|
|
|
246
330
|
injected = json.loads(json.dumps(body))
|
|
@@ -550,7 +634,12 @@ def test_broker_regrant_does_not_downgrade_ledger_state(
|
|
|
550
634
|
) -> None:
|
|
551
635
|
"""Guards PR #989 against a retry clearing an active validation lease."""
|
|
552
636
|
trial = _trial(tmp_path)
|
|
553
|
-
with stage_trajectory_capture(
|
|
637
|
+
with stage_trajectory_capture(
|
|
638
|
+
trial,
|
|
639
|
+
source_id="demo",
|
|
640
|
+
github_id="benchflow-ai",
|
|
641
|
+
email="contributor@benchflow.ai",
|
|
642
|
+
) as staged:
|
|
554
643
|
request = UploadRequest.model_validate(_request_from_manifest(staged.manifest))
|
|
555
644
|
digest = staged.traj_digest
|
|
556
645
|
table = FakeTable(
|
|
@@ -589,7 +678,12 @@ def test_broker_persists_declared_artifact_sizes(
|
|
|
589
678
|
) -> None:
|
|
590
679
|
"""Guards PR #989 against artifact-only uploads exceeding their declaration."""
|
|
591
680
|
trial = _trial(tmp_path)
|
|
592
|
-
with stage_trajectory_capture(
|
|
681
|
+
with stage_trajectory_capture(
|
|
682
|
+
trial,
|
|
683
|
+
source_id="demo",
|
|
684
|
+
github_id="benchflow-ai",
|
|
685
|
+
email="contributor@benchflow.ai",
|
|
686
|
+
) as staged:
|
|
593
687
|
request = UploadRequest.model_validate(_request_from_manifest(staged.manifest))
|
|
594
688
|
expected = {item.name: item.bytes for item in request.artifacts}
|
|
595
689
|
table = FakeTable()
|
|
@@ -619,7 +713,12 @@ def test_broker_reopens_rejected_digest_in_isolated_attempt(
|
|
|
619
713
|
) -> None:
|
|
620
714
|
"""Guards PR #989 against a rejected attempt poisoning a valid digest."""
|
|
621
715
|
trial = _trial(tmp_path)
|
|
622
|
-
with stage_trajectory_capture(
|
|
716
|
+
with stage_trajectory_capture(
|
|
717
|
+
trial,
|
|
718
|
+
source_id="demo",
|
|
719
|
+
github_id="benchflow-ai",
|
|
720
|
+
email="contributor@benchflow.ai",
|
|
721
|
+
) as staged:
|
|
623
722
|
request = UploadRequest.model_validate(_request_from_manifest(staged.manifest))
|
|
624
723
|
digest = staged.traj_digest
|
|
625
724
|
old_attempt = "u_" + "0" * 32
|
|
@@ -660,7 +759,12 @@ def test_terminal_digest_handshakes_consume_rate_limit(
|
|
|
660
759
|
) -> None:
|
|
661
760
|
"""Guards PR #989 against terminal ledger lookups bypassing broker quotas."""
|
|
662
761
|
trial = _trial(tmp_path)
|
|
663
|
-
with stage_trajectory_capture(
|
|
762
|
+
with stage_trajectory_capture(
|
|
763
|
+
trial,
|
|
764
|
+
source_id="demo",
|
|
765
|
+
github_id="benchflow-ai",
|
|
766
|
+
email="contributor@benchflow.ai",
|
|
767
|
+
) as staged:
|
|
664
768
|
request = UploadRequest.model_validate(_request_from_manifest(staged.manifest))
|
|
665
769
|
digest = staged.traj_digest
|
|
666
770
|
table = FakeTable(
|
|
@@ -683,7 +787,12 @@ def test_terminal_digest_handshakes_consume_rate_limit(
|
|
|
683
787
|
|
|
684
788
|
def _quarantine_capture(tmp_path: Path) -> tuple[str, dict[str, bytes]]:
|
|
685
789
|
trial = _trial(tmp_path)
|
|
686
|
-
with stage_trajectory_capture(
|
|
790
|
+
with stage_trajectory_capture(
|
|
791
|
+
trial,
|
|
792
|
+
source_id="demo",
|
|
793
|
+
github_id="benchflow-ai",
|
|
794
|
+
email="contributor@benchflow.ai",
|
|
795
|
+
) as staged:
|
|
687
796
|
digest = staged.traj_digest
|
|
688
797
|
prefix = f"inbox/{digest}/"
|
|
689
798
|
blobs = {
|
|
@@ -714,6 +823,13 @@ def test_queue_validator_promotes_manifest_last_and_cleans_quarantine(
|
|
|
714
823
|
|
|
715
824
|
assert validator.run_once() is True
|
|
716
825
|
assert container.uploaded[-1] == f"sources/community/{digest}/manifest.json"
|
|
826
|
+
promoted_manifest = json.loads(
|
|
827
|
+
container.blobs[f"sources/community/{digest}/manifest.json"]
|
|
828
|
+
)
|
|
829
|
+
assert promoted_manifest["contributor"] == {
|
|
830
|
+
"github_id": "benchflow-ai",
|
|
831
|
+
"email": "contributor@benchflow.ai",
|
|
832
|
+
}
|
|
717
833
|
assert not any(name.startswith(f"inbox/{digest}/") for name in container.blobs)
|
|
718
834
|
assert table.entities[-1]["status"] == "ingested"
|
|
719
835
|
assert queue.deleted == [("m1", "p1")]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|