benchflow 0.7.2.dev1994__tar.gz → 0.7.2.dev2000__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/CHANGELOG.md +41 -3
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/PKG-INFO +1 -1
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/pyproject.toml +1 -1
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/_traj_upload_ui.py +29 -1
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/main.py +15 -1
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/traj.py +47 -22
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/redact.py +138 -23
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/traj_capture.py +24 -4
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/traj_report.py +14 -1
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/types.py +86 -18
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/viewer.py +124 -44
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_edge_case_hardening.py +60 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traj_report.py +83 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traj_upload_cli.py +57 -3
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_viewer_confirm.py +78 -4
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_redaction.py +142 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/.gitignore +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/LICENSE +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/README.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_dotenv.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_paths.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_run.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_types.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/benchmark_repos.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/build_context_stage.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/config_override.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/content_address.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/dataset_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/env_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/evaluation_results.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/hf_datasets.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/json_safe.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/learner_memory.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/live_activity.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/result_metadata.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/reward_events.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/scoring.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/source_provenance.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/task_authoring/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/task_authoring/_evidence_paths.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/task_authoring/acceptance_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/task_authoring/scaffolding.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/task_authoring/structural_checks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/text.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/_utils/yaml_loader.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/client.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/container_transport.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/selection.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/session.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/transport.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/acp/types.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/_toolathlon.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/_toolathlon_container.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/_toolathlon_fake_mail.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/_toolathlon_poste_users.tsv +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/_toolathlon_services.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/harbor.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/inbound.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/inspect_ai.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/ors.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/resources/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/resources/mcp_atlas_bridge.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/resources/mcp_atlas_judge.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/adapters/source.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agent_router.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agent_router_guide.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agent_router_parity.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agent_router_scaffold.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/codex_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/credentials.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/deepagents_acp_shim.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/errors.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/harvey_lab_acp_shim.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/install.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/openclaw_acp_shim.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/pi_acp_launcher.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/providers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/agents/remote_manifests.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/branch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/_failure_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/_hosted_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/_live_progress.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/_options.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/_shared.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/adopt.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/agent.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/continue_cmd.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/environment.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/eval_artifacts.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/eval_lift.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/hub.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/monitor.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/review.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/sandbox.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/skills.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/tasks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/trace_import.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/cli/train.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/continue_run/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/continue_run/batch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/continue_run/orchestrator.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/continue_run/replay_proxy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/continue_run/run_folder.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/continue_run/sandbox_proxy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/contracts/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/contracts/planes.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/contracts/user.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/demo_task/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/demo_task/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/demo_task/task.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/demo_task/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/demo_task/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/diagnostics.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/_registry/env0@outage.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/_registry/env0@prod.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/manifest_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/environment/readiness.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/eval_artifacts.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/eval_lift.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/eval_plan.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/eval_sharding.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/eval_worker.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/evaluation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/experimental/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/experimental/mcp/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/experimental/mcp/hooks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/experimental/mcp/reviewer_server.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/hosted_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/hub/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/hub/harbor_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/integrations/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/integrations/trl/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/integrations/trl/spec.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/learner_skills.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/learner_store.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/loop_strategies.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/loop_sweep.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/metrics.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/models.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/monitor.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/litellm_bedrock_patch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/litellm_bedrock_preflight.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/litellm_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/litellm_logging.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/litellm_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/providers/runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/_progress.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/azure_blob.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/broker.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/publish/huggingface.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/py.typed +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/review/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/review/config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/review/default-rubric.json +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/review/prompts.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/review/runner.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/review/wrapper.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/README.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/builtins.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/events.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/file_readers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/llm.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/memory_scorer.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/node.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/rubric.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/rubric_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rewards/validation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_deadline.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_results.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_setup.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_skills.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_usage.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/_user_loop.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/session_factory_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout/task_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout_branch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/rollout_planes.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_base.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_cache_reclaim.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_compose.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_compose_files/docker-compose-base.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_compose_files/docker-compose-build.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_compose_files/docker-compose-no-network.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_compose_files/docker-compose-prebuilt.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/_sdk_ops.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/agentcore.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/agentcore_builder.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/agentcore_image.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/agentcore_provisioning.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/agentcore_reaper.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/agentcore_sealed.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/apple_container.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/daytona.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/daytona_dind.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/daytona_pty.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/daytona_reaper.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/daytona_strategies.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/docker.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/lockdown.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/metadata.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/modal_impl.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/process/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/process/_base.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/process/agentcore.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/process/apple.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/process/daytona.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/process/docker.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/providers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/services.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/setup.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/snapshot.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sandbox/user.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/scenes.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/sdk.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/self_gen.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/skill_eval/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/skill_eval/_core.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/skill_eval/gepa_export.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/skill_eval/schema.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/skill_policy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/skills.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/_document_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/_document_normalize.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/_document_parse.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/_document_profiles.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/acceptance_live.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/acceptance_live_model.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/acceptance_live_report.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/acceptance_live_validation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/discovery.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/document.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/export.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/imports.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/output_format.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/package.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/paths.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/prompts.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/runtime_capabilities.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/runtime_view.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/task.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_core.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_document.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_errors.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_judge_inputs.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_ors_episode.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_reward_kit.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_scan.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/task/verifier_script_strategy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/templates/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/templates/judge.py.tmpl +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/templates/test.sh.tmpl +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/traces/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/traces/huggingface.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/traces/local.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/traces/models.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/traces/parsers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/traces/task_gen.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/training/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/training/backends/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/training/backends/prime_rl.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/training/run_manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/_capture.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/_export_common.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/_llm_capture.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/call_purpose.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/export.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/export_adp.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/export_atif.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/export_prime_sft.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/export_trl_sft.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/metrics.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/results.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/sessions.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/tree.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/trl_sft_tokenization.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/src/benchflow/usage_tracking.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/acceptance_live_harness.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_manifest_dirscan.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_manifest_loader.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_manifest_parity.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_manifest_wiring.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_namespace_shorthand.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_plugin_entry_points.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/agents/test_remote_manifest_autoload.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/README.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/environment/docker-compose.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/environment/skills/conformance-writer/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/task.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/acp_smoke/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/conformance-results.json +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/proof_multi_agent.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/proof_snapshot.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/run_conformance.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conformance/self_gen_smoke_skills/skill-creator/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/conftest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/continue_run/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/continue_run/_helpers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/continue_run/test_batch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/continue_run/test_orchestrator.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/continue_run/test_replay_proxy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/continue_run/test_run_folder.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/test_chibench_manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/test_clawsbench_manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/test_manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/test_manifest_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/test_protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/environment/test_readiness.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/hello-world-task/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/hello-world-task/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/hello-world-task/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/hello-world-task/task.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/hello-world-task/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/hello-world-task/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/test_claude.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/test_codex.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/test_codex_custom_provider.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/test_gemini.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/test_openclaw.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/traces/minimal-claude.jsonl +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/examples/traces/minimal-opentraces.jsonl +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/README.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/alpha-task/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/alpha-task/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/alpha-task/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/alpha-task/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/beta-task/data/empty.bin +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/beta-task/data/unicode.txt +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/beta-task/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/beta-task/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/digest/not-a-task/notes.txt +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/mock_acp_agent.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/mock_acp_agent_interleaved.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/mock_acp_agent_multi_turn.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/mock_openai_responses_server.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/data/michigan_stations.txt +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/flood-detection/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/nws-flood-thresholds/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/environment/skills/usgs-data-download/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/flood-risk-analysis/tests/test_outputs.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/climate.csv +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/hydrology.csv +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/land_cover.csv +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/data/water_temperature.csv +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/contribution-analysis/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/meteorology-driver-classification/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/pca-decomposition/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/environment/skills/trend-analysis/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/lake-warming-attribution/tests/test_outputs.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/manifest.json +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/r2r_simulator.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/finite-horizon-lqr/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/integral-action-design/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/mpc-horizon-tuning/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/skills/state-space-linearization/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/environment/system_config.json +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/r2r-mpc-control/tests/test_outputs.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/generate_training_pcaps.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/local.rules +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/pcap-triage-tshark/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/pcap-triage-tshark/scripts/summarize_http_requests.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-offline-evejson/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-offline-evejson/scripts/run_suricata_offline.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/skills/suricata-rules-basics/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/environment/suricata.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/suricata-custom-exfil/tests/test_outputs.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/Dockerfile +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syz-extract-constants/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syzkaller-build-loop/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/environment/skills/syzlang-ioctl-basics/SKILL.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/instruction.md +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/solution/solve.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/task.toml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/tests/test.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/fixtures/skillsbench_slice/syzkaller-ppdev-syzlang/tests/test_outputs.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/_consistency.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/agent_judge.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/check_adapter_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/check_hosted_env_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/check_results.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/check_skillsbench_harbor_parity.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/check_trace_to_task_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/claude-agent-acp.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/codex-acp.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/gemini.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/harvey-lab-harness.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/mimo.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/openclaw.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/opencode.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/openhands.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/configs/pi-acp.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/deepagents_harness.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/rubric_checks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/run.sh +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/run_suite.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/scenarios.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/integration/suites/release.yaml +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acceptance_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acceptance_live.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acceptance_live_execution.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acp.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acp_capability_advertising.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acp_mcp_servers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acp_model_config_dispatch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acp_pinned_protocol_guard.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_acp_setup_failure_propagation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_adapter_scripts.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_adapters.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_env_resolution.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_gemini_defaults.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_idle_timeout_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_model_decouple.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_router.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_router_cli_e2e.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_router_scaffold.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_setup.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agent_spec.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agentcore_builder.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agentcore_parallel.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agentcore_reaper.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agentcore_sandbox.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_agentcore_transport.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_api_error_capture.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_apple_container_sandbox.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_bare_model_provider.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_base_install_imports.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_bedrock_thinking.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_benchflow_experiment_review_validator.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_benchmark_repos_resolve_repo_path.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_branch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_build_context_stage.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_build_review_pack.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_capture_trajectory.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_check_skillsbench_harbor_parity.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_clawsbench_slice.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_adopt_aliases.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_arg_validation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_daytona.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_docs_drift.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_hub_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_live_progress.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_cli_misc.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_codex_review.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_codex_self_write_auth.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_config_override.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_config_redaction.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_connect_as_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_console_error_truncation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_console_progress.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_continuallearningbench_adapter.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_dataset_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_daytona_command_polling.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_daytona_dind_compose_up.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_daytona_download.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_daytona_litellm_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_daytona_poll_deadline.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_daytona_reap.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_deepagents_agent.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_diagnostics.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_docker_prune_scoping.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_docker_uploads.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_docs_examples.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eng50_capabilities.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_env_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_env_setup.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_environment_manifest_controls.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_artifact_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_filters_applied.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_lift.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_sharding.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_single_task_summary.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_source_provenance.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_worker_retry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_eval_zero_task_guard.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_evaluation_environment_manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_exclude_tasks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_filter_credentialed_cells.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_harvey_lab_shim.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_hilbench_adapter.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_hosted_env.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_hosted_env_rollout_contract.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_hub_harbor_registry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_inbound_adapter_manifest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_inbound_adapters.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_agent_judge.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_check_results.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_consistency.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_matrix.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_provider_selection.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_run_suite.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_integration_suite.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_internet_policy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_job.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_job_sequential_shared.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_job_sequential_shared_resume.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_judge_robustness.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_judge_symlink_ingestion.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_learner_skills.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_learner_skills_traversal.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_learner_store.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_learner_store_persistence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_hardening.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_logging.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_model_via_env_generic.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_reasoning_passthrough.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_litellm_smoke.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_live_llm_trajectory.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_llm_judge.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_llm_judge_event_tags.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_llm_judge_verifier.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_loop_strategies.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_loop_summary.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_loop_sweep.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_memory_scorer.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_metrics.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_mle_bench_adapter.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_mock_openai_responses_server.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_monitor_scaffold.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_native_acp_usage.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_no_cross_provider_fallback.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_notification_order_real.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_opencode_family_proxy_tracking.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_oracle.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_oracle_chokepoint.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_paths_safe.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_paths_symlink_helpers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_pi_acp_launcher.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_pi_acp_proxy_routing.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_process.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_project_skills_layout.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_provider_auth_detection.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_providers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_publish_azure_blob.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_reexport.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_registry_invariants.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_release_hardening_r5.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_release_version.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_release_workflow_wiring.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_resolve_env_helpers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_review_boundaries.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_review_rubric.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_review_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_reward_lenient.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_reward_node.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_reward_range.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_reward_unified_contract.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rewards.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rewards_jsonl.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_architecture.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_branch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_config_path_coercion.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_environment.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_hard_deadline.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_import_no_side_effects.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_on_ask_user_wiring.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_planes_contract.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_probe_sandbox_health.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_session_factory_dispatch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_setup_commands.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rollout_upload.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rubric_checks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_rubric_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_runtime_capabilities.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_runtime_config_wired.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_runtime_live_sandbox.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_exec_secret_handling.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_hardening.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_isolation_copy_traversal.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_live_process.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_multi_service.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_prebuilt_validation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_protocol.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_provider_registry_drift.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_setup.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_snapshot_contract.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_upload_symlink.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sandbox_verifier_workspace.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_scene.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_scene_outbox_trial.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_scene_parallel_group.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_scene_result_aggregation.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_scoring.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sdk_internals.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sdk_lockdown.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sdk_run_alias.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_sealed_channel_container.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_self_gen_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_self_gen_export_error_channel.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_self_gen_export_failures.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_self_gen_orchestration.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_session_factory_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_session_request_permission_dispatch.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_eval.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_eval_dryrun.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_eval_integration.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_eval_sweep.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_eval_traversal.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_invocation_artifacts.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skill_policy.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skills.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skills_dir_agent_home_link.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skillsbench_conversion_conformance.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_skillsbench_harbor_run_suite.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_smoke.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_smoke_wiring.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_source_adapters.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_subscription_auth.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_check_eval_consistency.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_digest.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_document.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_download.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_export.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_output_format.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_package.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_task_runtime_primitive.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_tasks.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_token_usage_normalization.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trace_import_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trace_task_gen_traversal.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trace_to_task_evidence.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traces_huggingface.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traces_parsers.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traces_task_gen.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_train_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_train_mode_artifact_emission.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_train_trl_cli.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traj_staging_phases.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_traj_upload_skill.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trajectory_integration.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trajectory_streaming.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trajectory_upload_layout.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trajectory_upload_service.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trial_agent_timeout_verify.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trial_install_agent_timeout.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trial_litellm_runtime.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_trl_integration.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_usage_litellm.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_usage_required.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_usage_tracking.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_user.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_v06_guard_ports.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verifier_document.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verifier_multi_container.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verifier_output.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verifier_output_freshness.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verifier_strategies.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verifier_wedge_retry.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_verify.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_workflow_action_pinning.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/test_yaml_config.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/__init__.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_export.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_export_adp.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_export_atif.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_export_common.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_export_nan_handling.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_export_prime_sft.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_step_granularity.py +0 -0
- {benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev2000}/tests/trajectories/test_tree.py +0 -0
|
@@ -3,6 +3,23 @@
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
5
|
### Added
|
|
6
|
+
- **The upload preview itemizes what redaction masked, by kind.** The
|
|
7
|
+
redactor now categorizes every replacement by the rule that fired — API
|
|
8
|
+
keys, bearer tokens, private key blocks, passwords, URL credentials, and
|
|
9
|
+
credential-bearing field values — and the terminal trajectory report shows
|
|
10
|
+
a `Masked for you: 2 API keys, 1 bearer token — originals never leave this
|
|
11
|
+
machine` breakdown under the masked-count row, plus a reassurance that
|
|
12
|
+
redaction ran locally and the server independently rescans staged
|
|
13
|
+
artifacts (or `No secrets or personal identifiers detected — nothing
|
|
14
|
+
needed masking.` when nothing matched). `bench traj upload --dry-run`
|
|
15
|
+
prints the same breakdown as a plain `Masked for you:` line, and
|
|
16
|
+
`bench eval view --confirm` gains a display-only `--redaction-summary`
|
|
17
|
+
flag that renders it in the confirm bar next to the Approve button; the
|
|
18
|
+
`benchflow-traj-upload` skill stages a dry run first and passes the line
|
|
19
|
+
through. The total `redaction_replacements` count and the manifest
|
|
20
|
+
`trajectory_report` contract are unchanged (the server validates the
|
|
21
|
+
report with a closed schema and exact recompute equality, so per-category
|
|
22
|
+
counts stay display-only).
|
|
6
23
|
- **The trajectory viewer can collect the eval-prize confirmation in the
|
|
7
24
|
browser.** `bench eval view PATH --confirm` renders the normal page plus a
|
|
8
25
|
sticky site-styled bottom bar ("Submit this trajectory to the BenchFlow
|
|
@@ -17,13 +34,34 @@
|
|
|
17
34
|
- **Trajectory uploads are tagged with the session's repository by
|
|
18
35
|
default.** Unless `--source-id` is given, `bench traj upload` reads the
|
|
19
36
|
session's recorded working directory (Claude `cwd` events, Codex
|
|
20
|
-
`session_meta`), resolves its git `origin` remote
|
|
21
|
-
|
|
22
|
-
|
|
37
|
+
`session_meta`), resolves its git `origin` remote, and stores
|
|
38
|
+
`repo/<owner>/<name>` as the manifest source id, printing
|
|
39
|
+
`Repo: owner/name (from session cwd /path; use --no-repo to omit)` (the
|
|
40
|
+
local path is terminal output only, never uploaded). `--no-repo` opts out
|
|
23
41
|
— the `benchflow-traj-upload` skill now surfaces the detected tag during
|
|
24
42
|
the confirm step so contributors can decline it for private repos — and
|
|
25
43
|
undetectable repos fall back silently to the path-derived source id.
|
|
26
44
|
|
|
45
|
+
### Fixed
|
|
46
|
+
- **The repo tag derives only from the session's own recorded cwd.** The
|
|
47
|
+
initial repo-tagging implementation (#1015) fell back to the upload
|
|
48
|
+
invocation directory's git remote when the session cwd yielded nothing; a
|
|
49
|
+
collector-side audit showed this mis-attributes provenance — a session
|
|
50
|
+
recorded in a non-repo directory, uploaded from the benchflow checkout,
|
|
51
|
+
was tagged `repo/benchflow-ai/benchflow` (two community-dataset entries
|
|
52
|
+
carry the mis-tag). The fallback is removed: no session cwd, a missing
|
|
53
|
+
directory, or no GitHub remote now mean no repo tag, exactly like
|
|
54
|
+
`--no-repo`.
|
|
55
|
+
- **The trajectory viewer header no longer shows `?` badges on real Claude
|
|
56
|
+
Code sessions.** `bench eval view` on a `~/.claude` session JSONL rendered
|
|
57
|
+
`model: ?`, `session: ?...`, `claude code: ?`, and `total cost: $0.0000`
|
|
58
|
+
because the header only read a `type: system` event that real session
|
|
59
|
+
files don't contain. The header now derives its metadata from what the
|
|
60
|
+
file actually carries (first assistant event's `message.model`, per-event
|
|
61
|
+
`version` / `sessionId`, the filename stem as a session fallback) and
|
|
62
|
+
hides any badge whose value is unknown — including the cost badge when no
|
|
63
|
+
event carries cost data. Presentation only.
|
|
64
|
+
|
|
27
65
|
### Changed
|
|
28
66
|
- **Trajectory viewer tool calls are color-coded and backgrounds are light.**
|
|
29
67
|
Each tool kind now gets a muted GitHub-label-style accent on its name pill
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: benchflow
|
|
3
|
-
Version: 0.7.2.
|
|
3
|
+
Version: 0.7.2.dev2000
|
|
4
4
|
Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
|
|
5
5
|
Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
|
|
6
6
|
Project-URL: Repository, https://github.com/benchflow-ai/benchflow
|
|
@@ -20,10 +20,29 @@ from rich.progress import (
|
|
|
20
20
|
from rich.table import Table
|
|
21
21
|
from rich.text import Text
|
|
22
22
|
|
|
23
|
-
from benchflow.publish.redact import REDACTED
|
|
23
|
+
from benchflow.publish.redact import REDACTED, format_redaction_breakdown
|
|
24
24
|
from benchflow.publish.traj_capture import StagedFile
|
|
25
25
|
from benchflow.publish.traj_report import PREVIEW_WORD_LIMIT, TrajectoryReport
|
|
26
26
|
|
|
27
|
+
# Redaction-transparency copy shown with the masked-value count. The wording is
|
|
28
|
+
# deliberately honest: redaction is a local structural pass (no agents, no
|
|
29
|
+
# token spend), and the server independently rescans staged artifacts.
|
|
30
|
+
NO_MASKING_NEEDED_COPY = (
|
|
31
|
+
"No secrets or personal identifiers detected — nothing needed masking."
|
|
32
|
+
)
|
|
33
|
+
REDACTION_REASSURANCE_COPY = (
|
|
34
|
+
"Redaction ran locally before anything was staged; "
|
|
35
|
+
"the server independently rescans and rejects any survivor."
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def masked_for_you_line(report: TrajectoryReport) -> str:
|
|
40
|
+
"""Itemized ``Masked for you`` copy for a report with masked values."""
|
|
41
|
+
breakdown = format_redaction_breakdown(dict(report.masked_categories))
|
|
42
|
+
if not breakdown: # category data unavailable — fall back to the total
|
|
43
|
+
breakdown = f"{report.masked_values} secret-like values"
|
|
44
|
+
return f"{breakdown} — originals never leave this machine"
|
|
45
|
+
|
|
27
46
|
|
|
28
47
|
def render_trajectory_report(report: TrajectoryReport, *, console: Console) -> None:
|
|
29
48
|
"""Render upload facts and a bounded, already-redacted trajectory preview."""
|
|
@@ -43,8 +62,17 @@ def render_trajectory_report(report: TrajectoryReport, *, console: Console) -> N
|
|
|
43
62
|
"API keys / secrets masked",
|
|
44
63
|
Text(str(report.masked_values), style="bold green"),
|
|
45
64
|
)
|
|
65
|
+
if report.masked_values:
|
|
66
|
+
facts.add_row(
|
|
67
|
+
"Masked for you",
|
|
68
|
+
Text(masked_for_you_line(report), style="green"),
|
|
69
|
+
)
|
|
70
|
+
else:
|
|
71
|
+
facts.add_row("Masked for you", Text(NO_MASKING_NEEDED_COPY, style="dim"))
|
|
46
72
|
facts.add_row("Safe replacement", Text(REDACTED))
|
|
47
73
|
console.print(Panel(facts, title="Trajectory report", border_style="cyan"))
|
|
74
|
+
if report.masked_values:
|
|
75
|
+
console.print(f"[dim]{REDACTION_REASSURANCE_COPY}[/dim]")
|
|
48
76
|
|
|
49
77
|
if not report.preview:
|
|
50
78
|
return
|
|
@@ -1239,11 +1239,25 @@ def eval_view(
|
|
|
1239
1239
|
),
|
|
1240
1240
|
),
|
|
1241
1241
|
] = False,
|
|
1242
|
+
redaction_summary: Annotated[
|
|
1243
|
+
str | None,
|
|
1244
|
+
typer.Option(
|
|
1245
|
+
"--redaction-summary",
|
|
1246
|
+
help=(
|
|
1247
|
+
"Masked-secret breakdown shown in the --confirm bar, e.g. "
|
|
1248
|
+
"'2 API keys, 1 bearer token' (lift it from "
|
|
1249
|
+
"`bench traj upload PATH --dry-run`). Display-only; no effect "
|
|
1250
|
+
"without --confirm."
|
|
1251
|
+
),
|
|
1252
|
+
),
|
|
1253
|
+
] = None,
|
|
1242
1254
|
) -> None:
|
|
1243
1255
|
"""View a trial or session trajectory in the browser."""
|
|
1244
1256
|
from benchflow.trajectories import viewer
|
|
1245
1257
|
|
|
1246
|
-
decision = viewer.serve(
|
|
1258
|
+
decision = viewer.serve(
|
|
1259
|
+
str(rollout_dir), port, confirm=confirm, redaction_summary=redaction_summary
|
|
1260
|
+
)
|
|
1247
1261
|
# Exit-code contract for --confirm: 0 approved, 3 rejected. 3 is chosen
|
|
1248
1262
|
# so a rejection never collides with the CLI's existing error exits
|
|
1249
1263
|
# (1 = errors, 2 = Typer usage errors).
|
|
@@ -23,6 +23,7 @@ from benchflow.cli._traj_upload_ui import (
|
|
|
23
23
|
render_trajectory_report,
|
|
24
24
|
upload_progress,
|
|
25
25
|
)
|
|
26
|
+
from benchflow.publish.redact import format_redaction_breakdown
|
|
26
27
|
from benchflow.publish.traj_capture import (
|
|
27
28
|
StagedCapture,
|
|
28
29
|
default_source_id,
|
|
@@ -291,17 +292,23 @@ def register_traj(app: typer.Typer) -> None:
|
|
|
291
292
|
def _run_upload(options: _UploadOptions) -> None:
|
|
292
293
|
prompted = options.path is None
|
|
293
294
|
path = options.path or _prompt_for_path()
|
|
294
|
-
|
|
295
|
+
detected: _DetectedRepo | None = None
|
|
295
296
|
if options.source_id is not None:
|
|
296
297
|
source_id = options.source_id
|
|
297
298
|
else:
|
|
298
299
|
if options.repo:
|
|
299
|
-
|
|
300
|
-
source_id = f"repo/{
|
|
301
|
-
if
|
|
302
|
-
# Contributor-visible metadata: surface the tag
|
|
303
|
-
# sessions can opt out before anything leaves
|
|
304
|
-
|
|
300
|
+
detected = _detect_repo_slug(path)
|
|
301
|
+
source_id = f"repo/{detected.slug}" if detected else default_source_id(path)
|
|
302
|
+
if detected:
|
|
303
|
+
# Contributor-visible metadata: surface the tag and where it came
|
|
304
|
+
# from so private-repo sessions can opt out before anything leaves
|
|
305
|
+
# the machine. The local path stays terminal-only; the uploaded
|
|
306
|
+
# source id is just repo/<owner>/<name>. Plain print keeps the line
|
|
307
|
+
# one physical line (no Rich wrapping) so the path stays selectable.
|
|
308
|
+
print(
|
|
309
|
+
f"Repo: {detected.slug} "
|
|
310
|
+
f"(from session cwd {detected.session_cwd}; use --no-repo to omit)"
|
|
311
|
+
)
|
|
305
312
|
destination = _resolve_destination(options)
|
|
306
313
|
|
|
307
314
|
with (
|
|
@@ -314,6 +321,7 @@ def _run_upload(options: _UploadOptions) -> None:
|
|
|
314
321
|
artifacts.files,
|
|
315
322
|
masked_values=artifacts.redaction_replacements,
|
|
316
323
|
preview_steps=options.preview_steps,
|
|
324
|
+
masked_categories=artifacts.redaction_categories,
|
|
317
325
|
)
|
|
318
326
|
status.stop()
|
|
319
327
|
render_trajectory_report(report, console=console)
|
|
@@ -417,25 +425,35 @@ def _infer_email() -> str:
|
|
|
417
425
|
_SESSION_CWD_SCAN_LINES = 50
|
|
418
426
|
|
|
419
427
|
|
|
420
|
-
|
|
428
|
+
@dataclass(frozen=True)
|
|
429
|
+
class _DetectedRepo:
|
|
430
|
+
slug: str
|
|
431
|
+
session_cwd: Path
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _detect_repo_slug(path: Path) -> _DetectedRepo | None:
|
|
421
435
|
"""Best-effort ``owner/name`` for the repository the session was about.
|
|
422
436
|
|
|
423
437
|
Reads the working directory the session recorded (Claude events carry a
|
|
424
|
-
``cwd`` field; Codex ``session_meta`` payloads do too)
|
|
425
|
-
directory's git for the ``origin`` remote
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
438
|
+
``cwd`` field; Codex ``session_meta`` payloads do too) and asks that
|
|
439
|
+
directory's git for the ``origin`` remote. The trajectory's own recorded
|
|
440
|
+
cwd is the ONLY provenance source: there is deliberately no fallback to
|
|
441
|
+
the upload invocation directory, which mis-attributed sessions recorded
|
|
442
|
+
outside a repo to whatever checkout the contributor happened to upload
|
|
443
|
+
from. No session cwd, a missing directory, or no GitHub remote all mean
|
|
444
|
+
no tag. Every failure is silent — repo tagging must never break an
|
|
445
|
+
upload — and local-path remotes never produce a tag, so no local
|
|
446
|
+
absolute path can leak into the manifest.
|
|
429
447
|
"""
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
448
|
+
session_cwd = _session_cwd(path)
|
|
449
|
+
if session_cwd is None or not session_cwd.is_dir():
|
|
450
|
+
return None
|
|
451
|
+
remote = _command_stdout(
|
|
452
|
+
"git", "-C", str(session_cwd), "remote", "get-url", "origin"
|
|
453
|
+
)
|
|
454
|
+
slug = _repo_slug_from_remote(remote) if remote else None
|
|
455
|
+
if slug:
|
|
456
|
+
return _DetectedRepo(slug=slug, session_cwd=session_cwd)
|
|
439
457
|
return None
|
|
440
458
|
|
|
441
459
|
|
|
@@ -633,6 +651,13 @@ def _print_dry_run(staged: StagedCapture) -> None:
|
|
|
633
651
|
if staged.ignored:
|
|
634
652
|
console.print(f"Ignored: {escape(', '.join(staged.ignored))}")
|
|
635
653
|
console.print(f"Redactions: {staged.redaction_replacements}")
|
|
654
|
+
# One plain, greppable line so the upload skill can lift the breakdown into
|
|
655
|
+
# the viewer's confirm bar (bench eval view --redaction-summary "...").
|
|
656
|
+
breakdown = format_redaction_breakdown(dict(staged.artifact_redaction_categories))
|
|
657
|
+
if breakdown:
|
|
658
|
+
print(f"Masked for you: {breakdown}")
|
|
659
|
+
else:
|
|
660
|
+
print("Masked for you: nothing — no secrets detected")
|
|
636
661
|
|
|
637
662
|
|
|
638
663
|
def _print_contributor_prompt() -> None:
|
|
@@ -4,10 +4,19 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import re
|
|
6
6
|
import shlex
|
|
7
|
+
from collections import Counter
|
|
7
8
|
from collections.abc import Mapping, Sequence
|
|
8
9
|
from typing import Any, NamedTuple
|
|
9
10
|
|
|
10
|
-
from benchflow.trajectories.types import
|
|
11
|
+
from benchflow.trajectories.types import (
|
|
12
|
+
REDACTION_CATEGORY_API_KEY,
|
|
13
|
+
REDACTION_CATEGORY_BEARER_TOKEN,
|
|
14
|
+
REDACTION_CATEGORY_CREDENTIAL_FIELD,
|
|
15
|
+
REDACTION_CATEGORY_PASSWORD,
|
|
16
|
+
REDACTION_CATEGORY_PRIVATE_KEY,
|
|
17
|
+
REDACTION_CATEGORY_URL_CREDENTIAL,
|
|
18
|
+
redact_trajectory_text_with_categories,
|
|
19
|
+
)
|
|
11
20
|
|
|
12
21
|
REDACTED = "<XXX-benchflow-key-values-XXX>"
|
|
13
22
|
_CANONICAL_REDACTED = "***REDACTED***"
|
|
@@ -67,23 +76,81 @@ COMPACT_SENSITIVE_KEY_SUFFIXES = tuple(
|
|
|
67
76
|
class RedactionPattern(NamedTuple):
|
|
68
77
|
pattern: re.Pattern[str]
|
|
69
78
|
replacement: str
|
|
79
|
+
category: str
|
|
70
80
|
|
|
71
81
|
|
|
72
82
|
VALUE_PATTERNS = (
|
|
73
83
|
# Google AI Studio's newer token format is not covered by the canonical
|
|
74
84
|
# trajectory redactor yet.
|
|
75
|
-
RedactionPattern(
|
|
85
|
+
RedactionPattern(
|
|
86
|
+
re.compile(r"AQ\.[0-9A-Za-z_-]{20,}"), REDACTED, REDACTION_CATEGORY_API_KEY
|
|
87
|
+
),
|
|
76
88
|
RedactionPattern(
|
|
77
89
|
re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]{16,}", re.IGNORECASE),
|
|
78
90
|
f"Bearer {REDACTED}",
|
|
91
|
+
REDACTION_CATEGORY_BEARER_TOKEN,
|
|
79
92
|
),
|
|
80
93
|
)
|
|
81
94
|
|
|
95
|
+
# Display order and pluralized labels for the per-category breakdown shown to
|
|
96
|
+
# contributors ("Masked for you: 2 API keys, 1 bearer token"). Keys are the
|
|
97
|
+
# canonical category names from ``benchflow.trajectories.types``; the taxonomy
|
|
98
|
+
# names what the redaction rules actually detect — nothing more.
|
|
99
|
+
REDACTION_CATEGORY_LABELS: tuple[tuple[str, str], ...] = (
|
|
100
|
+
(REDACTION_CATEGORY_API_KEY, "API keys"),
|
|
101
|
+
(REDACTION_CATEGORY_BEARER_TOKEN, "bearer tokens"),
|
|
102
|
+
(REDACTION_CATEGORY_PRIVATE_KEY, "private key blocks"),
|
|
103
|
+
(REDACTION_CATEGORY_PASSWORD, "passwords"),
|
|
104
|
+
(REDACTION_CATEGORY_URL_CREDENTIAL, "URL credentials"),
|
|
105
|
+
(REDACTION_CATEGORY_CREDENTIAL_FIELD, "credential-bearing field values"),
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def redaction_breakdown(counts: Mapping[str, int]) -> tuple[tuple[str, int], ...]:
|
|
110
|
+
"""Return non-zero category counts in canonical display order."""
|
|
111
|
+
ordered = [
|
|
112
|
+
(category, counts[category])
|
|
113
|
+
for category, _ in REDACTION_CATEGORY_LABELS
|
|
114
|
+
if counts.get(category)
|
|
115
|
+
]
|
|
116
|
+
# Unknown categories cannot arise from this module's own accumulation, but
|
|
117
|
+
# persisted/forwarded counts must never be dropped silently.
|
|
118
|
+
ordered.extend(
|
|
119
|
+
(category, count)
|
|
120
|
+
for category, count in counts.items()
|
|
121
|
+
if count and category not in dict(REDACTION_CATEGORY_LABELS)
|
|
122
|
+
)
|
|
123
|
+
return tuple(ordered)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def format_redaction_breakdown(counts: Mapping[str, int]) -> str:
|
|
127
|
+
"""Render category counts as prose, e.g. ``2 API keys, 1 bearer token``."""
|
|
128
|
+
plurals = dict(REDACTION_CATEGORY_LABELS)
|
|
129
|
+
parts = []
|
|
130
|
+
for category, count in redaction_breakdown(counts):
|
|
131
|
+
label = category if count == 1 else plurals.get(category, f"{category}s")
|
|
132
|
+
parts.append(f"{count} {label}")
|
|
133
|
+
return ", ".join(parts)
|
|
82
134
|
|
|
83
|
-
|
|
84
|
-
|
|
135
|
+
|
|
136
|
+
def redact_value(
|
|
137
|
+
value: Any,
|
|
138
|
+
*,
|
|
139
|
+
field_name: str | None = None,
|
|
140
|
+
categories: Counter[str] | None = None,
|
|
141
|
+
) -> tuple[Any, int]:
|
|
142
|
+
"""Return a structurally redacted JSON value and replacement count.
|
|
143
|
+
|
|
144
|
+
When *categories* is given, every replacement also increments the matching
|
|
145
|
+
per-category counter (see ``REDACTION_CATEGORY_LABELS``); the counter's
|
|
146
|
+
total always equals the returned replacement count.
|
|
147
|
+
"""
|
|
85
148
|
if field_name is not None and _is_sensitive_key(field_name):
|
|
86
|
-
|
|
149
|
+
if _is_redacted_value(value):
|
|
150
|
+
return value, 0
|
|
151
|
+
if categories is not None:
|
|
152
|
+
categories[_field_category(field_name)] += 1
|
|
153
|
+
return REDACTED, 1
|
|
87
154
|
|
|
88
155
|
if isinstance(value, Mapping):
|
|
89
156
|
redacted: dict[Any, Any] = {}
|
|
@@ -104,7 +171,7 @@ def redact_value(value: Any, *, field_name: str | None = None) -> tuple[Any, int
|
|
|
104
171
|
for key, item in value.items():
|
|
105
172
|
clean_key = key
|
|
106
173
|
if isinstance(key, str):
|
|
107
|
-
_, key_replacements = _redact_text(key)
|
|
174
|
+
_, key_replacements = _redact_text(key, categories=categories)
|
|
108
175
|
if key_replacements:
|
|
109
176
|
secret_key_index += 1
|
|
110
177
|
clean_key = f"***REDACTED_KEY_{secret_key_index}***"
|
|
@@ -122,73 +189,102 @@ def redact_value(value: Any, *, field_name: str | None = None) -> tuple[Any, int
|
|
|
122
189
|
if isinstance(key, str)
|
|
123
190
|
else None
|
|
124
191
|
),
|
|
192
|
+
categories=categories,
|
|
125
193
|
)
|
|
126
194
|
redacted[clean_key] = clean
|
|
127
195
|
replacements += count
|
|
128
196
|
return redacted, replacements
|
|
129
197
|
|
|
130
198
|
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
|
131
|
-
return _redact_argv(value, redact_other_values=True)
|
|
199
|
+
return _redact_argv(value, redact_other_values=True, categories=categories)
|
|
132
200
|
|
|
133
201
|
if not isinstance(value, str):
|
|
134
202
|
return value, 0
|
|
135
203
|
|
|
136
|
-
return _redact_text(value)
|
|
204
|
+
return _redact_text(value, categories=categories)
|
|
137
205
|
|
|
138
206
|
|
|
139
|
-
def redact_value_to_stability(
|
|
207
|
+
def redact_value_to_stability(
|
|
208
|
+
value: Any, *, categories: Counter[str] | None = None
|
|
209
|
+
) -> tuple[Any, int]:
|
|
140
210
|
"""Redact until another server-side pass would make no replacements."""
|
|
141
211
|
redacted = value
|
|
142
212
|
replacements = 0
|
|
143
213
|
for _ in range(_MAX_REDACTION_PASSES):
|
|
144
|
-
redacted, count = redact_value(redacted)
|
|
214
|
+
redacted, count = redact_value(redacted, categories=categories)
|
|
145
215
|
replacements += count
|
|
146
216
|
if count == 0:
|
|
147
217
|
return redacted, replacements
|
|
148
218
|
raise ValueError("trajectory secret redaction did not converge")
|
|
149
219
|
|
|
150
220
|
|
|
151
|
-
def _redact_text(
|
|
221
|
+
def _redact_text(
|
|
222
|
+
value: str, *, categories: Counter[str] | None = None
|
|
223
|
+
) -> tuple[str, int]:
|
|
152
224
|
# The canonical carrier patterns intentionally ignore their asterisk marker.
|
|
153
225
|
# Protect our public-upload marker with that value during the scan so a second
|
|
154
226
|
# client/server pass is idempotent even inside text such as ``API_KEY=...``.
|
|
155
227
|
protected = value.replace(REDACTED, _CANONICAL_REDACTED)
|
|
156
|
-
redacted_text,
|
|
228
|
+
redacted_text, text_categories = redact_trajectory_text_with_categories(protected)
|
|
229
|
+
replacements = sum(text_categories.values())
|
|
230
|
+
if categories is not None:
|
|
231
|
+
categories.update(text_categories)
|
|
157
232
|
redacted_text = redacted_text.replace(_CANONICAL_REDACTED, REDACTED)
|
|
158
|
-
for pattern, replacement in VALUE_PATTERNS:
|
|
233
|
+
for pattern, replacement, category in VALUE_PATTERNS:
|
|
159
234
|
redacted_text, count = pattern.subn(replacement, redacted_text)
|
|
160
235
|
replacements += count
|
|
161
|
-
|
|
236
|
+
if count and categories is not None:
|
|
237
|
+
categories[category] += count
|
|
238
|
+
cli_redacted, cli_replacements = _redact_cli_text(
|
|
239
|
+
redacted_text, categories=categories
|
|
240
|
+
)
|
|
162
241
|
redacted_text = cli_redacted
|
|
163
242
|
replacements += cli_replacements
|
|
164
243
|
return redacted_text, replacements
|
|
165
244
|
|
|
166
245
|
|
|
167
|
-
def _redact_cli_text(
|
|
246
|
+
def _redact_cli_text(
|
|
247
|
+
value: str, *, categories: Counter[str] | None = None
|
|
248
|
+
) -> tuple[str, int]:
|
|
168
249
|
if "--" not in value:
|
|
169
250
|
return value, 0
|
|
170
251
|
try:
|
|
171
252
|
argv = shlex.split(value)
|
|
172
253
|
except ValueError:
|
|
173
254
|
return value, 0
|
|
174
|
-
|
|
175
|
-
|
|
255
|
+
# Count categories only for the accepted result: a zero-replacement parse
|
|
256
|
+
# returns the original text, so nothing may be attributed for it.
|
|
257
|
+
argv_categories: Counter[str] | None = None if categories is None else Counter()
|
|
258
|
+
redacted, replacements = _redact_argv(
|
|
259
|
+
argv, redact_other_values=False, categories=argv_categories
|
|
260
|
+
)
|
|
261
|
+
if not replacements:
|
|
262
|
+
return value, 0
|
|
263
|
+
if categories is not None and argv_categories is not None:
|
|
264
|
+
categories.update(argv_categories)
|
|
265
|
+
return shlex.join(redacted), replacements
|
|
176
266
|
|
|
177
267
|
|
|
178
268
|
def _redact_argv(
|
|
179
|
-
values: Sequence[Any],
|
|
269
|
+
values: Sequence[Any],
|
|
270
|
+
*,
|
|
271
|
+
redact_other_values: bool,
|
|
272
|
+
categories: Counter[str] | None = None,
|
|
180
273
|
) -> tuple[list[Any], int]:
|
|
181
274
|
redacted: list[Any] = []
|
|
182
275
|
replacements = 0
|
|
183
|
-
|
|
276
|
+
sensitive_option: str | None = None
|
|
184
277
|
for item in values:
|
|
185
|
-
if
|
|
278
|
+
if sensitive_option is not None:
|
|
279
|
+
option = sensitive_option
|
|
280
|
+
sensitive_option = None
|
|
186
281
|
if _is_redacted_value(item):
|
|
187
282
|
redacted.append(item)
|
|
188
283
|
else:
|
|
189
284
|
redacted.append(REDACTED)
|
|
190
285
|
replacements += 1
|
|
191
|
-
|
|
286
|
+
if categories is not None:
|
|
287
|
+
categories[_field_category(option)] += 1
|
|
192
288
|
continue
|
|
193
289
|
|
|
194
290
|
if isinstance(item, str) and item.startswith("-"):
|
|
@@ -200,13 +296,16 @@ def _redact_argv(
|
|
|
200
296
|
else:
|
|
201
297
|
redacted.append(f"{option}={REDACTED}")
|
|
202
298
|
replacements += 1
|
|
299
|
+
if categories is not None:
|
|
300
|
+
categories[_field_category(option)] += 1
|
|
203
301
|
else:
|
|
204
302
|
redacted.append(item)
|
|
205
|
-
|
|
303
|
+
if not separator:
|
|
304
|
+
sensitive_option = option
|
|
206
305
|
continue
|
|
207
306
|
|
|
208
307
|
if redact_other_values:
|
|
209
|
-
clean, count = redact_value(item)
|
|
308
|
+
clean, count = redact_value(item, categories=categories)
|
|
210
309
|
redacted.append(clean)
|
|
211
310
|
replacements += count
|
|
212
311
|
else:
|
|
@@ -220,6 +319,22 @@ def _is_redacted_value(value: Any) -> bool:
|
|
|
220
319
|
)
|
|
221
320
|
|
|
222
321
|
|
|
322
|
+
def _field_category(field_name: str) -> str:
|
|
323
|
+
"""Categorize a masked value by the credential-bearing name that matched.
|
|
324
|
+
|
|
325
|
+
Structural redaction fires on the field/option NAME, so the name is the
|
|
326
|
+
only honest signal for what kind of secret the value was.
|
|
327
|
+
"""
|
|
328
|
+
compact = _normalize_key(field_name).replace("_", "")
|
|
329
|
+
if compact.endswith(("password", "passwd")):
|
|
330
|
+
return REDACTION_CATEGORY_PASSWORD
|
|
331
|
+
if compact.endswith("apikey"):
|
|
332
|
+
return REDACTION_CATEGORY_API_KEY
|
|
333
|
+
if compact.endswith("authorization") or "bearertoken" in compact:
|
|
334
|
+
return REDACTION_CATEGORY_BEARER_TOKEN
|
|
335
|
+
return REDACTION_CATEGORY_CREDENTIAL_FIELD
|
|
336
|
+
|
|
337
|
+
|
|
223
338
|
def _is_sensitive_key(field_name: str) -> bool:
|
|
224
339
|
normalized = _normalize_key(field_name)
|
|
225
340
|
return (
|
|
@@ -7,6 +7,7 @@ import json
|
|
|
7
7
|
import re
|
|
8
8
|
import shutil
|
|
9
9
|
import tempfile
|
|
10
|
+
from collections import Counter
|
|
10
11
|
from collections.abc import Iterator, Mapping
|
|
11
12
|
from contextlib import contextmanager
|
|
12
13
|
from dataclasses import dataclass
|
|
@@ -15,7 +16,11 @@ from pathlib import Path
|
|
|
15
16
|
from typing import Any, Never
|
|
16
17
|
|
|
17
18
|
from benchflow import __version__
|
|
18
|
-
from benchflow.publish.redact import
|
|
19
|
+
from benchflow.publish.redact import (
|
|
20
|
+
redact_value,
|
|
21
|
+
redact_value_to_stability,
|
|
22
|
+
redaction_breakdown,
|
|
23
|
+
)
|
|
19
24
|
|
|
20
25
|
MAX_FILE_BYTES = 128 * 1024**2
|
|
21
26
|
MAX_ARTIFACTS = 8
|
|
@@ -81,6 +86,10 @@ class StagedTrajectoryArtifacts:
|
|
|
81
86
|
_metadata_dir: Path | None
|
|
82
87
|
_staging_dir: Path
|
|
83
88
|
_redact: bool
|
|
89
|
+
# Per-category artifact replacement counts in display order (display-only:
|
|
90
|
+
# the manifest/server contract stays untouched). Sums to
|
|
91
|
+
# ``redaction_replacements``.
|
|
92
|
+
redaction_categories: tuple[tuple[str, int], ...] = ()
|
|
84
93
|
|
|
85
94
|
|
|
86
95
|
@dataclass(frozen=True)
|
|
@@ -92,6 +101,8 @@ class StagedCapture:
|
|
|
92
101
|
ignored: tuple[str, ...]
|
|
93
102
|
artifact_redaction_replacements: int
|
|
94
103
|
redaction_replacements: int
|
|
104
|
+
# Display-only per-category counts for the artifact replacements.
|
|
105
|
+
artifact_redaction_categories: tuple[tuple[str, int], ...] = ()
|
|
95
106
|
|
|
96
107
|
|
|
97
108
|
@dataclass(frozen=True)
|
|
@@ -199,12 +210,15 @@ def stage_trajectory_artifacts(
|
|
|
199
210
|
staging_dir = Path(temp_name)
|
|
200
211
|
payloads: list[StagedFile] = []
|
|
201
212
|
replacement_count = 0
|
|
213
|
+
replacement_categories: Counter[str] = Counter()
|
|
202
214
|
for source in resolved.files:
|
|
203
215
|
relname = validate_artifact_name(f"trajectory/{source.name}")
|
|
204
216
|
target = staging_dir / relname
|
|
205
217
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
206
218
|
if redact:
|
|
207
|
-
replacements = _redact_jsonl(
|
|
219
|
+
replacements = _redact_jsonl(
|
|
220
|
+
source, target, categories=replacement_categories
|
|
221
|
+
)
|
|
208
222
|
replacement_count += replacements
|
|
209
223
|
else:
|
|
210
224
|
shutil.copyfile(source, target)
|
|
@@ -240,6 +254,7 @@ def stage_trajectory_artifacts(
|
|
|
240
254
|
_metadata_dir=resolved.metadata_dir,
|
|
241
255
|
_staging_dir=staging_dir,
|
|
242
256
|
_redact=redact,
|
|
257
|
+
redaction_categories=redaction_breakdown(replacement_categories),
|
|
243
258
|
)
|
|
244
259
|
|
|
245
260
|
|
|
@@ -305,6 +320,7 @@ def finalize_trajectory_capture(
|
|
|
305
320
|
ignored=artifacts.ignored,
|
|
306
321
|
artifact_redaction_replacements=artifacts.redaction_replacements,
|
|
307
322
|
redaction_replacements=replacement_count,
|
|
323
|
+
artifact_redaction_categories=artifacts.redaction_categories,
|
|
308
324
|
)
|
|
309
325
|
|
|
310
326
|
|
|
@@ -430,7 +446,9 @@ def _validate_jsonl(path: Path) -> None:
|
|
|
430
446
|
raise ValueError(f"trajectory JSONL has no records: {path}")
|
|
431
447
|
|
|
432
448
|
|
|
433
|
-
def _redact_jsonl(
|
|
449
|
+
def _redact_jsonl(
|
|
450
|
+
source: Path, target: Path, *, categories: Counter[str] | None = None
|
|
451
|
+
) -> int:
|
|
434
452
|
replacements = 0
|
|
435
453
|
with (
|
|
436
454
|
source.open(encoding="utf-8", newline="") as input_stream,
|
|
@@ -443,7 +461,9 @@ def _redact_jsonl(source: Path, target: Path) -> int:
|
|
|
443
461
|
continue
|
|
444
462
|
value = strict_json_loads(body)
|
|
445
463
|
try:
|
|
446
|
-
redacted, count = redact_value_to_stability(
|
|
464
|
+
redacted, count = redact_value_to_stability(
|
|
465
|
+
value, categories=categories
|
|
466
|
+
)
|
|
447
467
|
except RecursionError as exc: # defense in depth after complexity gate
|
|
448
468
|
raise ValueError(
|
|
449
469
|
f"{source}: line {line_number}: trajectory JSONL nesting "
|