recoverybench 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- recoverybench-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +100 -0
- recoverybench-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml +40 -0
- recoverybench-0.1.0/.github/ISSUE_TEMPLATE/integration_request.yml +43 -0
- recoverybench-0.1.0/.github/actions/recoverybench-test/action.yml +90 -0
- recoverybench-0.1.0/.github/workflows/publish.yml +99 -0
- recoverybench-0.1.0/.github/workflows/recoverybench-ci.yml +196 -0
- recoverybench-0.1.0/.github/workflows/recoverybench-pr-gate.yml +56 -0
- recoverybench-0.1.0/.gitignore +43 -0
- recoverybench-0.1.0/CHANGELOG.md +43 -0
- recoverybench-0.1.0/FINAL_RELEASE_READINESS_AUDIT.md +154 -0
- recoverybench-0.1.0/GITHUB_ACTIONS_DOCKER_DIGEST_REMEDIATION.md +177 -0
- recoverybench-0.1.0/GITHUB_ACTIONS_MATRIX_REMEDIATION.md +154 -0
- recoverybench-0.1.0/GITHUB_PUSH_PROTECTION_REMEDIATION.md +139 -0
- recoverybench-0.1.0/LAUNCH_PHASE_1_COMPLETION_REPORT.md +125 -0
- recoverybench-0.1.0/LICENSE +177 -0
- recoverybench-0.1.0/P0_3_FINAL_CONDITION_REMEDIATION.md +251 -0
- recoverybench-0.1.0/P0_3_GITHUB_ACTIONS_ROOT_CAUSE_REMEDIATION.md +167 -0
- recoverybench-0.1.0/P0_3_LINUX_WORKSPACE_PERMISSION_REMEDIATION.md +150 -0
- recoverybench-0.1.0/PHASE_2A_DISTRIBUTION_READINESS_REPORT.md +187 -0
- recoverybench-0.1.0/PHASE_2A_RELEASE_WORKFLOW_HARDENING_REPORT.md +116 -0
- recoverybench-0.1.0/PKG-INFO +496 -0
- recoverybench-0.1.0/README.md +459 -0
- recoverybench-0.1.0/docs/ARCHITECTURE.md +303 -0
- recoverybench-0.1.0/docs/CODEBASE_AUDIT.md +682 -0
- recoverybench-0.1.0/docs/COMPETITIVE_POSITIONING.md +100 -0
- recoverybench-0.1.0/docs/DEMO_WALKTHROUGH.md +204 -0
- recoverybench-0.1.0/docs/FIRST_USER_GUIDE.md +345 -0
- recoverybench-0.1.0/docs/GITHUB_ACTIONS_GUIDE.md +210 -0
- recoverybench-0.1.0/docs/IMPLEMENTATION_PLAN.md +128 -0
- recoverybench-0.1.0/docs/P0_1_FINAL_HARDENING.md +200 -0
- recoverybench-0.1.0/docs/P0_1_TRANSACTION_IDENTITY_REMEDIATION.md +203 -0
- recoverybench-0.1.0/docs/P0_2_CLOSEOUT_VERIFICATION.md +270 -0
- recoverybench-0.1.0/docs/P0_2_FALSE_SUCCESS_REMEDIATION.md +215 -0
- recoverybench-0.1.0/docs/P0_2_FINAL_HARDENING.md +214 -0
- recoverybench-0.1.0/docs/P0_3_FINAL_CONDITION_REMEDIATION.md +251 -0
- recoverybench-0.1.0/docs/P0_3_FINAL_SECURITY_CERTIFICATION.md +322 -0
- recoverybench-0.1.0/docs/P0_3_INDEPENDENT_CODE_REVIEW.md +267 -0
- recoverybench-0.1.0/docs/P0_3_NETWORK_ISOLATION_REMEDIATION.md +360 -0
- recoverybench-0.1.0/docs/P0_3_RUNTIME_VERIFICATION.md +308 -0
- recoverybench-0.1.0/docs/P0_3_TEST_FAILURE_REMEDIATION.md +190 -0
- recoverybench-0.1.0/docs/PRODUCT_SPEC.md +114 -0
- recoverybench-0.1.0/docs/SDK_INTEGRATION_GUIDE.md +514 -0
- recoverybench-0.1.0/docs/SECURITY_MODEL.md +131 -0
- recoverybench-0.1.0/docs/TEST_STRATEGY.md +143 -0
- recoverybench-0.1.0/examples/generic_callable_agent.py +100 -0
- recoverybench-0.1.0/examples/langgraph_recovery_agent.py +55 -0
- recoverybench-0.1.0/examples/sdk_quickstart.py +40 -0
- recoverybench-0.1.0/pyproject.toml +87 -0
- recoverybench-0.1.0/recoverybench.yaml +7 -0
- recoverybench-0.1.0/scenarios/payment_duplicate_delivery.yaml +26 -0
- recoverybench-0.1.0/scenarios/payment_http_500_transient.yaml +25 -0
- recoverybench-0.1.0/scenarios/payment_malformed_response.yaml +26 -0
- recoverybench-0.1.0/scenarios/payment_rate_limit_429.yaml +26 -0
- recoverybench-0.1.0/scenarios/payment_timeout_after_commit.yaml +27 -0
- recoverybench-0.1.0/scenarios/payment_timeout_before_commit.yaml +26 -0
- recoverybench-0.1.0/scripts/distribution_smoke_test.py +226 -0
- recoverybench-0.1.0/src/recoverybench/__init__.py +68 -0
- recoverybench-0.1.0/src/recoverybench/adapters/__init__.py +34 -0
- recoverybench-0.1.0/src/recoverybench/adapters/callable.py +205 -0
- recoverybench-0.1.0/src/recoverybench/adapters/langgraph.py +319 -0
- recoverybench-0.1.0/src/recoverybench/agent/__init__.py +21 -0
- recoverybench-0.1.0/src/recoverybench/agent/base.py +212 -0
- recoverybench-0.1.0/src/recoverybench/agent/idempotent.py +77 -0
- recoverybench-0.1.0/src/recoverybench/agent/naive.py +58 -0
- recoverybench-0.1.0/src/recoverybench/cli.py +829 -0
- recoverybench-0.1.0/src/recoverybench/clock.py +47 -0
- recoverybench-0.1.0/src/recoverybench/config.py +50 -0
- recoverybench-0.1.0/src/recoverybench/engine/__init__.py +6 -0
- recoverybench-0.1.0/src/recoverybench/engine/context.py +43 -0
- recoverybench-0.1.0/src/recoverybench/engine/runner.py +473 -0
- recoverybench-0.1.0/src/recoverybench/errors.py +88 -0
- recoverybench-0.1.0/src/recoverybench/evaluator/__init__.py +6 -0
- recoverybench-0.1.0/src/recoverybench/evaluator/evaluator.py +310 -0
- recoverybench-0.1.0/src/recoverybench/evaluator/rules.py +642 -0
- recoverybench-0.1.0/src/recoverybench/faults/__init__.py +6 -0
- recoverybench-0.1.0/src/recoverybench/faults/injector.py +138 -0
- recoverybench-0.1.0/src/recoverybench/faults/types.py +38 -0
- recoverybench-0.1.0/src/recoverybench/ledger/__init__.py +7 -0
- recoverybench-0.1.0/src/recoverybench/ledger/base.py +57 -0
- recoverybench-0.1.0/src/recoverybench/ledger/memory.py +48 -0
- recoverybench-0.1.0/src/recoverybench/ledger/sqlite.py +311 -0
- recoverybench-0.1.0/src/recoverybench/models.py +137 -0
- recoverybench-0.1.0/src/recoverybench/py.typed +0 -0
- recoverybench-0.1.0/src/recoverybench/replay/__init__.py +5 -0
- recoverybench-0.1.0/src/recoverybench/replay/engine.py +58 -0
- recoverybench-0.1.0/src/recoverybench/reporting/__init__.py +20 -0
- recoverybench-0.1.0/src/recoverybench/reporting/console.py +143 -0
- recoverybench-0.1.0/src/recoverybench/reporting/github_summary.py +83 -0
- recoverybench-0.1.0/src/recoverybench/reporting/html_reporter.py +260 -0
- recoverybench-0.1.0/src/recoverybench/reporting/json_reporter.py +88 -0
- recoverybench-0.1.0/src/recoverybench/reporting/junit_reporter.py +121 -0
- recoverybench-0.1.0/src/recoverybench/sample_project/__init__.py +13 -0
- recoverybench-0.1.0/src/recoverybench/sample_project/resources.py +67 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/__init__.py +34 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/backend.py +471 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/config.py +216 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/docker_client.py +175 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/protocol.py +135 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/spec.py +174 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/staging.py +472 -0
- recoverybench-0.1.0/src/recoverybench/sandbox/worker.py +509 -0
- recoverybench-0.1.0/src/recoverybench/scenarios/__init__.py +12 -0
- recoverybench-0.1.0/src/recoverybench/scenarios/registry.py +233 -0
- recoverybench-0.1.0/src/recoverybench/scenarios/schema.py +72 -0
- recoverybench-0.1.0/src/recoverybench/scenarios/validator.py +107 -0
- recoverybench-0.1.0/src/recoverybench/sdk/__init__.py +71 -0
- recoverybench-0.1.0/src/recoverybench/sdk/tester.py +188 -0
- recoverybench-0.1.0/src/recoverybench/security.py +69 -0
- recoverybench-0.1.0/src/recoverybench/tools/__init__.py +17 -0
- recoverybench-0.1.0/src/recoverybench/tools/base.py +74 -0
- recoverybench-0.1.0/src/recoverybench/tools/payment.py +138 -0
- recoverybench-0.1.0/tests/conftest.py +112 -0
- recoverybench-0.1.0/tests/security/__init__.py +1 -0
- recoverybench-0.1.0/tests/security/test_container_configuration.py +475 -0
- recoverybench-0.1.0/tests/security/test_container_lifecycle.py +128 -0
- recoverybench-0.1.0/tests/security/test_filesystem_isolation.py +611 -0
- recoverybench-0.1.0/tests/security/test_network_isolation.py +502 -0
- recoverybench-0.1.0/tests/security/test_worker_ipc.py +362 -0
- recoverybench-0.1.0/tests/test_all_fault_modes_recovery.py +246 -0
- recoverybench-0.1.0/tests/test_ci_integration.py +225 -0
- recoverybench-0.1.0/tests/test_cli.py +181 -0
- recoverybench-0.1.0/tests/test_demo_comparison.py +48 -0
- recoverybench-0.1.0/tests/test_effect_ledger.py +221 -0
- recoverybench-0.1.0/tests/test_execution_engine.py +125 -0
- recoverybench-0.1.0/tests/test_false_success_final_hardening.py +519 -0
- recoverybench-0.1.0/tests/test_false_success_prevention.py +524 -0
- recoverybench-0.1.0/tests/test_fault_injection.py +202 -0
- recoverybench-0.1.0/tests/test_mock_payment_service.py +74 -0
- recoverybench-0.1.0/tests/test_recovery_evaluator.py +339 -0
- recoverybench-0.1.0/tests/test_replay_engine.py +46 -0
- recoverybench-0.1.0/tests/test_reporting.py +126 -0
- recoverybench-0.1.0/tests/test_schema_validation.py +98 -0
- recoverybench-0.1.0/tests/test_sdk_generic_callable.py +105 -0
- recoverybench-0.1.0/tests/test_sdk_high_level_api.py +54 -0
- recoverybench-0.1.0/tests/test_sdk_langgraph_adapter.py +73 -0
- recoverybench-0.1.0/tests/test_transaction_identity.py +985 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
name: Bug Report
|
|
2
|
+
description: Report an issue, test anomaly, or harness error in RecoveryBench
|
|
3
|
+
title: "[Bug]: "
|
|
4
|
+
labels: ["bug"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
### Thank you for reporting an issue!
|
|
10
|
+
> **Security & Privacy Notice:** NEVER paste API keys, secret tokens, private credentials, or proprietary production payloads into GitHub issues. RecoveryBench redacts known secrets from logs, but always verify your logs before submitting.
|
|
11
|
+
|
|
12
|
+
- type: input
|
|
13
|
+
id: version
|
|
14
|
+
attributes:
|
|
15
|
+
label: RecoveryBench Version
|
|
16
|
+
description: Run `recoverybench --help` or check `pip show recoverybench`
|
|
17
|
+
placeholder: "0.1.0"
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
|
|
21
|
+
- type: input
|
|
22
|
+
id: python-version
|
|
23
|
+
attributes:
|
|
24
|
+
label: Python Version
|
|
25
|
+
description: Run `python --version`
|
|
26
|
+
placeholder: "3.11.8 / 3.12.3 / 3.13.0"
|
|
27
|
+
validations:
|
|
28
|
+
required: true
|
|
29
|
+
|
|
30
|
+
- type: dropdown
|
|
31
|
+
id: os
|
|
32
|
+
attributes:
|
|
33
|
+
label: Operating System
|
|
34
|
+
description: Select your host operating system
|
|
35
|
+
options:
|
|
36
|
+
- "Ubuntu / Debian Linux"
|
|
37
|
+
- "Fedora / RHEL / Arch Linux"
|
|
38
|
+
- "macOS (Apple Silicon)"
|
|
39
|
+
- "macOS (Intel)"
|
|
40
|
+
- "Windows 11 / 10 (PowerShell)"
|
|
41
|
+
- "Windows 11 / 10 (WSL2)"
|
|
42
|
+
- "Other"
|
|
43
|
+
validations:
|
|
44
|
+
required: true
|
|
45
|
+
|
|
46
|
+
- type: dropdown
|
|
47
|
+
id: isolation-mode
|
|
48
|
+
attributes:
|
|
49
|
+
label: Execution Isolation Mode
|
|
50
|
+
description: Which execution mode was used?
|
|
51
|
+
options:
|
|
52
|
+
- "isolated (Docker container with --network=none)"
|
|
53
|
+
- "trusted (in-process Python execution)"
|
|
54
|
+
validations:
|
|
55
|
+
required: true
|
|
56
|
+
|
|
57
|
+
- type: textarea
|
|
58
|
+
id: scenario-info
|
|
59
|
+
attributes:
|
|
60
|
+
label: Scenario & Agent Configuration
|
|
61
|
+
description: Which scenario (built-in YAML or custom) and agent adapter (Generic Callable, LangGraph, custom) were you running?
|
|
62
|
+
placeholder: "e.g. scenarios/payment_timeout_after_commit.yaml with custom LangGraph agent"
|
|
63
|
+
validations:
|
|
64
|
+
required: true
|
|
65
|
+
|
|
66
|
+
- type: textarea
|
|
67
|
+
id: repro
|
|
68
|
+
attributes:
|
|
69
|
+
label: Steps to Reproduce
|
|
70
|
+
description: Exact CLI commands or Python code used to reproduce the issue
|
|
71
|
+
placeholder: |
|
|
72
|
+
1. recoverybench run scenarios/my_scenario.yaml --agent my_agent --isolation isolated
|
|
73
|
+
2. ...
|
|
74
|
+
validations:
|
|
75
|
+
required: true
|
|
76
|
+
|
|
77
|
+
- type: textarea
|
|
78
|
+
id: expected
|
|
79
|
+
attributes:
|
|
80
|
+
label: Expected Behavior
|
|
81
|
+
description: What did you expect to happen?
|
|
82
|
+
validations:
|
|
83
|
+
required: true
|
|
84
|
+
|
|
85
|
+
- type: textarea
|
|
86
|
+
id: logs
|
|
87
|
+
attributes:
|
|
88
|
+
label: Sanitized Output & Traceback
|
|
89
|
+
description: Paste relevant terminal output or tracebacks. Ensure all proprietary URLs or secrets are removed!
|
|
90
|
+
render: shell
|
|
91
|
+
|
|
92
|
+
- type: checkboxes
|
|
93
|
+
id: confirmation
|
|
94
|
+
attributes:
|
|
95
|
+
label: Sanity Checks
|
|
96
|
+
options:
|
|
97
|
+
- label: "I have confirmed that no secrets, API keys, or private tokens are present in this issue."
|
|
98
|
+
required: true
|
|
99
|
+
- label: "I have checked existing GitHub issues to ensure this is not a duplicate."
|
|
100
|
+
required: true
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: Feature Request
|
|
2
|
+
description: Propose a new capability, fault injection mode, or invariant evaluation rule
|
|
3
|
+
title: "[Feature]: "
|
|
4
|
+
labels: ["enhancement"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
### Feature Request for RecoveryBench
|
|
10
|
+
Propose enhancements to fault injection, ledger capabilities, recovery evaluator rules, or reporting formats.
|
|
11
|
+
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: problem
|
|
14
|
+
attributes:
|
|
15
|
+
label: Is your feature request related to a problem or blindspot?
|
|
16
|
+
description: A clear and concise description of the limitation you encountered.
|
|
17
|
+
placeholder: "e.g. When testing webhook retries, I need to simulate payload corruptions with specific JSON schema drift..."
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
|
|
21
|
+
- type: textarea
|
|
22
|
+
id: proposal
|
|
23
|
+
attributes:
|
|
24
|
+
label: Proposed Solution or Behavior
|
|
25
|
+
description: Describe what you would like RecoveryBench to do.
|
|
26
|
+
validations:
|
|
27
|
+
required: true
|
|
28
|
+
|
|
29
|
+
- type: textarea
|
|
30
|
+
id: alternatives
|
|
31
|
+
attributes:
|
|
32
|
+
label: Alternative Solutions Considered
|
|
33
|
+
description: Any workarounds or alternative designs you have evaluated.
|
|
34
|
+
|
|
35
|
+
- type: checkboxes
|
|
36
|
+
id: community
|
|
37
|
+
attributes:
|
|
38
|
+
label: Contribution
|
|
39
|
+
options:
|
|
40
|
+
- label: "I would be interested in contributing or testing a pull request for this feature."
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: Framework Integration Request
|
|
2
|
+
description: Request or propose a new agent framework adapter (e.g. CrewAI, AutoGen, Model Context Protocol)
|
|
3
|
+
title: "[Integration]: "
|
|
4
|
+
labels: ["integration", "enhancement"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
### Framework Adapter Request
|
|
10
|
+
RecoveryBench currently provides native adapters for **Generic Python Callables** and **LangGraph**.
|
|
11
|
+
Use this template to propose integration with additional agent frameworks.
|
|
12
|
+
|
|
13
|
+
- type: input
|
|
14
|
+
id: framework
|
|
15
|
+
attributes:
|
|
16
|
+
label: Framework Name & Version
|
|
17
|
+
placeholder: "e.g. CrewAI >= 0.28.0, AutoGen >= 0.4.0, Model Context Protocol (MCP)"
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
|
|
21
|
+
- type: textarea
|
|
22
|
+
id: workflow
|
|
23
|
+
attributes:
|
|
24
|
+
label: Target Agent Workflow & Tool Execution Model
|
|
25
|
+
description: How does this framework invoke external tools and handle errors, retries, and state?
|
|
26
|
+
placeholder: "Describe how agents in this framework declare tool calls and handle ToolError exceptions..."
|
|
27
|
+
validations:
|
|
28
|
+
required: true
|
|
29
|
+
|
|
30
|
+
- type: textarea
|
|
31
|
+
id: integration-points
|
|
32
|
+
attributes:
|
|
33
|
+
label: Proposed Adapter Architecture
|
|
34
|
+
description: Where should RecoveryBench intercept tool execution and inject simulated faults?
|
|
35
|
+
placeholder: "e.g. Subclassing BaseTool / custom Hook / Middleware / Stdio IPC bridge..."
|
|
36
|
+
validations:
|
|
37
|
+
required: true
|
|
38
|
+
|
|
39
|
+
- type: textarea
|
|
40
|
+
id: references
|
|
41
|
+
attributes:
|
|
42
|
+
label: References & Documentation Links
|
|
43
|
+
description: Link to the framework repository, relevant documentation, or example implementations.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
name: "RecoveryBench Test Action"
|
|
2
|
+
description: "Run deterministic AI agent recovery testing and side-effect assertions in CI/CD"
|
|
3
|
+
author: "RecoveryBench Contributors"
|
|
4
|
+
|
|
5
|
+
inputs:
|
|
6
|
+
scenarios-dir:
|
|
7
|
+
description: "Directory containing recovery scenario YAML files"
|
|
8
|
+
required: false
|
|
9
|
+
default: "scenarios"
|
|
10
|
+
agent:
|
|
11
|
+
description: "Agent adapter to evaluate: 'idempotent', 'naive', 'langgraph-idempotent', 'langgraph-naive'"
|
|
12
|
+
required: false
|
|
13
|
+
default: "idempotent"
|
|
14
|
+
output-dir:
|
|
15
|
+
description: "Directory to save generated JSON, HTML, and JUnit XML reports"
|
|
16
|
+
required: false
|
|
17
|
+
default: "reports"
|
|
18
|
+
junit:
|
|
19
|
+
description: "Whether to emit JUnit XML reports"
|
|
20
|
+
required: false
|
|
21
|
+
default: "true"
|
|
22
|
+
isolation:
|
|
23
|
+
description: "Execution isolation mode: 'isolated' (Docker container) or 'trusted' (in-process)"
|
|
24
|
+
required: false
|
|
25
|
+
default: "isolated"
|
|
26
|
+
fail-on-failure:
|
|
27
|
+
description: "Whether to fail the step if any recovery assertion fails"
|
|
28
|
+
required: false
|
|
29
|
+
default: "true"
|
|
30
|
+
|
|
31
|
+
outputs:
|
|
32
|
+
passed:
|
|
33
|
+
description: "Whether all evaluated scenarios passed ('true' or 'false')"
|
|
34
|
+
value: ${{ steps.run-suite.outputs.passed }}
|
|
35
|
+
total-scenarios:
|
|
36
|
+
description: "Number of scenarios evaluated"
|
|
37
|
+
value: ${{ steps.run-suite.outputs.total_scenarios }}
|
|
38
|
+
duplicate-effects:
|
|
39
|
+
description: "Total duplicate side effects detected"
|
|
40
|
+
value: ${{ steps.run-suite.outputs.duplicate_effects }}
|
|
41
|
+
|
|
42
|
+
runs:
|
|
43
|
+
using: "composite"
|
|
44
|
+
steps:
|
|
45
|
+
- name: Validate Scenarios
|
|
46
|
+
shell: bash
|
|
47
|
+
run: |
|
|
48
|
+
echo "Validating scenarios in: ${{ inputs.scenarios-dir }}"
|
|
49
|
+
recoverybench validate "${{ inputs.scenarios-dir }}"
|
|
50
|
+
|
|
51
|
+
- name: Run RecoveryBench Suite
|
|
52
|
+
id: run-suite
|
|
53
|
+
shell: bash
|
|
54
|
+
run: |
|
|
55
|
+
JUNIT_FLAG="--junit"
|
|
56
|
+
if [ "${{ inputs.junit }}" = "false" ]; then
|
|
57
|
+
JUNIT_FLAG="--no-junit"
|
|
58
|
+
fi
|
|
59
|
+
|
|
60
|
+
echo "Executing RecoveryBench suite..."
|
|
61
|
+
set +e
|
|
62
|
+
recoverybench suite "${{ inputs.scenarios-dir }}" \
|
|
63
|
+
--agent "${{ inputs.agent }}" \
|
|
64
|
+
--isolation "${{ inputs.isolation }}" \
|
|
65
|
+
--output-dir "${{ inputs.output-dir }}" \
|
|
66
|
+
$JUNIT_FLAG
|
|
67
|
+
EXIT_CODE=$?
|
|
68
|
+
set -e
|
|
69
|
+
|
|
70
|
+
if [ $EXIT_CODE -eq 0 ]; then
|
|
71
|
+
echo "passed=true" >> "$GITHUB_OUTPUT"
|
|
72
|
+
else
|
|
73
|
+
echo "passed=false" >> "$GITHUB_OUTPUT"
|
|
74
|
+
fi
|
|
75
|
+
|
|
76
|
+
# Extract metrics from suite_summary.json if generated
|
|
77
|
+
if [ -f "${{ inputs.output-dir }}/suite_summary.json" ]; then
|
|
78
|
+
python -c "
|
|
79
|
+
import json
|
|
80
|
+
with open('${{ inputs.output-dir }}/suite_summary.json') as f:
|
|
81
|
+
d = json.load(f)
|
|
82
|
+
print(f'total_scenarios={d.get(\"total_scenarios\", 0)}')
|
|
83
|
+
print(f'duplicate_effects={sum(s.get(\"duplicate_effects\", 0) for s in d.get(\"scenarios\", []))}')
|
|
84
|
+
" >> "$GITHUB_OUTPUT"
|
|
85
|
+
fi
|
|
86
|
+
|
|
87
|
+
if [ $EXIT_CODE -ne 0 ] && [ "${{ inputs.fail-on-failure }}" = "true" ]; then
|
|
88
|
+
echo "::error::RecoveryBench evaluation failed with exit code $EXIT_CODE. One or more agent recovery invariants were violated!"
|
|
89
|
+
exit $EXIT_CODE
|
|
90
|
+
fi
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
name: Build & Validate Distribution Packages
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- name: Check out repository
|
|
16
|
+
uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.11"
|
|
22
|
+
cache: "pip"
|
|
23
|
+
|
|
24
|
+
- name: Validate Release Tag Matches pyproject.toml Version
|
|
25
|
+
run: |
|
|
26
|
+
python -c "
|
|
27
|
+
import os, sys, tomllib
|
|
28
|
+
|
|
29
|
+
tag_name = os.environ.get('RELEASE_TAG', '').strip()
|
|
30
|
+
if not tag_name:
|
|
31
|
+
sys.exit('Release tag validation failed: RELEASE_TAG environment variable is empty or missing!')
|
|
32
|
+
|
|
33
|
+
normalized_tag = tag_name[1:] if tag_name.startswith('v') else tag_name
|
|
34
|
+
|
|
35
|
+
with open('pyproject.toml', 'rb') as f:
|
|
36
|
+
pyproject = tomllib.load(f)
|
|
37
|
+
|
|
38
|
+
pkg_version = pyproject.get('project', {}).get('version', '').strip()
|
|
39
|
+
if not pkg_version:
|
|
40
|
+
sys.exit('Release tag validation failed: Could not read project.version from pyproject.toml!')
|
|
41
|
+
|
|
42
|
+
print(f'Release tag: \'{tag_name}\' (normalized: \'{normalized_tag}\')')
|
|
43
|
+
print(f'pyproject.toml version: \'{pkg_version}\'')
|
|
44
|
+
|
|
45
|
+
if normalized_tag != pkg_version:
|
|
46
|
+
sys.exit(
|
|
47
|
+
f'Release tag mismatch! Tag \'{tag_name}\' (normalized \'{normalized_tag}\') '
|
|
48
|
+
f'does not match pyproject.toml version \'{pkg_version}\'.'
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
print('Release tag matches pyproject.toml version successfully!')
|
|
52
|
+
"
|
|
53
|
+
env:
|
|
54
|
+
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
55
|
+
|
|
56
|
+
- name: Install Build and Distribution Validation Tools
|
|
57
|
+
run: |
|
|
58
|
+
python -m pip install --upgrade pip
|
|
59
|
+
python -m pip install --upgrade build twine
|
|
60
|
+
|
|
61
|
+
- name: Build Wheel and Source Distribution
|
|
62
|
+
run: python -m build
|
|
63
|
+
|
|
64
|
+
- name: Validate Package Metadata with Twine Check
|
|
65
|
+
run: twine check --strict dist/*
|
|
66
|
+
|
|
67
|
+
- name: Execute Distribution Smoke Test in Isolated Environment
|
|
68
|
+
run: python scripts/distribution_smoke_test.py
|
|
69
|
+
|
|
70
|
+
- name: Upload Built Package Artifacts
|
|
71
|
+
uses: actions/upload-artifact@v4
|
|
72
|
+
with:
|
|
73
|
+
name: python-package-distributions
|
|
74
|
+
path: dist/
|
|
75
|
+
retention-days: 1
|
|
76
|
+
|
|
77
|
+
publish-to-pypi:
|
|
78
|
+
name: Publish to PyPI via Trusted Publishing (OIDC)
|
|
79
|
+
if: github.event_name == 'release' && github.event.action == 'published'
|
|
80
|
+
needs: [build]
|
|
81
|
+
runs-on: ubuntu-latest
|
|
82
|
+
environment:
|
|
83
|
+
name: pypi
|
|
84
|
+
url: https://pypi.org/p/recoverybench
|
|
85
|
+
permissions:
|
|
86
|
+
id-token: write # MANDATORY for PyPI Trusted Publishing OIDC token exchange
|
|
87
|
+
|
|
88
|
+
steps:
|
|
89
|
+
- name: Download Built Artifacts
|
|
90
|
+
uses: actions/download-artifact@v4
|
|
91
|
+
with:
|
|
92
|
+
name: python-package-distributions
|
|
93
|
+
path: dist/
|
|
94
|
+
|
|
95
|
+
- name: Publish Package to PyPI
|
|
96
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
97
|
+
with:
|
|
98
|
+
packages-dir: dist/
|
|
99
|
+
verbose: true
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
name: RecoveryBench CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
- master
|
|
8
|
+
- fix/p0-audit-remediation
|
|
9
|
+
- phase-2/cli-distribution
|
|
10
|
+
pull_request:
|
|
11
|
+
branches:
|
|
12
|
+
- main
|
|
13
|
+
- master
|
|
14
|
+
- fix/p0-audit-remediation
|
|
15
|
+
- phase-2/cli-distribution
|
|
16
|
+
workflow_dispatch:
|
|
17
|
+
inputs:
|
|
18
|
+
agent:
|
|
19
|
+
description: "Agent adapter to evaluate"
|
|
20
|
+
required: true
|
|
21
|
+
default: "idempotent"
|
|
22
|
+
type: choice
|
|
23
|
+
options:
|
|
24
|
+
- idempotent
|
|
25
|
+
- langgraph-idempotent
|
|
26
|
+
- naive
|
|
27
|
+
- langgraph-naive
|
|
28
|
+
isolation:
|
|
29
|
+
description: "Execution isolation mode: 'isolated' (Docker container) or 'trusted' (in-process)"
|
|
30
|
+
required: true
|
|
31
|
+
default: "isolated"
|
|
32
|
+
type: choice
|
|
33
|
+
options:
|
|
34
|
+
- isolated
|
|
35
|
+
- trusted
|
|
36
|
+
|
|
37
|
+
permissions:
|
|
38
|
+
contents: read
|
|
39
|
+
checks: write
|
|
40
|
+
pull-requests: write
|
|
41
|
+
|
|
42
|
+
concurrency:
|
|
43
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
44
|
+
cancel-in-progress: true
|
|
45
|
+
|
|
46
|
+
jobs:
|
|
47
|
+
recovery-test:
|
|
48
|
+
name: Recovery Evaluation (${{ matrix.os }}, Python ${{ matrix.python-version }})
|
|
49
|
+
runs-on: ${{ matrix.os }}
|
|
50
|
+
strategy:
|
|
51
|
+
fail-fast: false
|
|
52
|
+
matrix:
|
|
53
|
+
os: ["ubuntu-22.04", "ubuntu-24.04"]
|
|
54
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
55
|
+
|
|
56
|
+
steps:
|
|
57
|
+
- name: Check out repository
|
|
58
|
+
uses: actions/checkout@v4
|
|
59
|
+
|
|
60
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
61
|
+
uses: actions/setup-python@v5
|
|
62
|
+
with:
|
|
63
|
+
python-version: ${{ matrix.python-version }}
|
|
64
|
+
cache: "pip"
|
|
65
|
+
|
|
66
|
+
- name: Install RecoveryBench & Dependencies
|
|
67
|
+
run: |
|
|
68
|
+
python -m pip install --upgrade pip setuptools wheel
|
|
69
|
+
python -m pip install -e ".[langgraph,dev]"
|
|
70
|
+
|
|
71
|
+
- name: Pull Pinned Docker Sandbox Runtime Image
|
|
72
|
+
run: |
|
|
73
|
+
docker pull python:3.11-slim@sha256:07d4824036af339fa1b6ef769cc99d9c10a141769538b52b3b597c84f7a7b535
|
|
74
|
+
|
|
75
|
+
- name: Environment Health Check (Doctor with Required Isolation)
|
|
76
|
+
run: recoverybench doctor --require-isolation
|
|
77
|
+
|
|
78
|
+
- name: Validate Scenario Definitions
|
|
79
|
+
run: recoverybench validate scenarios
|
|
80
|
+
|
|
81
|
+
- name: Execute RecoveryBench Scenario Suite
|
|
82
|
+
id: scenario_suite
|
|
83
|
+
run: |
|
|
84
|
+
AGENT="${{ github.event.inputs.agent || 'idempotent' }}"
|
|
85
|
+
ISOLATION="${{ github.event.inputs.isolation || 'isolated' }}"
|
|
86
|
+
echo "Running RecoveryBench suite against agent: $AGENT (isolation: $ISOLATION)"
|
|
87
|
+
recoverybench suite scenarios \
|
|
88
|
+
--agent "$AGENT" \
|
|
89
|
+
--isolation "$ISOLATION" \
|
|
90
|
+
--output-dir "reports" \
|
|
91
|
+
--junit
|
|
92
|
+
|
|
93
|
+
- name: Publish JUnit Test Results
|
|
94
|
+
uses: mikepenz/action-junit-report@v4
|
|
95
|
+
if: always() && (steps.scenario_suite.conclusion == 'success' || steps.scenario_suite.conclusion == 'failure')
|
|
96
|
+
with:
|
|
97
|
+
report_paths: "reports/*_report.xml"
|
|
98
|
+
check_name: "RecoveryBench Invariant Results (${{ matrix.os }}, py${{ matrix.python-version }})"
|
|
99
|
+
fail_on_failure: true
|
|
100
|
+
require_tests: true
|
|
101
|
+
|
|
102
|
+
- name: Upload Test Report Artifacts
|
|
103
|
+
uses: actions/upload-artifact@v4
|
|
104
|
+
if: always() && (steps.scenario_suite.conclusion == 'success' || steps.scenario_suite.conclusion == 'failure')
|
|
105
|
+
with:
|
|
106
|
+
name: recoverybench-reports-${{ matrix.os }}-py${{ matrix.python-version }}
|
|
107
|
+
path: reports/
|
|
108
|
+
retention-days: 30
|
|
109
|
+
|
|
110
|
+
- name: Run Lint and Static Type Verification
|
|
111
|
+
if: matrix.python-version == '3.11'
|
|
112
|
+
run: |
|
|
113
|
+
ruff check .
|
|
114
|
+
mypy src tests
|
|
115
|
+
|
|
116
|
+
- name: Verify Package Artifacts & Metadata (PEP 639)
|
|
117
|
+
if: matrix.python-version == '3.11'
|
|
118
|
+
run: |
|
|
119
|
+
python -m pip install --upgrade build
|
|
120
|
+
python -m build
|
|
121
|
+
python -c "
|
|
122
|
+
import zipfile, tarfile, pathlib, sys
|
|
123
|
+
|
|
124
|
+
dist_dir = pathlib.Path('dist')
|
|
125
|
+
wheels = list(dist_dir.glob('*.whl'))
|
|
126
|
+
sdists = list(dist_dir.glob('*.tar.gz'))
|
|
127
|
+
|
|
128
|
+
if not wheels:
|
|
129
|
+
sys.exit('Packaging verification failed: No wheel (.whl) artifact found in dist/')
|
|
130
|
+
if not sdists:
|
|
131
|
+
sys.exit('Packaging verification failed: No sdist (.tar.gz) artifact found in dist/')
|
|
132
|
+
|
|
133
|
+
wheel_path = wheels[0]
|
|
134
|
+
sdist_path = sdists[0]
|
|
135
|
+
|
|
136
|
+
print(f'Inspecting wheel: {wheel_path.name}')
|
|
137
|
+
with zipfile.ZipFile(wheel_path) as zf:
|
|
138
|
+
names = zf.namelist()
|
|
139
|
+
license_names = [n for n in names if n.endswith('.dist-info/licenses/LICENSE') or n.endswith('.dist-info/LICENSE')]
|
|
140
|
+
if not license_names:
|
|
141
|
+
sys.exit(f'Wheel verification failed: LICENSE file missing from dist-info. Found: {names}')
|
|
142
|
+
print(f'Found wheel license: {license_names}')
|
|
143
|
+
|
|
144
|
+
metadata_names = [n for n in names if n.endswith('.dist-info/METADATA')]
|
|
145
|
+
if not metadata_names:
|
|
146
|
+
sys.exit('Wheel verification failed: METADATA file missing from dist-info')
|
|
147
|
+
|
|
148
|
+
metadata_text = zf.read(metadata_names[0]).decode('utf-8', errors='replace')
|
|
149
|
+
if 'License-Expression: Apache-2.0' not in metadata_text:
|
|
150
|
+
sys.exit('Wheel verification failed: License-Expression: Apache-2.0 not in METADATA')
|
|
151
|
+
if 'License-File: LICENSE' not in metadata_text:
|
|
152
|
+
sys.exit('Wheel verification failed: License-File: LICENSE not in METADATA')
|
|
153
|
+
print('Wheel METADATA verified (PEP 639 License-Expression and License-File present).')
|
|
154
|
+
|
|
155
|
+
print(f'Inspecting sdist: {sdist_path.name}')
|
|
156
|
+
with tarfile.open(sdist_path, 'r:gz') as tf:
|
|
157
|
+
tar_names = tf.getnames()
|
|
158
|
+
sdist_licenses = [n for n in tar_names if n.endswith('/LICENSE') or n == 'LICENSE']
|
|
159
|
+
if not sdist_licenses:
|
|
160
|
+
sys.exit(f'Sdist verification failed: LICENSE missing in sdist. Found: {tar_names[:10]}')
|
|
161
|
+
print(f'Found sdist license: {sdist_licenses}')
|
|
162
|
+
|
|
163
|
+
pkg_info_names = [n for n in tar_names if n.endswith('/PKG-INFO') or n == 'PKG-INFO']
|
|
164
|
+
if not pkg_info_names:
|
|
165
|
+
sys.exit('Sdist verification failed: PKG-INFO missing in sdist')
|
|
166
|
+
|
|
167
|
+
pkg_info_f = tf.extractfile(pkg_info_names[0])
|
|
168
|
+
if pkg_info_f is None:
|
|
169
|
+
sys.exit('Sdist verification failed: could not extract PKG-INFO')
|
|
170
|
+
pkg_info_text = pkg_info_f.read().decode('utf-8', errors='replace')
|
|
171
|
+
if 'License-Expression: Apache-2.0' not in pkg_info_text:
|
|
172
|
+
sys.exit('Sdist verification failed: License-Expression: Apache-2.0 not in PKG-INFO')
|
|
173
|
+
if 'License-File: LICENSE' not in pkg_info_text:
|
|
174
|
+
sys.exit('Sdist verification failed: License-File: LICENSE not in PKG-INFO')
|
|
175
|
+
print('Sdist PKG-INFO verified (PEP 639 License-Expression and License-File present).')
|
|
176
|
+
|
|
177
|
+
print('Packaging and licensing verification successful!')
|
|
178
|
+
"
|
|
179
|
+
|
|
180
|
+
- name: Execute Clean Wheel-Install Distribution Smoke Test
|
|
181
|
+
run: |
|
|
182
|
+
python -m pip install --upgrade build
|
|
183
|
+
python scripts/distribution_smoke_test.py
|
|
184
|
+
|
|
185
|
+
- name: Run Unit & Adapter Test Suites
|
|
186
|
+
run: |
|
|
187
|
+
pytest -v tests/test_*.py
|
|
188
|
+
|
|
189
|
+
- name: Run Security & Sandbox Configuration Suites
|
|
190
|
+
run: |
|
|
191
|
+
pytest -v tests/security/test_container_configuration.py tests/security/test_filesystem_isolation.py tests/security/test_worker_ipc.py tests/security/test_container_lifecycle.py
|
|
192
|
+
|
|
193
|
+
- name: Run Live Docker Network Isolation Integration Tests
|
|
194
|
+
run: |
|
|
195
|
+
docker info
|
|
196
|
+
pytest -v -rs tests/security/test_network_isolation.py
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: RecoveryBench PR Quality Gate
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
types: [opened, synchronize, reopened]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
pull-requests: write
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
pr-recovery-gate:
|
|
13
|
+
name: Agent Recovery Invariant Verification
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- name: Check out PR code
|
|
18
|
+
uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Set up Python
|
|
21
|
+
uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.11"
|
|
24
|
+
cache: "pip"
|
|
25
|
+
|
|
26
|
+
- name: Install RecoveryBench
|
|
27
|
+
run: |
|
|
28
|
+
python -m pip install --upgrade pip
|
|
29
|
+
python -m pip install -e ".[langgraph,dev]"
|
|
30
|
+
|
|
31
|
+
- name: Verify Scenarios
|
|
32
|
+
run: recoverybench validate scenarios
|
|
33
|
+
|
|
34
|
+
- name: Pull Pinned Docker Sandbox Runtime Image
|
|
35
|
+
run: docker pull python:3.11-slim@sha256:07d4824036af339fa1b6ef769cc99d9c10a141769538b52b3b597c84f7a7b535
|
|
36
|
+
|
|
37
|
+
- name: Execute Full Recovery Suite
|
|
38
|
+
id: eval-suite
|
|
39
|
+
run: |
|
|
40
|
+
recoverybench suite scenarios \
|
|
41
|
+
--agent "idempotent" \
|
|
42
|
+
--isolation "isolated" \
|
|
43
|
+
--output-dir "pr-reports" \
|
|
44
|
+
--junit
|
|
45
|
+
|
|
46
|
+
- name: Run Live Docker Network Isolation Tests
|
|
47
|
+
run: |
|
|
48
|
+
pytest -v -rs tests/security/test_network_isolation.py
|
|
49
|
+
|
|
50
|
+
- name: Upload PR Recovery Artifacts
|
|
51
|
+
uses: actions/upload-artifact@v4
|
|
52
|
+
if: always() && (steps.eval-suite.conclusion == 'success' || steps.eval-suite.conclusion == 'failure')
|
|
53
|
+
with:
|
|
54
|
+
name: pr-recovery-reports
|
|
55
|
+
path: pr-reports/
|
|
56
|
+
retention-days: 14
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
develop-eggs/
|
|
9
|
+
dist/
|
|
10
|
+
downloads/
|
|
11
|
+
eggs/
|
|
12
|
+
.eggs/
|
|
13
|
+
lib/
|
|
14
|
+
lib64/
|
|
15
|
+
parts/
|
|
16
|
+
sdist/
|
|
17
|
+
var/
|
|
18
|
+
wheels/
|
|
19
|
+
share/python-wheels/
|
|
20
|
+
*.egg-info/
|
|
21
|
+
.installed.cfg
|
|
22
|
+
*.egg
|
|
23
|
+
MANIFEST
|
|
24
|
+
|
|
25
|
+
# Virtual environments
|
|
26
|
+
.venv/
|
|
27
|
+
venv/
|
|
28
|
+
ENV/
|
|
29
|
+
env/
|
|
30
|
+
|
|
31
|
+
# Testing & Coverage
|
|
32
|
+
.pytest_cache/
|
|
33
|
+
.coverage
|
|
34
|
+
.coverage.*
|
|
35
|
+
htmlcov/
|
|
36
|
+
.mypy_cache/
|
|
37
|
+
.ruff_cache/
|
|
38
|
+
|
|
39
|
+
# Local runtime data
|
|
40
|
+
.recoverybench/
|
|
41
|
+
reports/
|
|
42
|
+
pr-reports/
|
|
43
|
+
*.db
|