recoverybench 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. recoverybench-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +100 -0
  2. recoverybench-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml +40 -0
  3. recoverybench-0.1.0/.github/ISSUE_TEMPLATE/integration_request.yml +43 -0
  4. recoverybench-0.1.0/.github/actions/recoverybench-test/action.yml +90 -0
  5. recoverybench-0.1.0/.github/workflows/publish.yml +99 -0
  6. recoverybench-0.1.0/.github/workflows/recoverybench-ci.yml +196 -0
  7. recoverybench-0.1.0/.github/workflows/recoverybench-pr-gate.yml +56 -0
  8. recoverybench-0.1.0/.gitignore +43 -0
  9. recoverybench-0.1.0/CHANGELOG.md +43 -0
  10. recoverybench-0.1.0/FINAL_RELEASE_READINESS_AUDIT.md +154 -0
  11. recoverybench-0.1.0/GITHUB_ACTIONS_DOCKER_DIGEST_REMEDIATION.md +177 -0
  12. recoverybench-0.1.0/GITHUB_ACTIONS_MATRIX_REMEDIATION.md +154 -0
  13. recoverybench-0.1.0/GITHUB_PUSH_PROTECTION_REMEDIATION.md +139 -0
  14. recoverybench-0.1.0/LAUNCH_PHASE_1_COMPLETION_REPORT.md +125 -0
  15. recoverybench-0.1.0/LICENSE +177 -0
  16. recoverybench-0.1.0/P0_3_FINAL_CONDITION_REMEDIATION.md +251 -0
  17. recoverybench-0.1.0/P0_3_GITHUB_ACTIONS_ROOT_CAUSE_REMEDIATION.md +167 -0
  18. recoverybench-0.1.0/P0_3_LINUX_WORKSPACE_PERMISSION_REMEDIATION.md +150 -0
  19. recoverybench-0.1.0/PHASE_2A_DISTRIBUTION_READINESS_REPORT.md +187 -0
  20. recoverybench-0.1.0/PHASE_2A_RELEASE_WORKFLOW_HARDENING_REPORT.md +116 -0
  21. recoverybench-0.1.0/PKG-INFO +496 -0
  22. recoverybench-0.1.0/README.md +459 -0
  23. recoverybench-0.1.0/docs/ARCHITECTURE.md +303 -0
  24. recoverybench-0.1.0/docs/CODEBASE_AUDIT.md +682 -0
  25. recoverybench-0.1.0/docs/COMPETITIVE_POSITIONING.md +100 -0
  26. recoverybench-0.1.0/docs/DEMO_WALKTHROUGH.md +204 -0
  27. recoverybench-0.1.0/docs/FIRST_USER_GUIDE.md +345 -0
  28. recoverybench-0.1.0/docs/GITHUB_ACTIONS_GUIDE.md +210 -0
  29. recoverybench-0.1.0/docs/IMPLEMENTATION_PLAN.md +128 -0
  30. recoverybench-0.1.0/docs/P0_1_FINAL_HARDENING.md +200 -0
  31. recoverybench-0.1.0/docs/P0_1_TRANSACTION_IDENTITY_REMEDIATION.md +203 -0
  32. recoverybench-0.1.0/docs/P0_2_CLOSEOUT_VERIFICATION.md +270 -0
  33. recoverybench-0.1.0/docs/P0_2_FALSE_SUCCESS_REMEDIATION.md +215 -0
  34. recoverybench-0.1.0/docs/P0_2_FINAL_HARDENING.md +214 -0
  35. recoverybench-0.1.0/docs/P0_3_FINAL_CONDITION_REMEDIATION.md +251 -0
  36. recoverybench-0.1.0/docs/P0_3_FINAL_SECURITY_CERTIFICATION.md +322 -0
  37. recoverybench-0.1.0/docs/P0_3_INDEPENDENT_CODE_REVIEW.md +267 -0
  38. recoverybench-0.1.0/docs/P0_3_NETWORK_ISOLATION_REMEDIATION.md +360 -0
  39. recoverybench-0.1.0/docs/P0_3_RUNTIME_VERIFICATION.md +308 -0
  40. recoverybench-0.1.0/docs/P0_3_TEST_FAILURE_REMEDIATION.md +190 -0
  41. recoverybench-0.1.0/docs/PRODUCT_SPEC.md +114 -0
  42. recoverybench-0.1.0/docs/SDK_INTEGRATION_GUIDE.md +514 -0
  43. recoverybench-0.1.0/docs/SECURITY_MODEL.md +131 -0
  44. recoverybench-0.1.0/docs/TEST_STRATEGY.md +143 -0
  45. recoverybench-0.1.0/examples/generic_callable_agent.py +100 -0
  46. recoverybench-0.1.0/examples/langgraph_recovery_agent.py +55 -0
  47. recoverybench-0.1.0/examples/sdk_quickstart.py +40 -0
  48. recoverybench-0.1.0/pyproject.toml +87 -0
  49. recoverybench-0.1.0/recoverybench.yaml +7 -0
  50. recoverybench-0.1.0/scenarios/payment_duplicate_delivery.yaml +26 -0
  51. recoverybench-0.1.0/scenarios/payment_http_500_transient.yaml +25 -0
  52. recoverybench-0.1.0/scenarios/payment_malformed_response.yaml +26 -0
  53. recoverybench-0.1.0/scenarios/payment_rate_limit_429.yaml +26 -0
  54. recoverybench-0.1.0/scenarios/payment_timeout_after_commit.yaml +27 -0
  55. recoverybench-0.1.0/scenarios/payment_timeout_before_commit.yaml +26 -0
  56. recoverybench-0.1.0/scripts/distribution_smoke_test.py +226 -0
  57. recoverybench-0.1.0/src/recoverybench/__init__.py +68 -0
  58. recoverybench-0.1.0/src/recoverybench/adapters/__init__.py +34 -0
  59. recoverybench-0.1.0/src/recoverybench/adapters/callable.py +205 -0
  60. recoverybench-0.1.0/src/recoverybench/adapters/langgraph.py +319 -0
  61. recoverybench-0.1.0/src/recoverybench/agent/__init__.py +21 -0
  62. recoverybench-0.1.0/src/recoverybench/agent/base.py +212 -0
  63. recoverybench-0.1.0/src/recoverybench/agent/idempotent.py +77 -0
  64. recoverybench-0.1.0/src/recoverybench/agent/naive.py +58 -0
  65. recoverybench-0.1.0/src/recoverybench/cli.py +829 -0
  66. recoverybench-0.1.0/src/recoverybench/clock.py +47 -0
  67. recoverybench-0.1.0/src/recoverybench/config.py +50 -0
  68. recoverybench-0.1.0/src/recoverybench/engine/__init__.py +6 -0
  69. recoverybench-0.1.0/src/recoverybench/engine/context.py +43 -0
  70. recoverybench-0.1.0/src/recoverybench/engine/runner.py +473 -0
  71. recoverybench-0.1.0/src/recoverybench/errors.py +88 -0
  72. recoverybench-0.1.0/src/recoverybench/evaluator/__init__.py +6 -0
  73. recoverybench-0.1.0/src/recoverybench/evaluator/evaluator.py +310 -0
  74. recoverybench-0.1.0/src/recoverybench/evaluator/rules.py +642 -0
  75. recoverybench-0.1.0/src/recoverybench/faults/__init__.py +6 -0
  76. recoverybench-0.1.0/src/recoverybench/faults/injector.py +138 -0
  77. recoverybench-0.1.0/src/recoverybench/faults/types.py +38 -0
  78. recoverybench-0.1.0/src/recoverybench/ledger/__init__.py +7 -0
  79. recoverybench-0.1.0/src/recoverybench/ledger/base.py +57 -0
  80. recoverybench-0.1.0/src/recoverybench/ledger/memory.py +48 -0
  81. recoverybench-0.1.0/src/recoverybench/ledger/sqlite.py +311 -0
  82. recoverybench-0.1.0/src/recoverybench/models.py +137 -0
  83. recoverybench-0.1.0/src/recoverybench/py.typed +0 -0
  84. recoverybench-0.1.0/src/recoverybench/replay/__init__.py +5 -0
  85. recoverybench-0.1.0/src/recoverybench/replay/engine.py +58 -0
  86. recoverybench-0.1.0/src/recoverybench/reporting/__init__.py +20 -0
  87. recoverybench-0.1.0/src/recoverybench/reporting/console.py +143 -0
  88. recoverybench-0.1.0/src/recoverybench/reporting/github_summary.py +83 -0
  89. recoverybench-0.1.0/src/recoverybench/reporting/html_reporter.py +260 -0
  90. recoverybench-0.1.0/src/recoverybench/reporting/json_reporter.py +88 -0
  91. recoverybench-0.1.0/src/recoverybench/reporting/junit_reporter.py +121 -0
  92. recoverybench-0.1.0/src/recoverybench/sample_project/__init__.py +13 -0
  93. recoverybench-0.1.0/src/recoverybench/sample_project/resources.py +67 -0
  94. recoverybench-0.1.0/src/recoverybench/sandbox/__init__.py +34 -0
  95. recoverybench-0.1.0/src/recoverybench/sandbox/backend.py +471 -0
  96. recoverybench-0.1.0/src/recoverybench/sandbox/config.py +216 -0
  97. recoverybench-0.1.0/src/recoverybench/sandbox/docker_client.py +175 -0
  98. recoverybench-0.1.0/src/recoverybench/sandbox/protocol.py +135 -0
  99. recoverybench-0.1.0/src/recoverybench/sandbox/spec.py +174 -0
  100. recoverybench-0.1.0/src/recoverybench/sandbox/staging.py +472 -0
  101. recoverybench-0.1.0/src/recoverybench/sandbox/worker.py +509 -0
  102. recoverybench-0.1.0/src/recoverybench/scenarios/__init__.py +12 -0
  103. recoverybench-0.1.0/src/recoverybench/scenarios/registry.py +233 -0
  104. recoverybench-0.1.0/src/recoverybench/scenarios/schema.py +72 -0
  105. recoverybench-0.1.0/src/recoverybench/scenarios/validator.py +107 -0
  106. recoverybench-0.1.0/src/recoverybench/sdk/__init__.py +71 -0
  107. recoverybench-0.1.0/src/recoverybench/sdk/tester.py +188 -0
  108. recoverybench-0.1.0/src/recoverybench/security.py +69 -0
  109. recoverybench-0.1.0/src/recoverybench/tools/__init__.py +17 -0
  110. recoverybench-0.1.0/src/recoverybench/tools/base.py +74 -0
  111. recoverybench-0.1.0/src/recoverybench/tools/payment.py +138 -0
  112. recoverybench-0.1.0/tests/conftest.py +112 -0
  113. recoverybench-0.1.0/tests/security/__init__.py +1 -0
  114. recoverybench-0.1.0/tests/security/test_container_configuration.py +475 -0
  115. recoverybench-0.1.0/tests/security/test_container_lifecycle.py +128 -0
  116. recoverybench-0.1.0/tests/security/test_filesystem_isolation.py +611 -0
  117. recoverybench-0.1.0/tests/security/test_network_isolation.py +502 -0
  118. recoverybench-0.1.0/tests/security/test_worker_ipc.py +362 -0
  119. recoverybench-0.1.0/tests/test_all_fault_modes_recovery.py +246 -0
  120. recoverybench-0.1.0/tests/test_ci_integration.py +225 -0
  121. recoverybench-0.1.0/tests/test_cli.py +181 -0
  122. recoverybench-0.1.0/tests/test_demo_comparison.py +48 -0
  123. recoverybench-0.1.0/tests/test_effect_ledger.py +221 -0
  124. recoverybench-0.1.0/tests/test_execution_engine.py +125 -0
  125. recoverybench-0.1.0/tests/test_false_success_final_hardening.py +519 -0
  126. recoverybench-0.1.0/tests/test_false_success_prevention.py +524 -0
  127. recoverybench-0.1.0/tests/test_fault_injection.py +202 -0
  128. recoverybench-0.1.0/tests/test_mock_payment_service.py +74 -0
  129. recoverybench-0.1.0/tests/test_recovery_evaluator.py +339 -0
  130. recoverybench-0.1.0/tests/test_replay_engine.py +46 -0
  131. recoverybench-0.1.0/tests/test_reporting.py +126 -0
  132. recoverybench-0.1.0/tests/test_schema_validation.py +98 -0
  133. recoverybench-0.1.0/tests/test_sdk_generic_callable.py +105 -0
  134. recoverybench-0.1.0/tests/test_sdk_high_level_api.py +54 -0
  135. recoverybench-0.1.0/tests/test_sdk_langgraph_adapter.py +73 -0
  136. recoverybench-0.1.0/tests/test_transaction_identity.py +985 -0
@@ -0,0 +1,100 @@
1
+ name: Bug Report
2
+ description: Report an issue, test anomaly, or harness error in RecoveryBench
3
+ title: "[Bug]: "
4
+ labels: ["bug"]
5
+ body:
6
+ - type: markdown
7
+ attributes:
8
+ value: |
9
+ ### Thank you for reporting an issue!
10
+ > **Security & Privacy Notice:** NEVER paste API keys, secret tokens, private credentials, or proprietary production payloads into GitHub issues. RecoveryBench redacts known secrets from logs, but always verify your logs before submitting.
11
+
12
+ - type: input
13
+ id: version
14
+ attributes:
15
+ label: RecoveryBench Version
16
+ description: Run `recoverybench --help` or check `pip show recoverybench`
17
+ placeholder: "0.1.0"
18
+ validations:
19
+ required: true
20
+
21
+ - type: input
22
+ id: python-version
23
+ attributes:
24
+ label: Python Version
25
+ description: Run `python --version`
26
+ placeholder: "3.11.8 / 3.12.3 / 3.13.0"
27
+ validations:
28
+ required: true
29
+
30
+ - type: dropdown
31
+ id: os
32
+ attributes:
33
+ label: Operating System
34
+ description: Select your host operating system
35
+ options:
36
+ - "Ubuntu / Debian Linux"
37
+ - "Fedora / RHEL / Arch Linux"
38
+ - "macOS (Apple Silicon)"
39
+ - "macOS (Intel)"
40
+ - "Windows 11 / 10 (PowerShell)"
41
+ - "Windows 11 / 10 (WSL2)"
42
+ - "Other"
43
+ validations:
44
+ required: true
45
+
46
+ - type: dropdown
47
+ id: isolation-mode
48
+ attributes:
49
+ label: Execution Isolation Mode
50
+ description: Which execution mode was used?
51
+ options:
52
+ - "isolated (Docker container with --network=none)"
53
+ - "trusted (in-process Python execution)"
54
+ validations:
55
+ required: true
56
+
57
+ - type: textarea
58
+ id: scenario-info
59
+ attributes:
60
+ label: Scenario & Agent Configuration
61
+ description: Which scenario (built-in YAML or custom) and agent adapter (Generic Callable, LangGraph, custom) were you running?
62
+ placeholder: "e.g. scenarios/payment_timeout_after_commit.yaml with custom LangGraph agent"
63
+ validations:
64
+ required: true
65
+
66
+ - type: textarea
67
+ id: repro
68
+ attributes:
69
+ label: Steps to Reproduce
70
+ description: Exact CLI commands or Python code used to reproduce the issue
71
+ placeholder: |
72
+ 1. recoverybench run scenarios/my_scenario.yaml --agent my_agent --isolation isolated
73
+ 2. ...
74
+ validations:
75
+ required: true
76
+
77
+ - type: textarea
78
+ id: expected
79
+ attributes:
80
+ label: Expected Behavior
81
+ description: What did you expect to happen?
82
+ validations:
83
+ required: true
84
+
85
+ - type: textarea
86
+ id: logs
87
+ attributes:
88
+ label: Sanitized Output & Traceback
89
+ description: Paste relevant terminal output or tracebacks. Ensure all proprietary URLs or secrets are removed!
90
+ render: shell
91
+
92
+ - type: checkboxes
93
+ id: confirmation
94
+ attributes:
95
+ label: Sanity Checks
96
+ options:
97
+ - label: "I have confirmed that no secrets, API keys, or private tokens are present in this issue."
98
+ required: true
99
+ - label: "I have checked existing GitHub issues to ensure this is not a duplicate."
100
+ required: true
@@ -0,0 +1,40 @@
1
+ name: Feature Request
2
+ description: Propose a new capability, fault injection mode, or invariant evaluation rule
3
+ title: "[Feature]: "
4
+ labels: ["enhancement"]
5
+ body:
6
+ - type: markdown
7
+ attributes:
8
+ value: |
9
+ ### Feature Request for RecoveryBench
10
+ Propose enhancements to fault injection, ledger capabilities, recovery evaluator rules, or reporting formats.
11
+
12
+ - type: textarea
13
+ id: problem
14
+ attributes:
15
+ label: Is your feature request related to a problem or blindspot?
16
+ description: A clear and concise description of the limitation you encountered.
17
+ placeholder: "e.g. When testing webhook retries, I need to simulate payload corruptions with specific JSON schema drift..."
18
+ validations:
19
+ required: true
20
+
21
+ - type: textarea
22
+ id: proposal
23
+ attributes:
24
+ label: Proposed Solution or Behavior
25
+ description: Describe what you would like RecoveryBench to do.
26
+ validations:
27
+ required: true
28
+
29
+ - type: textarea
30
+ id: alternatives
31
+ attributes:
32
+ label: Alternative Solutions Considered
33
+ description: Any workarounds or alternative designs you have evaluated.
34
+
35
+ - type: checkboxes
36
+ id: community
37
+ attributes:
38
+ label: Contribution
39
+ options:
40
+ - label: "I would be interested in contributing or testing a pull request for this feature."
@@ -0,0 +1,43 @@
1
+ name: Framework Integration Request
2
+ description: Request or propose a new agent framework adapter (e.g. CrewAI, AutoGen, Model Context Protocol)
3
+ title: "[Integration]: "
4
+ labels: ["integration", "enhancement"]
5
+ body:
6
+ - type: markdown
7
+ attributes:
8
+ value: |
9
+ ### Framework Adapter Request
10
+ RecoveryBench currently provides native adapters for **Generic Python Callables** and **LangGraph**.
11
+ Use this template to propose integration with additional agent frameworks.
12
+
13
+ - type: input
14
+ id: framework
15
+ attributes:
16
+ label: Framework Name & Version
17
+ placeholder: "e.g. CrewAI >= 0.28.0, AutoGen >= 0.4.0, Model Context Protocol (MCP)"
18
+ validations:
19
+ required: true
20
+
21
+ - type: textarea
22
+ id: workflow
23
+ attributes:
24
+ label: Target Agent Workflow & Tool Execution Model
25
+ description: How does this framework invoke external tools and handle errors, retries, and state?
26
+ placeholder: "Describe how agents in this framework declare tool calls and handle ToolError exceptions..."
27
+ validations:
28
+ required: true
29
+
30
+ - type: textarea
31
+ id: integration-points
32
+ attributes:
33
+ label: Proposed Adapter Architecture
34
+ description: Where should RecoveryBench intercept tool execution and inject simulated faults?
35
+ placeholder: "e.g. Subclassing BaseTool / custom Hook / Middleware / Stdio IPC bridge..."
36
+ validations:
37
+ required: true
38
+
39
+ - type: textarea
40
+ id: references
41
+ attributes:
42
+ label: References & Documentation Links
43
+ description: Link to the framework repository, relevant documentation, or example implementations.
@@ -0,0 +1,90 @@
1
+ name: "RecoveryBench Test Action"
2
+ description: "Run deterministic AI agent recovery testing and side-effect assertions in CI/CD"
3
+ author: "RecoveryBench Contributors"
4
+
5
+ inputs:
6
+ scenarios-dir:
7
+ description: "Directory containing recovery scenario YAML files"
8
+ required: false
9
+ default: "scenarios"
10
+ agent:
11
+ description: "Agent adapter to evaluate: 'idempotent', 'naive', 'langgraph-idempotent', 'langgraph-naive'"
12
+ required: false
13
+ default: "idempotent"
14
+ output-dir:
15
+ description: "Directory to save generated JSON, HTML, and JUnit XML reports"
16
+ required: false
17
+ default: "reports"
18
+ junit:
19
+ description: "Whether to emit JUnit XML reports"
20
+ required: false
21
+ default: "true"
22
+ isolation:
23
+ description: "Execution isolation mode: 'isolated' (Docker container) or 'trusted' (in-process)"
24
+ required: false
25
+ default: "isolated"
26
+ fail-on-failure:
27
+ description: "Whether to fail the step if any recovery assertion fails"
28
+ required: false
29
+ default: "true"
30
+
31
+ outputs:
32
+ passed:
33
+ description: "Whether all evaluated scenarios passed ('true' or 'false')"
34
+ value: ${{ steps.run-suite.outputs.passed }}
35
+ total-scenarios:
36
+ description: "Number of scenarios evaluated"
37
+ value: ${{ steps.run-suite.outputs.total_scenarios }}
38
+ duplicate-effects:
39
+ description: "Total duplicate side effects detected"
40
+ value: ${{ steps.run-suite.outputs.duplicate_effects }}
41
+
42
+ runs:
43
+ using: "composite"
44
+ steps:
45
+ - name: Validate Scenarios
46
+ shell: bash
47
+ run: |
48
+ echo "Validating scenarios in: ${{ inputs.scenarios-dir }}"
49
+ recoverybench validate "${{ inputs.scenarios-dir }}"
50
+
51
+ - name: Run RecoveryBench Suite
52
+ id: run-suite
53
+ shell: bash
54
+ run: |
55
+ JUNIT_FLAG="--junit"
56
+ if [ "${{ inputs.junit }}" = "false" ]; then
57
+ JUNIT_FLAG="--no-junit"
58
+ fi
59
+
60
+ echo "Executing RecoveryBench suite..."
61
+ set +e
62
+ recoverybench suite "${{ inputs.scenarios-dir }}" \
63
+ --agent "${{ inputs.agent }}" \
64
+ --isolation "${{ inputs.isolation }}" \
65
+ --output-dir "${{ inputs.output-dir }}" \
66
+ $JUNIT_FLAG
67
+ EXIT_CODE=$?
68
+ set -e
69
+
70
+ if [ $EXIT_CODE -eq 0 ]; then
71
+ echo "passed=true" >> "$GITHUB_OUTPUT"
72
+ else
73
+ echo "passed=false" >> "$GITHUB_OUTPUT"
74
+ fi
75
+
76
+ # Extract metrics from suite_summary.json if generated
77
+ if [ -f "${{ inputs.output-dir }}/suite_summary.json" ]; then
78
+ python -c "
79
+ import json
80
+ with open('${{ inputs.output-dir }}/suite_summary.json') as f:
81
+ d = json.load(f)
82
+ print(f'total_scenarios={d.get(\"total_scenarios\", 0)}')
83
+ print(f'duplicate_effects={sum(s.get(\"duplicate_effects\", 0) for s in d.get(\"scenarios\", []))}')
84
+ " >> "$GITHUB_OUTPUT"
85
+ fi
86
+
87
+ if [ $EXIT_CODE -ne 0 ] && [ "${{ inputs.fail-on-failure }}" = "true" ]; then
88
+ echo "::error::RecoveryBench evaluation failed with exit code $EXIT_CODE. One or more agent recovery invariants were violated!"
89
+ exit $EXIT_CODE
90
+ fi
@@ -0,0 +1,99 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ build:
12
+ name: Build & Validate Distribution Packages
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - name: Check out repository
16
+ uses: actions/checkout@v4
17
+
18
+ - name: Set up Python
19
+ uses: actions/setup-python@v5
20
+ with:
21
+ python-version: "3.11"
22
+ cache: "pip"
23
+
24
+ - name: Validate Release Tag Matches pyproject.toml Version
25
+ run: |
26
+ python -c "
27
+ import os, sys, tomllib
28
+
29
+ tag_name = os.environ.get('RELEASE_TAG', '').strip()
30
+ if not tag_name:
31
+ sys.exit('Release tag validation failed: RELEASE_TAG environment variable is empty or missing!')
32
+
33
+ normalized_tag = tag_name[1:] if tag_name.startswith('v') else tag_name
34
+
35
+ with open('pyproject.toml', 'rb') as f:
36
+ pyproject = tomllib.load(f)
37
+
38
+ pkg_version = pyproject.get('project', {}).get('version', '').strip()
39
+ if not pkg_version:
40
+ sys.exit('Release tag validation failed: Could not read project.version from pyproject.toml!')
41
+
42
+ print(f'Release tag: \'{tag_name}\' (normalized: \'{normalized_tag}\')')
43
+ print(f'pyproject.toml version: \'{pkg_version}\'')
44
+
45
+ if normalized_tag != pkg_version:
46
+ sys.exit(
47
+ f'Release tag mismatch! Tag \'{tag_name}\' (normalized \'{normalized_tag}\') '
48
+ f'does not match pyproject.toml version \'{pkg_version}\'.'
49
+ )
50
+
51
+ print('Release tag matches pyproject.toml version successfully!')
52
+ "
53
+ env:
54
+ RELEASE_TAG: ${{ github.event.release.tag_name }}
55
+
56
+ - name: Install Build and Distribution Validation Tools
57
+ run: |
58
+ python -m pip install --upgrade pip
59
+ python -m pip install --upgrade build twine
60
+
61
+ - name: Build Wheel and Source Distribution
62
+ run: python -m build
63
+
64
+ - name: Validate Package Metadata with Twine Check
65
+ run: twine check --strict dist/*
66
+
67
+ - name: Execute Distribution Smoke Test in Isolated Environment
68
+ run: python scripts/distribution_smoke_test.py
69
+
70
+ - name: Upload Built Package Artifacts
71
+ uses: actions/upload-artifact@v4
72
+ with:
73
+ name: python-package-distributions
74
+ path: dist/
75
+ retention-days: 1
76
+
77
+ publish-to-pypi:
78
+ name: Publish to PyPI via Trusted Publishing (OIDC)
79
+ if: github.event_name == 'release' && github.event.action == 'published'
80
+ needs: [build]
81
+ runs-on: ubuntu-latest
82
+ environment:
83
+ name: pypi
84
+ url: https://pypi.org/p/recoverybench
85
+ permissions:
86
+ id-token: write # MANDATORY for PyPI Trusted Publishing OIDC token exchange
87
+
88
+ steps:
89
+ - name: Download Built Artifacts
90
+ uses: actions/download-artifact@v4
91
+ with:
92
+ name: python-package-distributions
93
+ path: dist/
94
+
95
+ - name: Publish Package to PyPI
96
+ uses: pypa/gh-action-pypi-publish@release/v1
97
+ with:
98
+ packages-dir: dist/
99
+ verbose: true
@@ -0,0 +1,196 @@
1
+ name: RecoveryBench CI
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ - master
8
+ - fix/p0-audit-remediation
9
+ - phase-2/cli-distribution
10
+ pull_request:
11
+ branches:
12
+ - main
13
+ - master
14
+ - fix/p0-audit-remediation
15
+ - phase-2/cli-distribution
16
+ workflow_dispatch:
17
+ inputs:
18
+ agent:
19
+ description: "Agent adapter to evaluate"
20
+ required: true
21
+ default: "idempotent"
22
+ type: choice
23
+ options:
24
+ - idempotent
25
+ - langgraph-idempotent
26
+ - naive
27
+ - langgraph-naive
28
+ isolation:
29
+ description: "Execution isolation mode: 'isolated' (Docker container) or 'trusted' (in-process)"
30
+ required: true
31
+ default: "isolated"
32
+ type: choice
33
+ options:
34
+ - isolated
35
+ - trusted
36
+
37
+ permissions:
38
+ contents: read
39
+ checks: write
40
+ pull-requests: write
41
+
42
+ concurrency:
43
+ group: ${{ github.workflow }}-${{ github.ref }}
44
+ cancel-in-progress: true
45
+
46
+ jobs:
47
+ recovery-test:
48
+ name: Recovery Evaluation (${{ matrix.os }}, Python ${{ matrix.python-version }})
49
+ runs-on: ${{ matrix.os }}
50
+ strategy:
51
+ fail-fast: false
52
+ matrix:
53
+ os: ["ubuntu-22.04", "ubuntu-24.04"]
54
+ python-version: ["3.11", "3.12", "3.13"]
55
+
56
+ steps:
57
+ - name: Check out repository
58
+ uses: actions/checkout@v4
59
+
60
+ - name: Set up Python ${{ matrix.python-version }}
61
+ uses: actions/setup-python@v5
62
+ with:
63
+ python-version: ${{ matrix.python-version }}
64
+ cache: "pip"
65
+
66
+ - name: Install RecoveryBench & Dependencies
67
+ run: |
68
+ python -m pip install --upgrade pip setuptools wheel
69
+ python -m pip install -e ".[langgraph,dev]"
70
+
71
+ - name: Pull Pinned Docker Sandbox Runtime Image
72
+ run: |
73
+ docker pull python:3.11-slim@sha256:07d4824036af339fa1b6ef769cc99d9c10a141769538b52b3b597c84f7a7b535
74
+
75
+ - name: Environment Health Check (Doctor with Required Isolation)
76
+ run: recoverybench doctor --require-isolation
77
+
78
+ - name: Validate Scenario Definitions
79
+ run: recoverybench validate scenarios
80
+
81
+ - name: Execute RecoveryBench Scenario Suite
82
+ id: scenario_suite
83
+ run: |
84
+ AGENT="${{ github.event.inputs.agent || 'idempotent' }}"
85
+ ISOLATION="${{ github.event.inputs.isolation || 'isolated' }}"
86
+ echo "Running RecoveryBench suite against agent: $AGENT (isolation: $ISOLATION)"
87
+ recoverybench suite scenarios \
88
+ --agent "$AGENT" \
89
+ --isolation "$ISOLATION" \
90
+ --output-dir "reports" \
91
+ --junit
92
+
93
+ - name: Publish JUnit Test Results
94
+ uses: mikepenz/action-junit-report@v4
95
+ if: always() && (steps.scenario_suite.conclusion == 'success' || steps.scenario_suite.conclusion == 'failure')
96
+ with:
97
+ report_paths: "reports/*_report.xml"
98
+ check_name: "RecoveryBench Invariant Results (${{ matrix.os }}, py${{ matrix.python-version }})"
99
+ fail_on_failure: true
100
+ require_tests: true
101
+
102
+ - name: Upload Test Report Artifacts
103
+ uses: actions/upload-artifact@v4
104
+ if: always() && (steps.scenario_suite.conclusion == 'success' || steps.scenario_suite.conclusion == 'failure')
105
+ with:
106
+ name: recoverybench-reports-${{ matrix.os }}-py${{ matrix.python-version }}
107
+ path: reports/
108
+ retention-days: 30
109
+
110
+ - name: Run Lint and Static Type Verification
111
+ if: matrix.python-version == '3.11'
112
+ run: |
113
+ ruff check .
114
+ mypy src tests
115
+
116
+ - name: Verify Package Artifacts & Metadata (PEP 639)
117
+ if: matrix.python-version == '3.11'
118
+ run: |
119
+ python -m pip install --upgrade build
120
+ python -m build
121
+ python -c "
122
+ import zipfile, tarfile, pathlib, sys
123
+
124
+ dist_dir = pathlib.Path('dist')
125
+ wheels = list(dist_dir.glob('*.whl'))
126
+ sdists = list(dist_dir.glob('*.tar.gz'))
127
+
128
+ if not wheels:
129
+ sys.exit('Packaging verification failed: No wheel (.whl) artifact found in dist/')
130
+ if not sdists:
131
+ sys.exit('Packaging verification failed: No sdist (.tar.gz) artifact found in dist/')
132
+
133
+ wheel_path = wheels[0]
134
+ sdist_path = sdists[0]
135
+
136
+ print(f'Inspecting wheel: {wheel_path.name}')
137
+ with zipfile.ZipFile(wheel_path) as zf:
138
+ names = zf.namelist()
139
+ license_names = [n for n in names if n.endswith('.dist-info/licenses/LICENSE') or n.endswith('.dist-info/LICENSE')]
140
+ if not license_names:
141
+ sys.exit(f'Wheel verification failed: LICENSE file missing from dist-info. Found: {names}')
142
+ print(f'Found wheel license: {license_names}')
143
+
144
+ metadata_names = [n for n in names if n.endswith('.dist-info/METADATA')]
145
+ if not metadata_names:
146
+ sys.exit('Wheel verification failed: METADATA file missing from dist-info')
147
+
148
+ metadata_text = zf.read(metadata_names[0]).decode('utf-8', errors='replace')
149
+ if 'License-Expression: Apache-2.0' not in metadata_text:
150
+ sys.exit('Wheel verification failed: License-Expression: Apache-2.0 not in METADATA')
151
+ if 'License-File: LICENSE' not in metadata_text:
152
+ sys.exit('Wheel verification failed: License-File: LICENSE not in METADATA')
153
+ print('Wheel METADATA verified (PEP 639 License-Expression and License-File present).')
154
+
155
+ print(f'Inspecting sdist: {sdist_path.name}')
156
+ with tarfile.open(sdist_path, 'r:gz') as tf:
157
+ tar_names = tf.getnames()
158
+ sdist_licenses = [n for n in tar_names if n.endswith('/LICENSE') or n == 'LICENSE']
159
+ if not sdist_licenses:
160
+ sys.exit(f'Sdist verification failed: LICENSE missing in sdist. Found: {tar_names[:10]}')
161
+ print(f'Found sdist license: {sdist_licenses}')
162
+
163
+ pkg_info_names = [n for n in tar_names if n.endswith('/PKG-INFO') or n == 'PKG-INFO']
164
+ if not pkg_info_names:
165
+ sys.exit('Sdist verification failed: PKG-INFO missing in sdist')
166
+
167
+ pkg_info_f = tf.extractfile(pkg_info_names[0])
168
+ if pkg_info_f is None:
169
+ sys.exit('Sdist verification failed: could not extract PKG-INFO')
170
+ pkg_info_text = pkg_info_f.read().decode('utf-8', errors='replace')
171
+ if 'License-Expression: Apache-2.0' not in pkg_info_text:
172
+ sys.exit('Sdist verification failed: License-Expression: Apache-2.0 not in PKG-INFO')
173
+ if 'License-File: LICENSE' not in pkg_info_text:
174
+ sys.exit('Sdist verification failed: License-File: LICENSE not in PKG-INFO')
175
+ print('Sdist PKG-INFO verified (PEP 639 License-Expression and License-File present).')
176
+
177
+ print('Packaging and licensing verification successful!')
178
+ "
179
+
180
+ - name: Execute Clean Wheel-Install Distribution Smoke Test
181
+ run: |
182
+ python -m pip install --upgrade build
183
+ python scripts/distribution_smoke_test.py
184
+
185
+ - name: Run Unit & Adapter Test Suites
186
+ run: |
187
+ pytest -v tests/test_*.py
188
+
189
+ - name: Run Security & Sandbox Configuration Suites
190
+ run: |
191
+ pytest -v tests/security/test_container_configuration.py tests/security/test_filesystem_isolation.py tests/security/test_worker_ipc.py tests/security/test_container_lifecycle.py
192
+
193
+ - name: Run Live Docker Network Isolation Integration Tests
194
+ run: |
195
+ docker info
196
+ pytest -v -rs tests/security/test_network_isolation.py
@@ -0,0 +1,56 @@
1
+ name: RecoveryBench PR Quality Gate
2
+
3
+ on:
4
+ pull_request:
5
+ types: [opened, synchronize, reopened]
6
+
7
+ permissions:
8
+ contents: read
9
+ pull-requests: write
10
+
11
+ jobs:
12
+ pr-recovery-gate:
13
+ name: Agent Recovery Invariant Verification
14
+ runs-on: ubuntu-latest
15
+
16
+ steps:
17
+ - name: Check out PR code
18
+ uses: actions/checkout@v4
19
+
20
+ - name: Set up Python
21
+ uses: actions/setup-python@v5
22
+ with:
23
+ python-version: "3.11"
24
+ cache: "pip"
25
+
26
+ - name: Install RecoveryBench
27
+ run: |
28
+ python -m pip install --upgrade pip
29
+ python -m pip install -e ".[langgraph,dev]"
30
+
31
+ - name: Verify Scenarios
32
+ run: recoverybench validate scenarios
33
+
34
+ - name: Pull Pinned Docker Sandbox Runtime Image
35
+ run: docker pull python:3.11-slim@sha256:07d4824036af339fa1b6ef769cc99d9c10a141769538b52b3b597c84f7a7b535
36
+
37
+ - name: Execute Full Recovery Suite
38
+ id: eval-suite
39
+ run: |
40
+ recoverybench suite scenarios \
41
+ --agent "idempotent" \
42
+ --isolation "isolated" \
43
+ --output-dir "pr-reports" \
44
+ --junit
45
+
46
+ - name: Run Live Docker Network Isolation Tests
47
+ run: |
48
+ pytest -v -rs tests/security/test_network_isolation.py
49
+
50
+ - name: Upload PR Recovery Artifacts
51
+ uses: actions/upload-artifact@v4
52
+ if: always() && (steps.eval-suite.conclusion == 'success' || steps.eval-suite.conclusion == 'failure')
53
+ with:
54
+ name: pr-recovery-reports
55
+ path: pr-reports/
56
+ retention-days: 14
@@ -0,0 +1,43 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ share/python-wheels/
20
+ *.egg-info/
21
+ .installed.cfg
22
+ *.egg
23
+ MANIFEST
24
+
25
+ # Virtual environments
26
+ .venv/
27
+ venv/
28
+ ENV/
29
+ env/
30
+
31
+ # Testing & Coverage
32
+ .pytest_cache/
33
+ .coverage
34
+ .coverage.*
35
+ htmlcov/
36
+ .mypy_cache/
37
+ .ruff_cache/
38
+
39
+ # Local runtime data
40
+ .recoverybench/
41
+ reports/
42
+ pr-reports/
43
+ *.db