tau2-faults 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tau2_faults-0.1.0/.gitignore +7 -0
- tau2_faults-0.1.0/CITATION.cff +26 -0
- tau2_faults-0.1.0/LICENSE +21 -0
- tau2_faults-0.1.0/PKG-INFO +108 -0
- tau2_faults-0.1.0/README.md +89 -0
- tau2_faults-0.1.0/examples/quickstart.py +83 -0
- tau2_faults-0.1.0/pyproject.toml +34 -0
- tau2_faults-0.1.0/src/tau2_faults/__init__.py +312 -0
- tau2_faults-0.1.0/tests/test_replay.py +116 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use tau2-faults, please cite the paper below."
|
|
3
|
+
title: "tau2-faults: tool-fault injection for tau2-bench with replay-exact grading"
|
|
4
|
+
type: software
|
|
5
|
+
version: 0.1.0
|
|
6
|
+
license: MIT
|
|
7
|
+
authors:
|
|
8
|
+
- family-names: Goel
|
|
9
|
+
given-names: Ankit
|
|
10
|
+
orcid: "https://orcid.org/0009-0000-8939-8745"
|
|
11
|
+
repository-code: "https://github.com/ankitgoelcmu/tau2-faults"
|
|
12
|
+
keywords:
|
|
13
|
+
- tau2-bench
|
|
14
|
+
- LLM agents
|
|
15
|
+
- fault injection
|
|
16
|
+
- idempotency
|
|
17
|
+
- agent evaluation
|
|
18
|
+
preferred-citation:
|
|
19
|
+
type: article
|
|
20
|
+
title: "Tool Contracts and Output Defenses Under Agent Faults: A Pre-Registered Replication and τ²-bench Extension"
|
|
21
|
+
authors:
|
|
22
|
+
- family-names: Goel
|
|
23
|
+
given-names: Ankit
|
|
24
|
+
orcid: "https://orcid.org/0009-0000-8939-8745"
|
|
25
|
+
year: 2026
|
|
26
|
+
url: "https://github.com/ankitgoelcmu/faultline-agent"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ankit Goel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tau2-faults
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tool-fault injection for tau2-bench (lost responses, transient errors, stale reads) with replay-exact grading
|
|
5
|
+
Project-URL: Homepage, https://github.com/ankitgoelcmu/tau2-faults
|
|
6
|
+
Project-URL: Paper, https://github.com/ankitgoelcmu/faultline-agent
|
|
7
|
+
Project-URL: Issues, https://github.com/ankitgoelcmu/tau2-faults/issues
|
|
8
|
+
Author: Ankit Goel
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agent-evaluation,benchmark,fault-injection,idempotency,llm-agents,tau2-bench
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# tau2-faults
|
|
21
|
+
|
|
22
|
+
Tool-fault injection for [τ²-bench](https://github.com/sierra-research/tau2-bench), without changing τ²-bench's code.
|
|
23
|
+
|
|
24
|
+
Real tools fail. A write can succeed while its reply is lost, a service can return a transient error, and a read can come from a replica that is one write behind. `tau2-faults` adds these faults to τ²-bench's airline and retail domains, so you can measure how agents recover:
|
|
25
|
+
|
|
26
|
+
| Fault | What the agent sees | What happened |
|
|
27
|
+
|---|---|---|
|
|
28
|
+
| transient error | `503 Service Unavailable` | nothing ran |
|
|
29
|
+
| lost response | `504 Gateway Timeout` | the write **ran**, but the reply was lost |
|
|
30
|
+
| stale read | an older value | the read came from a replica one write behind |
|
|
31
|
+
|
|
32
|
+
Faults can be **uniform** or **bursty** (a Gilbert–Elliott process: healthy and degraded periods). An optional **idempotent** condition deduplicates repeated identical writes, like an API with idempotency keys.
|
|
33
|
+
|
|
34
|
+
## Why a plug-in is needed: grading stays exact
|
|
35
|
+
|
|
36
|
+
τ²-bench grades a run by replaying the conversation into a fresh database and comparing hashes. A tool that fails after or before committing breaks that replay ([issue #608](https://github.com/sierra-research/tau2-bench/issues/608)): replay may count a write that never happened, or miss one that ran before its reply was lost.
|
|
37
|
+
|
|
38
|
+
`tau2-faults` records what each faulted call did and replays it the same way: a call that failed before committing is skipped, a write that ran but lost its reply is applied with the recorded reply, and a deduplicated write changes nothing the second time. Tests check that, under every fault kind at a 40% fault rate, replay rebuilds exactly the live final database (25 airline and 25 retail tasks).
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
τ²-bench is not on PyPI, so install it first. Python 3.12 or later.
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
git clone https://github.com/sierra-research/tau2-bench
|
|
46
|
+
git -C tau2-bench checkout 033bf082f48e4a3c6a1256208e93cf2ce9c3bb6f # the version tested
|
|
47
|
+
pip install -e tau2-bench
|
|
48
|
+
pip install tau2-faults
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Other τ²-bench versions may work, but the plug-in wraps `Environment.get_response` and `Environment.set_state`, so changes to those can break it. Run the tests below after upgrading.
|
|
52
|
+
|
|
53
|
+
## Try it free (no API keys)
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
python examples/quickstart.py
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
A scripted agent replays each airline task's reference actions while half of its tool calls fail. It prints how many tasks still end correct with and without idempotency, and confirms that replay grading matched the live database.
|
|
60
|
+
|
|
61
|
+
## Run a real agent under faults
|
|
62
|
+
|
|
63
|
+
τ²-bench simulates the customer with a second model (default `gpt-4.1-2025-04-14`), so set `OPENAI_API_KEY` as well as your agent's key.
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
tau2-faults --domain airline --condition baseline --fault-rate 0.2 \
|
|
67
|
+
--agent-llm claude-sonnet-5-5 --num-trials 3 --out runs/airline-baseline-p20
|
|
68
|
+
tau2-faults --domain airline --condition idempotent --fault-rate 0.2 \
|
|
69
|
+
--agent-llm claude-sonnet-5-5 --num-trials 3 --out runs/airline-idem-p20
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Both conditions draw the same fault sequence for the same task and trial, so results can be compared pair by pair. Useful options: `--fault-model bursty`, `--burst-len 4`, `--lost-share 0.5`, `--stale-rate 0.1`, `--task-ids`, `--seed`. Local models through Ollama: `--agent-llm openai/<model> --agent-api-base http://localhost:11434/v1`.
|
|
73
|
+
|
|
74
|
+
Each run writes τ²-bench's `results.json` and a `faults.json` with every injected fault per simulation. Rerunning the same command resumes, and retries simulations that ended in an infrastructure error.
|
|
75
|
+
|
|
76
|
+
## Use it from Python
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
import tau2_faults
|
|
80
|
+
tau2_faults.CONFIG = tau2_faults.FaultConfig(condition="idempotent", fault_rate=0.2, fault_model="bursty")
|
|
81
|
+
tau2_faults.install() # wraps tau2's Environment; then run tau2 as usual
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Tests
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
python tests/test_replay.py # 4 "ok" lines, no API calls
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Cite
|
|
91
|
+
|
|
92
|
+
If you use `tau2-faults`, please cite the paper it was built for:
|
|
93
|
+
|
|
94
|
+
```bibtex
|
|
95
|
+
@misc{goel2026toolcontracts,
|
|
96
|
+
author = {Goel, Ankit},
|
|
97
|
+
title = {Tool Contracts and Output Defenses Under Agent Faults:
|
|
98
|
+
A Pre-Registered Replication and $\tau^2$-bench Extension},
|
|
99
|
+
year = {2026},
|
|
100
|
+
note = {Preprint. Code: https://github.com/ankitgoelcmu/faultline-agent}
|
|
101
|
+
}
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
The paper's experiments, results on four models and redacted traces are in [faultline-agent](https://github.com/ankitgoelcmu/faultline-agent).
|
|
105
|
+
|
|
106
|
+
## License
|
|
107
|
+
|
|
108
|
+
MIT
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# tau2-faults
|
|
2
|
+
|
|
3
|
+
Tool-fault injection for [τ²-bench](https://github.com/sierra-research/tau2-bench), without changing τ²-bench's code.
|
|
4
|
+
|
|
5
|
+
Real tools fail. A write can succeed while its reply is lost, a service can return a transient error, and a read can come from a replica that is one write behind. `tau2-faults` adds these faults to τ²-bench's airline and retail domains, so you can measure how agents recover:
|
|
6
|
+
|
|
7
|
+
| Fault | What the agent sees | What happened |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| transient error | `503 Service Unavailable` | nothing ran |
|
|
10
|
+
| lost response | `504 Gateway Timeout` | the write **ran**, but the reply was lost |
|
|
11
|
+
| stale read | an older value | the read came from a replica one write behind |
|
|
12
|
+
|
|
13
|
+
Faults can be **uniform** or **bursty** (a Gilbert–Elliott process: healthy and degraded periods). An optional **idempotent** condition deduplicates repeated identical writes, like an API with idempotency keys.
|
|
14
|
+
|
|
15
|
+
## Why a plug-in is needed: grading stays exact
|
|
16
|
+
|
|
17
|
+
τ²-bench grades a run by replaying the conversation into a fresh database and comparing hashes. A tool that fails after or before committing breaks that replay ([issue #608](https://github.com/sierra-research/tau2-bench/issues/608)): replay may count a write that never happened, or miss one that ran before its reply was lost.
|
|
18
|
+
|
|
19
|
+
`tau2-faults` records what each faulted call did and replays it the same way: a call that failed before committing is skipped, a write that ran but lost its reply is applied with the recorded reply, and a deduplicated write changes nothing the second time. Tests check that, under every fault kind at a 40% fault rate, replay rebuilds exactly the live final database (25 airline and 25 retail tasks).
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
τ²-bench is not on PyPI, so install it first. Python 3.12 or later.
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
git clone https://github.com/sierra-research/tau2-bench
|
|
27
|
+
git -C tau2-bench checkout 033bf082f48e4a3c6a1256208e93cf2ce9c3bb6f # the version tested
|
|
28
|
+
pip install -e tau2-bench
|
|
29
|
+
pip install tau2-faults
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Other τ²-bench versions may work, but the plug-in wraps `Environment.get_response` and `Environment.set_state`, so changes to those can break it. Run the tests below after upgrading.
|
|
33
|
+
|
|
34
|
+
## Try it free (no API keys)
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
python examples/quickstart.py
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
A scripted agent replays each airline task's reference actions while half of its tool calls fail. It prints how many tasks still end correct with and without idempotency, and confirms that replay grading matched the live database.
|
|
41
|
+
|
|
42
|
+
## Run a real agent under faults
|
|
43
|
+
|
|
44
|
+
τ²-bench simulates the customer with a second model (default `gpt-4.1-2025-04-14`), so set `OPENAI_API_KEY` as well as your agent's key.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
tau2-faults --domain airline --condition baseline --fault-rate 0.2 \
|
|
48
|
+
--agent-llm claude-sonnet-5-5 --num-trials 3 --out runs/airline-baseline-p20
|
|
49
|
+
tau2-faults --domain airline --condition idempotent --fault-rate 0.2 \
|
|
50
|
+
--agent-llm claude-sonnet-5-5 --num-trials 3 --out runs/airline-idem-p20
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Both conditions draw the same fault sequence for the same task and trial, so results can be compared pair by pair. Useful options: `--fault-model bursty`, `--burst-len 4`, `--lost-share 0.5`, `--stale-rate 0.1`, `--task-ids`, `--seed`. Local models through Ollama: `--agent-llm openai/<model> --agent-api-base http://localhost:11434/v1`.
|
|
54
|
+
|
|
55
|
+
Each run writes τ²-bench's `results.json` and a `faults.json` with every injected fault per simulation. Rerunning the same command resumes, and retries simulations that ended in an infrastructure error.
|
|
56
|
+
|
|
57
|
+
## Use it from Python
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import tau2_faults
|
|
61
|
+
tau2_faults.CONFIG = tau2_faults.FaultConfig(condition="idempotent", fault_rate=0.2, fault_model="bursty")
|
|
62
|
+
tau2_faults.install() # wraps tau2's Environment; then run tau2 as usual
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Tests
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
python tests/test_replay.py # 4 "ok" lines, no API calls
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Cite
|
|
72
|
+
|
|
73
|
+
If you use `tau2-faults`, please cite the paper it was built for:
|
|
74
|
+
|
|
75
|
+
```bibtex
|
|
76
|
+
@misc{goel2026toolcontracts,
|
|
77
|
+
author = {Goel, Ankit},
|
|
78
|
+
title = {Tool Contracts and Output Defenses Under Agent Faults:
|
|
79
|
+
A Pre-Registered Replication and $\tau^2$-bench Extension},
|
|
80
|
+
year = {2026},
|
|
81
|
+
note = {Preprint. Code: https://github.com/ankitgoelcmu/faultline-agent}
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The paper's experiments, results on four models and redacted traces are in [faultline-agent](https://github.com/ankitgoelcmu/faultline-agent).
|
|
86
|
+
|
|
87
|
+
## License
|
|
88
|
+
|
|
89
|
+
MIT
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Quick start for the tau2-bench fault plug-in. Free: no model or API calls.
|
|
2
|
+
|
|
3
|
+
python examples/quickstart.py
|
|
4
|
+
|
|
5
|
+
A scripted agent replays each airline task's reference actions through the
|
|
6
|
+
plug-in. Half of its tool calls fail, and every failed write is a lost response:
|
|
7
|
+
the write ran, but the agent sees a timeout. The agent retries a failed call
|
|
8
|
+
up to twice, as a careful agent might. It prints, per condition:
|
|
9
|
+
|
|
10
|
+
- how many tasks still end in the correct state (tau2's own grader),
|
|
11
|
+
- whether tau2's replay grading rebuilt exactly the database the live run left.
|
|
12
|
+
|
|
13
|
+
Without idempotency, some retries repeat a write that already ran, and the task
|
|
14
|
+
fails. With idempotency, the repeat is deduplicated. Replay matches live in both.
|
|
15
|
+
"""
|
|
16
|
+
from loguru import logger # noqa: E402
|
|
17
|
+
import tau2_faults as tf
|
|
18
|
+
from tau2.data_model.message import AssistantMessage, ToolCall # noqa: E402
|
|
19
|
+
from tau2.evaluator.evaluator_env import EnvironmentEvaluator # noqa: E402
|
|
20
|
+
from tau2.registry import registry # noqa: E402
|
|
21
|
+
|
|
22
|
+
DOMAIN, N_TASKS, SEED = "airline", 30, 7
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def run(task, cfg, retries=2):
|
|
26
|
+
"""Reference actions through the fault layer. Returns (live env, trajectory, episode)."""
|
|
27
|
+
tf.CONFIG = cfg
|
|
28
|
+
env = registry.get_env_constructor(DOMAIN)()
|
|
29
|
+
if task.initial_state is not None:
|
|
30
|
+
env.set_state(task.initial_state.initialization_data,
|
|
31
|
+
task.initial_state.initialization_actions,
|
|
32
|
+
task.initial_state.message_history or [])
|
|
33
|
+
ep = tf.make_schedule(cfg, task.id, SEED)
|
|
34
|
+
token = tf._EPISODE.set(ep)
|
|
35
|
+
traj, n = [], 0
|
|
36
|
+
try:
|
|
37
|
+
for action in task.evaluation_criteria.actions or []:
|
|
38
|
+
for _ in range(retries + 1):
|
|
39
|
+
n += 1
|
|
40
|
+
call = ToolCall(id=f"call_{n}", name=action.name, arguments=action.arguments,
|
|
41
|
+
requestor="assistant")
|
|
42
|
+
traj.append(AssistantMessage(role="assistant", tool_calls=[call]))
|
|
43
|
+
out = env.get_response(call)
|
|
44
|
+
traj.append(out)
|
|
45
|
+
if not out.error:
|
|
46
|
+
break
|
|
47
|
+
finally:
|
|
48
|
+
tf._EPISODE.reset(token)
|
|
49
|
+
return env, traj, ep
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def replay_hash(task, traj):
|
|
53
|
+
env = registry.get_env_constructor(DOMAIN)()
|
|
54
|
+
init = task.initial_state
|
|
55
|
+
env.set_state(init.initialization_data if init else None,
|
|
56
|
+
init.initialization_actions if init else None,
|
|
57
|
+
list((init.message_history or []) if init else []) + traj)
|
|
58
|
+
return env.get_db_hash()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def main():
|
|
62
|
+
logger.remove()
|
|
63
|
+
tf.install()
|
|
64
|
+
ctor = registry.get_env_constructor(DOMAIN)
|
|
65
|
+
tasks = [t for t in registry.get_tasks_loader(DOMAIN)()
|
|
66
|
+
if t.evaluation_criteria and t.evaluation_criteria.actions][:N_TASKS]
|
|
67
|
+
print(f"tau2-bench {DOMAIN}, {len(tasks)} tasks, scripted agent, 50% faults, every write fault a lost response\n")
|
|
68
|
+
for cond in tf.CONDITIONS:
|
|
69
|
+
cfg = tf.FaultConfig(cond, fault_rate=0.5, fault_model="uniform", lost_share=1.0)
|
|
70
|
+
solved = matched = lost = dedup = 0
|
|
71
|
+
for task in tasks:
|
|
72
|
+
env, traj, ep = run(task, cfg)
|
|
73
|
+
solved += EnvironmentEvaluator.calculate_reward(ctor, task, traj).reward >= 1.0
|
|
74
|
+
matched += replay_hash(task, traj) == env.get_db_hash()
|
|
75
|
+
lost += sum(e["kind"].startswith("lost_response") for e in ep.log)
|
|
76
|
+
dedup += sum(e["kind"] == "dedup" for e in ep.log)
|
|
77
|
+
print(f"{cond:<11} tasks correct {solved}/{len(tasks)} lost responses {lost:>3} "
|
|
78
|
+
f"deduplicated writes {dedup:>3} replay matched live {matched}/{len(tasks)}")
|
|
79
|
+
print("\nSame fault draws in both conditions; only deduplication differs.")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
main()
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tau2-faults"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Tool-fault injection for tau2-bench (lost responses, transient errors, stale reads) with replay-exact grading"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.12"
|
|
13
|
+
authors = [{ name = "Ankit Goel" }]
|
|
14
|
+
keywords = ["tau2-bench", "llm-agents", "fault-injection", "idempotency", "agent-evaluation", "benchmark"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.12",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
"Topic :: Software Development :: Testing",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
]
|
|
22
|
+
# tau2-bench is not on PyPI: install it from GitHub first (see README).
|
|
23
|
+
dependencies = []
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/ankitgoelcmu/tau2-faults"
|
|
27
|
+
Paper = "https://github.com/ankitgoelcmu/faultline-agent"
|
|
28
|
+
Issues = "https://github.com/ankitgoelcmu/tau2-faults/issues"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
tau2-faults = "tau2_faults:main"
|
|
32
|
+
|
|
33
|
+
[tool.hatch.build.targets.wheel]
|
|
34
|
+
packages = ["src/tau2_faults"]
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Tool-fault injection for tau2-bench (τ²), without modifying tau2.
|
|
2
|
+
|
|
3
|
+
Needs tau2-bench installed in the same Python 3.12+ environment:
|
|
4
|
+
|
|
5
|
+
tau2-faults --domain airline --condition baseline \\
|
|
6
|
+
--fault-rate 0.1 --agent-llm claude-sonnet-5-5 --num-trials 3 --out runs/airline-baseline-p10
|
|
7
|
+
|
|
8
|
+
or from Python: import tau2_faults; tau2_faults.CONFIG = tau2_faults.FaultConfig(...); tau2_faults.install()
|
|
9
|
+
|
|
10
|
+
How it works. tau2 sends every tool call through Environment.get_response()
|
|
11
|
+
and grades a run by replaying the conversation's tool calls into a fresh
|
|
12
|
+
environment (Environment.set_state), then comparing database hashes. This
|
|
13
|
+
module wraps both methods:
|
|
14
|
+
|
|
15
|
+
get_response injects faults into the agent's tool calls:
|
|
16
|
+
transient "503". Nothing ran. Marked execution_status
|
|
17
|
+
"rejected", which tau2's replay skips.
|
|
18
|
+
lost_response write tools only. The write ran, but the agent
|
|
19
|
+
gets a timeout. Replay runs the write again
|
|
20
|
+
and accepts the recorded timeout as its reply.
|
|
21
|
+
stale_read read tools only. Served from the database as
|
|
22
|
+
it was before the latest write.
|
|
23
|
+
Under the "idempotent" condition, a repeated write (same tool,
|
|
24
|
+
same arguments) returns the first result without running
|
|
25
|
+
again, like an API with idempotency keys.
|
|
26
|
+
set_state replays with faults off and the same dedup rule, so grading
|
|
27
|
+
sees exactly the writes that happened live. Everything replay
|
|
28
|
+
needs is in the conversation itself, so saved runs can be
|
|
29
|
+
re-graded later (set CONFIG from the run's .faults.json).
|
|
30
|
+
|
|
31
|
+
Conditions. tau2 replays each tool call once, so harness-level retries
|
|
32
|
+
(several executions behind one message) can't be graded faithfully. The
|
|
33
|
+
conditions therefore test the agent's own handling of errors:
|
|
34
|
+
baseline faults reach the agent; every write it sends runs.
|
|
35
|
+
idempotent faults reach the agent; repeated identical writes are deduplicated.
|
|
36
|
+
|
|
37
|
+
Pairing. Each episode's fault sequence is seeded by (task id, trial seed), and
|
|
38
|
+
the k-th tool call always sees the k-th fault draw, so both conditions face the
|
|
39
|
+
same dice. tau2 fixes trial seeds from --seed.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
import argparse
|
|
43
|
+
import contextvars
|
|
44
|
+
import copy
|
|
45
|
+
import hashlib
|
|
46
|
+
import json
|
|
47
|
+
import os
|
|
48
|
+
import random
|
|
49
|
+
import sys
|
|
50
|
+
from dataclasses import dataclass, field
|
|
51
|
+
|
|
52
|
+
FAULT_MODELS = ["uniform", "bursty"]
|
|
53
|
+
BURST_FAULT_RATE = 0.8 # share of attempts that fail while the service is degraded (bursty model)
|
|
54
|
+
__version__ = "0.1.0"
|
|
55
|
+
|
|
56
|
+
CONDITIONS = ["baseline", "idempotent"]
|
|
57
|
+
TRANSIENT_MSG = "Error: 503 Service Unavailable. Please retry."
|
|
58
|
+
LOST_MSG = ("Error: 504 Gateway Timeout: no response from service. "
|
|
59
|
+
"The request may or may not have been applied.")
|
|
60
|
+
SCHEDULE_LEN = 1000 # far more tool calls than any tau2 episode makes
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class FaultConfig:
|
|
65
|
+
condition: str = "baseline"
|
|
66
|
+
fault_rate: float = 0.0
|
|
67
|
+
fault_model: str = "uniform"
|
|
68
|
+
burst_len: float = 4.0
|
|
69
|
+
lost_share: float = 0.5
|
|
70
|
+
stale_rate: float = 0.0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class Episode:
|
|
75
|
+
"""Fault draws for one episode, fixed in advance so both conditions share them."""
|
|
76
|
+
faulted: list
|
|
77
|
+
lost_roll: list
|
|
78
|
+
stale_roll: list
|
|
79
|
+
calls: int = 0
|
|
80
|
+
log: list = field(default_factory=list)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def make_schedule(cfg, task_id, seed):
|
|
84
|
+
h = hashlib.sha256(f"tau2|{task_id}|{seed}|{cfg.fault_rate}".encode()).hexdigest()
|
|
85
|
+
rng = random.Random(int(h[:12], 16))
|
|
86
|
+
faulted = []
|
|
87
|
+
if cfg.fault_model == "uniform":
|
|
88
|
+
faulted = [rng.random() < cfg.fault_rate for _ in range(SCHEDULE_LEN)]
|
|
89
|
+
else: # Gilbert-Elliott, started in its long-run state mix
|
|
90
|
+
share = cfg.fault_rate / BURST_FAULT_RATE
|
|
91
|
+
p_recover = 1.0 / cfg.burst_len
|
|
92
|
+
p_degrade = share * p_recover / (1 - share) if share < 1 else 1.0
|
|
93
|
+
degraded = rng.random() < share
|
|
94
|
+
for _ in range(SCHEDULE_LEN):
|
|
95
|
+
degraded = (rng.random() >= p_recover) if degraded else (rng.random() < p_degrade)
|
|
96
|
+
faulted.append(degraded and rng.random() < BURST_FAULT_RATE)
|
|
97
|
+
return Episode(faulted=faulted,
|
|
98
|
+
lost_roll=[rng.random() for _ in range(SCHEDULE_LEN)],
|
|
99
|
+
stale_roll=[rng.random() for _ in range(SCHEDULE_LEN)])
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
CONFIG = FaultConfig()
|
|
103
|
+
_EPISODE = contextvars.ContextVar("tau2_fault_episode", default=None)
|
|
104
|
+
EPISODE_LOGS = {} # (task id, seed) -> list of injected events, for the run summary
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _write_key(call):
|
|
108
|
+
blob = json.dumps({"t": call.name, "a": call.arguments}, sort_keys=True, default=str)
|
|
109
|
+
return hashlib.sha256(blob.encode()).hexdigest()[:16]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def install():
|
|
113
|
+
"""Wrap tau2's Environment and per-task runner. Safe to call twice."""
|
|
114
|
+
from tau2.data_model.message import ToolMessage
|
|
115
|
+
from tau2.environment.environment import Environment
|
|
116
|
+
from tau2.runner import batch
|
|
117
|
+
|
|
118
|
+
if getattr(Environment, "_faults_installed", False):
|
|
119
|
+
return
|
|
120
|
+
orig_get_response = Environment.get_response
|
|
121
|
+
orig_set_state = Environment.set_state
|
|
122
|
+
orig_run_single_task = batch.run_single_task
|
|
123
|
+
|
|
124
|
+
def run_single_task(config, task, *args, seed=None, **kwargs):
|
|
125
|
+
ep = make_schedule(CONFIG, task.id, seed)
|
|
126
|
+
EPISODE_LOGS[(task.id, seed)] = ep.log
|
|
127
|
+
token = _EPISODE.set(ep)
|
|
128
|
+
try:
|
|
129
|
+
return orig_run_single_task(config, task, *args, seed=seed, **kwargs)
|
|
130
|
+
finally:
|
|
131
|
+
_EPISODE.reset(token)
|
|
132
|
+
|
|
133
|
+
def msg(call, content, error, status):
|
|
134
|
+
return ToolMessage(id=call.id, role="tool", content=content, requestor=call.requestor,
|
|
135
|
+
error=error, execution_status=status)
|
|
136
|
+
|
|
137
|
+
def get_response(self, call):
|
|
138
|
+
replay = getattr(self, "_fault_replay", None)
|
|
139
|
+
if replay is not None:
|
|
140
|
+
# Grading replay (tau2 calls this only for writes that weren't
|
|
141
|
+
# rejected). Apply the same dedup rule as the live run, and hand
|
|
142
|
+
# back the recorded message for lost responses and dedups so
|
|
143
|
+
# tau2's strict comparison matches.
|
|
144
|
+
recorded = replay["msgs"].get(call.id)
|
|
145
|
+
if CONFIG.condition == "idempotent":
|
|
146
|
+
if _write_key(call) in replay["cache"]:
|
|
147
|
+
return recorded
|
|
148
|
+
replay["cache"].add(_write_key(call))
|
|
149
|
+
real = orig_get_response(self, call)
|
|
150
|
+
return recorded if recorded is not None and recorded.content == LOST_MSG else real
|
|
151
|
+
ep = getattr(self, "_fault_episode", None) or _EPISODE.get()
|
|
152
|
+
if ep is None or call.requestor != "assistant":
|
|
153
|
+
return orig_get_response(self, call)
|
|
154
|
+
self._fault_episode = ep
|
|
155
|
+
k = ep.calls
|
|
156
|
+
ep.calls += 1
|
|
157
|
+
known = self._has_tool(call.name)
|
|
158
|
+
writes = known and self._is_mutating_tool(call.name)
|
|
159
|
+
cache = self.__dict__.setdefault("_fault_cache", {})
|
|
160
|
+
|
|
161
|
+
def event(kind):
|
|
162
|
+
ep.log.append({"call": k, "tool": call.name, "kind": kind})
|
|
163
|
+
|
|
164
|
+
if ep.faulted[k % SCHEDULE_LEN] and known:
|
|
165
|
+
if writes and ep.lost_roll[k % SCHEDULE_LEN] < CONFIG.lost_share:
|
|
166
|
+
real = self._run_write(call, cache)
|
|
167
|
+
event("lost_response" if real is not None else "lost_response_dedup")
|
|
168
|
+
return msg(call, LOST_MSG, True, "executed")
|
|
169
|
+
event("transient")
|
|
170
|
+
return msg(call, TRANSIENT_MSG, True, "rejected")
|
|
171
|
+
if writes:
|
|
172
|
+
if CONFIG.condition == "idempotent" and _write_key(call) in cache:
|
|
173
|
+
event("dedup")
|
|
174
|
+
done = cache[_write_key(call)]
|
|
175
|
+
return msg(call, done.content, done.error, "executed")
|
|
176
|
+
return self._run_write(call, cache)
|
|
177
|
+
if (known and CONFIG.stale_rate and getattr(self, "_fault_replica", None) is not None
|
|
178
|
+
and ep.stale_roll[k % SCHEDULE_LEN] < CONFIG.stale_rate):
|
|
179
|
+
event("stale_read")
|
|
180
|
+
live = self.tools.db
|
|
181
|
+
self.tools.db = self._fault_replica
|
|
182
|
+
try:
|
|
183
|
+
return orig_get_response(self, call)
|
|
184
|
+
finally:
|
|
185
|
+
self.tools.db = live
|
|
186
|
+
return orig_get_response(self, call)
|
|
187
|
+
|
|
188
|
+
def _run_write(self, call, cache):
|
|
189
|
+
"""Run a write (unless deduplicated). Returns its ToolMessage, or None if deduplicated."""
|
|
190
|
+
if CONFIG.condition == "idempotent" and _write_key(call) in cache:
|
|
191
|
+
return None
|
|
192
|
+
self._fault_replica = copy.deepcopy(self.tools.db)
|
|
193
|
+
real = orig_get_response(self, call)
|
|
194
|
+
if CONFIG.condition == "idempotent":
|
|
195
|
+
cache[_write_key(call)] = real
|
|
196
|
+
return real
|
|
197
|
+
|
|
198
|
+
def set_state(self, initialization_data, initialization_actions, message_history, strict=True):
|
|
199
|
+
self._fault_replay = {"cache": set(),
|
|
200
|
+
"msgs": {m.id: m for m in message_history
|
|
201
|
+
if getattr(m, "role", None) == "tool"}}
|
|
202
|
+
try:
|
|
203
|
+
return orig_set_state(self, initialization_data, initialization_actions,
|
|
204
|
+
message_history, strict=strict)
|
|
205
|
+
finally:
|
|
206
|
+
self._fault_replay = None
|
|
207
|
+
|
|
208
|
+
Environment.get_response = get_response
|
|
209
|
+
Environment._run_write = _run_write
|
|
210
|
+
Environment.set_state = set_state
|
|
211
|
+
Environment._faults_installed = True
|
|
212
|
+
batch.run_single_task = run_single_task
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def main():
|
|
216
|
+
p = argparse.ArgumentParser(description="Run tau2-bench with tool-fault injection")
|
|
217
|
+
p.add_argument("--domain", default="airline", choices=["airline", "retail"])
|
|
218
|
+
p.add_argument("--condition", choices=CONDITIONS, default="baseline")
|
|
219
|
+
p.add_argument("--fault-rate", type=float, default=0.0)
|
|
220
|
+
p.add_argument("--fault-model", choices=FAULT_MODELS, default="uniform")
|
|
221
|
+
p.add_argument("--burst-len", type=float, default=4.0)
|
|
222
|
+
p.add_argument("--lost-share", type=float, default=0.5)
|
|
223
|
+
p.add_argument("--stale-rate", type=float, default=0.0)
|
|
224
|
+
p.add_argument("--agent-llm", required=True, help="litellm model name, e.g. claude-sonnet-5-5")
|
|
225
|
+
p.add_argument("--agent-api-base", default=None,
|
|
226
|
+
help="OpenAI-compatible server for the agent, e.g. http://localhost:11434/v1 for "
|
|
227
|
+
"Ollama (use --agent-llm openai/<model>). litellm's ollama_chat route doesn't "
|
|
228
|
+
"return tool results to the model correctly.")
|
|
229
|
+
p.add_argument("--user-llm", default="gpt-4.1-2025-04-14", help="tau2's default user simulator")
|
|
230
|
+
p.add_argument("--num-trials", type=int, default=1)
|
|
231
|
+
p.add_argument("--num-tasks", type=int, default=None)
|
|
232
|
+
p.add_argument("--task-ids", default=None, help="comma-separated tau2 task ids")
|
|
233
|
+
p.add_argument("--seed", type=int, default=300, help="tau2 trial seeds; keep fixed across conditions")
|
|
234
|
+
p.add_argument("--max-errors", type=int, default=30,
|
|
235
|
+
help="tau2 ends a run after this many tool errors (its default is 10)")
|
|
236
|
+
p.add_argument("--max-steps", type=int, default=200, help="tau2's step limit per simulation")
|
|
237
|
+
p.add_argument("--concurrency", type=int, default=3)
|
|
238
|
+
p.add_argument("--out", required=True,
|
|
239
|
+
help="results folder: tau2 writes results.json there (resumes if present), "
|
|
240
|
+
"this plug-in writes faults.json")
|
|
241
|
+
a = p.parse_args()
|
|
242
|
+
out = os.path.abspath(a.out) # tau2 joins this onto its data dir; absolute keeps it here
|
|
243
|
+
os.makedirs(out, exist_ok=True)
|
|
244
|
+
# Current Claude models reject `temperature`; other agents run at temperature 0.
|
|
245
|
+
agent_args = {} if "claude" in a.agent_llm else {"temperature": 0.0}
|
|
246
|
+
if a.agent_api_base:
|
|
247
|
+
# Local (Ollama) agents: cap each reply at 8,192 tokens. Without a cap, a
|
|
248
|
+
# looping reply can run ~67k tokens per turn and starve the other episodes.
|
|
249
|
+
agent_args.update(api_base=a.agent_api_base, api_key="local", max_tokens=8192)
|
|
250
|
+
|
|
251
|
+
global CONFIG
|
|
252
|
+
CONFIG = FaultConfig(a.condition, a.fault_rate, a.fault_model, a.burst_len,
|
|
253
|
+
a.lost_share, a.stale_rate)
|
|
254
|
+
install()
|
|
255
|
+
from tau2.data_model.simulation import TextRunConfig
|
|
256
|
+
from tau2.runner.batch import run_domain
|
|
257
|
+
|
|
258
|
+
config = TextRunConfig(
|
|
259
|
+
domain=a.domain, llm_agent=a.agent_llm, llm_args_agent=agent_args,
|
|
260
|
+
llm_user=a.user_llm, llm_args_user={"temperature": 0.0},
|
|
261
|
+
num_trials=a.num_trials, num_tasks=a.num_tasks,
|
|
262
|
+
task_ids=a.task_ids.split(",") if a.task_ids else None,
|
|
263
|
+
seed=a.seed, max_errors=a.max_errors, max_steps=a.max_steps, max_concurrency=a.concurrency,
|
|
264
|
+
hallucination_retries=0, auto_resume=True, save_to=out)
|
|
265
|
+
meta_path = os.path.join(out, "faults.json")
|
|
266
|
+
meta = {"fault_config": CONFIG.__dict__, "agent_llm": a.agent_llm, "user_llm": a.user_llm,
|
|
267
|
+
"seed": a.seed, "max_errors": a.max_errors, "max_steps": a.max_steps, "faults": {}}
|
|
268
|
+
if os.path.exists(meta_path): # resuming: refuse different settings, keep earlier episodes
|
|
269
|
+
prev = json.load(open(meta_path))
|
|
270
|
+
if prev["fault_config"] != meta["fault_config"]:
|
|
271
|
+
raise SystemExit(f"{out} was run with {prev['fault_config']}. Use a new --out folder.")
|
|
272
|
+
meta["faults"] = prev["faults"]
|
|
273
|
+
meta["infra_errors_rerun"] = prev.get("infra_errors_rerun", 0) if os.path.exists(meta_path) else 0
|
|
274
|
+
meta["infra_errors_rerun"] += drop_infra_errors(os.path.join(out, "results.json"))
|
|
275
|
+
results = run_domain(config)
|
|
276
|
+
meta["faults"].update({f"{t}|{s}": log for (t, s), log in EPISODE_LOGS.items()})
|
|
277
|
+
with open(meta_path, "w") as f:
|
|
278
|
+
json.dump(meta, f, indent=2)
|
|
279
|
+
errs = [s for s in results.simulations if str(s.termination_reason).endswith("infrastructure_error")]
|
|
280
|
+
rewards = [s.reward_info.reward for s in results.simulations if s.reward_info]
|
|
281
|
+
print(f"\n{len(rewards)} graded simulations, mean reward {sum(rewards) / max(len(rewards), 1):.3f}. "
|
|
282
|
+
f"Results in {out}", file=sys.stderr)
|
|
283
|
+
if errs:
|
|
284
|
+
print(f"WARNING: {len(errs)} simulations ended in an infrastructure error (e.g. "
|
|
285
|
+
f"{str((errs[0].info or {}).get('error', ''))[:150]}). They have no reward; rerun the "
|
|
286
|
+
f"same command to retry them.", file=sys.stderr)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def drop_infra_errors(results_path):
|
|
290
|
+
"""Remove simulations that ended in an infrastructure error (API failures,
|
|
291
|
+
bad requests) from a results file, so a resumed run retries them instead of
|
|
292
|
+
keeping them as done. Returns how many were removed."""
|
|
293
|
+
if not os.path.exists(results_path):
|
|
294
|
+
return 0
|
|
295
|
+
with open(results_path) as f:
|
|
296
|
+
data = json.load(f)
|
|
297
|
+
sims = data.get("simulations", [])
|
|
298
|
+
keep = [s for s in sims if s.get("termination_reason") != "infrastructure_error"]
|
|
299
|
+
if len(keep) == len(sims):
|
|
300
|
+
return 0
|
|
301
|
+
with open(results_path + ".bak", "w") as f:
|
|
302
|
+
json.dump(data, f)
|
|
303
|
+
data["simulations"] = keep
|
|
304
|
+
with open(results_path, "w") as f:
|
|
305
|
+
json.dump(data, f)
|
|
306
|
+
print(f"Retrying {len(sims) - len(keep)} simulations that ended in an infrastructure error.",
|
|
307
|
+
file=sys.stderr)
|
|
308
|
+
return len(sims) - len(keep)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
if __name__ == "__main__":
|
|
312
|
+
main()
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Checks for the tau2-bench fault plug-in. No model calls.
|
|
2
|
+
|
|
3
|
+
Run with the tau2 environment: python tests/test_replay.py
|
|
4
|
+
|
|
5
|
+
The key property: tau2 grades by replaying the conversation into a fresh
|
|
6
|
+
database, so that replay must rebuild exactly the database the live run left
|
|
7
|
+
behind, whatever faults were injected. Otherwise every reward is wrong.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import tau2_faults as tf
|
|
11
|
+
from tau2.data_model.message import AssistantMessage, ToolCall
|
|
12
|
+
from tau2.evaluator.evaluator_env import EnvironmentEvaluator
|
|
13
|
+
from tau2.registry import registry
|
|
14
|
+
|
|
15
|
+
tf.install()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def scripted_episode(domain, task, cfg, seed, retries=2):
|
|
19
|
+
"""Run a task's reference actions through the fault layer, like an agent that
|
|
20
|
+
retries a failed call up to `retries` times. Returns (live env, trajectory, episode)."""
|
|
21
|
+
tf.CONFIG = cfg
|
|
22
|
+
env = registry.get_env_constructor(domain)()
|
|
23
|
+
if task.initial_state is not None:
|
|
24
|
+
env.set_state(task.initial_state.initialization_data,
|
|
25
|
+
task.initial_state.initialization_actions,
|
|
26
|
+
task.initial_state.message_history or [])
|
|
27
|
+
ep = tf.make_schedule(cfg, task.id, seed)
|
|
28
|
+
token = tf._EPISODE.set(ep)
|
|
29
|
+
traj, n = [], 0
|
|
30
|
+
try:
|
|
31
|
+
for action in task.evaluation_criteria.actions or []:
|
|
32
|
+
for _ in range(retries + 1):
|
|
33
|
+
n += 1
|
|
34
|
+
call = ToolCall(id=f"call_{n}", name=action.name, arguments=action.arguments,
|
|
35
|
+
requestor="assistant")
|
|
36
|
+
traj.append(AssistantMessage(role="assistant", tool_calls=[call]))
|
|
37
|
+
out = env.get_response(call)
|
|
38
|
+
traj.append(out)
|
|
39
|
+
if not out.error:
|
|
40
|
+
break
|
|
41
|
+
finally:
|
|
42
|
+
tf._EPISODE.reset(token)
|
|
43
|
+
return env, traj, ep
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def replayed_db_hash(domain, task, traj):
|
|
47
|
+
"""The database tau2's grader rebuilds from the trajectory."""
|
|
48
|
+
env = registry.get_env_constructor(domain)()
|
|
49
|
+
env.set_state(task.initial_state.initialization_data if task.initial_state else None,
|
|
50
|
+
task.initial_state.initialization_actions if task.initial_state else None,
|
|
51
|
+
list((task.initial_state.message_history or []) if task.initial_state else []) + traj)
|
|
52
|
+
return env.get_db_hash()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _tasks(domain, n):
|
|
56
|
+
tasks = registry.get_tasks_loader(domain)()
|
|
57
|
+
return [t for t in tasks if t.evaluation_criteria and t.evaluation_criteria.actions][:n]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_replay_matches_live_under_every_fault():
|
|
61
|
+
kinds = set()
|
|
62
|
+
for domain in ("airline", "retail"):
|
|
63
|
+
for task in _tasks(domain, 25):
|
|
64
|
+
for cond in tf.CONDITIONS:
|
|
65
|
+
for model in ("uniform", "bursty"):
|
|
66
|
+
cfg = tf.FaultConfig(cond, 0.4, model, 4.0, 0.5, 0.3)
|
|
67
|
+
for seed in (1, 2, 3):
|
|
68
|
+
env, traj, ep = scripted_episode(domain, task, cfg, seed)
|
|
69
|
+
kinds |= {e["kind"] for e in ep.log}
|
|
70
|
+
assert replayed_db_hash(domain, task, traj) == env.get_db_hash(), \
|
|
71
|
+
(domain, task.id, cond, model, seed, ep.log)
|
|
72
|
+
# The test must actually exercise every fault type.
|
|
73
|
+
assert {"transient", "lost_response", "dedup"} <= kinds, kinds
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_no_faults_gives_full_reward_on_reference_actions():
|
|
77
|
+
for domain in ("airline", "retail"):
|
|
78
|
+
ctor = registry.get_env_constructor(domain)
|
|
79
|
+
for task in _tasks(domain, 15):
|
|
80
|
+
for cond in tf.CONDITIONS:
|
|
81
|
+
_, traj, ep = scripted_episode(domain, task, tf.FaultConfig(cond), 1)
|
|
82
|
+
assert not ep.log
|
|
83
|
+
r = EnvironmentEvaluator.calculate_reward(ctor, task, traj)
|
|
84
|
+
assert r.reward == 1.0, (domain, task.id, cond)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_lost_response_duplicates_only_without_idempotency():
|
|
88
|
+
"""Find an airline task where a lost write plus a retry changes the outcome,
|
|
89
|
+
and check the idempotent condition prevents it."""
|
|
90
|
+
ctor = registry.get_env_constructor("airline")
|
|
91
|
+
rewards = {c: [] for c in tf.CONDITIONS}
|
|
92
|
+
for task in _tasks("airline", 50):
|
|
93
|
+
for cond in tf.CONDITIONS:
|
|
94
|
+
cfg = tf.FaultConfig(cond, 0.5, "uniform", 4.0, 1.0, 0.0) # every write fault is lost
|
|
95
|
+
_, traj, ep = scripted_episode("airline", task, cfg, 7)
|
|
96
|
+
rewards[cond].append(EnvironmentEvaluator.calculate_reward(ctor, task, traj).reward)
|
|
97
|
+
# Same fault dice; only the dedup differs. Idempotency must never do worse,
|
|
98
|
+
# and must rescue at least one task the duplicate broke.
|
|
99
|
+
pairs = list(zip(rewards["baseline"], rewards["idempotent"]))
|
|
100
|
+
assert all(i >= b for b, i in pairs), pairs
|
|
101
|
+
assert any(i > b for b, i in pairs), pairs
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_same_dice_for_both_conditions():
|
|
105
|
+
t = _tasks("airline", 1)[0]
|
|
106
|
+
a = tf.make_schedule(tf.FaultConfig("baseline", 0.2), t.id, 5)
|
|
107
|
+
b = tf.make_schedule(tf.FaultConfig("idempotent", 0.2), t.id, 5)
|
|
108
|
+
assert a.faulted == b.faulted and a.lost_roll == b.lost_roll
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
from loguru import logger
|
|
113
|
+
logger.remove()
|
|
114
|
+
for name, fn in list(globals().items()):
|
|
115
|
+
if name.startswith("test_"):
|
|
116
|
+
fn(); print("ok", name)
|