spillway-guard 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spillway_guard-0.1.0/.gitignore +58 -0
- spillway_guard-0.1.0/CHANGELOG.md +114 -0
- spillway_guard-0.1.0/CITATION.cff +35 -0
- spillway_guard-0.1.0/CONTRIBUTING.md +87 -0
- spillway_guard-0.1.0/LICENSE +201 -0
- spillway_guard-0.1.0/Makefile +50 -0
- spillway_guard-0.1.0/NOTICE +81 -0
- spillway_guard-0.1.0/PKG-INFO +263 -0
- spillway_guard-0.1.0/PYPI_README.md +225 -0
- spillway_guard-0.1.0/README.md +279 -0
- spillway_guard-0.1.0/SECURITY.md +133 -0
- spillway_guard-0.1.0/benchmarks/TABLES.md +101 -0
- spillway_guard-0.1.0/benchmarks/results/README.md +16 -0
- spillway_guard-0.1.0/benchmarks/results/ablation_and_chatinject.json +7739 -0
- spillway_guard-0.1.0/benchmarks/results/ablation_no_span_veto.json +34 -0
- spillway_guard-0.1.0/benchmarks/results/breadth_grid_cells.json +1962 -0
- spillway_guard-0.1.0/benchmarks/results/breadth_grid_summary.json +870 -0
- spillway_guard-0.1.0/benchmarks/results/delegated_routing_agent_router.json +1022 -0
- spillway_guard-0.1.0/benchmarks/results/delegated_routing_opus_router.json +1060 -0
- spillway_guard-0.1.0/benchmarks/results/fig_cost_values.json +38 -0
- spillway_guard-0.1.0/benchmarks/results/pismith_a4/README.md +17 -0
- spillway_guard-0.1.0/benchmarks/results/pismith_a4/drift_a25f.md +6 -0
- spillway_guard-0.1.0/benchmarks/results/pismith_a4/fast_3119.md +26 -0
- spillway_guard-0.1.0/benchmarks/results/pismith_a4/progent_133f.md +6 -0
- spillway_guard-0.1.0/benchmarks/results/pismith_a4/rope_9b4e.md +19 -0
- spillway_guard-0.1.0/benchmarks/scripts/appendix_asr_levels.py +29 -0
- spillway_guard-0.1.0/benchmarks/scripts/make_tables.py +624 -0
- spillway_guard-0.1.0/pyproject.toml +70 -0
- spillway_guard-0.1.0/reference/BUILD_PROVENANCE.md +39 -0
- spillway_guard-0.1.0/reference/harness/21_agentdojo_full_benchmark.py +907 -0
- spillway_guard-0.1.0/reference/harness/24_agentdojo_gemini_backbone.py +201 -0
- spillway_guard-0.1.0/reference/harness/27_agentdojo_v8_action_boundary.py +293 -0
- spillway_guard-0.1.0/reference/harness/30_agentdojo_v12_taint_gate.py +616 -0
- spillway_guard-0.1.0/reference/spillway_gate.py +3674 -0
- spillway_guard-0.1.0/reproduce/README.md +309 -0
- spillway_guard-0.1.0/reproduce/run_cell.sh +90 -0
- spillway_guard-0.1.0/ruff.toml +20 -0
- spillway_guard-0.1.0/src/spillway_guard/__init__.py +36 -0
- spillway_guard-0.1.0/src/spillway_guard/agentdojo/__init__.py +28 -0
- spillway_guard-0.1.0/src/spillway_guard/agentdojo/agentdojo_default_prompt.txt +7 -0
- spillway_guard-0.1.0/src/spillway_guard/agentdojo/runner.py +373 -0
- spillway_guard-0.1.0/src/spillway_guard/floor.py +1903 -0
- spillway_guard-0.1.0/src/spillway_guard/guard.py +225 -0
- spillway_guard-0.1.0/src/spillway_guard/judge.py +311 -0
- spillway_guard-0.1.0/src/spillway_guard/lexicon.py +273 -0
- spillway_guard-0.1.0/src/spillway_guard/sanitizer.py +577 -0
- spillway_guard-0.1.0/tests/_path.py +7 -0
- spillway_guard-0.1.0/tests/test_differential.py +332 -0
- spillway_guard-0.1.0/tests/test_floor.py +285 -0
- spillway_guard-0.1.0/tests/test_guard.py +313 -0
- spillway_guard-0.1.0/tests/test_hardening.py +364 -0
- spillway_guard-0.1.0/tests/test_sanitizer.py +320 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
*.egg
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.eggs/
|
|
9
|
+
|
|
10
|
+
# Build artifacts
|
|
11
|
+
build/
|
|
12
|
+
dist/
|
|
13
|
+
wheels/
|
|
14
|
+
sdist/
|
|
15
|
+
pip-wheel-metadata/
|
|
16
|
+
*.whl
|
|
17
|
+
|
|
18
|
+
# Virtual environments
|
|
19
|
+
.venv/
|
|
20
|
+
.venv-test/
|
|
21
|
+
venv/
|
|
22
|
+
env/
|
|
23
|
+
ENV/
|
|
24
|
+
.python-version
|
|
25
|
+
|
|
26
|
+
# Tooling caches
|
|
27
|
+
.pytest_cache/
|
|
28
|
+
.ruff_cache/
|
|
29
|
+
.mypy_cache/
|
|
30
|
+
.hypothesis/
|
|
31
|
+
.coverage
|
|
32
|
+
.coverage.*
|
|
33
|
+
htmlcov/
|
|
34
|
+
.tox/
|
|
35
|
+
.nox/
|
|
36
|
+
|
|
37
|
+
# Logs and run output
|
|
38
|
+
*.log
|
|
39
|
+
logs/
|
|
40
|
+
spillway_runs/
|
|
41
|
+
|
|
42
|
+
# Local scratch (never committed)
|
|
43
|
+
_work/
|
|
44
|
+
|
|
45
|
+
# Secrets and local config
|
|
46
|
+
.env
|
|
47
|
+
.env.*
|
|
48
|
+
*.pem
|
|
49
|
+
|
|
50
|
+
# Editors and OS
|
|
51
|
+
.DS_Store
|
|
52
|
+
.idea/
|
|
53
|
+
.vscode/
|
|
54
|
+
*.swp
|
|
55
|
+
*.swo
|
|
56
|
+
|
|
57
|
+
# local preview builds
|
|
58
|
+
docs/_preview.html
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to SPILLWAY are recorded here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
|
|
5
|
+
[semantic versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0]
|
|
10
|
+
|
|
11
|
+
Initial release. `reference/spillway_gate.py` is the final build of the
|
|
12
|
+
gate the paper evaluated, and every number in the paper comes from it or from
|
|
13
|
+
an earlier build. The `spillway_guard` package ports it, and differential tests
|
|
14
|
+
compare their decisions on synthetic inputs. The library checks and their regression coverage are recorded below.
|
|
15
|
+
`reference/BUILD_PROVENANCE.md` pins the evaluation artifacts used for the reported tables.
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- `spillway_guard` package: the typed-identity egress floor (identity index read
|
|
20
|
+
from typed tool returns, taint-novel check, grant and destruction checks,
|
|
21
|
+
fixed delegation rules), the rule-based ingress sanitizer that removes
|
|
22
|
+
instruction-like regions and retains the removed spans as evidence, and the
|
|
23
|
+
optional two-view judge with the span veto (disabled by default, enabled in the paper's
|
|
24
|
+
results), which may restore a refused call but never one whose target
|
|
25
|
+
lies in the removed text. `Policy(delegation_span_veto=True)`, off by default
|
|
26
|
+
(the paper's setting), extends the span and co-residence veto to the
|
|
27
|
+
delegation rules and to grants to a named principal.
|
|
28
|
+
- `spillway-agentdojo` (the `[agentdojo]` extra): runs the gate as a drop-in
|
|
29
|
+
AgentDojo pipeline element with the switches used in every run of the paper
|
|
30
|
+
(`--auto --access-grant --chatml-guard --exfil-provenance` and the `SPW_*`
|
|
31
|
+
environment switches).
|
|
32
|
+
- `benchmarks/results/`: the per-cell numbers behind most of the paper's
|
|
33
|
+
tables (its README says which), and
|
|
34
|
+
`benchmarks/scripts/appendix_asr_levels.py` (`make asr-levels`), which
|
|
35
|
+
prints the raw and in-scope attack-success levels of the paper's Table 5
|
|
36
|
+
without its ROPE column.
|
|
37
|
+
- Packaging and community files: `pyproject.toml`, `CITATION.cff`,
|
|
38
|
+
`CONTRIBUTING.md`, `SECURITY.md`, a bypass-report issue template, and CI
|
|
39
|
+
(lint, unit tests on Python 3.10 to 3.13, differential tests against the
|
|
40
|
+
reference gate, a build check).
|
|
41
|
+
|
|
42
|
+
### Library checks and regression coverage
|
|
43
|
+
|
|
44
|
+
`spillway-agentdojo` runs the archived evaluation gate in
|
|
45
|
+
`reference/spillway_gate.py`. The reusable library API maintains the checks
|
|
46
|
+
below. Each path has a test in `tests/test_hardening.py`, and `SECURITY.md`
|
|
47
|
+
lists the integration conditions and remaining admission paths.
|
|
48
|
+
Replaying recorded tool calls through the package before and after these
|
|
49
|
+
changes, without a model, changed none of 7,831 AgentDojo decisions. On 4,259
|
|
50
|
+
AgentDyn calls from an earlier run it refused 2 more with the judge fixed at yes
|
|
51
|
+
and 284 more with no judge. This replay checks fixed-call decisions. Task-level utility and attack success
|
|
52
|
+
are the paper's evaluation results, whose build lineage is recorded in
|
|
53
|
+
`reference/BUILD_PROVENANCE.md`.
|
|
54
|
+
|
|
55
|
+
- Delegation rules: every target of a call, a number included, must be bound to
|
|
56
|
+
the task, a value that holds an address or a URL no longer passes on its first
|
|
57
|
+
word, and a digit run no longer makes a value an identity attribute.
|
|
58
|
+
- Task-named fetches: a task word must name the URL's host, a parent domain counts
|
|
59
|
+
only when written in full, and only URL arguments count. `fetch_*` and
|
|
60
|
+
`browse_*` tools get the content check.
|
|
61
|
+
- Grants: a principal must be named in the task as a whole word, numeric member
|
|
62
|
+
ids count as principals, and a `transfer` tool counts as a grant only when it
|
|
63
|
+
moves an owner or a repository.
|
|
64
|
+
- Free text: an IBAN or another opaque identifier repeated inside a sentence
|
|
65
|
+
counts as repeated in free text.
|
|
66
|
+
- Decoy spans: a harmless removed span no longer opens the injection-span rule.
|
|
67
|
+
`Policy(injspan_clean_rule=True)` restores the evaluated rule.
|
|
68
|
+
- Span veto: forged chat turns join the removed spans, a sign-off removes the
|
|
69
|
+
sender's name only at the claim, and the grant rule honors removed text.
|
|
70
|
+
- Taint: returns are also compared as raw strings, so a target with a character
|
|
71
|
+
the renderer escapes still matches.
|
|
72
|
+
- Arguments and fields: target values are read at any depth, camelCase names
|
|
73
|
+
split into words, and free-text names such as `email_body` are never targets
|
|
74
|
+
or identity roles. `attendees`, `invitees`, `guests` and `shared_with` are
|
|
75
|
+
target arguments.
|
|
76
|
+
- Deletions: an intent word must be a whole word that no negation covers, so
|
|
77
|
+
"clearly", "cancellation policy" and "do not delete" no longer authorize a
|
|
78
|
+
deletion.
|
|
79
|
+
- Judge: only a true verdict restores a call, an exception counts as a no,
|
|
80
|
+
`Judge(threshold=...)` takes effect, a panel counts every model, and a long
|
|
81
|
+
argument no longer pushes a target out of the prompt.
|
|
82
|
+
|
|
83
|
+
### Results in the paper (AgentDojo suite v1.2.2 and AgentDyn)
|
|
84
|
+
|
|
85
|
+
- Standard attack, three backbones x four suites: in-scope attack success
|
|
86
|
+
`.000` on all twelve cells, at the highest twelve-cell under-attack utility of
|
|
87
|
+
the defenses run (`.484`) and 96% of the undefended agent's benign utility.
|
|
88
|
+
- ChatInject (forged chat template, qwen3:8b, 665 pairs): raw zero.
|
|
89
|
+
- AutoDojo (optimizer-generated injections): in-scope attack success `.000` on
|
|
90
|
+
the 389 released pairs (descriptive). On two held-out Banking sets, `.000` and
|
|
91
|
+
`.007` by AgentDojo's check, `.014` counting a second transfer that check
|
|
92
|
+
misses.
|
|
93
|
+
- PISmith (released RL attacker): mean in-scope attack success `.007`.
|
|
94
|
+
- Delegated routing against ROPE, seven suites on gpt-4o-mini: in-scope attack
|
|
95
|
+
success zero on all seven.
|
|
96
|
+
|
|
97
|
+
Some of these results, among them SPILLWAY's cells in Table 1, ran on an earlier
|
|
98
|
+
build without the address and payee-change rules. Appendix B of the paper
|
|
99
|
+
reports that these rules change no sanitizer output on the 27,392 tool messages
|
|
100
|
+
behind Tables 1 and 3, the ChatInject panel and Figures 3 and 4, and that
|
|
101
|
+
rerunning Table 1's 949 qwen3:8b pairs with the address rules reproduces each
|
|
102
|
+
suite's utility and attack success to the third decimal.
|
|
103
|
+
|
|
104
|
+
### Scope
|
|
105
|
+
|
|
106
|
+
- The stated property is scoped to identity targets: a recipient, account, participant,
|
|
107
|
+
channel or URL that the user never named and that no read-only tool returned
|
|
108
|
+
in a typed field. A spoofed typed field and content-only effects are out of
|
|
109
|
+
scope. See `SECURITY.md`.
|
|
110
|
+
- Target roles come from one lexicon per tool set. A new tool set needs its roles
|
|
111
|
+
declared before the floor covers it. See `CONTRIBUTING.md`.
|
|
112
|
+
|
|
113
|
+
[Unreleased]: https://github.com/saintmachines/spillway/compare/v0.1.0...HEAD
|
|
114
|
+
[0.1.0]: https://github.com/saintmachines/spillway/releases/tag/v0.1.0
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use SPILLWAY in your work, please cite the paper listed under preferred-citation."
|
|
3
|
+
type: software
|
|
4
|
+
title: "SPILLWAY"
|
|
5
|
+
abstract: "Prompt-injection defense for tool-using LLM agents: a typed-identity egress floor, a rule-based ingress sanitizer, and an optional judge that reviews refused calls."
|
|
6
|
+
authors:
|
|
7
|
+
- family-names: Lim
|
|
8
|
+
given-names: Woosang
|
|
9
|
+
- family-names: Chang
|
|
10
|
+
given-names: Yeseul
|
|
11
|
+
- family-names: Lee
|
|
12
|
+
given-names: Jaewoo
|
|
13
|
+
version: 0.1.0
|
|
14
|
+
license: Apache-2.0
|
|
15
|
+
repository-code: "https://github.com/saintmachines/spillway"
|
|
16
|
+
url: "https://saintmachines.github.io/spillway/"
|
|
17
|
+
keywords:
|
|
18
|
+
- prompt injection
|
|
19
|
+
- LLM agents
|
|
20
|
+
- tool use
|
|
21
|
+
- agent security
|
|
22
|
+
- AgentDojo
|
|
23
|
+
preferred-citation:
|
|
24
|
+
type: unpublished
|
|
25
|
+
title: "SPILLWAY: Separating Target Admission from Sanitization to Defend Tool-Using Agents against Prompt Injection"
|
|
26
|
+
authors:
|
|
27
|
+
- family-names: Lim
|
|
28
|
+
given-names: Woosang
|
|
29
|
+
- family-names: Chang
|
|
30
|
+
given-names: Yeseul
|
|
31
|
+
- family-names: Lee
|
|
32
|
+
given-names: Jaewoo
|
|
33
|
+
year: 2026
|
|
34
|
+
url: "https://github.com/saintmachines/spillway/blob/main/docs/SPILLWAY_preprint.pdf"
|
|
35
|
+
notes: "Preprint."
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# Contributing to SPILLWAY
|
|
2
|
+
|
|
3
|
+
Thanks for helping. Three kinds of contribution are most useful: a bypass report
|
|
4
|
+
(an injection that gets an effect to an attacker's target through the floor), a
|
|
5
|
+
tool-role lexicon for a tool set the floor does not cover yet, and fixes with a
|
|
6
|
+
test. Keep the floor deterministic: no model call decides admission.
|
|
7
|
+
|
|
8
|
+
## Setup and tests
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
git clone https://github.com/saintmachines/spillway
|
|
12
|
+
cd spillway
|
|
13
|
+
python -m venv .venv && source .venv/bin/activate
|
|
14
|
+
pip install -e ".[dev]"
|
|
15
|
+
ruff check .
|
|
16
|
+
pytest -q -m "not differential"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The differential tests run synthetic tool messages and typed models through the
|
|
20
|
+
packaged floor and through `reference/spillway_gate.py` (the final build the paper
|
|
21
|
+
evaluated) and require identical decisions on those inputs. The library checks
|
|
22
|
+
and their regression cases are recorded in `CHANGELOG.md` and
|
|
23
|
+
`tests/test_hardening.py`. A deliberate behavioral difference from the archived
|
|
24
|
+
gate belongs in both records. The differential tests
|
|
25
|
+
need the AgentDojo extra. Set `SPW_REQUIRE_DIFFERENTIAL=1` so that a missing
|
|
26
|
+
dependency fails the run instead of skipping it:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install -e ".[dev,agentdojo]"
|
|
30
|
+
SPW_REQUIRE_DIFFERENTIAL=1 pytest -q -m differential
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Add such a test to `tests/test_differential.py`, whose tests carry the
|
|
34
|
+
`differential` mark. CI runs both groups on pushes to `main` and on pull
|
|
35
|
+
requests.
|
|
36
|
+
|
|
37
|
+
## Adding a tool-role lexicon for a new tool set
|
|
38
|
+
|
|
39
|
+
The floor reads where a call goes from the tools' declared types, never from
|
|
40
|
+
text. What it needs per tool set is a role lexicon, written once:
|
|
41
|
+
|
|
42
|
+
1. **Verb prefixes** that mark a tool as read-only, writing, destructive, or
|
|
43
|
+
granting (the AgentDojo lexicon has 11, 22, 10, and 11 of them).
|
|
44
|
+
2. **Target-argument names**: the arguments that name where an effect goes
|
|
45
|
+
(recipient, account, participant, channel, URL, with 26 names for AgentDojo and
|
|
46
|
+
6 more for AgentDyn).
|
|
47
|
+
3. **Principal-argument names**: the arguments that name who receives a grant.
|
|
48
|
+
4. **Self-owned tools**: the short list of tools that write only to the user's
|
|
49
|
+
own calendar or drive.
|
|
50
|
+
5. **Identity field types and routing roles** on the tools' return models: a
|
|
51
|
+
field is indexed when its declared type is an identity format (email, URI,
|
|
52
|
+
UUID, hostname, IP) or its field has a routing role (recipient, sender,
|
|
53
|
+
participant, channel).
|
|
54
|
+
|
|
55
|
+
Then:
|
|
56
|
+
|
|
57
|
+
- Add a classmethod to `Lexicon` in `src/spillway_guard/lexicon.py`, like
|
|
58
|
+
`for_agentdyn`, and pass the result to `Guard` through `lexicon`. Keep an existing
|
|
59
|
+
lexicon as it is when you cover a new tool set.
|
|
60
|
+
- Add a test that runs the new tool set's benign tasks with the gate on and off
|
|
61
|
+
and shows which calls the floor refuses and why. A refusal on benign traffic
|
|
62
|
+
indicates a lexicon gap. Correct the role declarations and rerun the check.
|
|
63
|
+
- Confirm the change is a no-op on the AgentDojo suites: the differential tests
|
|
64
|
+
must still pass unchanged.
|
|
65
|
+
- Say in the pull request which tool set, how many tools in each role, and which
|
|
66
|
+
identity fields the index picks up.
|
|
67
|
+
|
|
68
|
+
## Reporting an attack that bypasses the floor
|
|
69
|
+
|
|
70
|
+
An injection that routes an effect to an identity target the user never named is
|
|
71
|
+
a security report. Follow [SECURITY.md](SECURITY.md) and send it through the
|
|
72
|
+
private channel described there. The fields a report needs are listed in the
|
|
73
|
+
[bypass report template](.github/ISSUE_TEMPLATE/bypass_report.md). A utility
|
|
74
|
+
regression, a sanitizer rule that removes benign text, or a content-only effect
|
|
75
|
+
is a normal issue.
|
|
76
|
+
|
|
77
|
+
## Conventions
|
|
78
|
+
|
|
79
|
+
- Python 3.10 or newer, standard library first. `pydantic` and `PyYAML` are the only
|
|
80
|
+
hard runtime dependencies. `agentdojo` stays optional.
|
|
81
|
+
- Environment switches use the `SPW_` prefix.
|
|
82
|
+
- Every result in the documentation traces to `benchmarks/results/*.json` or to
|
|
83
|
+
the paper. Do not add a result that neither source carries. Operational figures
|
|
84
|
+
from the paper's runs, such as GPU memory and API cost, are labelled as such.
|
|
85
|
+
- Claims stay within the threat model: zero in-scope attack success on the
|
|
86
|
+
AgentDojo benchmarks the paper runs, nothing wider than that.
|
|
87
|
+
- `ruff check .` and `ruff format .` before you push (line length 120).
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2026 SPILLWAY authors
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# SPILLWAY release: developer targets. `make help` lists them.
|
|
2
|
+
PY ?= python3
|
|
3
|
+
TABLES := benchmarks/scripts/make_tables.py
|
|
4
|
+
ASR_LEVELS := benchmarks/scripts/appendix_asr_levels.py
|
|
5
|
+
|
|
6
|
+
.PHONY: help install install-agentdojo test tables asr-levels lint selftest cell
|
|
7
|
+
|
|
8
|
+
help:
|
|
9
|
+
@echo "make install editable install with the dev extra (pytest, ruff, email-validator)"
|
|
10
|
+
@echo "make install-agentdojo editable install with the dev and agentdojo extras (runs the gate on AgentDojo)"
|
|
11
|
+
@echo "make test check benchmarks/results against the paper's numbers, then the unit tests"
|
|
12
|
+
@echo "make tables write benchmarks/TABLES.md from benchmarks/results/*.json"
|
|
13
|
+
@echo "make asr-levels print Appendix Table 5 (raw / in-scope ASR per cell) from the breadth-grid files, without its ROPE column"
|
|
14
|
+
@echo "make lint ruff check . (the configuration in ruff.toml, the same as CI)"
|
|
15
|
+
@echo "make selftest the reference gate's own self-tests (needs the agentdojo extra, no server)"
|
|
16
|
+
@echo "make cell SUITE=banking BACKBONE=qwen3:8b run one AgentDojo cell with the paper's settings"
|
|
17
|
+
|
|
18
|
+
# A venv made by `uv venv` ships no pip: run `$(PY) -m ensurepip --upgrade` first, or use `uv pip install -e ".[dev]"`.
|
|
19
|
+
install:
|
|
20
|
+
$(PY) -m pip install -e ".[dev]"
|
|
21
|
+
|
|
22
|
+
install-agentdojo:
|
|
23
|
+
$(PY) -m pip install -e ".[dev,agentdojo]"
|
|
24
|
+
|
|
25
|
+
# make_tables.py exits non-zero when a printed value differs from the paper, so it is a test.
|
|
26
|
+
test:
|
|
27
|
+
$(PY) $(TABLES) --no-write
|
|
28
|
+
@$(PY) -m pytest -q -m "not differential" tests
|
|
29
|
+
|
|
30
|
+
tables:
|
|
31
|
+
$(PY) $(TABLES) --out benchmarks/TABLES.md
|
|
32
|
+
|
|
33
|
+
# appendix_asr_levels.py is pinned byte for byte in reference/BUILD_PROVENANCE.md and opens its two input files under
|
|
34
|
+
# fixed names in the current directory. This target runs it unchanged in a temporary directory that links the shipped files under
|
|
35
|
+
# those names. It reads the per-cell aggregates in those two files, not per-pair records.
|
|
36
|
+
asr-levels:
|
|
37
|
+
@tmp=$$(mktemp -d) && trap 'rm -rf "$$tmp"' EXIT && \
|
|
38
|
+
ln -s "$(CURDIR)/benchmarks/results/breadth_grid_summary.json" "$$tmp/final_v122.json" && \
|
|
39
|
+
ln -s "$(CURDIR)/benchmarks/results/breadth_grid_cells.json" "$$tmp/tables_v122.json" && \
|
|
40
|
+
$(PY) -c 'import os, runpy, sys; os.chdir(sys.argv[1]); runpy.run_path(sys.argv[2], run_name="__main__")' "$$tmp" "$(CURDIR)/$(ASR_LEVELS)"
|
|
41
|
+
|
|
42
|
+
lint:
|
|
43
|
+
ruff check .
|
|
44
|
+
|
|
45
|
+
selftest:
|
|
46
|
+
$(PY) -m spillway_guard.agentdojo.runner --self-test
|
|
47
|
+
|
|
48
|
+
cell:
|
|
49
|
+
@test -n "$(SUITE)" -a -n "$(BACKBONE)" || { echo "usage: make cell SUITE=<suite> BACKBONE=<model>"; exit 2; }
|
|
50
|
+
bash reproduce/run_cell.sh "$(SUITE)" "$(BACKBONE)"
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
SPILLWAY (spillway-guard)
|
|
2
|
+
Copyright 2026 SPILLWAY authors
|
|
3
|
+
|
|
4
|
+
This product is licensed under the Apache License, Version 2.0. See LICENSE.
|
|
5
|
+
|
|
6
|
+
This product includes material from AgentDojo
|
|
7
|
+
(https://github.com/ethz-spylab/agentdojo), which is licensed under the MIT
|
|
8
|
+
License. The AgentDojo copyright notice and license text are reproduced at the
|
|
9
|
+
end of this file, copied from licenses/LICENSE in the agentdojo 0.1.35
|
|
10
|
+
distribution.
|
|
11
|
+
|
|
12
|
+
Files that derive from AgentDojo:
|
|
13
|
+
|
|
14
|
+
src/spillway_guard/agentdojo/agentdojo_default_prompt.txt
|
|
15
|
+
AgentDojo's default system prompt, the "default" entry of
|
|
16
|
+
agentdojo/data/system_messages.yaml, byte for byte.
|
|
17
|
+
Ships in the wheel and in the source distribution.
|
|
18
|
+
|
|
19
|
+
reference/harness/21_agentdojo_full_benchmark.py
|
|
20
|
+
Contains code copied and adapted from
|
|
21
|
+
agentdojo/agent_pipeline/tool_execution.py and
|
|
22
|
+
agentdojo/agent_pipeline/llms/google_llm.py.
|
|
23
|
+
Ships in the source distribution.
|
|
24
|
+
|
|
25
|
+
reference/harness/24_agentdojo_gemini_backbone.py
|
|
26
|
+
Contains code copied and adapted from
|
|
27
|
+
agentdojo/agent_pipeline/llms/google_llm.py.
|
|
28
|
+
Ships in the source distribution.
|
|
29
|
+
|
|
30
|
+
Test inputs in tests/, the self-test in reference/spillway_gate.py and the
|
|
31
|
+
template patterns in src/spillway_guard/sanitizer.py reuse AgentDojo's
|
|
32
|
+
important_instructions injection template, addresses and tool names. The typed
|
|
33
|
+
fixtures in tests/test_floor.py are shaped like AgentDojo's return models, and
|
|
34
|
+
the prompt listed above is also copied into
|
|
35
|
+
reference/harness/21_agentdojo_full_benchmark.py.
|
|
36
|
+
|
|
37
|
+
The project page under docs/ shows one recorded AgentDojo episode. Its task,
|
|
38
|
+
tool returns and injection text come from AgentDojo's task suites.
|
|
39
|
+
|
|
40
|
+
The lexicon in src/spillway_guard/lexicon.py and the harness under
|
|
41
|
+
reference/harness/ use tool-argument names and one sentence of the default
|
|
42
|
+
prompt of AgentDyn (https://github.com/SaFo-Lab/AgentDyn), which is under the
|
|
43
|
+
MIT License with the same copyright line as AgentDojo.
|
|
44
|
+
|
|
45
|
+
Other licenses. The Apache License does not cover these files:
|
|
46
|
+
|
|
47
|
+
docs/assets/fonts/
|
|
48
|
+
Fraunces and IBM Plex, under the SIL Open Font License 1.1. The license
|
|
49
|
+
texts sit beside the fonts.
|
|
50
|
+
|
|
51
|
+
docs/assets/saintmachines_lockup.svg, docs/assets/saintmachines_lockup_white.svg
|
|
52
|
+
Logos. No license to these or to the SPILLWAY and SAINT Machines names is
|
|
53
|
+
granted, as section 6 of the Apache License says of trademarks.
|
|
54
|
+
|
|
55
|
+
docs/SPILLWAY_preprint.pdf
|
|
56
|
+
The paper, under the license it carries on arXiv.
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
AgentDojo license (agentdojo 0.1.35, licenses/LICENSE):
|
|
60
|
+
|
|
61
|
+
MIT License
|
|
62
|
+
|
|
63
|
+
Copyright (c) 2024 Edoardo Debenedetti, Jie Zhang, Mislav Balunovic, Luca Beurer-Kellner, Marc Fischer, and Florian Tramèr
|
|
64
|
+
|
|
65
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
66
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
67
|
+
in the Software without restriction, including without limitation the rights
|
|
68
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
69
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
70
|
+
furnished to do so, subject to the following conditions:
|
|
71
|
+
|
|
72
|
+
The above copyright notice and this permission notice shall be included in all
|
|
73
|
+
copies or substantial portions of the Software.
|
|
74
|
+
|
|
75
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
76
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
77
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
78
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
79
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
80
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
81
|
+
SOFTWARE.
|