spillway-guard 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. spillway_guard-0.1.0/.gitignore +58 -0
  2. spillway_guard-0.1.0/CHANGELOG.md +114 -0
  3. spillway_guard-0.1.0/CITATION.cff +35 -0
  4. spillway_guard-0.1.0/CONTRIBUTING.md +87 -0
  5. spillway_guard-0.1.0/LICENSE +201 -0
  6. spillway_guard-0.1.0/Makefile +50 -0
  7. spillway_guard-0.1.0/NOTICE +81 -0
  8. spillway_guard-0.1.0/PKG-INFO +263 -0
  9. spillway_guard-0.1.0/PYPI_README.md +225 -0
  10. spillway_guard-0.1.0/README.md +279 -0
  11. spillway_guard-0.1.0/SECURITY.md +133 -0
  12. spillway_guard-0.1.0/benchmarks/TABLES.md +101 -0
  13. spillway_guard-0.1.0/benchmarks/results/README.md +16 -0
  14. spillway_guard-0.1.0/benchmarks/results/ablation_and_chatinject.json +7739 -0
  15. spillway_guard-0.1.0/benchmarks/results/ablation_no_span_veto.json +34 -0
  16. spillway_guard-0.1.0/benchmarks/results/breadth_grid_cells.json +1962 -0
  17. spillway_guard-0.1.0/benchmarks/results/breadth_grid_summary.json +870 -0
  18. spillway_guard-0.1.0/benchmarks/results/delegated_routing_agent_router.json +1022 -0
  19. spillway_guard-0.1.0/benchmarks/results/delegated_routing_opus_router.json +1060 -0
  20. spillway_guard-0.1.0/benchmarks/results/fig_cost_values.json +38 -0
  21. spillway_guard-0.1.0/benchmarks/results/pismith_a4/README.md +17 -0
  22. spillway_guard-0.1.0/benchmarks/results/pismith_a4/drift_a25f.md +6 -0
  23. spillway_guard-0.1.0/benchmarks/results/pismith_a4/fast_3119.md +26 -0
  24. spillway_guard-0.1.0/benchmarks/results/pismith_a4/progent_133f.md +6 -0
  25. spillway_guard-0.1.0/benchmarks/results/pismith_a4/rope_9b4e.md +19 -0
  26. spillway_guard-0.1.0/benchmarks/scripts/appendix_asr_levels.py +29 -0
  27. spillway_guard-0.1.0/benchmarks/scripts/make_tables.py +624 -0
  28. spillway_guard-0.1.0/pyproject.toml +70 -0
  29. spillway_guard-0.1.0/reference/BUILD_PROVENANCE.md +39 -0
  30. spillway_guard-0.1.0/reference/harness/21_agentdojo_full_benchmark.py +907 -0
  31. spillway_guard-0.1.0/reference/harness/24_agentdojo_gemini_backbone.py +201 -0
  32. spillway_guard-0.1.0/reference/harness/27_agentdojo_v8_action_boundary.py +293 -0
  33. spillway_guard-0.1.0/reference/harness/30_agentdojo_v12_taint_gate.py +616 -0
  34. spillway_guard-0.1.0/reference/spillway_gate.py +3674 -0
  35. spillway_guard-0.1.0/reproduce/README.md +309 -0
  36. spillway_guard-0.1.0/reproduce/run_cell.sh +90 -0
  37. spillway_guard-0.1.0/ruff.toml +20 -0
  38. spillway_guard-0.1.0/src/spillway_guard/__init__.py +36 -0
  39. spillway_guard-0.1.0/src/spillway_guard/agentdojo/__init__.py +28 -0
  40. spillway_guard-0.1.0/src/spillway_guard/agentdojo/agentdojo_default_prompt.txt +7 -0
  41. spillway_guard-0.1.0/src/spillway_guard/agentdojo/runner.py +373 -0
  42. spillway_guard-0.1.0/src/spillway_guard/floor.py +1903 -0
  43. spillway_guard-0.1.0/src/spillway_guard/guard.py +225 -0
  44. spillway_guard-0.1.0/src/spillway_guard/judge.py +311 -0
  45. spillway_guard-0.1.0/src/spillway_guard/lexicon.py +273 -0
  46. spillway_guard-0.1.0/src/spillway_guard/sanitizer.py +577 -0
  47. spillway_guard-0.1.0/tests/_path.py +7 -0
  48. spillway_guard-0.1.0/tests/test_differential.py +332 -0
  49. spillway_guard-0.1.0/tests/test_floor.py +285 -0
  50. spillway_guard-0.1.0/tests/test_guard.py +313 -0
  51. spillway_guard-0.1.0/tests/test_hardening.py +364 -0
  52. spillway_guard-0.1.0/tests/test_sanitizer.py +320 -0
@@ -0,0 +1,58 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ *.egg
7
+ *.egg-info/
8
+ .eggs/
9
+
10
+ # Build artifacts
11
+ build/
12
+ dist/
13
+ wheels/
14
+ sdist/
15
+ pip-wheel-metadata/
16
+ *.whl
17
+
18
+ # Virtual environments
19
+ .venv/
20
+ .venv-test/
21
+ venv/
22
+ env/
23
+ ENV/
24
+ .python-version
25
+
26
+ # Tooling caches
27
+ .pytest_cache/
28
+ .ruff_cache/
29
+ .mypy_cache/
30
+ .hypothesis/
31
+ .coverage
32
+ .coverage.*
33
+ htmlcov/
34
+ .tox/
35
+ .nox/
36
+
37
+ # Logs and run output
38
+ *.log
39
+ logs/
40
+ spillway_runs/
41
+
42
+ # Local scratch (never committed)
43
+ _work/
44
+
45
+ # Secrets and local config
46
+ .env
47
+ .env.*
48
+ *.pem
49
+
50
+ # Editors and OS
51
+ .DS_Store
52
+ .idea/
53
+ .vscode/
54
+ *.swp
55
+ *.swo
56
+
57
+ # local preview builds
58
+ docs/_preview.html
@@ -0,0 +1,114 @@
1
+ # Changelog
2
+
3
+ All notable changes to SPILLWAY are recorded here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
5
+ [semantic versioning](https://semver.org/).
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [0.1.0]
10
+
11
+ Initial release. `reference/spillway_gate.py` is the final build of the
12
+ gate the paper evaluated, and every number in the paper comes from it or from
13
+ an earlier build. The `spillway_guard` package ports it, and differential tests
14
+ compare their decisions on synthetic inputs. The library checks and their regression coverage are recorded below.
15
+ `reference/BUILD_PROVENANCE.md` pins the evaluation artifacts used for the reported tables.
16
+
17
+ ### Added
18
+
19
+ - `spillway_guard` package: the typed-identity egress floor (identity index read
20
+ from typed tool returns, taint-novel check, grant and destruction checks,
21
+ fixed delegation rules), the rule-based ingress sanitizer that removes
22
+ instruction-like regions and retains the removed spans as evidence, and the
23
+ optional two-view judge with the span veto (disabled by default, enabled in the paper's
24
+ results), which may restore a refused call but never one whose target
25
+ lies in the removed text. `Policy(delegation_span_veto=True)`, off by default
26
+ (the paper's setting), extends the span and co-residence veto to the
27
+ delegation rules and to grants to a named principal.
28
+ - `spillway-agentdojo` (the `[agentdojo]` extra): runs the gate as a drop-in
29
+ AgentDojo pipeline element with the switches used in every run of the paper
30
+ (`--auto --access-grant --chatml-guard --exfil-provenance` and the `SPW_*`
31
+ environment switches).
32
+ - `benchmarks/results/`: the per-cell numbers behind most of the paper's
33
+ tables (its README says which), and
34
+ `benchmarks/scripts/appendix_asr_levels.py` (`make asr-levels`), which
35
+ prints the raw and in-scope attack-success levels of the paper's Table 5
36
+ without its ROPE column.
37
+ - Packaging and community files: `pyproject.toml`, `CITATION.cff`,
38
+ `CONTRIBUTING.md`, `SECURITY.md`, a bypass-report issue template, and CI
39
+ (lint, unit tests on Python 3.10 to 3.13, differential tests against the
40
+ reference gate, a build check).
41
+
42
+ ### Library checks and regression coverage
43
+
44
+ `spillway-agentdojo` runs the archived evaluation gate in
45
+ `reference/spillway_gate.py`. The reusable library API maintains the checks
46
+ below. Each path has a test in `tests/test_hardening.py`, and `SECURITY.md`
47
+ lists the integration conditions and remaining admission paths.
48
+ Replaying recorded tool calls through the package before and after these
49
+ changes, without a model, changed none of 7,831 AgentDojo decisions. On 4,259
50
+ AgentDyn calls from an earlier run it refused 2 more with the judge fixed at yes
51
+ and 284 more with no judge. This replay checks fixed-call decisions. Task-level utility and attack success
52
+ are the paper's evaluation results, whose build lineage is recorded in
53
+ `reference/BUILD_PROVENANCE.md`.
54
+
55
+ - Delegation rules: every target of a call, a number included, must be bound to
56
+ the task, a value that holds an address or a URL no longer passes on its first
57
+ word, and a digit run no longer makes a value an identity attribute.
58
+ - Task-named fetches: a task word must name the URL's host, a parent domain counts
59
+ only when written in full, and only URL arguments count. `fetch_*` and
60
+ `browse_*` tools get the content check.
61
+ - Grants: a principal must be named in the task as a whole word, numeric member
62
+ ids count as principals, and a `transfer` tool counts as a grant only when it
63
+ moves an owner or a repository.
64
+ - Free text: an IBAN or another opaque identifier repeated inside a sentence
65
+ counts as repeated in free text.
66
+ - Decoy spans: a harmless removed span no longer opens the injection-span rule.
67
+ `Policy(injspan_clean_rule=True)` restores the evaluated rule.
68
+ - Span veto: forged chat turns join the removed spans, a sign-off removes the
69
+ sender's name only at the claim, and the grant rule honors removed text.
70
+ - Taint: returns are also compared as raw strings, so a target with a character
71
+ the renderer escapes still matches.
72
+ - Arguments and fields: target values are read at any depth, camelCase names
73
+ split into words, and free-text names such as `email_body` are never targets
74
+ or identity roles. `attendees`, `invitees`, `guests` and `shared_with` are
75
+ target arguments.
76
+ - Deletions: an intent word must be a whole word that no negation covers, so
77
+ "clearly", "cancellation policy" and "do not delete" no longer authorize a
78
+ deletion.
79
+ - Judge: only a true verdict restores a call, an exception counts as a no,
80
+ `Judge(threshold=...)` takes effect, a panel counts every model, and a long
81
+ argument no longer pushes a target out of the prompt.
82
+
83
+ ### Results in the paper (AgentDojo suite v1.2.2 and AgentDyn)
84
+
85
+ - Standard attack, three backbones x four suites: in-scope attack success
86
+ `.000` on all twelve cells, at the highest twelve-cell under-attack utility of
87
+ the defenses run (`.484`) and 96% of the undefended agent's benign utility.
88
+ - ChatInject (forged chat template, qwen3:8b, 665 pairs): raw zero.
89
+ - AutoDojo (optimizer-generated injections): in-scope attack success `.000` on
90
+ the 389 released pairs (descriptive). On two held-out Banking sets, `.000` and
91
+ `.007` by AgentDojo's check, `.014` counting a second transfer that check
92
+ misses.
93
+ - PISmith (released RL attacker): mean in-scope attack success `.007`.
94
+ - Delegated routing against ROPE, seven suites on gpt-4o-mini: in-scope attack
95
+ success zero on all seven.
96
+
97
+ Some of these results, among them SPILLWAY's cells in Table 1, ran on an earlier
98
+ build without the address and payee-change rules. Appendix B of the paper
99
+ reports that these rules change no sanitizer output on the 27,392 tool messages
100
+ behind Tables 1 and 3, the ChatInject panel and Figures 3 and 4, and that
101
+ rerunning Table 1's 949 qwen3:8b pairs with the address rules reproduces each
102
+ suite's utility and attack success to the third decimal.
103
+
104
+ ### Scope
105
+
106
+ - The stated property is scoped to identity targets: a recipient, account, participant,
107
+ channel or URL that the user never named and that no read-only tool returned
108
+ in a typed field. A spoofed typed field and content-only effects are out of
109
+ scope. See `SECURITY.md`.
110
+ - Target roles come from one lexicon per tool set. A new tool set needs its roles
111
+ declared before the floor covers it. See `CONTRIBUTING.md`.
112
+
113
+ [Unreleased]: https://github.com/saintmachines/spillway/compare/v0.1.0...HEAD
114
+ [0.1.0]: https://github.com/saintmachines/spillway/releases/tag/v0.1.0
@@ -0,0 +1,35 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use SPILLWAY in your work, please cite the paper listed under preferred-citation."
3
+ type: software
4
+ title: "SPILLWAY"
5
+ abstract: "Prompt-injection defense for tool-using LLM agents: a typed-identity egress floor, a rule-based ingress sanitizer, and an optional judge that reviews refused calls."
6
+ authors:
7
+ - family-names: Lim
8
+ given-names: Woosang
9
+ - family-names: Chang
10
+ given-names: Yeseul
11
+ - family-names: Lee
12
+ given-names: Jaewoo
13
+ version: 0.1.0
14
+ license: Apache-2.0
15
+ repository-code: "https://github.com/saintmachines/spillway"
16
+ url: "https://saintmachines.github.io/spillway/"
17
+ keywords:
18
+ - prompt injection
19
+ - LLM agents
20
+ - tool use
21
+ - agent security
22
+ - AgentDojo
23
+ preferred-citation:
24
+ type: unpublished
25
+ title: "SPILLWAY: Separating Target Admission from Sanitization to Defend Tool-Using Agents against Prompt Injection"
26
+ authors:
27
+ - family-names: Lim
28
+ given-names: Woosang
29
+ - family-names: Chang
30
+ given-names: Yeseul
31
+ - family-names: Lee
32
+ given-names: Jaewoo
33
+ year: 2026
34
+ url: "https://github.com/saintmachines/spillway/blob/main/docs/SPILLWAY_preprint.pdf"
35
+ notes: "Preprint."
@@ -0,0 +1,87 @@
1
+ # Contributing to SPILLWAY
2
+
3
+ Thanks for helping. Three kinds of contribution are most useful: a bypass report
4
+ (an injection that gets an effect to an attacker's target through the floor), a
5
+ tool-role lexicon for a tool set the floor does not cover yet, and fixes with a
6
+ test. Keep the floor deterministic: no model call decides admission.
7
+
8
+ ## Setup and tests
9
+
10
+ ```bash
11
+ git clone https://github.com/saintmachines/spillway
12
+ cd spillway
13
+ python -m venv .venv && source .venv/bin/activate
14
+ pip install -e ".[dev]"
15
+ ruff check .
16
+ pytest -q -m "not differential"
17
+ ```
18
+
19
+ The differential tests run synthetic tool messages and typed models through the
20
+ packaged floor and through `reference/spillway_gate.py` (the final build the paper
21
+ evaluated) and require identical decisions on those inputs. The library checks
22
+ and their regression cases are recorded in `CHANGELOG.md` and
23
+ `tests/test_hardening.py`. A deliberate behavioral difference from the archived
24
+ gate belongs in both records. The differential tests
25
+ need the AgentDojo extra. Set `SPW_REQUIRE_DIFFERENTIAL=1` so that a missing
26
+ dependency fails the run instead of skipping it:
27
+
28
+ ```bash
29
+ pip install -e ".[dev,agentdojo]"
30
+ SPW_REQUIRE_DIFFERENTIAL=1 pytest -q -m differential
31
+ ```
32
+
33
+ Add such a test to `tests/test_differential.py`, whose tests carry the
34
+ `differential` mark. CI runs both groups on pushes to `main` and on pull
35
+ requests.
36
+
37
+ ## Adding a tool-role lexicon for a new tool set
38
+
39
+ The floor reads where a call goes from the tools' declared types, never from
40
+ text. What it needs per tool set is a role lexicon, written once:
41
+
42
+ 1. **Verb prefixes** that mark a tool as read-only, writing, destructive, or
43
+ granting (the AgentDojo lexicon has 11, 22, 10, and 11 of them).
44
+ 2. **Target-argument names**: the arguments that name where an effect goes
45
+ (recipient, account, participant, channel, URL, with 26 names for AgentDojo and
46
+ 6 more for AgentDyn).
47
+ 3. **Principal-argument names**: the arguments that name who receives a grant.
48
+ 4. **Self-owned tools**: the short list of tools that write only to the user's
49
+ own calendar or drive.
50
+ 5. **Identity field types and routing roles** on the tools' return models: a
51
+ field is indexed when its declared type is an identity format (email, URI,
52
+ UUID, hostname, IP) or its field has a routing role (recipient, sender,
53
+ participant, channel).
54
+
55
+ Then:
56
+
57
+ - Add a classmethod to `Lexicon` in `src/spillway_guard/lexicon.py`, like
58
+ `for_agentdyn`, and pass the result to `Guard` through `lexicon`. Keep an existing
59
+ lexicon as it is when you cover a new tool set.
60
+ - Add a test that runs the new tool set's benign tasks with the gate on and off
61
+ and shows which calls the floor refuses and why. A refusal on benign traffic
62
+ indicates a lexicon gap. Correct the role declarations and rerun the check.
63
+ - Confirm the change is a no-op on the AgentDojo suites: the differential tests
64
+ must still pass unchanged.
65
+ - Say in the pull request which tool set, how many tools in each role, and which
66
+ identity fields the index picks up.
67
+
68
+ ## Reporting an attack that bypasses the floor
69
+
70
+ An injection that routes an effect to an identity target the user never named is
71
+ a security report. Follow [SECURITY.md](SECURITY.md) and send it through the
72
+ private channel described there. The fields a report needs are listed in the
73
+ [bypass report template](.github/ISSUE_TEMPLATE/bypass_report.md). A utility
74
+ regression, a sanitizer rule that removes benign text, or a content-only effect
75
+ is a normal issue.
76
+
77
+ ## Conventions
78
+
79
+ - Python 3.10 or newer, standard library first. `pydantic` and `PyYAML` are the only
80
+ hard runtime dependencies. `agentdojo` stays optional.
81
+ - Environment switches use the `SPW_` prefix.
82
+ - Every result in the documentation traces to `benchmarks/results/*.json` or to
83
+ the paper. Do not add a result that neither source carries. Operational figures
84
+ from the paper's runs, such as GPU memory and API cost, are labelled as such.
85
+ - Claims stay within the threat model: zero in-scope attack success on the
86
+ AgentDojo benchmarks the paper runs, nothing wider than that.
87
+ - `ruff check .` and `ruff format .` before you push (line length 120).
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright 2026 SPILLWAY authors
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
@@ -0,0 +1,50 @@
1
+ # SPILLWAY release: developer targets. `make help` lists them.
2
+ PY ?= python3
3
+ TABLES := benchmarks/scripts/make_tables.py
4
+ ASR_LEVELS := benchmarks/scripts/appendix_asr_levels.py
5
+
6
+ .PHONY: help install install-agentdojo test tables asr-levels lint selftest cell
7
+
8
+ help:
9
+ @echo "make install editable install with the dev extra (pytest, ruff, email-validator)"
10
+ @echo "make install-agentdojo editable install with the dev and agentdojo extras (runs the gate on AgentDojo)"
11
+ @echo "make test check benchmarks/results against the paper's numbers, then the unit tests"
12
+ @echo "make tables write benchmarks/TABLES.md from benchmarks/results/*.json"
13
+ @echo "make asr-levels print Appendix Table 5 (raw / in-scope ASR per cell) from the breadth-grid files, without its ROPE column"
14
+ @echo "make lint ruff check . (the configuration in ruff.toml, the same as CI)"
15
+ @echo "make selftest the reference gate's own self-tests (needs the agentdojo extra, no server)"
16
+ @echo "make cell SUITE=banking BACKBONE=qwen3:8b run one AgentDojo cell with the paper's settings"
17
+
18
+ # A venv made by `uv venv` ships no pip: run `$(PY) -m ensurepip --upgrade` first, or use `uv pip install -e ".[dev]"`.
19
+ install:
20
+ $(PY) -m pip install -e ".[dev]"
21
+
22
+ install-agentdojo:
23
+ $(PY) -m pip install -e ".[dev,agentdojo]"
24
+
25
+ # make_tables.py exits non-zero when a printed value differs from the paper, so it is a test.
26
+ test:
27
+ $(PY) $(TABLES) --no-write
28
+ @$(PY) -m pytest -q -m "not differential" tests
29
+
30
+ tables:
31
+ $(PY) $(TABLES) --out benchmarks/TABLES.md
32
+
33
+ # appendix_asr_levels.py is pinned byte for byte in reference/BUILD_PROVENANCE.md and opens its two input files under
34
+ # fixed names in the current directory. This target runs it unchanged in a temporary directory that links the shipped files under
35
+ # those names. It reads the per-cell aggregates in those two files, not per-pair records.
36
+ asr-levels:
37
+ @tmp=$$(mktemp -d) && trap 'rm -rf "$$tmp"' EXIT && \
38
+ ln -s "$(CURDIR)/benchmarks/results/breadth_grid_summary.json" "$$tmp/final_v122.json" && \
39
+ ln -s "$(CURDIR)/benchmarks/results/breadth_grid_cells.json" "$$tmp/tables_v122.json" && \
40
+ $(PY) -c 'import os, runpy, sys; os.chdir(sys.argv[1]); runpy.run_path(sys.argv[2], run_name="__main__")' "$$tmp" "$(CURDIR)/$(ASR_LEVELS)"
41
+
42
+ lint:
43
+ ruff check .
44
+
45
+ selftest:
46
+ $(PY) -m spillway_guard.agentdojo.runner --self-test
47
+
48
+ cell:
49
+ @test -n "$(SUITE)" -a -n "$(BACKBONE)" || { echo "usage: make cell SUITE=<suite> BACKBONE=<model>"; exit 2; }
50
+ bash reproduce/run_cell.sh "$(SUITE)" "$(BACKBONE)"
@@ -0,0 +1,81 @@
1
+ SPILLWAY (spillway-guard)
2
+ Copyright 2026 SPILLWAY authors
3
+
4
+ This product is licensed under the Apache License, Version 2.0. See LICENSE.
5
+
6
+ This product includes material from AgentDojo
7
+ (https://github.com/ethz-spylab/agentdojo), which is licensed under the MIT
8
+ License. The AgentDojo copyright notice and license text are reproduced at the
9
+ end of this file, copied from licenses/LICENSE in the agentdojo 0.1.35
10
+ distribution.
11
+
12
+ Files that derive from AgentDojo:
13
+
14
+ src/spillway_guard/agentdojo/agentdojo_default_prompt.txt
15
+ AgentDojo's default system prompt, the "default" entry of
16
+ agentdojo/data/system_messages.yaml, byte for byte.
17
+ Ships in the wheel and in the source distribution.
18
+
19
+ reference/harness/21_agentdojo_full_benchmark.py
20
+ Contains code copied and adapted from
21
+ agentdojo/agent_pipeline/tool_execution.py and
22
+ agentdojo/agent_pipeline/llms/google_llm.py.
23
+ Ships in the source distribution.
24
+
25
+ reference/harness/24_agentdojo_gemini_backbone.py
26
+ Contains code copied and adapted from
27
+ agentdojo/agent_pipeline/llms/google_llm.py.
28
+ Ships in the source distribution.
29
+
30
+ Test inputs in tests/, the self-test in reference/spillway_gate.py and the
31
+ template patterns in src/spillway_guard/sanitizer.py reuse AgentDojo's
32
+ important_instructions injection template, addresses and tool names. The typed
33
+ fixtures in tests/test_floor.py are shaped like AgentDojo's return models, and
34
+ the prompt listed above is also copied into
35
+ reference/harness/21_agentdojo_full_benchmark.py.
36
+
37
+ The project page under docs/ shows one recorded AgentDojo episode. Its task,
38
+ tool returns and injection text come from AgentDojo's task suites.
39
+
40
+ The lexicon in src/spillway_guard/lexicon.py and the harness under
41
+ reference/harness/ use tool-argument names and one sentence of the default
42
+ prompt of AgentDyn (https://github.com/SaFo-Lab/AgentDyn), which is under the
43
+ MIT License with the same copyright line as AgentDojo.
44
+
45
+ Other licenses. The Apache License does not cover these files:
46
+
47
+ docs/assets/fonts/
48
+ Fraunces and IBM Plex, under the SIL Open Font License 1.1. The license
49
+ texts sit beside the fonts.
50
+
51
+ docs/assets/saintmachines_lockup.svg, docs/assets/saintmachines_lockup_white.svg
52
+ Logos. No license to these or to the SPILLWAY and SAINT Machines names is
53
+ granted, as section 6 of the Apache License says of trademarks.
54
+
55
+ docs/SPILLWAY_preprint.pdf
56
+ The paper, under the license it carries on arXiv.
57
+
58
+
59
+ AgentDojo license (agentdojo 0.1.35, licenses/LICENSE):
60
+
61
+ MIT License
62
+
63
+ Copyright (c) 2024 Edoardo Debenedetti, Jie Zhang, Mislav Balunovic, Luca Beurer-Kellner, Marc Fischer, and Florian Tramèr
64
+
65
+ Permission is hereby granted, free of charge, to any person obtaining a copy
66
+ of this software and associated documentation files (the "Software"), to deal
67
+ in the Software without restriction, including without limitation the rights
68
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
69
+ copies of the Software, and to permit persons to whom the Software is
70
+ furnished to do so, subject to the following conditions:
71
+
72
+ The above copyright notice and this permission notice shall be included in all
73
+ copies or substantial portions of the Software.
74
+
75
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
76
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
77
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
78
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
79
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
80
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
81
+ SOFTWARE.