sandbox-drift 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. sandbox_drift-0.1.0/.github/ISSUE_TEMPLATE/bug.yml +22 -0
  2. sandbox_drift-0.1.0/.github/ISSUE_TEMPLATE/question.yml +11 -0
  3. sandbox_drift-0.1.0/.github/workflows/ci.yml +21 -0
  4. sandbox_drift-0.1.0/.gitignore +41 -0
  5. sandbox_drift-0.1.0/CHANGELOG.md +18 -0
  6. sandbox_drift-0.1.0/LICENSE +21 -0
  7. sandbox_drift-0.1.0/PKG-INFO +367 -0
  8. sandbox_drift-0.1.0/README.ko.md +350 -0
  9. sandbox_drift-0.1.0/README.md +350 -0
  10. sandbox_drift-0.1.0/SECURITY.md +7 -0
  11. sandbox_drift-0.1.0/docs/images/baseline-flip.ko.png +0 -0
  12. sandbox_drift-0.1.0/docs/images/baseline-flip.png +0 -0
  13. sandbox_drift-0.1.0/docs/images/cannot-run.ko.png +0 -0
  14. sandbox_drift-0.1.0/docs/images/cannot-run.png +0 -0
  15. sandbox_drift-0.1.0/docs/images/diff-flip.ko.png +0 -0
  16. sandbox_drift-0.1.0/docs/images/diff-flip.png +0 -0
  17. sandbox_drift-0.1.0/docs/images/fingerprint.ko.png +0 -0
  18. sandbox_drift-0.1.0/docs/images/fingerprint.png +0 -0
  19. sandbox_drift-0.1.0/docs/images/init-policy.ko.png +0 -0
  20. sandbox_drift-0.1.0/docs/images/init-policy.png +0 -0
  21. sandbox_drift-0.1.0/docs/images/self-table.ko.png +0 -0
  22. sandbox_drift-0.1.0/docs/images/self-table.png +0 -0
  23. sandbox_drift-0.1.0/examples/policies/claude-srt.toml +11 -0
  24. sandbox_drift-0.1.0/examples/policies/self-basic.toml +11 -0
  25. sandbox_drift-0.1.0/examples/replay/after.json +73 -0
  26. sandbox_drift-0.1.0/examples/replay/before.json +75 -0
  27. sandbox_drift-0.1.0/pyproject.toml +36 -0
  28. sandbox_drift-0.1.0/src/sandbox_drift/__init__.py +3 -0
  29. sandbox_drift-0.1.0/src/sandbox_drift/__main__.py +4 -0
  30. sandbox_drift-0.1.0/src/sandbox_drift/canary.py +62 -0
  31. sandbox_drift-0.1.0/src/sandbox_drift/classify.py +69 -0
  32. sandbox_drift-0.1.0/src/sandbox_drift/cli.py +340 -0
  33. sandbox_drift-0.1.0/src/sandbox_drift/convert.py +77 -0
  34. sandbox_drift-0.1.0/src/sandbox_drift/errors.py +16 -0
  35. sandbox_drift-0.1.0/src/sandbox_drift/fingerprint.py +43 -0
  36. sandbox_drift-0.1.0/src/sandbox_drift/i18n.py +61 -0
  37. sandbox_drift-0.1.0/src/sandbox_drift/listener.py +47 -0
  38. sandbox_drift-0.1.0/src/sandbox_drift/locales/en.json +132 -0
  39. sandbox_drift-0.1.0/src/sandbox_drift/locales/ko.json +132 -0
  40. sandbox_drift-0.1.0/src/sandbox_drift/policy.py +79 -0
  41. sandbox_drift-0.1.0/src/sandbox_drift/report.py +260 -0
  42. sandbox_drift-0.1.0/src/sandbox_drift/snapshot.py +115 -0
  43. sandbox_drift-0.1.0/src/sandbox_drift/targets.py +162 -0
  44. sandbox_drift-0.1.0/src/sandbox_drift/worker.py +254 -0
  45. sandbox_drift-0.1.0/tests/conftest.py +67 -0
  46. sandbox_drift-0.1.0/tests/fixtures/claude-settings.json +16 -0
  47. sandbox_drift-0.1.0/tests/fixtures/old-snapshot.json +46 -0
  48. sandbox_drift-0.1.0/tests/fixtures/profile-a1.sb +6 -0
  49. sandbox_drift-0.1.0/tests/fixtures/profile-a2.sb +6 -0
  50. sandbox_drift-0.1.0/tests/fixtures/profile-b.sb +6 -0
  51. sandbox_drift-0.1.0/tests/test_canary.py +58 -0
  52. sandbox_drift-0.1.0/tests/test_classify.py +81 -0
  53. sandbox_drift-0.1.0/tests/test_fingerprint.py +33 -0
  54. sandbox_drift-0.1.0/tests/test_i18n.py +306 -0
  55. sandbox_drift-0.1.0/tests/test_no_real_paths.py +50 -0
  56. sandbox_drift-0.1.0/tests/test_report.py +55 -0
  57. sandbox_drift-0.1.0/tests/test_self_mode.py +30 -0
  58. sandbox_drift-0.1.0/tests/test_snapshot.py +72 -0
  59. sandbox_drift-0.1.0/tests/test_srt_target.py +78 -0
  60. sandbox_drift-0.1.0/tests/test_targets.py +79 -0
  61. sandbox_drift-0.1.0/uv.lock +79 -0
@@ -0,0 +1,22 @@
1
+ name: 버그 / Bug
2
+ description: README대로 했는데 다르게 동작해요 / Something does not work as the README says
3
+ labels: [bug]
4
+ body:
5
+ - type: textarea
6
+ id: what
7
+ attributes:
8
+ label: 무엇이 일어났나요? / What happened?
9
+ description: 실행한 명령과 나온 메시지를 그대로 붙여 주세요. / Paste the command and the output.
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: expected
14
+ attributes:
15
+ label: 기대한 결과 / Expected
16
+ - type: input
17
+ id: env
18
+ attributes:
19
+ label: 환경 / Environment
20
+ placeholder: "macOS 15 (M3) · Ubuntu 24.04 · Windows 11 / uv 0.x / mock or real mode"
21
+ validations:
22
+ required: true
@@ -0,0 +1,11 @@
1
+ name: 질문 / Question
2
+ description: 이해가 안 되는 부분이나 더 알고 싶은 점 / Something unclear or worth explaining
3
+ labels: [question]
4
+ body:
5
+ - type: textarea
6
+ id: question
7
+ attributes:
8
+ label: 질문 / Question
9
+ description: README의 어느 부분(절 제목)에 대한 질문인지 알려 주세요. / Which README section is this about?
10
+ validations:
11
+ required: true
@@ -0,0 +1,21 @@
1
+ # Keyless: every test uses canaries, fake runners or saved snapshots.
2
+ name: ci
3
+ on:
4
+ push:
5
+ pull_request:
6
+ schedule:
7
+ - cron: "17 3 * * 1" # weekly: catches runtime changes that break the tests
8
+ workflow_dispatch:
9
+ jobs:
10
+ test:
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ os: [ubuntu-latest, macos-latest]
15
+ runs-on: ${{ matrix.os }}
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: astral-sh/setup-uv@v10.3.3
19
+ - run: uv python install
20
+ - run: uv sync --locked
21
+ - run: uv run pytest -q
@@ -0,0 +1,41 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ .venv/
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ .mypy_cache/
8
+ *.egg-info/
9
+
10
+ # Node
11
+ node_modules/
12
+ dist/
13
+ .next/
14
+
15
+ # 실행 산출물·모델·데이터 캐시
16
+ runs/
17
+ outputs/
18
+ .cache/
19
+ *.safetensors
20
+ *.gguf
21
+ *.bin
22
+
23
+ # 환경·비밀
24
+ .env
25
+ .env.*
26
+ !.env.example
27
+
28
+ # OS·에디터
29
+ .DS_Store
30
+ .vscode/
31
+ .idea/
32
+
33
+ # Flutter·Dart
34
+ .dart_tool/
35
+ build/
36
+ .flutter-plugins
37
+ .flutter-plugins-dependencies
38
+ coverage/
39
+
40
+ # sandbox-drift local output
41
+ .sandbox-drift/
@@ -0,0 +1,18 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
5
+ [Semantic Versioning](https://semver.org/).
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [0.1.0] - 2026-11-12
10
+
11
+ ### Added
12
+ - `self`, `run` (`wrap`, `srt`, `replay` targets), `diff`, `init`, `fingerprint` and `convert` commands.
13
+ - Eight harmless canary probes: file writes inside and outside the work dir, a fake-HOME credential canary,
14
+ a localhost listener, a proxy reserved-host request, a direct TEST-NET-1 connection, the docker socket
15
+ and nested sandbox support.
16
+ - Expected-policy TOML (`block` / `allow` / `info`), snapshots, baseline comparison and `FLIP` detection.
17
+ - Exit codes: 0 matches, 1 mismatch or regression, 2 could not run.
18
+ - English and Korean output (`--lang`, `SANDBOX_DRIFT_LANG`, `LC_ALL`, `LC_MESSAGES`, `LANG`).
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 LeeGyeongPil
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,367 @@
1
+ Metadata-Version: 2.5
2
+ Name: sandbox-drift
3
+ Version: 0.1.0
4
+ Summary: Probe the OS sandbox of an AI coding agent with harmless canaries and catch silent regressions between versions.
5
+ Project-URL: Homepage, https://github.com/LeeGyeongPil/sandbox-drift
6
+ Author: LeeGyeongPil
7
+ License: MIT
8
+ License-File: LICENSE
9
+ Keywords: agent-cli,ai-agent,regression,sandbox,sandbox-runtime,security
10
+ Classifier: Environment :: Console
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Topic :: Security
14
+ Classifier: Topic :: Software Development :: Testing
15
+ Requires-Python: >=3.11
16
+ Description-Content-Type: text/markdown
17
+
18
+ # sandbox-drift
19
+
20
+ **English** | [한국어](https://github.com/LeeGyeongPil/sandbox-drift/blob/v0.1.0/README.ko.md)
21
+
22
+ [![ci](https://github.com/LeeGyeongPil/sandbox-drift/actions/workflows/ci.yml/badge.svg)](https://github.com/LeeGyeongPil/sandbox-drift/actions/workflows/ci.yml) ![license](https://img.shields.io/badge/license-MIT-blue)
23
+
24
+ Probe the OS sandbox of an AI coding agent with harmless canaries, compare the result with the previous run, and exit 1 when something that used to be blocked is now allowed. Python 3.11+, no runtime dependencies.
25
+
26
+ ![A FLIP row: fs.write_outside_workdir changed from blocked to allowed](https://raw.githubusercontent.com/LeeGyeongPil/sandbox-drift/v0.1.0/docs/images/diff-flip.png)
27
+
28
+ *Two saved results compared. The `FLIP` row is a regression: writing outside the work dir used to be blocked and is now allowed. Exit code 1.*
29
+
30
+ ## Why
31
+
32
+ Coding agents run shell commands inside an OS sandbox (Seatbelt on macOS, bubblewrap on Linux, plus a proxy allowlist for the network). The boundary is software, so it can change between versions, and the release notes do not always say so. Public reports (all checked 2026-11-12):
33
+
34
+ - A network allowlist that did not actually restrict anything: an upstream agent CLI issue report (issue #78608 in the upstream agent CLI repository).
35
+ - A sandbox that opened silently after a settings change during a session: an upstream agent CLI issue report (issue #81572 in the same repository).
36
+ - A bypass that was fixed without a release note: a security write-up titled "Second time, same sandbox" by the oddguan.com blog (search for the title).
37
+ - The opposite problem, things that should work being blocked or settings being ignored: an upstream agent CLI issue report (issue #56959 in the same repository), [codex#10390](https://github.com/openai/codex/issues/10390).
38
+ - Escaping a sandbox through the docker socket: [Pillar Security write-up](https://www.pillar.security/blog/one-docker-socket-to-rule-them-all-escaping-codex-cursor-and-gemini-clis-sandboxes).
39
+
40
+ Re-checking by hand after every upgrade ("can it still not write outside the project?") is easy to forget. sandbox-drift does it with a few harmless probes and tells you what changed since last time.
41
+
42
+ ## Quick start
43
+
44
+ Needs `git` and [uv](https://docs.astral.sh/uv/) (`curl -LsSf https://astral.sh/uv/install.sh | sh`). The tool has no runtime dependencies (Python standard library only). When you run it from a clone with `uv run`, uv also installs the dev packages (pytest) into `.venv` on the first run.
45
+
46
+ ```bash
47
+ git clone https://github.com/LeeGyeongPil/sandbox-drift
48
+ cd sandbox-drift
49
+ uv run sandbox-drift diff examples/replay/before.json examples/replay/after.json --lang en
50
+ echo "exit=$?"
51
+ ```
52
+
53
+ Output:
54
+
55
+ ```
56
+ sandbox-drift diff
57
+ agent_version: example-1.0.0 -> example-1.1.0
58
+
59
+ Probe Verdict Change
60
+ ------------------------ ------- -----------------
61
+ fs.write_outside_workdir FLIP blocked → allowed
62
+
63
+ 1 cell opened (regression), 0 cells tightened.
64
+ exit=1
65
+ ```
66
+
67
+ The two files in `examples/replay/` are a **practice mock**. The shape of each observation comes from a real probe run, but the version change and the regression in `after.json` were put in on purpose so you can see what a FLIP looks like. Nothing is executed.
68
+
69
+ Install as a command instead of running from a clone:
70
+
71
+ ```bash
72
+ uv tool install git+https://github.com/LeeGyeongPil/sandbox-drift
73
+ sandbox-drift --version
74
+ ```
75
+
76
+ ### Measure a real sandbox
77
+
78
+ Ask the agent to run this in its own shell, so the probes run inside the sandbox the agent really uses:
79
+
80
+ ```bash
81
+ uv run sandbox-drift self --policy examples/policies/self-basic.toml --baseline .sandbox-drift/ --lang en
82
+ ```
83
+
84
+ Run inside an agent sandbox, the table looks like this (measured on macOS, 2026-11-12):
85
+
86
+ ```
87
+ sandbox-drift report
88
+ target=self agent=unknown engine=unknown platform=darwin/arm64
89
+
90
+ Probe Expect Observed Verdict Change Detail
91
+ -------------------------- ------ -------- ------- ------ ---------------------------------------------------------
92
+ fs.write_inside_workdir allow allowed PASS file written and removed
93
+ fs.write_outside_workdir block blocked PASS EPERM: Operation not permitted
94
+ cred.read_fake_home_canary block n/a SKIP self mode cannot swap HOME; run with --target wrap or srt
95
+ net.localhost_listener allow allowed PASS connected to the localhost canary listener
96
+ net.proxy_reserved_host block blocked PASS HTTP 403
97
+ net.direct_egress_testnet block blocked PASS EPERM: Operation not permitted
98
+ proc.docker_socket block blocked PASS EPERM: Operation not permitted
99
+ proc.nested_sandbox info blocked INFO sandbox-exec: sandbox_apply: Operation not permitted
100
+
101
+ PASS 6 FAIL 0 FLIP 0 ERROR 0 SKIP 1 INFO 1
102
+ Baseline saved: .sandbox-drift/baseline-self.json
103
+
104
+ Measures the Bash/command boundary only. Tools that run outside the sandbox (file editors, MCP, hooks) are not covered.
105
+ ```
106
+
107
+ ![The self table, run outside any sandbox](https://raw.githubusercontent.com/LeeGyeongPil/sandbox-drift/v0.1.0/docs/images/self-table.png)
108
+
109
+ *The same command run in a plain terminal with no sandbox. The "block" rows come out as FAIL because nothing blocks them. That is the negative control: if a sandbox were present they would be PASS.*
110
+
111
+ ## How it works
112
+
113
+ ```mermaid
114
+ flowchart LR
115
+ P[policy.toml<br/>block / allow / info] --> CLI[sandbox-drift CLI]
116
+ CLI --> C[canaries<br/>temp workdir, fake HOME, token]
117
+ CLI --> L[localhost listener]
118
+ CLI --> T{target}
119
+ T -->|self| W0[worker in this process]
120
+ T -->|wrap| W1[runner + worker]
121
+ T -->|srt| W2[srt --settings + worker<br/>HOME = fake HOME]
122
+ T -->|replay| W3[saved observations<br/>practice mock]
123
+ W0 & W1 & W2 --> O[observations<br/>blocked / allowed / error / n/a]
124
+ W3 --> O
125
+ W1 & W2 -.runner failed.-> E[cannot run, exit 2]
126
+ L -.hit.-> O
127
+ O --> V[match policy: PASS / FAIL]
128
+ O --> S[compare snapshot: FLIP]
129
+ V & S --> R[table / Markdown / JSON<br/>exit 0 / 1 / 2]
130
+ ```
131
+
132
+ 1. You write what the sandbox **must block**, **must allow** and what is just **info** in a small TOML policy.
133
+ 2. A single-file worker runs eight probes inside the target and prints what happened.
134
+ 3. Each result is classified as `blocked` (EPERM, EACCES, HTTP 403/407), `allowed` (success, or the packet left the sandbox), `error` or `n/a`.
135
+ 4. The result is matched against the policy (PASS/FAIL) and against the previous snapshot (FLIP).
136
+
137
+ **Why the probes are harmless.** Nothing real is touched.
138
+
139
+ | Probe | What it does | Why it is safe |
140
+ |---|---|---|
141
+ | `fs.write_inside_workdir` | writes and removes a file in the work dir | positive control: this must work |
142
+ | `fs.write_outside_workdir` | writes and removes a randomly named file in your HOME (change the directory with `--outside-dir`) | the file is removed right away |
143
+ | `cred.read_fake_home_canary` | reads credential-shaped files in a temporary fake HOME | contents are a random token; the real HOME is never read or stat'ed |
144
+ | `net.localhost_listener` | connects to a throwaway listener on `127.0.0.1` | stays on your machine |
145
+ | `net.proxy_reserved_host` | requests `canary.invalid` through the sandbox proxy | `.invalid` is reserved ([RFC 6761](https://www.rfc-editor.org/rfc/rfc6761)) and never resolves |
146
+ | `net.direct_egress_testnet` | connects directly to `192.0.2.1:9` | TEST-NET-1 ([RFC 5737](https://www.rfc-editor.org/rfc/rfc5737)) is not routed |
147
+ | `proc.docker_socket` | tries to connect to the docker socket, nothing more | no API request is sent |
148
+ | `proc.nested_sandbox` | tries to start a nested `sandbox-exec` (macOS) | information only |
149
+
150
+ A connect that times out counts as `allowed`: the packet left the sandbox instead of being refused locally.
151
+
152
+ ## Targets
153
+
154
+ | Target | Command | Use it for |
155
+ |---|---|---|
156
+ | `self` | `sandbox-drift self ...` | The sandbox you are already inside. Have the agent run it. No nested sandbox needed. |
157
+ | `srt` | `sandbox-drift run --target srt --settings .claude/settings.json ...` | Re-create an agent CLI `sandbox` block under [sandbox-runtime](https://github.com/anthropics/sandbox-runtime) (`srt`) with HOME set to a fake HOME. |
158
+ | `wrap` | `sandbox-drift run --target wrap --wrap "my-runner {}" ...` | Any other runner. `{}` is replaced by the worker command. |
159
+ | `replay` | `sandbox-drift run --target replay --replay snapshot.json ...` | Practice mock: replays a saved result through the real policy and diff logic. |
160
+
161
+ Convert an agent CLI `sandbox` block to a `srt` settings file (unknown keys are warned about and dropped):
162
+
163
+ ```bash
164
+ uv run sandbox-drift convert --from-claude tests/fixtures/claude-settings.json
165
+ ```
166
+
167
+ ```
168
+ warning: sandbox.enabled has no sandbox-runtime equivalent and was left out
169
+ warning: sandbox.autoAllowBashIfSandboxed has no sandbox-runtime equivalent and was left out
170
+ warning: ignored unknown setting sandbox.network.futureOption
171
+ {
172
+ "network": {
173
+ "allowedDomains": [
174
+ "example.com"
175
+ ],
176
+ "allowLocalBinding": true
177
+ },
178
+ "filesystem": {
179
+ "allowWrite": [
180
+ "~/scratch",
181
+ "/opt/shared"
182
+ ],
183
+ "denyRead": [
184
+ "~/.canary-secrets"
185
+ ]
186
+ }
187
+ }
188
+ ```
189
+
190
+ The fake HOME canary only means something when the target is started with HOME set to the fake HOME. That is why `self` skips it (it cannot swap HOME). With the identity wrapper `--wrap "{}"` there is no sandbox at all, so the canary is readable and the row is FAIL, a negative control you can use to check that the probe works.
191
+
192
+ Roadmap: `codex` and `gemini` targets and more probes (milestone M2, not in 0.1.0).
193
+
194
+ ## Policy file
195
+
196
+ ```bash
197
+ uv run sandbox-drift init --preset claude-srt
198
+ ```
199
+
200
+ ```toml
201
+ # Expected results for `sandbox-drift run --target srt` with an agent settings style
202
+ # sandbox block (workdir writable, everything else closed, only localhost open).
203
+ [expect]
204
+ "fs.write_inside_workdir" = "allow"
205
+ "fs.write_outside_workdir" = "block"
206
+ "cred.read_fake_home_canary" = "block"
207
+ "net.localhost_listener" = "allow"
208
+ "net.proxy_reserved_host" = "block"
209
+ "net.direct_egress_testnet" = "block"
210
+ "proc.docker_socket" = "block"
211
+ "proc.nested_sandbox" = "info"
212
+ ```
213
+
214
+ - `block`: the sandbox must refuse it. Observed `allowed` is a FAIL.
215
+ - `allow`: it must work. Observed `blocked` is a FAIL.
216
+ - `info`: recorded, not judged. `n/a` is shown as SKIP.
217
+
218
+ Presets: `self-basic` and `claude-srt` (also in `examples/policies/`). Use `--out FILE` to write the file instead of printing it.
219
+
220
+ ![The claude-srt preset printed by init](https://raw.githubusercontent.com/LeeGyeongPil/sandbox-drift/v0.1.0/docs/images/init-policy.png)
221
+
222
+ *The generated policy: one line per probe.*
223
+
224
+ ## Snapshots & diff
225
+
226
+ With `--baseline DIR` the first run saves a snapshot, and later runs compare against it. Each snapshot records `target`, `agent_version`, `engine_version`, `os`, `arch`, `policy_hash` and `settings_fingerprint`. Running the practice mock twice shows it:
227
+
228
+ ```bash
229
+ uv run sandbox-drift run --target replay --replay examples/replay/before.json --policy examples/policies/claude-srt.toml --baseline .sandbox-drift/
230
+ uv run sandbox-drift run --target replay --replay examples/replay/after.json --policy examples/policies/claude-srt.toml --baseline .sandbox-drift/
231
+ ```
232
+
233
+ The second run prints (exit 1):
234
+
235
+ ```
236
+ sandbox-drift report
237
+ target=srt agent=example-1.1.0 engine=0.0.82 platform=darwin/arm64
238
+
239
+ Probe Expect Observed Verdict Change Detail
240
+ -------------------------- ------ -------- ------- ----------------- ----------------------------------------------------
241
+ fs.write_inside_workdir allow allowed PASS file written and removed
242
+ fs.write_outside_workdir block allowed FLIP blocked → allowed file written and removed
243
+ cred.read_fake_home_canary block blocked PASS read denied for 4 canary file(s)
244
+ net.localhost_listener allow allowed PASS connected to the localhost canary listener
245
+ net.proxy_reserved_host block blocked PASS HTTP 403
246
+ net.direct_egress_testnet block blocked PASS EPERM: Operation not permitted
247
+ proc.docker_socket block blocked PASS EPERM: Operation not permitted
248
+ proc.nested_sandbox info blocked INFO sandbox-exec: sandbox_apply: Operation not permitted
249
+
250
+ PASS 6 FAIL 0 FLIP 1 ERROR 0 SKIP 0 INFO 1
251
+ Compared with baseline: .sandbox-drift/baseline-srt.json
252
+ agent_version: example-1.0.0 -> example-1.1.0
253
+ Practice mock: replayed from a saved file, no sandbox was exercised.
254
+ Practice mock: the observation shapes come from a real probe run, but the version change and the regression in the 'after' file were put in on purpose to show how a FLIP looks.
255
+
256
+ Measures the Bash/command boundary only. Tools that run outside the sandbox (file editors, MCP, hooks) are not covered.
257
+ ```
258
+
259
+ The fixed wording in the Detail column follows the output language; raw OS errors such as `EPERM: Operation not permitted`, `HTTP 403` and `sandbox-exec` messages are shown as the OS returned them.
260
+
261
+ ![Baseline comparison with a FLIP row](https://raw.githubusercontent.com/LeeGyeongPil/sandbox-drift/v0.1.0/docs/images/baseline-flip.png)
262
+
263
+ *A second run against the saved baseline.*
264
+
265
+ - **FLIP**: blocked in the baseline, allowed now. Exit 1. If your policy says `allow` for that probe it is a PASS instead, because something you wanted open is not a failure.
266
+ - **tightened**: allowed before, blocked now. Reported, but fails only with `--strict`.
267
+ - `--exit-zero`: print the same report but exit 0 (cannot-run still exits 2).
268
+ - `--update-baseline`: overwrite the saved baseline after comparing. Without it the baseline is never overwritten.
269
+ - `sandbox-drift diff OLD.json NEW.json` compares two saved files without running anything.
270
+
271
+ Two fingerprints help you tell whether the policy or the settings changed. SBPL (Seatbelt profile) files are normalised first, so random log tags, proxy ports and temp paths do not change the hash:
272
+
273
+ ```bash
274
+ uv run sandbox-drift fingerprint --sbpl tests/fixtures/profile-a1.sb
275
+ uv run sandbox-drift fingerprint --sbpl tests/fixtures/profile-a2.sb
276
+ uv run sandbox-drift fingerprint --sbpl tests/fixtures/profile-b.sb
277
+ ```
278
+
279
+ ```
280
+ sha256:1ccf6994407702ae9ec37889cc7a3f9c7be4cccf0a9e1c3cf137b8bbba83e0b3
281
+ sha256:1ccf6994407702ae9ec37889cc7a3f9c7be4cccf0a9e1c3cf137b8bbba83e0b3
282
+ sha256:e59afa598a1f93858e6dd8617750bf3c9be45942965160e87507aac63325a682
283
+ ```
284
+
285
+ `profile-a1` and `profile-a2` differ only in the random tag and ports. `profile-b` differs by one rule.
286
+
287
+ ![Fingerprints of three Seatbelt profiles](https://raw.githubusercontent.com/LeeGyeongPil/sandbox-drift/v0.1.0/docs/images/fingerprint.png)
288
+
289
+ *The first two profiles get the same hash; the third, with one rule changed, does not.*
290
+
291
+ ## Exit codes
292
+
293
+ | Code | Meaning |
294
+ |---|---|
295
+ | 0 | every judged cell matches the policy and nothing flipped |
296
+ | 1 | a mismatch (FAIL), a regression (FLIP) or a probe that could not tell (ERROR) |
297
+ | 2 | the probes could not run |
298
+
299
+ "Cannot run" is not "blocked". If the runner is missing, or a nested sandbox refuses to start (`sandbox_apply: Operation not permitted`, `listen EPERM`), nothing was measured. Reporting that as "blocked, PASS" would be a false pass, so sandbox-drift exits 2, reports no cells, and recommends `self`:
300
+
301
+ ```bash
302
+ uv run sandbox-drift run --target wrap --wrap "/nonexistent/runner {}" --policy examples/policies/self-basic.toml
303
+ echo "exit=$?"
304
+ ```
305
+
306
+ ```
307
+ error: Could not start the runner /nonexistent/runner (No such file or directory). This is not a 'blocked' result. If you are already inside an agent sandbox, use `sandbox-drift self` instead.
308
+ exit=2
309
+ ```
310
+
311
+ ![The cannot-run message](https://raw.githubusercontent.com/LeeGyeongPil/sandbox-drift/v0.1.0/docs/images/cannot-run.png)
312
+
313
+ *Exit 2 and a pointer to `self`. Known limitation: in this screenshot the `exit=2` line appears before the error text because the two streams were captured separately; the text block above shows the real order.*
314
+
315
+ `--exit-zero` never hides exit 2.
316
+
317
+ ## Using in CI
318
+
319
+ The repo's own CI (`.github/workflows/ci.yml`) runs `uv run pytest -q` on Ubuntu and macOS. To watch your own settings, run the tool on a schedule and keep the baseline directory between runs (commit it or use a cache). The step below is a sketch; it was not run against a real `srt` in this repository's docs:
320
+
321
+ ```yaml
322
+ - name: Sandbox drift
323
+ run: |
324
+ uvx --from git+https://github.com/LeeGyeongPil/sandbox-drift sandbox-drift run \
325
+ --target srt --settings .claude/settings.json \
326
+ --policy .sandbox-drift/policy.toml --baseline .sandbox-drift/ \
327
+ --format md >> "$GITHUB_STEP_SUMMARY"
328
+ ```
329
+
330
+ The job fails (exit 1) when a cell flipped from blocked to allowed. On Ubuntu 24.04 runners, AppArmor can restrict unprivileged user namespaces, so bubblewrap and `srt` may fail to start. sandbox-drift reports that as exit 2, not as a blocked cell (see [sandbox-runtime#429](https://github.com/anthropics/sandbox-runtime/issues/429), checked 2026-11-12).
331
+
332
+ Real `srt` runs are not possible inside another sandbox (nested Seatbelt/`listen EPERM`). This repository's tests cover that path with a fake `srt` script that checks the arguments, HOME and converted settings. Running a real `srt` is for your terminal or CI runner.
333
+
334
+ ## What it does NOT check
335
+
336
+ - **Only the Bash/command boundary.** File editors, `WebFetch`, MCP tools and hooks run outside the sandbox and are not measured. A clean result does not mean they are safe.
337
+ - It does not enumerate everything visible inside a sandbox. For that, use [controlplaneio/sandbox-probe](https://github.com/controlplaneio/sandbox-probe) next to this tool.
338
+ - No real attacks, no prompt-injection tests, no model behaviour.
339
+ - No Windows, no container/Compose isolation checks (only indirectly through `--wrap`).
340
+ - Codex and Gemini CLI targets are not in 0.1.0.
341
+
342
+ ## Language
343
+
344
+ Output and help are available in English and Korean. The order is `--lang en|ko`, then `SANDBOX_DRIFT_LANG`, then `LC_ALL`, `LC_MESSAGES`, `LANG`. Anything that is not Korean falls back to English.
345
+
346
+ ```bash
347
+ SANDBOX_DRIFT_LANG=ko uv run sandbox-drift diff examples/replay/before.json examples/replay/after.json --exit-zero
348
+ ```
349
+
350
+ Verdict codes (`PASS`, `FAIL`, `FLIP`, `ERROR`, `SKIP`, `INFO`), observed values and probe ids are not translated.
351
+
352
+ The fixed Detail wording and the notes of the bundled example files follow the output language, including older snapshots saved without a message code. A `note` you write yourself in a replay file is data, so it is printed as written after the label `Note from the file (not translated):`.
353
+
354
+ OS error text (`EPERM: Operation not permitted`, `HTTP 403`, `sandbox-exec` stderr) is shown as the OS gave it and is not translated.
355
+
356
+ ## Safety notes
357
+
358
+ - No API keys, no accounts, no outbound requests. Network probes go to localhost, `canary.invalid` and TEST-NET-1 only.
359
+ - Credential-shaped canary files exist only inside a temporary fake HOME, with a random token as content, and are deleted afterwards. The real HOME is never read.
360
+ - The docker socket probe only calls `connect`.
361
+ - `examples/replay/` is a practice mock, and every report from it says so.
362
+ - `fs.write_outside_workdir` writes a randomly named file in your real HOME (or the `--outside-dir` directory) for a moment and deletes it. Run it inside a sandbox to see it blocked.
363
+ - Tests: `uv run pytest -q` (105 tests; no outbound traffic except the reserved `canary.invalid` request through a proxy, if one is set).
364
+
365
+ ## License
366
+
367
+ MIT, see [LICENSE](https://github.com/LeeGyeongPil/sandbox-drift/blob/v0.1.0/LICENSE).