sandbox-drift 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sandbox_drift-0.1.0/.github/ISSUE_TEMPLATE/bug.yml +22 -0
- sandbox_drift-0.1.0/.github/ISSUE_TEMPLATE/question.yml +11 -0
- sandbox_drift-0.1.0/.github/workflows/ci.yml +21 -0
- sandbox_drift-0.1.0/.gitignore +41 -0
- sandbox_drift-0.1.0/CHANGELOG.md +18 -0
- sandbox_drift-0.1.0/LICENSE +21 -0
- sandbox_drift-0.1.0/PKG-INFO +367 -0
- sandbox_drift-0.1.0/README.ko.md +350 -0
- sandbox_drift-0.1.0/README.md +350 -0
- sandbox_drift-0.1.0/SECURITY.md +7 -0
- sandbox_drift-0.1.0/docs/images/baseline-flip.ko.png +0 -0
- sandbox_drift-0.1.0/docs/images/baseline-flip.png +0 -0
- sandbox_drift-0.1.0/docs/images/cannot-run.ko.png +0 -0
- sandbox_drift-0.1.0/docs/images/cannot-run.png +0 -0
- sandbox_drift-0.1.0/docs/images/diff-flip.ko.png +0 -0
- sandbox_drift-0.1.0/docs/images/diff-flip.png +0 -0
- sandbox_drift-0.1.0/docs/images/fingerprint.ko.png +0 -0
- sandbox_drift-0.1.0/docs/images/fingerprint.png +0 -0
- sandbox_drift-0.1.0/docs/images/init-policy.ko.png +0 -0
- sandbox_drift-0.1.0/docs/images/init-policy.png +0 -0
- sandbox_drift-0.1.0/docs/images/self-table.ko.png +0 -0
- sandbox_drift-0.1.0/docs/images/self-table.png +0 -0
- sandbox_drift-0.1.0/examples/policies/claude-srt.toml +11 -0
- sandbox_drift-0.1.0/examples/policies/self-basic.toml +11 -0
- sandbox_drift-0.1.0/examples/replay/after.json +73 -0
- sandbox_drift-0.1.0/examples/replay/before.json +75 -0
- sandbox_drift-0.1.0/pyproject.toml +36 -0
- sandbox_drift-0.1.0/src/sandbox_drift/__init__.py +3 -0
- sandbox_drift-0.1.0/src/sandbox_drift/__main__.py +4 -0
- sandbox_drift-0.1.0/src/sandbox_drift/canary.py +62 -0
- sandbox_drift-0.1.0/src/sandbox_drift/classify.py +69 -0
- sandbox_drift-0.1.0/src/sandbox_drift/cli.py +340 -0
- sandbox_drift-0.1.0/src/sandbox_drift/convert.py +77 -0
- sandbox_drift-0.1.0/src/sandbox_drift/errors.py +16 -0
- sandbox_drift-0.1.0/src/sandbox_drift/fingerprint.py +43 -0
- sandbox_drift-0.1.0/src/sandbox_drift/i18n.py +61 -0
- sandbox_drift-0.1.0/src/sandbox_drift/listener.py +47 -0
- sandbox_drift-0.1.0/src/sandbox_drift/locales/en.json +132 -0
- sandbox_drift-0.1.0/src/sandbox_drift/locales/ko.json +132 -0
- sandbox_drift-0.1.0/src/sandbox_drift/policy.py +79 -0
- sandbox_drift-0.1.0/src/sandbox_drift/report.py +260 -0
- sandbox_drift-0.1.0/src/sandbox_drift/snapshot.py +115 -0
- sandbox_drift-0.1.0/src/sandbox_drift/targets.py +162 -0
- sandbox_drift-0.1.0/src/sandbox_drift/worker.py +254 -0
- sandbox_drift-0.1.0/tests/conftest.py +67 -0
- sandbox_drift-0.1.0/tests/fixtures/claude-settings.json +16 -0
- sandbox_drift-0.1.0/tests/fixtures/old-snapshot.json +46 -0
- sandbox_drift-0.1.0/tests/fixtures/profile-a1.sb +6 -0
- sandbox_drift-0.1.0/tests/fixtures/profile-a2.sb +6 -0
- sandbox_drift-0.1.0/tests/fixtures/profile-b.sb +6 -0
- sandbox_drift-0.1.0/tests/test_canary.py +58 -0
- sandbox_drift-0.1.0/tests/test_classify.py +81 -0
- sandbox_drift-0.1.0/tests/test_fingerprint.py +33 -0
- sandbox_drift-0.1.0/tests/test_i18n.py +306 -0
- sandbox_drift-0.1.0/tests/test_no_real_paths.py +50 -0
- sandbox_drift-0.1.0/tests/test_report.py +55 -0
- sandbox_drift-0.1.0/tests/test_self_mode.py +30 -0
- sandbox_drift-0.1.0/tests/test_snapshot.py +72 -0
- sandbox_drift-0.1.0/tests/test_srt_target.py +78 -0
- sandbox_drift-0.1.0/tests/test_targets.py +79 -0
- sandbox_drift-0.1.0/uv.lock +79 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: 버그 / Bug
|
|
2
|
+
description: README대로 했는데 다르게 동작해요 / Something does not work as the README says
|
|
3
|
+
labels: [bug]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: what
|
|
7
|
+
attributes:
|
|
8
|
+
label: 무엇이 일어났나요? / What happened?
|
|
9
|
+
description: 실행한 명령과 나온 메시지를 그대로 붙여 주세요. / Paste the command and the output.
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: expected
|
|
14
|
+
attributes:
|
|
15
|
+
label: 기대한 결과 / Expected
|
|
16
|
+
- type: input
|
|
17
|
+
id: env
|
|
18
|
+
attributes:
|
|
19
|
+
label: 환경 / Environment
|
|
20
|
+
placeholder: "macOS 15 (M3) · Ubuntu 24.04 · Windows 11 / uv 0.x / mock or real mode"
|
|
21
|
+
validations:
|
|
22
|
+
required: true
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
name: 질문 / Question
|
|
2
|
+
description: 이해가 안 되는 부분이나 더 알고 싶은 점 / Something unclear or worth explaining
|
|
3
|
+
labels: [question]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: question
|
|
7
|
+
attributes:
|
|
8
|
+
label: 질문 / Question
|
|
9
|
+
description: README의 어느 부분(절 제목)에 대한 질문인지 알려 주세요. / Which README section is this about?
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Keyless: every test uses canaries, fake runners or saved snapshots.
|
|
2
|
+
name: ci
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
schedule:
|
|
7
|
+
- cron: "17 3 * * 1" # weekly: catches runtime changes that break the tests
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
os: [ubuntu-latest, macos-latest]
|
|
15
|
+
runs-on: ${{ matrix.os }}
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: astral-sh/setup-uv@v10.3.3
|
|
19
|
+
- run: uv python install
|
|
20
|
+
- run: uv sync --locked
|
|
21
|
+
- run: uv run pytest -q
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
.venv/
|
|
5
|
+
.pytest_cache/
|
|
6
|
+
.ruff_cache/
|
|
7
|
+
.mypy_cache/
|
|
8
|
+
*.egg-info/
|
|
9
|
+
|
|
10
|
+
# Node
|
|
11
|
+
node_modules/
|
|
12
|
+
dist/
|
|
13
|
+
.next/
|
|
14
|
+
|
|
15
|
+
# 실행 산출물·모델·데이터 캐시
|
|
16
|
+
runs/
|
|
17
|
+
outputs/
|
|
18
|
+
.cache/
|
|
19
|
+
*.safetensors
|
|
20
|
+
*.gguf
|
|
21
|
+
*.bin
|
|
22
|
+
|
|
23
|
+
# 환경·비밀
|
|
24
|
+
.env
|
|
25
|
+
.env.*
|
|
26
|
+
!.env.example
|
|
27
|
+
|
|
28
|
+
# OS·에디터
|
|
29
|
+
.DS_Store
|
|
30
|
+
.vscode/
|
|
31
|
+
.idea/
|
|
32
|
+
|
|
33
|
+
# Flutter·Dart
|
|
34
|
+
.dart_tool/
|
|
35
|
+
build/
|
|
36
|
+
.flutter-plugins
|
|
37
|
+
.flutter-plugins-dependencies
|
|
38
|
+
coverage/
|
|
39
|
+
|
|
40
|
+
# sandbox-drift local output
|
|
41
|
+
.sandbox-drift/
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] - 2026-11-12
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- `self`, `run` (`wrap`, `srt`, `replay` targets), `diff`, `init`, `fingerprint` and `convert` commands.
|
|
13
|
+
- Eight harmless canary probes: file writes inside and outside the work dir, a fake-HOME credential canary,
|
|
14
|
+
a localhost listener, a proxy reserved-host request, a direct TEST-NET-1 connection, the docker socket
|
|
15
|
+
and nested sandbox support.
|
|
16
|
+
- Expected-policy TOML (`block` / `allow` / `info`), snapshots, baseline comparison and `FLIP` detection.
|
|
17
|
+
- Exit codes: 0 matches, 1 mismatch or regression, 2 could not run.
|
|
18
|
+
- English and Korean output (`--lang`, `SANDBOX_DRIFT_LANG`, `LC_ALL`, `LC_MESSAGES`, `LANG`).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LeeGyeongPil
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sandbox-drift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Probe the OS sandbox of an AI coding agent with harmless canaries and catch silent regressions between versions.
|
|
5
|
+
Project-URL: Homepage, https://github.com/LeeGyeongPil/sandbox-drift
|
|
6
|
+
Author: LeeGyeongPil
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: agent-cli,ai-agent,regression,sandbox,sandbox-runtime,security
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Security
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# sandbox-drift
|
|
19
|
+
|
|
20
|
+
**English** | [한국어](https://github.com/LeeGyeongPil/sandbox-drift/blob/v0.1.0/README.ko.md)
|
|
21
|
+
|
|
22
|
+
[](https://github.com/LeeGyeongPil/sandbox-drift/actions/workflows/ci.yml) 
|
|
23
|
+
|
|
24
|
+
Probe the OS sandbox of an AI coding agent with harmless canaries, compare the result with the previous run, and exit 1 when something that used to be blocked is now allowed. Python 3.11+, no runtime dependencies.
|
|
25
|
+
|
|
26
|
+

|
|
27
|
+
|
|
28
|
+
*Two saved results compared. The `FLIP` row is a regression: writing outside the work dir used to be blocked and is now allowed. Exit code 1.*
|
|
29
|
+
|
|
30
|
+
## Why
|
|
31
|
+
|
|
32
|
+
Coding agents run shell commands inside an OS sandbox (Seatbelt on macOS, bubblewrap on Linux, plus a proxy allowlist for the network). The boundary is software, so it can change between versions, and the release notes do not always say so. Public reports (all checked 2026-11-12):
|
|
33
|
+
|
|
34
|
+
- A network allowlist that did not actually restrict anything: an upstream agent CLI issue report (issue #78608 in the upstream agent CLI repository).
|
|
35
|
+
- A sandbox that opened silently after a settings change during a session: an upstream agent CLI issue report (issue #81572 in the same repository).
|
|
36
|
+
- A bypass that was fixed without a release note: a security write-up titled "Second time, same sandbox" by the oddguan.com blog (search for the title).
|
|
37
|
+
- The opposite problem, things that should work being blocked or settings being ignored: an upstream agent CLI issue report (issue #56959 in the same repository), [codex#10390](https://github.com/openai/codex/issues/10390).
|
|
38
|
+
- Escaping a sandbox through the docker socket: [Pillar Security write-up](https://www.pillar.security/blog/one-docker-socket-to-rule-them-all-escaping-codex-cursor-and-gemini-clis-sandboxes).
|
|
39
|
+
|
|
40
|
+
Re-checking by hand after every upgrade ("can it still not write outside the project?") is easy to forget. sandbox-drift does it with a few harmless probes and tells you what changed since last time.
|
|
41
|
+
|
|
42
|
+
## Quick start
|
|
43
|
+
|
|
44
|
+
Needs `git` and [uv](https://docs.astral.sh/uv/) (`curl -LsSf https://astral.sh/uv/install.sh | sh`). The tool has no runtime dependencies (Python standard library only). When you run it from a clone with `uv run`, uv also installs the dev packages (pytest) into `.venv` on the first run.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
git clone https://github.com/LeeGyeongPil/sandbox-drift
|
|
48
|
+
cd sandbox-drift
|
|
49
|
+
uv run sandbox-drift diff examples/replay/before.json examples/replay/after.json --lang en
|
|
50
|
+
echo "exit=$?"
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Output:
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
sandbox-drift diff
|
|
57
|
+
agent_version: example-1.0.0 -> example-1.1.0
|
|
58
|
+
|
|
59
|
+
Probe Verdict Change
|
|
60
|
+
------------------------ ------- -----------------
|
|
61
|
+
fs.write_outside_workdir FLIP blocked → allowed
|
|
62
|
+
|
|
63
|
+
1 cell opened (regression), 0 cells tightened.
|
|
64
|
+
exit=1
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The two files in `examples/replay/` are a **practice mock**. The shape of each observation comes from a real probe run, but the version change and the regression in `after.json` were put in on purpose so you can see what a FLIP looks like. Nothing is executed.
|
|
68
|
+
|
|
69
|
+
Install as a command instead of running from a clone:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
uv tool install git+https://github.com/LeeGyeongPil/sandbox-drift
|
|
73
|
+
sandbox-drift --version
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Measure a real sandbox
|
|
77
|
+
|
|
78
|
+
Ask the agent to run this in its own shell, so the probes run inside the sandbox the agent really uses:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
uv run sandbox-drift self --policy examples/policies/self-basic.toml --baseline .sandbox-drift/ --lang en
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Run inside an agent sandbox, the table looks like this (measured on macOS, 2026-11-12):
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
sandbox-drift report
|
|
88
|
+
target=self agent=unknown engine=unknown platform=darwin/arm64
|
|
89
|
+
|
|
90
|
+
Probe Expect Observed Verdict Change Detail
|
|
91
|
+
-------------------------- ------ -------- ------- ------ ---------------------------------------------------------
|
|
92
|
+
fs.write_inside_workdir allow allowed PASS file written and removed
|
|
93
|
+
fs.write_outside_workdir block blocked PASS EPERM: Operation not permitted
|
|
94
|
+
cred.read_fake_home_canary block n/a SKIP self mode cannot swap HOME; run with --target wrap or srt
|
|
95
|
+
net.localhost_listener allow allowed PASS connected to the localhost canary listener
|
|
96
|
+
net.proxy_reserved_host block blocked PASS HTTP 403
|
|
97
|
+
net.direct_egress_testnet block blocked PASS EPERM: Operation not permitted
|
|
98
|
+
proc.docker_socket block blocked PASS EPERM: Operation not permitted
|
|
99
|
+
proc.nested_sandbox info blocked INFO sandbox-exec: sandbox_apply: Operation not permitted
|
|
100
|
+
|
|
101
|
+
PASS 6 FAIL 0 FLIP 0 ERROR 0 SKIP 1 INFO 1
|
|
102
|
+
Baseline saved: .sandbox-drift/baseline-self.json
|
|
103
|
+
|
|
104
|
+
Measures the Bash/command boundary only. Tools that run outside the sandbox (file editors, MCP, hooks) are not covered.
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+

|
|
108
|
+
|
|
109
|
+
*The same command run in a plain terminal with no sandbox. The "block" rows come out as FAIL because nothing blocks them. That is the negative control: if a sandbox were present they would be PASS.*
|
|
110
|
+
|
|
111
|
+
## How it works
|
|
112
|
+
|
|
113
|
+
```mermaid
|
|
114
|
+
flowchart LR
|
|
115
|
+
P[policy.toml<br/>block / allow / info] --> CLI[sandbox-drift CLI]
|
|
116
|
+
CLI --> C[canaries<br/>temp workdir, fake HOME, token]
|
|
117
|
+
CLI --> L[localhost listener]
|
|
118
|
+
CLI --> T{target}
|
|
119
|
+
T -->|self| W0[worker in this process]
|
|
120
|
+
T -->|wrap| W1[runner + worker]
|
|
121
|
+
T -->|srt| W2[srt --settings + worker<br/>HOME = fake HOME]
|
|
122
|
+
T -->|replay| W3[saved observations<br/>practice mock]
|
|
123
|
+
W0 & W1 & W2 --> O[observations<br/>blocked / allowed / error / n/a]
|
|
124
|
+
W3 --> O
|
|
125
|
+
W1 & W2 -.runner failed.-> E[cannot run, exit 2]
|
|
126
|
+
L -.hit.-> O
|
|
127
|
+
O --> V[match policy: PASS / FAIL]
|
|
128
|
+
O --> S[compare snapshot: FLIP]
|
|
129
|
+
V & S --> R[table / Markdown / JSON<br/>exit 0 / 1 / 2]
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
1. You write what the sandbox **must block**, **must allow** and what is just **info** in a small TOML policy.
|
|
133
|
+
2. A single-file worker runs eight probes inside the target and prints what happened.
|
|
134
|
+
3. Each result is classified as `blocked` (EPERM, EACCES, HTTP 403/407), `allowed` (success, or the packet left the sandbox), `error` or `n/a`.
|
|
135
|
+
4. The result is matched against the policy (PASS/FAIL) and against the previous snapshot (FLIP).
|
|
136
|
+
|
|
137
|
+
**Why the probes are harmless.** Nothing real is touched.
|
|
138
|
+
|
|
139
|
+
| Probe | What it does | Why it is safe |
|
|
140
|
+
|---|---|---|
|
|
141
|
+
| `fs.write_inside_workdir` | writes and removes a file in the work dir | positive control: this must work |
|
|
142
|
+
| `fs.write_outside_workdir` | writes and removes a randomly named file in your HOME (change the directory with `--outside-dir`) | the file is removed right away |
|
|
143
|
+
| `cred.read_fake_home_canary` | reads credential-shaped files in a temporary fake HOME | contents are a random token; the real HOME is never read or stat'ed |
|
|
144
|
+
| `net.localhost_listener` | connects to a throwaway listener on `127.0.0.1` | stays on your machine |
|
|
145
|
+
| `net.proxy_reserved_host` | requests `canary.invalid` through the sandbox proxy | `.invalid` is reserved ([RFC 6761](https://www.rfc-editor.org/rfc/rfc6761)) and never resolves |
|
|
146
|
+
| `net.direct_egress_testnet` | connects directly to `192.0.2.1:9` | TEST-NET-1 ([RFC 5737](https://www.rfc-editor.org/rfc/rfc5737)) is not routed |
|
|
147
|
+
| `proc.docker_socket` | tries to connect to the docker socket, nothing more | no API request is sent |
|
|
148
|
+
| `proc.nested_sandbox` | tries to start a nested `sandbox-exec` (macOS) | information only |
|
|
149
|
+
|
|
150
|
+
A connect that times out counts as `allowed`: the packet left the sandbox instead of being refused locally.
|
|
151
|
+
|
|
152
|
+
## Targets
|
|
153
|
+
|
|
154
|
+
| Target | Command | Use it for |
|
|
155
|
+
|---|---|---|
|
|
156
|
+
| `self` | `sandbox-drift self ...` | The sandbox you are already inside. Have the agent run it. No nested sandbox needed. |
|
|
157
|
+
| `srt` | `sandbox-drift run --target srt --settings .claude/settings.json ...` | Re-create an agent CLI `sandbox` block under [sandbox-runtime](https://github.com/anthropics/sandbox-runtime) (`srt`) with HOME set to a fake HOME. |
|
|
158
|
+
| `wrap` | `sandbox-drift run --target wrap --wrap "my-runner {}" ...` | Any other runner. `{}` is replaced by the worker command. |
|
|
159
|
+
| `replay` | `sandbox-drift run --target replay --replay snapshot.json ...` | Practice mock: replays a saved result through the real policy and diff logic. |
|
|
160
|
+
|
|
161
|
+
Convert an agent CLI `sandbox` block to a `srt` settings file (unknown keys are warned about and dropped):
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
uv run sandbox-drift convert --from-claude tests/fixtures/claude-settings.json
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
```
|
|
168
|
+
warning: sandbox.enabled has no sandbox-runtime equivalent and was left out
|
|
169
|
+
warning: sandbox.autoAllowBashIfSandboxed has no sandbox-runtime equivalent and was left out
|
|
170
|
+
warning: ignored unknown setting sandbox.network.futureOption
|
|
171
|
+
{
|
|
172
|
+
"network": {
|
|
173
|
+
"allowedDomains": [
|
|
174
|
+
"example.com"
|
|
175
|
+
],
|
|
176
|
+
"allowLocalBinding": true
|
|
177
|
+
},
|
|
178
|
+
"filesystem": {
|
|
179
|
+
"allowWrite": [
|
|
180
|
+
"~/scratch",
|
|
181
|
+
"/opt/shared"
|
|
182
|
+
],
|
|
183
|
+
"denyRead": [
|
|
184
|
+
"~/.canary-secrets"
|
|
185
|
+
]
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
The fake HOME canary only means something when the target is started with HOME set to the fake HOME. That is why `self` skips it (it cannot swap HOME). With the identity wrapper `--wrap "{}"` there is no sandbox at all, so the canary is readable and the row is FAIL, a negative control you can use to check that the probe works.
|
|
191
|
+
|
|
192
|
+
Roadmap: `codex` and `gemini` targets and more probes (milestone M2, not in 0.1.0).
|
|
193
|
+
|
|
194
|
+
## Policy file
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
uv run sandbox-drift init --preset claude-srt
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
```toml
|
|
201
|
+
# Expected results for `sandbox-drift run --target srt` with an agent settings style
|
|
202
|
+
# sandbox block (workdir writable, everything else closed, only localhost open).
|
|
203
|
+
[expect]
|
|
204
|
+
"fs.write_inside_workdir" = "allow"
|
|
205
|
+
"fs.write_outside_workdir" = "block"
|
|
206
|
+
"cred.read_fake_home_canary" = "block"
|
|
207
|
+
"net.localhost_listener" = "allow"
|
|
208
|
+
"net.proxy_reserved_host" = "block"
|
|
209
|
+
"net.direct_egress_testnet" = "block"
|
|
210
|
+
"proc.docker_socket" = "block"
|
|
211
|
+
"proc.nested_sandbox" = "info"
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
- `block`: the sandbox must refuse it. Observed `allowed` is a FAIL.
|
|
215
|
+
- `allow`: it must work. Observed `blocked` is a FAIL.
|
|
216
|
+
- `info`: recorded, not judged. `n/a` is shown as SKIP.
|
|
217
|
+
|
|
218
|
+
Presets: `self-basic` and `claude-srt` (also in `examples/policies/`). Use `--out FILE` to write the file instead of printing it.
|
|
219
|
+
|
|
220
|
+

|
|
221
|
+
|
|
222
|
+
*The generated policy: one line per probe.*
|
|
223
|
+
|
|
224
|
+
## Snapshots & diff
|
|
225
|
+
|
|
226
|
+
With `--baseline DIR` the first run saves a snapshot, and later runs compare against it. Each snapshot records `target`, `agent_version`, `engine_version`, `os`, `arch`, `policy_hash` and `settings_fingerprint`. Running the practice mock twice shows it:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
uv run sandbox-drift run --target replay --replay examples/replay/before.json --policy examples/policies/claude-srt.toml --baseline .sandbox-drift/
|
|
230
|
+
uv run sandbox-drift run --target replay --replay examples/replay/after.json --policy examples/policies/claude-srt.toml --baseline .sandbox-drift/
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
The second run prints (exit 1):
|
|
234
|
+
|
|
235
|
+
```
|
|
236
|
+
sandbox-drift report
|
|
237
|
+
target=srt agent=example-1.1.0 engine=0.0.82 platform=darwin/arm64
|
|
238
|
+
|
|
239
|
+
Probe Expect Observed Verdict Change Detail
|
|
240
|
+
-------------------------- ------ -------- ------- ----------------- ----------------------------------------------------
|
|
241
|
+
fs.write_inside_workdir allow allowed PASS file written and removed
|
|
242
|
+
fs.write_outside_workdir block allowed FLIP blocked → allowed file written and removed
|
|
243
|
+
cred.read_fake_home_canary block blocked PASS read denied for 4 canary file(s)
|
|
244
|
+
net.localhost_listener allow allowed PASS connected to the localhost canary listener
|
|
245
|
+
net.proxy_reserved_host block blocked PASS HTTP 403
|
|
246
|
+
net.direct_egress_testnet block blocked PASS EPERM: Operation not permitted
|
|
247
|
+
proc.docker_socket block blocked PASS EPERM: Operation not permitted
|
|
248
|
+
proc.nested_sandbox info blocked INFO sandbox-exec: sandbox_apply: Operation not permitted
|
|
249
|
+
|
|
250
|
+
PASS 6 FAIL 0 FLIP 1 ERROR 0 SKIP 0 INFO 1
|
|
251
|
+
Compared with baseline: .sandbox-drift/baseline-srt.json
|
|
252
|
+
agent_version: example-1.0.0 -> example-1.1.0
|
|
253
|
+
Practice mock: replayed from a saved file, no sandbox was exercised.
|
|
254
|
+
Practice mock: the observation shapes come from a real probe run, but the version change and the regression in the 'after' file were put in on purpose to show how a FLIP looks.
|
|
255
|
+
|
|
256
|
+
Measures the Bash/command boundary only. Tools that run outside the sandbox (file editors, MCP, hooks) are not covered.
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
The fixed wording in the Detail column follows the output language; raw OS errors such as `EPERM: Operation not permitted`, `HTTP 403` and `sandbox-exec` messages are shown as the OS returned them.
|
|
260
|
+
|
|
261
|
+

|
|
262
|
+
|
|
263
|
+
*A second run against the saved baseline.*
|
|
264
|
+
|
|
265
|
+
- **FLIP**: blocked in the baseline, allowed now. Exit 1. If your policy says `allow` for that probe it is a PASS instead, because something you wanted open is not a failure.
|
|
266
|
+
- **tightened**: allowed before, blocked now. Reported, but fails only with `--strict`.
|
|
267
|
+
- `--exit-zero`: print the same report but exit 0 (cannot-run still exits 2).
|
|
268
|
+
- `--update-baseline`: overwrite the saved baseline after comparing. Without it the baseline is never overwritten.
|
|
269
|
+
- `sandbox-drift diff OLD.json NEW.json` compares two saved files without running anything.
|
|
270
|
+
|
|
271
|
+
Two fingerprints help you tell whether the policy or the settings changed. SBPL (Seatbelt profile) files are normalised first, so random log tags, proxy ports and temp paths do not change the hash:
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
uv run sandbox-drift fingerprint --sbpl tests/fixtures/profile-a1.sb
|
|
275
|
+
uv run sandbox-drift fingerprint --sbpl tests/fixtures/profile-a2.sb
|
|
276
|
+
uv run sandbox-drift fingerprint --sbpl tests/fixtures/profile-b.sb
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
```
|
|
280
|
+
sha256:1ccf6994407702ae9ec37889cc7a3f9c7be4cccf0a9e1c3cf137b8bbba83e0b3
|
|
281
|
+
sha256:1ccf6994407702ae9ec37889cc7a3f9c7be4cccf0a9e1c3cf137b8bbba83e0b3
|
|
282
|
+
sha256:e59afa598a1f93858e6dd8617750bf3c9be45942965160e87507aac63325a682
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
`profile-a1` and `profile-a2` differ only in the random tag and ports. `profile-b` differs by one rule.
|
|
286
|
+
|
|
287
|
+

|
|
288
|
+
|
|
289
|
+
*The first two profiles get the same hash; the third, with one rule changed, does not.*
|
|
290
|
+
|
|
291
|
+
## Exit codes
|
|
292
|
+
|
|
293
|
+
| Code | Meaning |
|
|
294
|
+
|---|---|
|
|
295
|
+
| 0 | every judged cell matches the policy and nothing flipped |
|
|
296
|
+
| 1 | a mismatch (FAIL), a regression (FLIP) or a probe that could not tell (ERROR) |
|
|
297
|
+
| 2 | the probes could not run |
|
|
298
|
+
|
|
299
|
+
"Cannot run" is not "blocked". If the runner is missing, or a nested sandbox refuses to start (`sandbox_apply: Operation not permitted`, `listen EPERM`), nothing was measured. Reporting that as "blocked, PASS" would be a false pass, so sandbox-drift exits 2, reports no cells, and recommends `self`:
|
|
300
|
+
|
|
301
|
+
```bash
|
|
302
|
+
uv run sandbox-drift run --target wrap --wrap "/nonexistent/runner {}" --policy examples/policies/self-basic.toml
|
|
303
|
+
echo "exit=$?"
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
```
|
|
307
|
+
error: Could not start the runner /nonexistent/runner (No such file or directory). This is not a 'blocked' result. If you are already inside an agent sandbox, use `sandbox-drift self` instead.
|
|
308
|
+
exit=2
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+

|
|
312
|
+
|
|
313
|
+
*Exit 2 and a pointer to `self`. Known limitation: in this screenshot the `exit=2` line appears before the error text because the two streams were captured separately; the text block above shows the real order.*
|
|
314
|
+
|
|
315
|
+
`--exit-zero` never hides exit 2.
|
|
316
|
+
|
|
317
|
+
## Using in CI
|
|
318
|
+
|
|
319
|
+
The repo's own CI (`.github/workflows/ci.yml`) runs `uv run pytest -q` on Ubuntu and macOS. To watch your own settings, run the tool on a schedule and keep the baseline directory between runs (commit it or use a cache). The step below is a sketch; it was not run against a real `srt` in this repository's docs:
|
|
320
|
+
|
|
321
|
+
```yaml
|
|
322
|
+
- name: Sandbox drift
|
|
323
|
+
run: |
|
|
324
|
+
uvx --from git+https://github.com/LeeGyeongPil/sandbox-drift sandbox-drift run \
|
|
325
|
+
--target srt --settings .claude/settings.json \
|
|
326
|
+
--policy .sandbox-drift/policy.toml --baseline .sandbox-drift/ \
|
|
327
|
+
--format md >> "$GITHUB_STEP_SUMMARY"
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
The job fails (exit 1) when a cell flipped from blocked to allowed. On Ubuntu 24.04 runners, AppArmor can restrict unprivileged user namespaces, so bubblewrap and `srt` may fail to start. sandbox-drift reports that as exit 2, not as a blocked cell (see [sandbox-runtime#429](https://github.com/anthropics/sandbox-runtime/issues/429), checked 2026-11-12).
|
|
331
|
+
|
|
332
|
+
Real `srt` runs are not possible inside another sandbox (nested Seatbelt/`listen EPERM`). This repository's tests cover that path with a fake `srt` script that checks the arguments, HOME and converted settings. Running a real `srt` is for your terminal or CI runner.
|
|
333
|
+
|
|
334
|
+
## What it does NOT check
|
|
335
|
+
|
|
336
|
+
- **Only the Bash/command boundary.** File editors, `WebFetch`, MCP tools and hooks run outside the sandbox and are not measured. A clean result does not mean they are safe.
|
|
337
|
+
- It does not enumerate everything visible inside a sandbox. For that, use [controlplaneio/sandbox-probe](https://github.com/controlplaneio/sandbox-probe) next to this tool.
|
|
338
|
+
- No real attacks, no prompt-injection tests, no model behaviour.
|
|
339
|
+
- No Windows, no container/Compose isolation checks (only indirectly through `--wrap`).
|
|
340
|
+
- Codex and Gemini CLI targets are not in 0.1.0.
|
|
341
|
+
|
|
342
|
+
## Language
|
|
343
|
+
|
|
344
|
+
Output and help are available in English and Korean. The order is `--lang en|ko`, then `SANDBOX_DRIFT_LANG`, then `LC_ALL`, `LC_MESSAGES`, `LANG`. Anything that is not Korean falls back to English.
|
|
345
|
+
|
|
346
|
+
```bash
|
|
347
|
+
SANDBOX_DRIFT_LANG=ko uv run sandbox-drift diff examples/replay/before.json examples/replay/after.json --exit-zero
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
Verdict codes (`PASS`, `FAIL`, `FLIP`, `ERROR`, `SKIP`, `INFO`), observed values and probe ids are not translated.
|
|
351
|
+
|
|
352
|
+
The fixed Detail wording and the notes of the bundled example files follow the output language, including older snapshots saved without a message code. A `note` you write yourself in a replay file is data, so it is printed as written after the label `Note from the file (not translated):`.
|
|
353
|
+
|
|
354
|
+
OS error text (`EPERM: Operation not permitted`, `HTTP 403`, `sandbox-exec` stderr) is shown as the OS gave it and is not translated.
|
|
355
|
+
|
|
356
|
+
## Safety notes
|
|
357
|
+
|
|
358
|
+
- No API keys, no accounts, no outbound requests. Network probes go to localhost, `canary.invalid` and TEST-NET-1 only.
|
|
359
|
+
- Credential-shaped canary files exist only inside a temporary fake HOME, with a random token as content, and are deleted afterwards. The real HOME is never read.
|
|
360
|
+
- The docker socket probe only calls `connect`.
|
|
361
|
+
- `examples/replay/` is a practice mock, and every report from it says so.
|
|
362
|
+
- `fs.write_outside_workdir` writes a randomly named file in your real HOME (or the `--outside-dir` directory) for a moment and deletes it. Run it inside a sandbox to see it blocked.
|
|
363
|
+
- Tests: `uv run pytest -q` (105 tests; no outbound traffic except the reserved `canary.invalid` request through a proxy, if one is set).
|
|
364
|
+
|
|
365
|
+
## License
|
|
366
|
+
|
|
367
|
+
MIT, see [LICENSE](https://github.com/LeeGyeongPil/sandbox-drift/blob/v0.1.0/LICENSE).
|