storage-leak-diff-detector 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- storage_leak_diff_detector-0.1.2/PKG-INFO +338 -0
- storage_leak_diff_detector-0.1.2/README.md +313 -0
- storage_leak_diff_detector-0.1.2/pyproject.toml +59 -0
- storage_leak_diff_detector-0.1.2/setup.cfg +4 -0
- storage_leak_diff_detector-0.1.2/src/sldd/__init__.py +6 -0
- storage_leak_diff_detector-0.1.2/src/sldd/adaptive.py +326 -0
- storage_leak_diff_detector-0.1.2/src/sldd/api.py +564 -0
- storage_leak_diff_detector-0.1.2/src/sldd/cli.py +673 -0
- storage_leak_diff_detector-0.1.2/src/sldd/delete.py +177 -0
- storage_leak_diff_detector-0.1.2/src/sldd/detect.py +244 -0
- storage_leak_diff_detector-0.1.2/src/sldd/diff.py +123 -0
- storage_leak_diff_detector-0.1.2/src/sldd/models.py +241 -0
- storage_leak_diff_detector-0.1.2/src/sldd/platform_utils.py +102 -0
- storage_leak_diff_detector-0.1.2/src/sldd/playback.py +102 -0
- storage_leak_diff_detector-0.1.2/src/sldd/process_io.py +146 -0
- storage_leak_diff_detector-0.1.2/src/sldd/report.py +352 -0
- storage_leak_diff_detector-0.1.2/src/sldd/scheduler.py +87 -0
- storage_leak_diff_detector-0.1.2/src/sldd/server.py +1637 -0
- storage_leak_diff_detector-0.1.2/src/sldd/snapshot.py +205 -0
- storage_leak_diff_detector-0.1.2/src/sldd/storage.py +872 -0
- storage_leak_diff_detector-0.1.2/src/sldd/web/assets/index-AMvNMnJO.css +1 -0
- storage_leak_diff_detector-0.1.2/src/sldd/web/assets/index-Cys4a6nx.js +48 -0
- storage_leak_diff_detector-0.1.2/src/sldd/web/favicon.svg +1 -0
- storage_leak_diff_detector-0.1.2/src/sldd/web/icons.svg +24 -0
- storage_leak_diff_detector-0.1.2/src/sldd/web/index.html +14 -0
- storage_leak_diff_detector-0.1.2/src/storage_leak_diff_detector.egg-info/PKG-INFO +338 -0
- storage_leak_diff_detector-0.1.2/src/storage_leak_diff_detector.egg-info/SOURCES.txt +42 -0
- storage_leak_diff_detector-0.1.2/src/storage_leak_diff_detector.egg-info/dependency_links.txt +1 -0
- storage_leak_diff_detector-0.1.2/src/storage_leak_diff_detector.egg-info/entry_points.txt +2 -0
- storage_leak_diff_detector-0.1.2/src/storage_leak_diff_detector.egg-info/requires.txt +18 -0
- storage_leak_diff_detector-0.1.2/src/storage_leak_diff_detector.egg-info/top_level.txt +1 -0
- storage_leak_diff_detector-0.1.2/tests/test_api.py +98 -0
- storage_leak_diff_detector-0.1.2/tests/test_cli.py +127 -0
- storage_leak_diff_detector-0.1.2/tests/test_delete.py +128 -0
- storage_leak_diff_detector-0.1.2/tests/test_detect.py +169 -0
- storage_leak_diff_detector-0.1.2/tests/test_diff.py +118 -0
- storage_leak_diff_detector-0.1.2/tests/test_models.py +84 -0
- storage_leak_diff_detector-0.1.2/tests/test_platform_utils.py +81 -0
- storage_leak_diff_detector-0.1.2/tests/test_playback.py +106 -0
- storage_leak_diff_detector-0.1.2/tests/test_process_io_e2e.py +255 -0
- storage_leak_diff_detector-0.1.2/tests/test_report.py +92 -0
- storage_leak_diff_detector-0.1.2/tests/test_server.py +189 -0
- storage_leak_diff_detector-0.1.2/tests/test_snapshot.py +100 -0
- storage_leak_diff_detector-0.1.2/tests/test_storage.py +141 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: storage-leak-diff-detector
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Cross-platform storage leak diff detector — find what's eating your disk
|
|
5
|
+
Author: sldd contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: click>=8.1
|
|
10
|
+
Requires-Dist: rich>=13.0
|
|
11
|
+
Requires-Dist: humanize>=4.9
|
|
12
|
+
Requires-Dist: psutil>=5.9
|
|
13
|
+
Provides-Extra: web
|
|
14
|
+
Requires-Dist: fastapi>=0.115; extra == "web"
|
|
15
|
+
Requires-Dist: uvicorn[standard]>=0.34; extra == "web"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
18
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
19
|
+
Requires-Dist: pytest-tmp-files>=0.0.2; extra == "dev"
|
|
20
|
+
Requires-Dist: mypy>=1.8; extra == "dev"
|
|
21
|
+
Requires-Dist: ruff>=0.3; extra == "dev"
|
|
22
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
23
|
+
Requires-Dist: fastapi>=0.115; extra == "dev"
|
|
24
|
+
Requires-Dist: uvicorn[standard]>=0.34; extra == "dev"
|
|
25
|
+
|
|
26
|
+
# sldd — Storage Leak Diff Detector
|
|
27
|
+
|
|
28
|
+
Cross-platform tool that finds what's eating your disk space by taking filesystem snapshots and comparing them over time. Designed for the scenario where your system runs out of space every few hours and you need to find the culprit fast.
|
|
29
|
+
|
|
30
|
+
## Features
|
|
31
|
+
|
|
32
|
+
- **Snapshot & diff** — capture directory sizes, compare any two snapshots, see exactly what grew
|
|
33
|
+
- **Anomaly detection** — flags abnormal growth using absolute thresholds, growth rate, relative change, and statistical deviation
|
|
34
|
+
- **Depth-aware attribution** — traces growth from `/` down to the deepest directory responsible
|
|
35
|
+
- **Adaptive scanning** — starts shallow (depth 3), focuses on what changes, discards the rest. Keeps DB small automatically
|
|
36
|
+
- **Web dashboard** — real-time UI with charts, drill-down explorer, playback animation, deletion manager, and settings
|
|
37
|
+
- **CLI** — full-featured terminal interface with Rich tables and color output
|
|
38
|
+
- **Safe deletion** — preview impact before deleting, blocklist protects system paths, full audit log
|
|
39
|
+
- **Playback** — animate filesystem changes over time like a video, with speed controls
|
|
40
|
+
|
|
41
|
+
## Prerequisites
|
|
42
|
+
|
|
43
|
+
- **Python 3.10+**
|
|
44
|
+
- **Node.js 18+** — only when [installing from source](#quick-start-clone-install-run--one-command); not needed for pip install (frontend is bundled)
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
### Install without building (recommended)
|
|
49
|
+
|
|
50
|
+
No clone, no Node.js, no build. The frontend is bundled in the package.
|
|
51
|
+
|
|
52
|
+
**From PyPI** (when published):
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install storage-leak-diff-detector[web]
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
**From GitHub release** (pre-built wheel):
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install https://github.com/datazinc/storage-leak-detector/releases/download/v0.1.2/storage_leak_diff_detector-0.1.2-py3-none-any.whl
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Then run:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
python -m sldd.cli web
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Quick start (clone, install, run — one command)
|
|
71
|
+
|
|
72
|
+
[**→ Open in GitHub**](https://github.com/datazinc/storage-leak-detector) | [**→ Download ZIP**](https://github.com/datazinc/storage-leak-detector/archive/refs/heads/main.zip)
|
|
73
|
+
|
|
74
|
+
**Bash** (Linux, macOS, Git Bash):
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
([ -d storage-leak-detector ] || git clone https://github.com/datazinc/storage-leak-detector.git) && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location && python -m sldd.cli web
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**Windows cmd**:
|
|
81
|
+
|
|
82
|
+
```cmd
|
|
83
|
+
if not exist storage-leak-detector git clone https://github.com/datazinc/storage-leak-detector.git && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location && python -m sldd.cli web
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
**Windows PowerShell**:
|
|
87
|
+
|
|
88
|
+
```powershell
|
|
89
|
+
if (-not (Test-Path storage-leak-detector)) { git clone https://github.com/datazinc/storage-leak-detector.git }; cd storage-leak-detector; pip install ".[web]" --no-warn-script-location; python -m sldd.cli web
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Skips cloning if the directory already exists. Uses `python -m` so it works without PATH setup.
|
|
93
|
+
|
|
94
|
+
### Other options
|
|
95
|
+
|
|
96
|
+
**From source** (clone + install; requires Node.js for first-time frontend build):
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
# Bash
|
|
100
|
+
([ -d storage-leak-detector ] || git clone https://github.com/datazinc/storage-leak-detector.git) && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location
|
|
101
|
+
|
|
102
|
+
# Windows cmd
|
|
103
|
+
if not exist storage-leak-detector git clone https://github.com/datazinc/storage-leak-detector.git && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
**Development:** `pip install -e ".[dev,web]"`
|
|
107
|
+
|
|
108
|
+
### Verify
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
sldd --help
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
If `sldd` is not found, use `python -m sldd.cli --help` instead.
|
|
115
|
+
|
|
116
|
+
### PATH setup
|
|
117
|
+
|
|
118
|
+
To use `sldd` instead of `python -m sldd.cli`:
|
|
119
|
+
|
|
120
|
+
**Windows:** Add `Python\Scripts` to PATH (e.g. `C:\Users\<you>\AppData\Local\Programs\Python\Python311\Scripts`). Restart the terminal.
|
|
121
|
+
|
|
122
|
+
**macOS / Linux:** Add the pip user bin or venv bin to PATH. Restart the terminal or run `source ~/.bashrc` / `source ~/.zshrc`.
|
|
123
|
+
|
|
124
|
+
**Check:** `which sldd` (macOS/Linux) or `where sldd` (Windows)
|
|
125
|
+
|
|
126
|
+
## Platform support
|
|
127
|
+
|
|
128
|
+
| Feature | Linux | macOS | Windows |
|
|
129
|
+
| ------------------------------------------------- | :----------: | :-------------------------: | :----------------------: |
|
|
130
|
+
| Snapshot, diff, watch, drill, history | ✓ | ✓ | ✓ |
|
|
131
|
+
| Duplicate file detection | ✓ | ✓ | ✓ |
|
|
132
|
+
| Web dashboard | ✓ | ✓ | ✓ |
|
|
133
|
+
| Open in file manager | ✓ (xdg-open) | ✓ (Finder) | ✓ (Explorer) |
|
|
134
|
+
| Process I/O (open files, read/write bytes) | ✓ | Partial (I/O bytes often 0) | ✓ |
|
|
135
|
+
| Port fallback when in use | ✓ | ✓ | ✓ |
|
|
136
|
+
| Kill previous sldd on port before start | ✓ (lsof) | ✓ (lsof) | ✓ (psutil) |
|
|
137
|
+
| Graceful SIGINT/SIGTERM (kill child on Ctrl-C) | ✓ | ✓ | ✓ |
|
|
138
|
+
| Run as root detection | ✓ | ✓ | ✓ |
|
|
139
|
+
| Restart as regular user (sudo → drop privileges) | ✓ | ✓ | — |
|
|
140
|
+
| Restart as administrator (elevate when not admin) | ✓ (pkexec) | ✓ (osascript) | ✓ (UAC) |
|
|
141
|
+
| Symlink following | ✓ | ✓ | Partial (may need admin) |
|
|
142
|
+
|
|
143
|
+
## Usage
|
|
144
|
+
|
|
145
|
+
### Web dashboard
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
sldd web
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Try `sldd` first; if not found, use `python -m sldd.cli web`.
|
|
152
|
+
|
|
153
|
+
First run will:
|
|
154
|
+
|
|
155
|
+
1. Install frontend dependencies if needed (`npm install`)
|
|
156
|
+
2. Build the frontend if missing or stale (`npm run build`)
|
|
157
|
+
3. Start the server on http://localhost:8080
|
|
158
|
+
4. Open your browser automatically
|
|
159
|
+
|
|
160
|
+
If Node.js is not installed, you'll see instructions to install it. The CLI (snapshot, diff, watch) works without Node.
|
|
161
|
+
|
|
162
|
+
### Watch mode (CLI)
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
sldd watch -r / -i 300
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
(Or `python -m sldd.cli watch -r / -i 300` if `sldd` is not found.)
|
|
169
|
+
|
|
170
|
+
Scans `/` every 5 minutes with adaptive mode on by default. Prints a report whenever anomalies are detected. Press Ctrl-C to stop.
|
|
171
|
+
|
|
172
|
+
### Web dashboard options
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
sldd web --port 8080 --db snapshots.db
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
(Or `python -m sldd.cli web ...` if `sldd` is not found.)
|
|
179
|
+
|
|
180
|
+
The dashboard includes:
|
|
181
|
+
|
|
182
|
+
- **Dashboard** — stats, anomaly table, top growers chart, disk usage timeline
|
|
183
|
+
- **Playback** — animate changes between any two snapshots with speed controls
|
|
184
|
+
- **Explorer** — navigate the directory tree, see size history for any path
|
|
185
|
+
- **Deletion** — safely delete files/directories or prune old snapshots
|
|
186
|
+
- **Settings** — configure scan depth, thresholds, adaptive mode, database
|
|
187
|
+
|
|
188
|
+
For development with hot-reload:
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
# Terminal 1: backend
|
|
192
|
+
sldd web --port 8080 --db snapshots.db
|
|
193
|
+
|
|
194
|
+
# Terminal 2: frontend dev server
|
|
195
|
+
cd frontend && npx vite --port 5173
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Then open http://localhost:5173 (proxies API calls to the backend).
|
|
199
|
+
|
|
200
|
+
### Manual snapshots
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
# Take snapshots at different times
|
|
204
|
+
sldd snapshot -r / --db snapshots.db
|
|
205
|
+
# ... wait ...
|
|
206
|
+
sldd snapshot -r / --db snapshots.db
|
|
207
|
+
|
|
208
|
+
# Compare the two most recent
|
|
209
|
+
sldd diff --db snapshots.db
|
|
210
|
+
|
|
211
|
+
# Compare specific snapshots
|
|
212
|
+
sldd diff --from 1 --to 5 --db snapshots.db
|
|
213
|
+
|
|
214
|
+
# Output as JSON
|
|
215
|
+
sldd diff --json --db snapshots.db
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
## CLI Reference
|
|
219
|
+
|
|
220
|
+
Use `python -m sldd.cli` instead of `sldd` if the command is not found.
|
|
221
|
+
|
|
222
|
+
| Command | Description |
|
|
223
|
+
| ----------------------- | ----------------------------------------- |
|
|
224
|
+
| `sldd snapshot` | Take a filesystem snapshot |
|
|
225
|
+
| `sldd diff` | Compare two snapshots and show what grew |
|
|
226
|
+
| `sldd watch` | Periodic snapshots with anomaly alerts |
|
|
227
|
+
| `sldd web` | Launch the web dashboard |
|
|
228
|
+
| `sldd ls` | List saved snapshots |
|
|
229
|
+
| `sldd drill -p /path` | Drill into a directory's children |
|
|
230
|
+
| `sldd history -p /path` | Size history of a path across snapshots |
|
|
231
|
+
| `sldd compact` | Run compaction (collapse stable subtrees) |
|
|
232
|
+
| `sldd prune -k N` | Keep only the N most recent snapshots |
|
|
233
|
+
| `sldd rm <id>` | Delete a specific snapshot |
|
|
234
|
+
|
|
235
|
+
### Watch mode options
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
sldd watch \
|
|
239
|
+
-r / # root path to scan
|
|
240
|
+
-i 120 # scan interval in seconds
|
|
241
|
+
--threshold 200MB # absolute growth alert threshold
|
|
242
|
+
--rate-threshold 100MB # growth rate alert threshold (per hour)
|
|
243
|
+
--initial-depth 4 # depth for discovery scans
|
|
244
|
+
--stability-scans 5 # scans before marking a path stable
|
|
245
|
+
--keep 10 # snapshots to retain
|
|
246
|
+
--no-adaptive # disable adaptive mode (full deep scan)
|
|
247
|
+
--json # output reports as JSON
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
### Web server options
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
sldd web \
|
|
254
|
+
--port 8080 # port to listen on
|
|
255
|
+
--host 127.0.0.1 # host to bind to
|
|
256
|
+
--db snapshots.db # database file path
|
|
257
|
+
-r / # scan root for safety checks
|
|
258
|
+
--no-auto-restart # disable auto-restart on crash
|
|
259
|
+
--max-restarts 10 # max consecutive auto-restarts
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## Adaptive Scanning
|
|
263
|
+
|
|
264
|
+
The default `auto` mode dramatically reduces storage usage by scanning smart:
|
|
265
|
+
|
|
266
|
+
| Phase | What happens | Storage cost |
|
|
267
|
+
| ------------------------ | -------------------------------------------------------------- | ------------------ |
|
|
268
|
+
| Discovery (scan 0) | Scans at depth 3 (~20K entries) | ~6 MB/snapshot |
|
|
269
|
+
| Tracking (scans 1-2) | Compares snapshots, identifies growers | ~6 MB/snapshot |
|
|
270
|
+
| Focused (scan 3+) | Only scans growing paths at full depth, skips stable paths | ~0.5-3 MB/snapshot |
|
|
271
|
+
| Compaction (every 3rd) | Deletes child entries of stable subtrees, prunes old snapshots | Reclaims 50-90% |
|
|
272
|
+
| Rediscovery (every 10th) | Full depth-3 scan to catch new growth | ~6 MB |
|
|
273
|
+
|
|
274
|
+
**Comparison**: naive full `/` scans produce ~58 MB per snapshot (705 MB for 7 snapshots). Adaptive mode keeps the DB under ~25 MB across 100+ scans.
|
|
275
|
+
|
|
276
|
+
Configure in the web UI under Settings > Adaptive Scanning, or via CLI flags.
|
|
277
|
+
|
|
278
|
+
## Architecture
|
|
279
|
+
|
|
280
|
+
```
|
|
281
|
+
src/sldd/
|
|
282
|
+
models.py Pure dataclasses — Snapshot, DirDiff, Anomaly, configs
|
|
283
|
+
snapshot.py Filesystem walker — os.scandir + size aggregation
|
|
284
|
+
storage.py SQLite repository — snapshots, entries, path tracking
|
|
285
|
+
diff.py Diff engine — SQL join to compare snapshots
|
|
286
|
+
detect.py Anomaly detection — threshold, statistical, attribution
|
|
287
|
+
adaptive.py Adaptive scan engine — plan, track, compact
|
|
288
|
+
api.py Public API facade (SLDD class)
|
|
289
|
+
cli.py Click CLI
|
|
290
|
+
server.py FastAPI web server + REST endpoints
|
|
291
|
+
scheduler.py Watch mode scheduler
|
|
292
|
+
report.py Terminal report formatting
|
|
293
|
+
delete.py Safe deletion with blocklist
|
|
294
|
+
playback.py Playback frame generation
|
|
295
|
+
|
|
296
|
+
frontend/
|
|
297
|
+
src/api.ts TypeScript API client
|
|
298
|
+
src/App.tsx Router + layout + toast system
|
|
299
|
+
src/views/ Dashboard, Playback, Explorer, Deletion, Settings
|
|
300
|
+
src/components/ Card, ResizableTable, DepthFilter, Toast
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
## Development
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
# Install dev dependencies
|
|
307
|
+
pip install -e ".[dev,web]"
|
|
308
|
+
|
|
309
|
+
# Run tests (136 tests)
|
|
310
|
+
pytest
|
|
311
|
+
|
|
312
|
+
# Lint
|
|
313
|
+
ruff check src/
|
|
314
|
+
|
|
315
|
+
# Type check
|
|
316
|
+
mypy src/sldd/
|
|
317
|
+
|
|
318
|
+
# Frontend type check
|
|
319
|
+
cd frontend && npx tsc --noEmit
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
## Distribution
|
|
323
|
+
|
|
324
|
+
| Method | Use case |
|
|
325
|
+
| -------------------------------------- | ------------------------------------------------------------------- |
|
|
326
|
+
| `pip install storage-leak-diff-detector[web]` | PyPI (no build, frontend bundled) |
|
|
327
|
+
| `pip install <wheel URL>` | GitHub release wheel (no build) |
|
|
328
|
+
| `pip install storage-leak-diff-detector` | CLI only (no web dashboard) |
|
|
329
|
+
| Source + `pip install ".[web]"` | From clone (requires Node.js for first build) |
|
|
330
|
+
| Source + `pip install -e ".[dev,web]"` | Development, contributions |
|
|
331
|
+
| PyInstaller / Nuitka | Standalone executable — build scripts TBD |
|
|
332
|
+
| Docker | Isolated environment — Dockerfile TBD |
|
|
333
|
+
|
|
334
|
+
**Publishing a release:** Run `cd frontend && npm run build`, then `python scripts/prepare_build.py` to copy the built frontend into the package, then `python -m build`.
|
|
335
|
+
|
|
336
|
+
## License
|
|
337
|
+
|
|
338
|
+
MIT
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
# sldd — Storage Leak Diff Detector
|
|
2
|
+
|
|
3
|
+
Cross-platform tool that finds what's eating your disk space by taking filesystem snapshots and comparing them over time. Designed for the scenario where your system runs out of space every few hours and you need to find the culprit fast.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Snapshot & diff** — capture directory sizes, compare any two snapshots, see exactly what grew
|
|
8
|
+
- **Anomaly detection** — flags abnormal growth using absolute thresholds, growth rate, relative change, and statistical deviation
|
|
9
|
+
- **Depth-aware attribution** — traces growth from `/` down to the deepest directory responsible
|
|
10
|
+
- **Adaptive scanning** — starts shallow (depth 3), focuses on what changes, discards the rest. Keeps DB small automatically
|
|
11
|
+
- **Web dashboard** — real-time UI with charts, drill-down explorer, playback animation, deletion manager, and settings
|
|
12
|
+
- **CLI** — full-featured terminal interface with Rich tables and color output
|
|
13
|
+
- **Safe deletion** — preview impact before deleting, blocklist protects system paths, full audit log
|
|
14
|
+
- **Playback** — animate filesystem changes over time like a video, with speed controls
|
|
15
|
+
|
|
16
|
+
## Prerequisites
|
|
17
|
+
|
|
18
|
+
- **Python 3.10+**
|
|
19
|
+
- **Node.js 18+** — only when [installing from source](#quick-start-clone-install-run--one-command); not needed for pip install (frontend is bundled)
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
### Install without building (recommended)
|
|
24
|
+
|
|
25
|
+
No clone, no Node.js, no build. The frontend is bundled in the package.
|
|
26
|
+
|
|
27
|
+
**From PyPI** (when published):
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install storage-leak-diff-detector[web]
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**From GitHub release** (pre-built wheel):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install https://github.com/datazinc/storage-leak-detector/releases/download/v0.1.2/storage_leak_diff_detector-0.1.2-py3-none-any.whl
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Then run:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
python -m sldd.cli web
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Quick start (clone, install, run — one command)
|
|
46
|
+
|
|
47
|
+
[**→ Open in GitHub**](https://github.com/datazinc/storage-leak-detector) | [**→ Download ZIP**](https://github.com/datazinc/storage-leak-detector/archive/refs/heads/main.zip)
|
|
48
|
+
|
|
49
|
+
**Bash** (Linux, macOS, Git Bash):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
([ -d storage-leak-detector ] || git clone https://github.com/datazinc/storage-leak-detector.git) && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location && python -m sldd.cli web
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Windows cmd**:
|
|
56
|
+
|
|
57
|
+
```cmd
|
|
58
|
+
if not exist storage-leak-detector git clone https://github.com/datazinc/storage-leak-detector.git && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location && python -m sldd.cli web
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**Windows PowerShell**:
|
|
62
|
+
|
|
63
|
+
```powershell
|
|
64
|
+
if (-not (Test-Path storage-leak-detector)) { git clone https://github.com/datazinc/storage-leak-detector.git }; cd storage-leak-detector; pip install ".[web]" --no-warn-script-location; python -m sldd.cli web
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Skips cloning if the directory already exists. Uses `python -m` so it works without PATH setup.
|
|
68
|
+
|
|
69
|
+
### Other options
|
|
70
|
+
|
|
71
|
+
**From source** (clone + install; requires Node.js for first-time frontend build):
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Bash
|
|
75
|
+
([ -d storage-leak-detector ] || git clone https://github.com/datazinc/storage-leak-detector.git) && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location
|
|
76
|
+
|
|
77
|
+
# Windows cmd
|
|
78
|
+
if not exist storage-leak-detector git clone https://github.com/datazinc/storage-leak-detector.git && cd storage-leak-detector && pip install ".[web]" --no-warn-script-location
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**Development:** `pip install -e ".[dev,web]"`
|
|
82
|
+
|
|
83
|
+
### Verify
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
sldd --help
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
If `sldd` is not found, use `python -m sldd.cli --help` instead.
|
|
90
|
+
|
|
91
|
+
### PATH setup
|
|
92
|
+
|
|
93
|
+
To use `sldd` instead of `python -m sldd.cli`:
|
|
94
|
+
|
|
95
|
+
**Windows:** Add `Python\Scripts` to PATH (e.g. `C:\Users\<you>\AppData\Local\Programs\Python\Python311\Scripts`). Restart the terminal.
|
|
96
|
+
|
|
97
|
+
**macOS / Linux:** Add the pip user bin or venv bin to PATH. Restart the terminal or run `source ~/.bashrc` / `source ~/.zshrc`.
|
|
98
|
+
|
|
99
|
+
**Check:** `which sldd` (macOS/Linux) or `where sldd` (Windows)
|
|
100
|
+
|
|
101
|
+
## Platform support
|
|
102
|
+
|
|
103
|
+
| Feature | Linux | macOS | Windows |
|
|
104
|
+
| ------------------------------------------------- | :----------: | :-------------------------: | :----------------------: |
|
|
105
|
+
| Snapshot, diff, watch, drill, history | ✓ | ✓ | ✓ |
|
|
106
|
+
| Duplicate file detection | ✓ | ✓ | ✓ |
|
|
107
|
+
| Web dashboard | ✓ | ✓ | ✓ |
|
|
108
|
+
| Open in file manager | ✓ (xdg-open) | ✓ (Finder) | ✓ (Explorer) |
|
|
109
|
+
| Process I/O (open files, read/write bytes) | ✓ | Partial (I/O bytes often 0) | ✓ |
|
|
110
|
+
| Port fallback when in use | ✓ | ✓ | ✓ |
|
|
111
|
+
| Kill previous sldd on port before start | ✓ (lsof) | ✓ (lsof) | ✓ (psutil) |
|
|
112
|
+
| Graceful SIGINT/SIGTERM (kill child on Ctrl-C) | ✓ | ✓ | ✓ |
|
|
113
|
+
| Run as root detection | ✓ | ✓ | ✓ |
|
|
114
|
+
| Restart as regular user (sudo → drop privileges) | ✓ | ✓ | — |
|
|
115
|
+
| Restart as administrator (elevate when not admin) | ✓ (pkexec) | ✓ (osascript) | ✓ (UAC) |
|
|
116
|
+
| Symlink following | ✓ | ✓ | Partial (may need admin) |
|
|
117
|
+
|
|
118
|
+
## Usage
|
|
119
|
+
|
|
120
|
+
### Web dashboard
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
sldd web
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Try `sldd` first; if not found, use `python -m sldd.cli web`.
|
|
127
|
+
|
|
128
|
+
First run will:
|
|
129
|
+
|
|
130
|
+
1. Install frontend dependencies if needed (`npm install`)
|
|
131
|
+
2. Build the frontend if missing or stale (`npm run build`)
|
|
132
|
+
3. Start the server on http://localhost:8080
|
|
133
|
+
4. Open your browser automatically
|
|
134
|
+
|
|
135
|
+
If Node.js is not installed, you'll see instructions to install it. The CLI (snapshot, diff, watch) works without Node.
|
|
136
|
+
|
|
137
|
+
### Watch mode (CLI)
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
sldd watch -r / -i 300
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
(Or `python -m sldd.cli watch -r / -i 300` if `sldd` is not found.)
|
|
144
|
+
|
|
145
|
+
Scans `/` every 5 minutes with adaptive mode on by default. Prints a report whenever anomalies are detected. Press Ctrl-C to stop.
|
|
146
|
+
|
|
147
|
+
### Web dashboard options
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
sldd web --port 8080 --db snapshots.db
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
(Or `python -m sldd.cli web ...` if `sldd` is not found.)
|
|
154
|
+
|
|
155
|
+
The dashboard includes:
|
|
156
|
+
|
|
157
|
+
- **Dashboard** — stats, anomaly table, top growers chart, disk usage timeline
|
|
158
|
+
- **Playback** — animate changes between any two snapshots with speed controls
|
|
159
|
+
- **Explorer** — navigate the directory tree, see size history for any path
|
|
160
|
+
- **Deletion** — safely delete files/directories or prune old snapshots
|
|
161
|
+
- **Settings** — configure scan depth, thresholds, adaptive mode, database
|
|
162
|
+
|
|
163
|
+
For development with hot-reload:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
# Terminal 1: backend
|
|
167
|
+
sldd web --port 8080 --db snapshots.db
|
|
168
|
+
|
|
169
|
+
# Terminal 2: frontend dev server
|
|
170
|
+
cd frontend && npx vite --port 5173
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Then open http://localhost:5173 (proxies API calls to the backend).
|
|
174
|
+
|
|
175
|
+
### Manual snapshots
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
# Take snapshots at different times
|
|
179
|
+
sldd snapshot -r / --db snapshots.db
|
|
180
|
+
# ... wait ...
|
|
181
|
+
sldd snapshot -r / --db snapshots.db
|
|
182
|
+
|
|
183
|
+
# Compare the two most recent
|
|
184
|
+
sldd diff --db snapshots.db
|
|
185
|
+
|
|
186
|
+
# Compare specific snapshots
|
|
187
|
+
sldd diff --from 1 --to 5 --db snapshots.db
|
|
188
|
+
|
|
189
|
+
# Output as JSON
|
|
190
|
+
sldd diff --json --db snapshots.db
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
## CLI Reference
|
|
194
|
+
|
|
195
|
+
Use `python -m sldd.cli` instead of `sldd` if the command is not found.
|
|
196
|
+
|
|
197
|
+
| Command | Description |
|
|
198
|
+
| ----------------------- | ----------------------------------------- |
|
|
199
|
+
| `sldd snapshot` | Take a filesystem snapshot |
|
|
200
|
+
| `sldd diff` | Compare two snapshots and show what grew |
|
|
201
|
+
| `sldd watch` | Periodic snapshots with anomaly alerts |
|
|
202
|
+
| `sldd web` | Launch the web dashboard |
|
|
203
|
+
| `sldd ls` | List saved snapshots |
|
|
204
|
+
| `sldd drill -p /path` | Drill into a directory's children |
|
|
205
|
+
| `sldd history -p /path` | Size history of a path across snapshots |
|
|
206
|
+
| `sldd compact` | Run compaction (collapse stable subtrees) |
|
|
207
|
+
| `sldd prune -k N` | Keep only the N most recent snapshots |
|
|
208
|
+
| `sldd rm <id>` | Delete a specific snapshot |
|
|
209
|
+
|
|
210
|
+
### Watch mode options
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
sldd watch \
|
|
214
|
+
-r / # root path to scan
|
|
215
|
+
-i 120 # scan interval in seconds
|
|
216
|
+
--threshold 200MB # absolute growth alert threshold
|
|
217
|
+
--rate-threshold 100MB # growth rate alert threshold (per hour)
|
|
218
|
+
--initial-depth 4 # depth for discovery scans
|
|
219
|
+
--stability-scans 5 # scans before marking a path stable
|
|
220
|
+
--keep 10 # snapshots to retain
|
|
221
|
+
--no-adaptive # disable adaptive mode (full deep scan)
|
|
222
|
+
--json # output reports as JSON
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
### Web server options
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
sldd web \
|
|
229
|
+
--port 8080 # port to listen on
|
|
230
|
+
--host 127.0.0.1 # host to bind to
|
|
231
|
+
--db snapshots.db # database file path
|
|
232
|
+
-r / # scan root for safety checks
|
|
233
|
+
--no-auto-restart # disable auto-restart on crash
|
|
234
|
+
--max-restarts 10 # max consecutive auto-restarts
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
## Adaptive Scanning
|
|
238
|
+
|
|
239
|
+
The default `auto` mode dramatically reduces storage usage by scanning smart:
|
|
240
|
+
|
|
241
|
+
| Phase | What happens | Storage cost |
|
|
242
|
+
| ------------------------ | -------------------------------------------------------------- | ------------------ |
|
|
243
|
+
| Discovery (scan 0) | Scans at depth 3 (~20K entries) | ~6 MB/snapshot |
|
|
244
|
+
| Tracking (scans 1-2) | Compares snapshots, identifies growers | ~6 MB/snapshot |
|
|
245
|
+
| Focused (scan 3+) | Only scans growing paths at full depth, skips stable paths | ~0.5-3 MB/snapshot |
|
|
246
|
+
| Compaction (every 3rd) | Deletes child entries of stable subtrees, prunes old snapshots | Reclaims 50-90% |
|
|
247
|
+
| Rediscovery (every 10th) | Full depth-3 scan to catch new growth | ~6 MB |
|
|
248
|
+
|
|
249
|
+
**Comparison**: naive full `/` scans produce ~58 MB per snapshot (705 MB for 7 snapshots). Adaptive mode keeps the DB under ~25 MB across 100+ scans.
|
|
250
|
+
|
|
251
|
+
Configure in the web UI under Settings > Adaptive Scanning, or via CLI flags.
|
|
252
|
+
|
|
253
|
+
## Architecture
|
|
254
|
+
|
|
255
|
+
```
|
|
256
|
+
src/sldd/
|
|
257
|
+
models.py Pure dataclasses — Snapshot, DirDiff, Anomaly, configs
|
|
258
|
+
snapshot.py Filesystem walker — os.scandir + size aggregation
|
|
259
|
+
storage.py SQLite repository — snapshots, entries, path tracking
|
|
260
|
+
diff.py Diff engine — SQL join to compare snapshots
|
|
261
|
+
detect.py Anomaly detection — threshold, statistical, attribution
|
|
262
|
+
adaptive.py Adaptive scan engine — plan, track, compact
|
|
263
|
+
api.py Public API facade (SLDD class)
|
|
264
|
+
cli.py Click CLI
|
|
265
|
+
server.py FastAPI web server + REST endpoints
|
|
266
|
+
scheduler.py Watch mode scheduler
|
|
267
|
+
report.py Terminal report formatting
|
|
268
|
+
delete.py Safe deletion with blocklist
|
|
269
|
+
playback.py Playback frame generation
|
|
270
|
+
|
|
271
|
+
frontend/
|
|
272
|
+
src/api.ts TypeScript API client
|
|
273
|
+
src/App.tsx Router + layout + toast system
|
|
274
|
+
src/views/ Dashboard, Playback, Explorer, Deletion, Settings
|
|
275
|
+
src/components/ Card, ResizableTable, DepthFilter, Toast
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## Development
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
# Install dev dependencies
|
|
282
|
+
pip install -e ".[dev,web]"
|
|
283
|
+
|
|
284
|
+
# Run tests (136 tests)
|
|
285
|
+
pytest
|
|
286
|
+
|
|
287
|
+
# Lint
|
|
288
|
+
ruff check src/
|
|
289
|
+
|
|
290
|
+
# Type check
|
|
291
|
+
mypy src/sldd/
|
|
292
|
+
|
|
293
|
+
# Frontend type check
|
|
294
|
+
cd frontend && npx tsc --noEmit
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
## Distribution
|
|
298
|
+
|
|
299
|
+
| Method | Use case |
|
|
300
|
+
| -------------------------------------- | ------------------------------------------------------------------- |
|
|
301
|
+
| `pip install storage-leak-diff-detector[web]` | PyPI (no build, frontend bundled) |
|
|
302
|
+
| `pip install <wheel URL>` | GitHub release wheel (no build) |
|
|
303
|
+
| `pip install storage-leak-diff-detector` | CLI only (no web dashboard) |
|
|
304
|
+
| Source + `pip install ".[web]"` | From clone (requires Node.js for first build) |
|
|
305
|
+
| Source + `pip install -e ".[dev,web]"` | Development, contributions |
|
|
306
|
+
| PyInstaller / Nuitka | Standalone executable — build scripts TBD |
|
|
307
|
+
| Docker | Isolated environment — Dockerfile TBD |
|
|
308
|
+
|
|
309
|
+
**Publishing a release:** Run `cd frontend && npm run build`, then `python scripts/prepare_build.py` to copy the built frontend into the package, then `python -m build`.
|
|
310
|
+
|
|
311
|
+
## License
|
|
312
|
+
|
|
313
|
+
MIT
|