diskuh 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diskuh/__init__.py +1 -0
- diskuh/__main__.py +4 -0
- diskuh/cache.py +452 -0
- diskuh/cli.py +210 -0
- diskuh/format.py +223 -0
- diskuh/scanner.py +134 -0
- diskuh/tui.py +409 -0
- diskuh-0.1.0.dist-info/METADATA +59 -0
- diskuh-0.1.0.dist-info/RECORD +12 -0
- diskuh-0.1.0.dist-info/WHEEL +4 -0
- diskuh-0.1.0.dist-info/entry_points.txt +5 -0
- diskuh-0.1.0.dist-info/licenses/LICENSE +21 -0
diskuh/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
diskuh/__main__.py
ADDED
diskuh/cache.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
"""SQLite-backed scan cache.
|
|
2
|
+
|
|
3
|
+
Design
|
|
4
|
+
------
|
|
5
|
+
Scan results are cached per-directory in a SQLite database at
|
|
6
|
+
``~/.diskuh/cache.sqlite3``. Only directories are cached (not individual
|
|
7
|
+
files) -- files are cheap to `stat` once their parent directory is already
|
|
8
|
+
being visited, so caching them separately would bloat the database for no
|
|
9
|
+
benefit.
|
|
10
|
+
|
|
11
|
+
Staleness check ("is this directory's cached size still good?") uses a
|
|
12
|
+
**metadata fingerprint**: a hash of the immediate children's
|
|
13
|
+
``(name, kind, size, mtime_ns)``, computed from a single ``os.scandir()``
|
|
14
|
+
pass over that directory (never its full subtree). If a directory's own
|
|
15
|
+
``(mtime_ns, entry_count, fingerprint)`` still match what's cached, its
|
|
16
|
+
*entire* cached subtree aggregate is trusted and **we do not recurse into
|
|
17
|
+
it at all** -- this is the actual speedup: an unchanged directory costs one
|
|
18
|
+
`stat` + one shallow `scandir`, regardless of how large or deep the subtree
|
|
19
|
+
beneath it is. A mismatch means something in that directory changed
|
|
20
|
+
(entry added/removed/renamed/resized), so it's rescanned -- but each of
|
|
21
|
+
*its* children still gets an independent trust check, so one changed
|
|
22
|
+
directory doesn't invalidate unrelated subtrees elsewhere in the tree.
|
|
23
|
+
|
|
24
|
+
Known limitation (accepted trade-off, same class of heuristic used by tools
|
|
25
|
+
like ``ncdu``/``duc``/``make``): editing a file's *contents* in place (same
|
|
26
|
+
name, changed size/mtime) does not change its *parent* directory's own
|
|
27
|
+
mtime -- only the file's own entry does. Our fingerprint *does* catch this
|
|
28
|
+
for the file's direct parent (the file's own size/mtime is part of that
|
|
29
|
+
directory's immediate-children hash). But if some ancestor further up is
|
|
30
|
+
trusted and we stop recursing there, we never reach down far enough to see
|
|
31
|
+
it. So an in-place edit below an otherwise-untouched ancestor can go
|
|
32
|
+
unnoticed until something else in that ancestor's chain changes. Use
|
|
33
|
+
``--no-cache``/``--rescan`` for a guaranteed-fresh, guaranteed-accurate
|
|
34
|
+
scan when that matters.
|
|
35
|
+
|
|
36
|
+
Note on an earlier draft of this algorithm: a version that always recurses
|
|
37
|
+
into every child directory regardless of trust (to "be extra safe") was
|
|
38
|
+
considered, but it means every directory gets a full `scandir` on every
|
|
39
|
+
single scan either way -- identical total cost to an uncached scan, so it
|
|
40
|
+
provides no actual caching speedup at all. The short-circuit implemented
|
|
41
|
+
here (skip recursion entirely on a trust hit) is what makes the cache
|
|
42
|
+
actually pay for itself on repeat scans, at the cost of the documented
|
|
43
|
+
blind spot above.
|
|
44
|
+
|
|
45
|
+
Traversal is iterative (an explicit stack, post-order aggregation pass),
|
|
46
|
+
mirroring `scanner.scan`, so it doesn't rely on Python-level recursion and
|
|
47
|
+
can't blow the recursion limit on very deep trees.
|
|
48
|
+
|
|
49
|
+
A directory that can't be scanned at all (e.g. permission denied) still
|
|
50
|
+
gets a row stored, carrying its error message in the `error` column --
|
|
51
|
+
without this, the error would only ever be reported on the very first,
|
|
52
|
+
uncached scan: the moment its *parent* becomes a trusted cache hit, the
|
|
53
|
+
parent stops re-visiting it and the problem would silently disappear from
|
|
54
|
+
future reports even though it's still just as inaccessible. A trusted
|
|
55
|
+
parent checks its immediate children's stored `error` and re-surfaces it.
|
|
56
|
+
This only bridges one level per trusted hop, though: an error several
|
|
57
|
+
levels below multiple nested trusted ancestors may not resurface until
|
|
58
|
+
something along that chain changes (or `--no-cache` is used) -- the same
|
|
59
|
+
class of trade-off as the staleness blind spot above.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
from __future__ import annotations
|
|
63
|
+
|
|
64
|
+
import hashlib
|
|
65
|
+
import os
|
|
66
|
+
import sqlite3
|
|
67
|
+
import time
|
|
68
|
+
from pathlib import Path
|
|
69
|
+
from typing import Callable
|
|
70
|
+
|
|
71
|
+
from diskuh.scanner import DirNode, Entry, FileNode, disk_size
|
|
72
|
+
|
|
73
|
+
DB_DIR = Path.home() / ".diskuh"
|
|
74
|
+
DB_PATH = DB_DIR / "cache.sqlite3"
|
|
75
|
+
|
|
76
|
+
SCHEMA = """
|
|
77
|
+
CREATE TABLE IF NOT EXISTS scan_roots (
|
|
78
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
79
|
+
path TEXT NOT NULL UNIQUE,
|
|
80
|
+
last_scanned REAL NOT NULL
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
CREATE TABLE IF NOT EXISTS directories (
|
|
84
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
85
|
+
root_id INTEGER NOT NULL REFERENCES scan_roots(id) ON DELETE CASCADE,
|
|
86
|
+
path TEXT NOT NULL,
|
|
87
|
+
parent_path TEXT,
|
|
88
|
+
dir_mtime_ns INTEGER NOT NULL,
|
|
89
|
+
entry_count INTEGER NOT NULL,
|
|
90
|
+
fingerprint TEXT NOT NULL,
|
|
91
|
+
agg_size INTEGER NOT NULL,
|
|
92
|
+
agg_file_count INTEGER NOT NULL,
|
|
93
|
+
last_scanned REAL NOT NULL,
|
|
94
|
+
-- Set when this directory itself couldn't be scanned (e.g. permission
|
|
95
|
+
-- denied). Without this, a directory that fails to scan never gets a
|
|
96
|
+
-- row stored at all, so the moment its *parent* becomes a cache hit,
|
|
97
|
+
-- the error silently stops being reported even though the underlying
|
|
98
|
+
-- problem hasn't gone away. NULL means no error.
|
|
99
|
+
error TEXT,
|
|
100
|
+
UNIQUE(root_id, path)
|
|
101
|
+
);
|
|
102
|
+
CREATE INDEX IF NOT EXISTS idx_directories_root_path ON directories(root_id, path);
|
|
103
|
+
CREATE INDEX IF NOT EXISTS idx_directories_parent ON directories(root_id, parent_path);
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def connect(db_path: Path | None = None) -> sqlite3.Connection:
|
|
108
|
+
if db_path is None:
|
|
109
|
+
db_path = DB_PATH # read at call time so tests can monkeypatch it
|
|
110
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
# check_same_thread=False: the TUI runs scans on a worker thread (to
|
|
112
|
+
# keep the UI responsive) while the connection is created on the main
|
|
113
|
+
# thread. We never use it from two threads *concurrently* -- scans are
|
|
114
|
+
# exclusive/sequenced -- so a single connection without the default's
|
|
115
|
+
# same-thread restriction is safe here.
|
|
116
|
+
conn = sqlite3.connect(db_path, check_same_thread=False)
|
|
117
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
118
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
119
|
+
conn.row_factory = sqlite3.Row
|
|
120
|
+
return conn
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def init_schema(conn: sqlite3.Connection) -> None:
|
|
124
|
+
conn.executescript(SCHEMA)
|
|
125
|
+
conn.commit()
|
|
126
|
+
_migrate(conn)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _migrate(conn: sqlite3.Connection) -> None:
|
|
130
|
+
"""`CREATE TABLE IF NOT EXISTS` doesn't retrofit new columns onto a
|
|
131
|
+
database file created by an older version of diskuh, so do that by
|
|
132
|
+
hand for anything added after the initial schema."""
|
|
133
|
+
cols = {row["name"] for row in conn.execute("PRAGMA table_info(directories)")}
|
|
134
|
+
if "error" not in cols:
|
|
135
|
+
conn.execute("ALTER TABLE directories ADD COLUMN error TEXT")
|
|
136
|
+
conn.commit()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def compute_fingerprint(children_meta: list[tuple[str, str, int, int]]) -> str:
|
|
140
|
+
"""Hash the sorted (name, kind, size, mtime_ns) of a directory's
|
|
141
|
+
immediate children. `kind` is 'f' for file, 'd' for dir."""
|
|
142
|
+
h = hashlib.blake2b(digest_size=16)
|
|
143
|
+
for name, kind, size, mtime_ns in sorted(children_meta):
|
|
144
|
+
h.update(name.encode("utf-8", "surrogateescape"))
|
|
145
|
+
h.update(kind.encode("ascii"))
|
|
146
|
+
h.update(int(size).to_bytes(8, "little", signed=False))
|
|
147
|
+
h.update(int(mtime_ns).to_bytes(8, "little", signed=True))
|
|
148
|
+
return h.hexdigest()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _get_root_id(conn: sqlite3.Connection, root_path: Path) -> int:
|
|
152
|
+
now = time.time()
|
|
153
|
+
row = conn.execute(
|
|
154
|
+
"SELECT id FROM scan_roots WHERE path = ?", (str(root_path),)
|
|
155
|
+
).fetchone()
|
|
156
|
+
if row:
|
|
157
|
+
conn.execute(
|
|
158
|
+
"UPDATE scan_roots SET last_scanned = ? WHERE id = ?", (now, row["id"])
|
|
159
|
+
)
|
|
160
|
+
conn.commit()
|
|
161
|
+
return row["id"]
|
|
162
|
+
cur = conn.execute(
|
|
163
|
+
"INSERT INTO scan_roots (path, last_scanned) VALUES (?, ?)",
|
|
164
|
+
(str(root_path), now),
|
|
165
|
+
)
|
|
166
|
+
conn.commit()
|
|
167
|
+
return cur.lastrowid
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _fetch_row(conn: sqlite3.Connection, root_id: int, path: Path) -> sqlite3.Row | None:
|
|
171
|
+
return conn.execute(
|
|
172
|
+
"SELECT * FROM directories WHERE root_id = ? AND path = ?",
|
|
173
|
+
(root_id, str(path)),
|
|
174
|
+
).fetchone()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _store_row(
|
|
178
|
+
conn: sqlite3.Connection,
|
|
179
|
+
root_id: int,
|
|
180
|
+
path: Path,
|
|
181
|
+
dir_mtime_ns: int,
|
|
182
|
+
entry_count: int,
|
|
183
|
+
fingerprint: str,
|
|
184
|
+
agg_size: int,
|
|
185
|
+
agg_file_count: int,
|
|
186
|
+
error: str | None = None,
|
|
187
|
+
) -> None:
|
|
188
|
+
now = time.time()
|
|
189
|
+
conn.execute(
|
|
190
|
+
"""
|
|
191
|
+
INSERT INTO directories
|
|
192
|
+
(root_id, path, parent_path, dir_mtime_ns, entry_count,
|
|
193
|
+
fingerprint, agg_size, agg_file_count, last_scanned, error)
|
|
194
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
195
|
+
ON CONFLICT(root_id, path) DO UPDATE SET
|
|
196
|
+
parent_path=excluded.parent_path,
|
|
197
|
+
dir_mtime_ns=excluded.dir_mtime_ns,
|
|
198
|
+
entry_count=excluded.entry_count,
|
|
199
|
+
fingerprint=excluded.fingerprint,
|
|
200
|
+
agg_size=excluded.agg_size,
|
|
201
|
+
agg_file_count=excluded.agg_file_count,
|
|
202
|
+
last_scanned=excluded.last_scanned,
|
|
203
|
+
error=excluded.error
|
|
204
|
+
""",
|
|
205
|
+
(
|
|
206
|
+
root_id,
|
|
207
|
+
str(path),
|
|
208
|
+
str(path.parent),
|
|
209
|
+
dir_mtime_ns,
|
|
210
|
+
entry_count,
|
|
211
|
+
fingerprint,
|
|
212
|
+
agg_size,
|
|
213
|
+
agg_file_count,
|
|
214
|
+
now,
|
|
215
|
+
error,
|
|
216
|
+
),
|
|
217
|
+
)
|
|
218
|
+
conn.commit()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _check_dir(
|
|
222
|
+
conn: sqlite3.Connection, root_id: int, path: Path, *, force_rescan: bool
|
|
223
|
+
):
|
|
224
|
+
"""Do the one-directory work needed to decide cache trust: stat the
|
|
225
|
+
directory, look up its cached row, and do a single shallow `scandir` to
|
|
226
|
+
compute a fresh fingerprint. No recursion, no writes -- shared by
|
|
227
|
+
`get_or_refresh`'s traversal and the standalone `is_cached` peek."""
|
|
228
|
+
st = path.lstat() # may raise OSError; caller handles it
|
|
229
|
+
row = None if force_rescan else _fetch_row(conn, root_id, path)
|
|
230
|
+
|
|
231
|
+
children_meta: list[tuple[str, str, int, int]] = []
|
|
232
|
+
live_entries: list[tuple[bool, os.stat_result, Path]] = []
|
|
233
|
+
errors: list[str] = []
|
|
234
|
+
with os.scandir(path) as it: # may raise OSError; caller handles it
|
|
235
|
+
for entry in it:
|
|
236
|
+
try:
|
|
237
|
+
if entry.is_symlink():
|
|
238
|
+
continue
|
|
239
|
+
cst = entry.stat(follow_symlinks=False)
|
|
240
|
+
except OSError as e:
|
|
241
|
+
errors.append(f"{entry.path}: {e}")
|
|
242
|
+
continue
|
|
243
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
244
|
+
kind = "d" if is_dir else "f"
|
|
245
|
+
children_meta.append((entry.name, kind, cst.st_size, cst.st_mtime_ns))
|
|
246
|
+
live_entries.append((is_dir, cst, Path(entry.path)))
|
|
247
|
+
|
|
248
|
+
fingerprint = compute_fingerprint(children_meta)
|
|
249
|
+
entry_count = len(children_meta)
|
|
250
|
+
trusted = (
|
|
251
|
+
row is not None
|
|
252
|
+
and row["fingerprint"] == fingerprint
|
|
253
|
+
and row["dir_mtime_ns"] == st.st_mtime_ns
|
|
254
|
+
and row["entry_count"] == entry_count
|
|
255
|
+
)
|
|
256
|
+
return st, row, live_entries, fingerprint, entry_count, trusted, errors
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def is_cached(conn: sqlite3.Connection, root_path: Path, path: Path) -> bool:
|
|
260
|
+
"""Cheap check: would `get_or_refresh(path)` hit the cache (fast) or
|
|
261
|
+
need a real (re)scan (potentially expensive)? Does one shallow
|
|
262
|
+
`scandir` of `path` itself -- same cost as the real trust check -- but
|
|
263
|
+
never recurses and never writes to the cache. Returns True if `path`
|
|
264
|
+
doesn't exist or can't be read (nothing to scan, so nothing "expensive"
|
|
265
|
+
would happen)."""
|
|
266
|
+
root_path = Path(root_path).resolve()
|
|
267
|
+
path = Path(path).resolve()
|
|
268
|
+
root_id = _get_root_id(conn, root_path)
|
|
269
|
+
try:
|
|
270
|
+
*_, trusted, _errors = _check_dir(conn, root_id, path, force_rescan=False)
|
|
271
|
+
except OSError:
|
|
272
|
+
return True
|
|
273
|
+
return trusted
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def get_or_refresh(
|
|
277
|
+
conn: sqlite3.Connection,
|
|
278
|
+
root_path: Path,
|
|
279
|
+
path: Path,
|
|
280
|
+
*,
|
|
281
|
+
force_rescan: bool = False,
|
|
282
|
+
on_progress: Callable[[Path], None] | None = None,
|
|
283
|
+
) -> DirNode:
|
|
284
|
+
"""Return an aggregated `DirNode` for `path`, using the cache when safe.
|
|
285
|
+
|
|
286
|
+
Directories whose fingerprint still matches the cache are returned with
|
|
287
|
+
their cached aggregate size/file_count and are NOT recursed into further
|
|
288
|
+
(their immediate children are still listed, as lightweight stubs for
|
|
289
|
+
subdirectories, so callers can display or lazily expand one more level).
|
|
290
|
+
Directories that changed (or aren't cached yet, or `force_rescan=True`)
|
|
291
|
+
are fully rescanned, recursing into each child directory independently.
|
|
292
|
+
|
|
293
|
+
`on_progress`, if given, is called with the path of each directory as
|
|
294
|
+
it's visited (whether trusted or rescanned) -- useful for showing live
|
|
295
|
+
progress on a scan that may touch a great many directories.
|
|
296
|
+
"""
|
|
297
|
+
root_path = Path(root_path).resolve()
|
|
298
|
+
path = Path(path).resolve()
|
|
299
|
+
root_id = _get_root_id(conn, root_path)
|
|
300
|
+
|
|
301
|
+
root_node = DirNode(path=path)
|
|
302
|
+
stack: list[DirNode] = [root_node]
|
|
303
|
+
# Nodes that needed a (re)scan, in pop order (parent before its own
|
|
304
|
+
# children, since children are pushed after their parent is popped).
|
|
305
|
+
# Reversing gives child-before-parent, which is what post-order
|
|
306
|
+
# aggregation needs -- same trick as scanner.scan.
|
|
307
|
+
rescanned_order: list[DirNode] = []
|
|
308
|
+
fingerprints: dict[str, str] = {}
|
|
309
|
+
|
|
310
|
+
while stack:
|
|
311
|
+
node = stack.pop()
|
|
312
|
+
if on_progress is not None:
|
|
313
|
+
on_progress(node.path)
|
|
314
|
+
try:
|
|
315
|
+
st, row, live_entries, fingerprint, entry_count, trusted, errors = _check_dir(
|
|
316
|
+
conn, root_id, node.path, force_rescan=force_rescan
|
|
317
|
+
)
|
|
318
|
+
except OSError as e:
|
|
319
|
+
err_msg = f"{node.path}: {e}"
|
|
320
|
+
node.errors.append(err_msg)
|
|
321
|
+
# Persist this, not just report it in-memory this one time --
|
|
322
|
+
# otherwise the moment this node's *parent* becomes a cache
|
|
323
|
+
# hit, the error silently stops being reported even though
|
|
324
|
+
# the underlying problem (e.g. permission denied) hasn't
|
|
325
|
+
# gone away. Best-effort mtime: if even lstat() failed, 0 is
|
|
326
|
+
# fine -- this row exists purely to carry the error forward.
|
|
327
|
+
try:
|
|
328
|
+
mtime_ns = node.path.lstat().st_mtime_ns
|
|
329
|
+
except OSError:
|
|
330
|
+
mtime_ns = 0
|
|
331
|
+
node.mtime_ns = mtime_ns
|
|
332
|
+
_store_row(conn, root_id, node.path, mtime_ns, 0, "", 0, 0, error=err_msg)
|
|
333
|
+
continue
|
|
334
|
+
node.mtime_ns = st.st_mtime_ns
|
|
335
|
+
node.entry_count = entry_count
|
|
336
|
+
node.errors.extend(errors)
|
|
337
|
+
|
|
338
|
+
if trusted:
|
|
339
|
+
node.size = row["agg_size"]
|
|
340
|
+
node.file_count = row["agg_file_count"]
|
|
341
|
+
for is_dir, cst, full_path in live_entries:
|
|
342
|
+
if is_dir:
|
|
343
|
+
# Seed the stub's size/file_count from ITS OWN cached
|
|
344
|
+
# row (a cheap indexed lookup) so it displays correctly
|
|
345
|
+
# even if a caller never expands it further -- e.g.
|
|
346
|
+
# `--depth 1` only ever sees these stubs.
|
|
347
|
+
child_row = _fetch_row(conn, root_id, full_path)
|
|
348
|
+
stub = DirNode(path=full_path, mtime_ns=cst.st_mtime_ns, expanded=False)
|
|
349
|
+
if child_row is not None:
|
|
350
|
+
stub.size = child_row["agg_size"]
|
|
351
|
+
stub.file_count = child_row["agg_file_count"]
|
|
352
|
+
if child_row["error"]:
|
|
353
|
+
# Surface a previously-recorded error even
|
|
354
|
+
# though we're not re-descending into this
|
|
355
|
+
# child (see the "error" column comment in
|
|
356
|
+
# SCHEMA). Known limitation: this only
|
|
357
|
+
# reaches one level up per trusted hop, so an
|
|
358
|
+
# error many levels below several *nested*
|
|
359
|
+
# trusted ancestors may not resurface until
|
|
360
|
+
# something along that chain changes, or
|
|
361
|
+
# --no-cache is used.
|
|
362
|
+
stub.errors.append(child_row["error"])
|
|
363
|
+
node.errors.append(child_row["error"])
|
|
364
|
+
node.children.append(stub)
|
|
365
|
+
else:
|
|
366
|
+
node.children.append(
|
|
367
|
+
FileNode(path=full_path, size=disk_size(cst), mtime_ns=cst.st_mtime_ns)
|
|
368
|
+
)
|
|
369
|
+
continue # cache win: nothing below this directory is touched
|
|
370
|
+
|
|
371
|
+
fingerprints[str(node.path)] = fingerprint
|
|
372
|
+
rescanned_order.append(node)
|
|
373
|
+
for is_dir, cst, full_path in live_entries:
|
|
374
|
+
if is_dir:
|
|
375
|
+
child = DirNode(path=full_path, mtime_ns=cst.st_mtime_ns)
|
|
376
|
+
node.children.append(child)
|
|
377
|
+
stack.append(child)
|
|
378
|
+
else:
|
|
379
|
+
fsize = disk_size(cst)
|
|
380
|
+
fnode = FileNode(path=full_path, size=fsize, mtime_ns=cst.st_mtime_ns)
|
|
381
|
+
node.children.append(fnode)
|
|
382
|
+
node.size += fsize
|
|
383
|
+
node.file_count += 1
|
|
384
|
+
|
|
385
|
+
for node in reversed(rescanned_order):
|
|
386
|
+
for child in node.children:
|
|
387
|
+
if isinstance(child, DirNode):
|
|
388
|
+
node.size += child.size
|
|
389
|
+
node.file_count += child.file_count
|
|
390
|
+
node.errors.extend(child.errors)
|
|
391
|
+
_store_row(
|
|
392
|
+
conn,
|
|
393
|
+
root_id,
|
|
394
|
+
node.path,
|
|
395
|
+
node.mtime_ns,
|
|
396
|
+
node.entry_count,
|
|
397
|
+
fingerprints[str(node.path)],
|
|
398
|
+
node.size,
|
|
399
|
+
node.file_count,
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
return root_node
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def expand(
|
|
406
|
+
conn: sqlite3.Connection,
|
|
407
|
+
root_path: Path,
|
|
408
|
+
node: DirNode,
|
|
409
|
+
*,
|
|
410
|
+
force_rescan: bool = False,
|
|
411
|
+
on_progress: Callable[[Path], None] | None = None,
|
|
412
|
+
) -> None:
|
|
413
|
+
"""Replace a cache-collapsed stub `node` (see `DirNode.expanded`) in
|
|
414
|
+
place with its real children, one level down. A renderer walks the tree
|
|
415
|
+
and calls this on any stub it needs to descend past (e.g. because
|
|
416
|
+
`--depth` asks for rows below it); nodes it never descends into stay
|
|
417
|
+
collapsed and cost nothing further."""
|
|
418
|
+
if node.expanded:
|
|
419
|
+
return
|
|
420
|
+
fresh = get_or_refresh(
|
|
421
|
+
conn, root_path, node.path, force_rescan=force_rescan, on_progress=on_progress
|
|
422
|
+
)
|
|
423
|
+
node.children = fresh.children
|
|
424
|
+
node.entry_count = fresh.entry_count
|
|
425
|
+
node.errors = fresh.errors
|
|
426
|
+
node.size = fresh.size
|
|
427
|
+
node.file_count = fresh.file_count
|
|
428
|
+
node.mtime_ns = fresh.mtime_ns
|
|
429
|
+
node.expanded = True
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def evict_path(conn: sqlite3.Connection, root_path: Path, path: Path) -> None:
|
|
433
|
+
"""Drop cached rows for `path` (and everything under it) plus its
|
|
434
|
+
parent, so the parent's next fingerprint check notices the missing
|
|
435
|
+
entry. Call this after deleting a file/directory from the TUI."""
|
|
436
|
+
root_path = Path(root_path).resolve()
|
|
437
|
+
path = Path(path).resolve()
|
|
438
|
+
root_row = conn.execute(
|
|
439
|
+
"SELECT id FROM scan_roots WHERE path = ?", (str(root_path),)
|
|
440
|
+
).fetchone()
|
|
441
|
+
if root_row is None:
|
|
442
|
+
return
|
|
443
|
+
root_id = root_row["id"]
|
|
444
|
+
conn.execute(
|
|
445
|
+
"DELETE FROM directories WHERE root_id = ? AND (path = ? OR path LIKE ? || '/%')",
|
|
446
|
+
(root_id, str(path), str(path)),
|
|
447
|
+
)
|
|
448
|
+
conn.execute(
|
|
449
|
+
"DELETE FROM directories WHERE root_id = ? AND path = ?",
|
|
450
|
+
(root_id, str(path.parent)),
|
|
451
|
+
)
|
|
452
|
+
conn.commit()
|
diskuh/cli.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from contextlib import contextmanager
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
|
|
10
|
+
from diskuh.format import DEFAULT_HEAD
|
|
11
|
+
|
|
12
|
+
# How often the progress display's *content* actually redraws. It's called
|
|
13
|
+
# once per directory visited -- possibly tens of thousands of times on a
|
|
14
|
+
# big tree -- so redrawing on every single call would make the progress
|
|
15
|
+
# display itself a meaningful chunk of the scan's cost. The count is still
|
|
16
|
+
# tracked exactly; only the (relatively expensive) repaint is throttled.
|
|
17
|
+
_PROGRESS_UPDATE_INTERVAL = 1.0
|
|
18
|
+
|
|
19
|
+
# Rich's own spinner animation redraws independently of our content
|
|
20
|
+
# throttling above -- by default at ~12.5Hz, for the entire scan duration,
|
|
21
|
+
# regardless of whether the text changed. On a huge tree (hundreds of
|
|
22
|
+
# thousands of directories, possibly minutes of scanning) that's a lot of
|
|
23
|
+
# terminal writes competing for CPU with the actual scan. Match it to our
|
|
24
|
+
# own update cadence instead.
|
|
25
|
+
_SPINNER_REFRESH_PER_SECOND = 1 / _PROGRESS_UPDATE_INTERVAL
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@contextmanager
|
|
29
|
+
def _scan_progress(console: Console):
|
|
30
|
+
"""Show a live "Scanning..." spinner (directory count + current path)
|
|
31
|
+
on `console` while the body runs, so a slow scan doesn't look like the
|
|
32
|
+
tool has just hung. Silent when `console` isn't an interactive terminal
|
|
33
|
+
(piped/redirected), matching how du/curl/pip etc. behave -- and always
|
|
34
|
+
on stderr, so it never mixes into piped stdout output."""
|
|
35
|
+
if not console.is_terminal:
|
|
36
|
+
yield None
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
count = 0
|
|
40
|
+
last_update = 0.0
|
|
41
|
+
with console.status("Scanning...", refresh_per_second=_SPINNER_REFRESH_PER_SECOND) as status:
|
|
42
|
+
|
|
43
|
+
def on_progress(path):
|
|
44
|
+
nonlocal count, last_update
|
|
45
|
+
count += 1
|
|
46
|
+
now = time.monotonic()
|
|
47
|
+
if now - last_update >= _PROGRESS_UPDATE_INTERVAL:
|
|
48
|
+
last_update = now
|
|
49
|
+
status.update(f"Scanning... {count} directories [dim]{path}[/dim]")
|
|
50
|
+
|
|
51
|
+
yield on_progress
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _report_errors(errors: list[str], *, show_errors: bool) -> None:
|
|
55
|
+
"""By default, a scan hitting hundreds of permission-denied
|
|
56
|
+
subdirectories (common under ~/Library on macOS, say) would otherwise
|
|
57
|
+
dump one line per error -- easily swamping the actual report. Print a
|
|
58
|
+
one-line summary instead unless the caller asked to see them all."""
|
|
59
|
+
if not errors:
|
|
60
|
+
return
|
|
61
|
+
if show_errors:
|
|
62
|
+
for err in errors:
|
|
63
|
+
click.echo(f"diskuh: {err}", err=True)
|
|
64
|
+
else:
|
|
65
|
+
click.echo(
|
|
66
|
+
f"diskuh: {len(errors)} error(s) while scanning (e.g. permission denied); "
|
|
67
|
+
"rerun with --show-errors to see them.",
|
|
68
|
+
err=True,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@click.command()
|
|
73
|
+
@click.argument(
|
|
74
|
+
"path",
|
|
75
|
+
type=click.Path(exists=True, file_okay=True, dir_okay=True, path_type=Path),
|
|
76
|
+
default=".",
|
|
77
|
+
)
|
|
78
|
+
@click.option(
|
|
79
|
+
"--depth",
|
|
80
|
+
"-d",
|
|
81
|
+
"depth",
|
|
82
|
+
type=int,
|
|
83
|
+
default=1,
|
|
84
|
+
show_default=True,
|
|
85
|
+
help="Limit reported directory depth (like du --max-depth); 0 = unlimited. "
|
|
86
|
+
"Sizes are always accurate regardless of depth.",
|
|
87
|
+
)
|
|
88
|
+
@click.option(
|
|
89
|
+
"--terse",
|
|
90
|
+
is_flag=True,
|
|
91
|
+
default=False,
|
|
92
|
+
help="Plain script-friendly output: size<TAB>path, no colors/bars.",
|
|
93
|
+
)
|
|
94
|
+
@click.option(
|
|
95
|
+
"--no-cache",
|
|
96
|
+
"--rescan",
|
|
97
|
+
"no_cache",
|
|
98
|
+
is_flag=True,
|
|
99
|
+
default=False,
|
|
100
|
+
help="Bypass the cache; force a fresh full scan.",
|
|
101
|
+
)
|
|
102
|
+
@click.option(
|
|
103
|
+
"--tui",
|
|
104
|
+
is_flag=True,
|
|
105
|
+
default=False,
|
|
106
|
+
help="Launch the full-screen TUI instead of printing to stdout.",
|
|
107
|
+
)
|
|
108
|
+
@click.option(
|
|
109
|
+
"--head",
|
|
110
|
+
"head",
|
|
111
|
+
type=int,
|
|
112
|
+
default=None,
|
|
113
|
+
metavar="N",
|
|
114
|
+
help=f"Show only the N largest entries (default: {DEFAULT_HEAD}); 0 = unlimited.",
|
|
115
|
+
)
|
|
116
|
+
@click.option(
|
|
117
|
+
"--tail",
|
|
118
|
+
"tail",
|
|
119
|
+
type=int,
|
|
120
|
+
default=None,
|
|
121
|
+
metavar="N",
|
|
122
|
+
help="Show only the N smallest entries. Overrides the default --head.",
|
|
123
|
+
)
|
|
124
|
+
@click.option(
|
|
125
|
+
"--show-errors",
|
|
126
|
+
is_flag=True,
|
|
127
|
+
default=False,
|
|
128
|
+
help="Print every scan error individually instead of a one-line summary.",
|
|
129
|
+
)
|
|
130
|
+
def main(
|
|
131
|
+
path: Path,
|
|
132
|
+
depth: int | None,
|
|
133
|
+
terse: bool,
|
|
134
|
+
no_cache: bool,
|
|
135
|
+
tui: bool,
|
|
136
|
+
head: int | None,
|
|
137
|
+
tail: int | None,
|
|
138
|
+
show_errors: bool,
|
|
139
|
+
) -> None:
|
|
140
|
+
"""du, but with attractive human-readable output and an optional TUI."""
|
|
141
|
+
resolved = path.resolve()
|
|
142
|
+
|
|
143
|
+
if depth < 0:
|
|
144
|
+
raise click.UsageError("--depth must be 0 (unlimited) or a positive integer.")
|
|
145
|
+
if head is not None and head < 0:
|
|
146
|
+
raise click.UsageError("--head must be 0 (unlimited) or a positive integer.")
|
|
147
|
+
if tail is not None and tail <= 0:
|
|
148
|
+
raise click.UsageError("--tail must be a positive integer.")
|
|
149
|
+
if head is not None and head > 0 and tail is not None:
|
|
150
|
+
raise click.UsageError("--head and --tail are mutually exclusive.")
|
|
151
|
+
|
|
152
|
+
max_depth = None if depth == 0 else depth
|
|
153
|
+
if head is None and tail is None:
|
|
154
|
+
head = DEFAULT_HEAD # show only the biggest entries unless told otherwise
|
|
155
|
+
elif head == 0:
|
|
156
|
+
head = None # explicit opt-out of the default limit
|
|
157
|
+
|
|
158
|
+
if tui:
|
|
159
|
+
from diskuh.tui import run_tui
|
|
160
|
+
|
|
161
|
+
run_tui(resolved)
|
|
162
|
+
return
|
|
163
|
+
|
|
164
|
+
from diskuh import cache, format as fmt
|
|
165
|
+
|
|
166
|
+
conn = cache.connect()
|
|
167
|
+
cache.init_schema(conn)
|
|
168
|
+
|
|
169
|
+
progress_console = Console(stderr=True)
|
|
170
|
+
with _scan_progress(progress_console) as on_progress:
|
|
171
|
+
root_node = cache.get_or_refresh(
|
|
172
|
+
conn, resolved, resolved, force_rescan=no_cache, on_progress=on_progress
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def _expand(node):
|
|
176
|
+
cache.expand(conn, resolved, node, force_rescan=no_cache, on_progress=on_progress)
|
|
177
|
+
|
|
178
|
+
# Collecting entries can still trigger real scanning (expanding a
|
|
179
|
+
# cache-collapsed stub), so it happens *inside* the progress
|
|
180
|
+
# context. Rendering happens only after that context has fully
|
|
181
|
+
# torn down its live display below -- doing it while the spinner
|
|
182
|
+
# was still active (a separate Console, on stderr) used to garble
|
|
183
|
+
# the very first line of output where the two writes interleaved.
|
|
184
|
+
entries, hidden = fmt.collect_entries(
|
|
185
|
+
root_node, max_depth, expand=_expand, head=head, tail=tail
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
if terse:
|
|
189
|
+
fmt.render_terse(root_node, entries=entries, hidden=hidden)
|
|
190
|
+
else:
|
|
191
|
+
fmt.render_tree(Console(), root_node, entries=entries, hidden=hidden)
|
|
192
|
+
|
|
193
|
+
_report_errors(root_node.errors, show_errors=show_errors)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@click.command()
|
|
197
|
+
@click.argument(
|
|
198
|
+
"path",
|
|
199
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
200
|
+
default=".",
|
|
201
|
+
)
|
|
202
|
+
def tui_main(path: Path) -> None:
|
|
203
|
+
"""Launch the diskuh TUI directly."""
|
|
204
|
+
from diskuh.tui import run_tui
|
|
205
|
+
|
|
206
|
+
run_tui(path.resolve())
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
main()
|