diskuh 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diskuh-0.1.0/.github/workflows/publish.yml +54 -0
- diskuh-0.1.0/.gitignore +11 -0
- diskuh-0.1.0/LICENSE +21 -0
- diskuh-0.1.0/PKG-INFO +59 -0
- diskuh-0.1.0/README.md +43 -0
- diskuh-0.1.0/pyproject.toml +35 -0
- diskuh-0.1.0/src/diskuh/__init__.py +1 -0
- diskuh-0.1.0/src/diskuh/__main__.py +4 -0
- diskuh-0.1.0/src/diskuh/cache.py +452 -0
- diskuh-0.1.0/src/diskuh/cli.py +210 -0
- diskuh-0.1.0/src/diskuh/format.py +223 -0
- diskuh-0.1.0/src/diskuh/scanner.py +134 -0
- diskuh-0.1.0/src/diskuh/tui.py +409 -0
- diskuh-0.1.0/tests/conftest.py +50 -0
- diskuh-0.1.0/tests/test_cache.py +196 -0
- diskuh-0.1.0/tests/test_cli.py +209 -0
- diskuh-0.1.0/tests/test_format.py +82 -0
- diskuh-0.1.0/tests/test_scanner.py +58 -0
- diskuh-0.1.0/tests/test_tui.py +242 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Runs whenever a version tag (v0.1.0, v0.1.1, ...) is pushed. Publishing
|
|
4
|
+
# itself uses PyPI's Trusted Publisher (OIDC) mechanism -- no API token is
|
|
5
|
+
# stored in this repo; PyPI trusts this exact workflow file, running in
|
|
6
|
+
# this exact repo, under the "pypi" environment. See the PyPI project's
|
|
7
|
+
# "Publishing" settings (registered as a pending publisher before the
|
|
8
|
+
# project exists there) for the values that must match this file.
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
tags:
|
|
13
|
+
- "v*"
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
build:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.12"
|
|
24
|
+
|
|
25
|
+
- name: Install build tooling
|
|
26
|
+
run: python -m pip install --upgrade build
|
|
27
|
+
|
|
28
|
+
- name: Build sdist and wheel
|
|
29
|
+
run: python -m build
|
|
30
|
+
|
|
31
|
+
- name: Check distribution metadata
|
|
32
|
+
run: |
|
|
33
|
+
python -m pip install --upgrade twine
|
|
34
|
+
python -m twine check dist/*
|
|
35
|
+
|
|
36
|
+
- uses: actions/upload-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: dist
|
|
39
|
+
path: dist/
|
|
40
|
+
|
|
41
|
+
publish:
|
|
42
|
+
needs: build
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
environment: pypi
|
|
45
|
+
permissions:
|
|
46
|
+
id-token: write # required for PyPI Trusted Publishing
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/download-artifact@v4
|
|
49
|
+
with:
|
|
50
|
+
name: dist
|
|
51
|
+
path: dist/
|
|
52
|
+
|
|
53
|
+
- name: Publish to PyPI
|
|
54
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
diskuh-0.1.0/.gitignore
ADDED
diskuh-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ploskon
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
diskuh-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: diskuh
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: du, but with attractive human-readable CLI output and an optional TUI
|
|
5
|
+
Author-email: ploskon <ploskon@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Requires-Dist: click>=8.1
|
|
10
|
+
Requires-Dist: rich>=13.7
|
|
11
|
+
Requires-Dist: textual>=0.58
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# diskuh
|
|
18
|
+
|
|
19
|
+
`du`, but with attractive human-readable output — numeric sizes plus a
|
|
20
|
+
proportional graphical bar per entry — and an optional full-screen TUI.
|
|
21
|
+
|
|
22
|
+
## Install (dev)
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
henv --name duh-dev -x python -m pip install -e ".[dev]"
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
diskuh [PATH] # human-readable sizes + bars, sorted by size desc
|
|
32
|
+
# (top-level entries only by default: depth 1)
|
|
33
|
+
diskuh --depth 2 [PATH] # go deeper (sizes are always accurate regardless of depth)
|
|
34
|
+
diskuh --depth 0 [PATH] # unlimited depth: every directory, recursively (the old default)
|
|
35
|
+
diskuh --head 10 [PATH] # show only the 10 largest entries
|
|
36
|
+
diskuh --tail 10 [PATH] # show only the 10 smallest entries
|
|
37
|
+
diskuh --terse [PATH] # plain "size<TAB>path" output, script/pipe friendly
|
|
38
|
+
diskuh --no-cache [PATH] # bypass the cache; force a fresh full scan
|
|
39
|
+
diskuh --tui [PATH] # launch the full-screen TUI
|
|
40
|
+
diskuh-tui [PATH] # same TUI, dedicated entry point
|
|
41
|
+
|
|
42
|
+
dhx [PATH] # short alias for diskuh (identical, same flags)
|
|
43
|
+
dhx-tui [PATH] # short alias for diskuh-tui
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Scan results are cached in a SQLite database at `~/.diskuh/cache.sqlite3`,
|
|
47
|
+
keyed on a per-directory metadata fingerprint (name/type/size/mtime of each
|
|
48
|
+
directory's immediate children) so repeat scans skip re-reading unchanged
|
|
49
|
+
subtrees. See `src/diskuh/cache.py` for the exact staleness algorithm and its
|
|
50
|
+
documented limitations.
|
|
51
|
+
|
|
52
|
+
In the TUI, press `d` on a selected entry to delete it (after confirmation).
|
|
53
|
+
Deletion is **permanent** — there is no trash/undo.
|
|
54
|
+
|
|
55
|
+
## Development
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
henv --name duh-dev -x pytest -v
|
|
59
|
+
```
|
diskuh-0.1.0/README.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# diskuh
|
|
2
|
+
|
|
3
|
+
`du`, but with attractive human-readable output — numeric sizes plus a
|
|
4
|
+
proportional graphical bar per entry — and an optional full-screen TUI.
|
|
5
|
+
|
|
6
|
+
## Install (dev)
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
henv --name duh-dev -x python -m pip install -e ".[dev]"
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Usage
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
diskuh [PATH] # human-readable sizes + bars, sorted by size desc
|
|
16
|
+
# (top-level entries only by default: depth 1)
|
|
17
|
+
diskuh --depth 2 [PATH] # go deeper (sizes are always accurate regardless of depth)
|
|
18
|
+
diskuh --depth 0 [PATH] # unlimited depth: every directory, recursively (the old default)
|
|
19
|
+
diskuh --head 10 [PATH] # show only the 10 largest entries
|
|
20
|
+
diskuh --tail 10 [PATH] # show only the 10 smallest entries
|
|
21
|
+
diskuh --terse [PATH] # plain "size<TAB>path" output, script/pipe friendly
|
|
22
|
+
diskuh --no-cache [PATH] # bypass the cache; force a fresh full scan
|
|
23
|
+
diskuh --tui [PATH] # launch the full-screen TUI
|
|
24
|
+
diskuh-tui [PATH] # same TUI, dedicated entry point
|
|
25
|
+
|
|
26
|
+
dhx [PATH] # short alias for diskuh (identical, same flags)
|
|
27
|
+
dhx-tui [PATH] # short alias for diskuh-tui
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Scan results are cached in a SQLite database at `~/.diskuh/cache.sqlite3`,
|
|
31
|
+
keyed on a per-directory metadata fingerprint (name/type/size/mtime of each
|
|
32
|
+
directory's immediate children) so repeat scans skip re-reading unchanged
|
|
33
|
+
subtrees. See `src/diskuh/cache.py` for the exact staleness algorithm and its
|
|
34
|
+
documented limitations.
|
|
35
|
+
|
|
36
|
+
In the TUI, press `d` on a selected entry to delete it (after confirmation).
|
|
37
|
+
Deletion is **permanent** — there is no trash/undo.
|
|
38
|
+
|
|
39
|
+
## Development
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
henv --name duh-dev -x pytest -v
|
|
43
|
+
```
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "diskuh"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "du, but with attractive human-readable CLI output and an optional TUI"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "ploskon", email = "ploskon@gmail.com" }]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"click>=8.1",
|
|
15
|
+
"rich>=13.7",
|
|
16
|
+
"textual>=0.58",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.optional-dependencies]
|
|
20
|
+
dev = [
|
|
21
|
+
"pytest>=8.0",
|
|
22
|
+
"pytest-asyncio>=0.24",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
diskuh = "diskuh.cli:main"
|
|
27
|
+
diskuh-tui = "diskuh.cli:tui_main"
|
|
28
|
+
dhx = "diskuh.cli:main"
|
|
29
|
+
dhx-tui = "diskuh.cli:tui_main"
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.wheel]
|
|
32
|
+
packages = ["src/diskuh"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
asyncio_mode = "auto"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
"""SQLite-backed scan cache.
|
|
2
|
+
|
|
3
|
+
Design
|
|
4
|
+
------
|
|
5
|
+
Scan results are cached per-directory in a SQLite database at
|
|
6
|
+
``~/.diskuh/cache.sqlite3``. Only directories are cached (not individual
|
|
7
|
+
files) -- files are cheap to `stat` once their parent directory is already
|
|
8
|
+
being visited, so caching them separately would bloat the database for no
|
|
9
|
+
benefit.
|
|
10
|
+
|
|
11
|
+
Staleness check ("is this directory's cached size still good?") uses a
|
|
12
|
+
**metadata fingerprint**: a hash of the immediate children's
|
|
13
|
+
``(name, kind, size, mtime_ns)``, computed from a single ``os.scandir()``
|
|
14
|
+
pass over that directory (never its full subtree). If a directory's own
|
|
15
|
+
``(mtime_ns, entry_count, fingerprint)`` still match what's cached, its
|
|
16
|
+
*entire* cached subtree aggregate is trusted and **we do not recurse into
|
|
17
|
+
it at all** -- this is the actual speedup: an unchanged directory costs one
|
|
18
|
+
`stat` + one shallow `scandir`, regardless of how large or deep the subtree
|
|
19
|
+
beneath it is. A mismatch means something in that directory changed
|
|
20
|
+
(entry added/removed/renamed/resized), so it's rescanned -- but each of
|
|
21
|
+
*its* children still gets an independent trust check, so one changed
|
|
22
|
+
directory doesn't invalidate unrelated subtrees elsewhere in the tree.
|
|
23
|
+
|
|
24
|
+
Known limitation (accepted trade-off, same class of heuristic used by tools
|
|
25
|
+
like ``ncdu``/``duc``/``make``): editing a file's *contents* in place (same
|
|
26
|
+
name, changed size/mtime) does not change its *parent* directory's own
|
|
27
|
+
mtime -- only the file's own entry does. Our fingerprint *does* catch this
|
|
28
|
+
for the file's direct parent (the file's own size/mtime is part of that
|
|
29
|
+
directory's immediate-children hash). But if some ancestor further up is
|
|
30
|
+
trusted and we stop recursing there, we never reach down far enough to see
|
|
31
|
+
it. So an in-place edit below an otherwise-untouched ancestor can go
|
|
32
|
+
unnoticed until something else in that ancestor's chain changes. Use
|
|
33
|
+
``--no-cache``/``--rescan`` for a guaranteed-fresh, guaranteed-accurate
|
|
34
|
+
scan when that matters.
|
|
35
|
+
|
|
36
|
+
Note on an earlier draft of this algorithm: a version that always recurses
|
|
37
|
+
into every child directory regardless of trust (to "be extra safe") was
|
|
38
|
+
considered, but it means every directory gets a full `scandir` on every
|
|
39
|
+
single scan either way -- identical total cost to an uncached scan, so it
|
|
40
|
+
provides no actual caching speedup at all. The short-circuit implemented
|
|
41
|
+
here (skip recursion entirely on a trust hit) is what makes the cache
|
|
42
|
+
actually pay for itself on repeat scans, at the cost of the documented
|
|
43
|
+
blind spot above.
|
|
44
|
+
|
|
45
|
+
Traversal is iterative (an explicit stack, post-order aggregation pass),
|
|
46
|
+
mirroring `scanner.scan`, so it doesn't rely on Python-level recursion and
|
|
47
|
+
can't blow the recursion limit on very deep trees.
|
|
48
|
+
|
|
49
|
+
A directory that can't be scanned at all (e.g. permission denied) still
|
|
50
|
+
gets a row stored, carrying its error message in the `error` column --
|
|
51
|
+
without this, the error would only ever be reported on the very first,
|
|
52
|
+
uncached scan: the moment its *parent* becomes a trusted cache hit, the
|
|
53
|
+
parent stops re-visiting it and the problem would silently disappear from
|
|
54
|
+
future reports even though it's still just as inaccessible. A trusted
|
|
55
|
+
parent checks its immediate children's stored `error` and re-surfaces it.
|
|
56
|
+
This only bridges one level per trusted hop, though: an error several
|
|
57
|
+
levels below multiple nested trusted ancestors may not resurface until
|
|
58
|
+
something along that chain changes (or `--no-cache` is used) -- the same
|
|
59
|
+
class of trade-off as the staleness blind spot above.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
from __future__ import annotations
|
|
63
|
+
|
|
64
|
+
import hashlib
|
|
65
|
+
import os
|
|
66
|
+
import sqlite3
|
|
67
|
+
import time
|
|
68
|
+
from pathlib import Path
|
|
69
|
+
from typing import Callable
|
|
70
|
+
|
|
71
|
+
from diskuh.scanner import DirNode, Entry, FileNode, disk_size
|
|
72
|
+
|
|
73
|
+
DB_DIR = Path.home() / ".diskuh"
|
|
74
|
+
DB_PATH = DB_DIR / "cache.sqlite3"
|
|
75
|
+
|
|
76
|
+
SCHEMA = """
|
|
77
|
+
CREATE TABLE IF NOT EXISTS scan_roots (
|
|
78
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
79
|
+
path TEXT NOT NULL UNIQUE,
|
|
80
|
+
last_scanned REAL NOT NULL
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
CREATE TABLE IF NOT EXISTS directories (
|
|
84
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
85
|
+
root_id INTEGER NOT NULL REFERENCES scan_roots(id) ON DELETE CASCADE,
|
|
86
|
+
path TEXT NOT NULL,
|
|
87
|
+
parent_path TEXT,
|
|
88
|
+
dir_mtime_ns INTEGER NOT NULL,
|
|
89
|
+
entry_count INTEGER NOT NULL,
|
|
90
|
+
fingerprint TEXT NOT NULL,
|
|
91
|
+
agg_size INTEGER NOT NULL,
|
|
92
|
+
agg_file_count INTEGER NOT NULL,
|
|
93
|
+
last_scanned REAL NOT NULL,
|
|
94
|
+
-- Set when this directory itself couldn't be scanned (e.g. permission
|
|
95
|
+
-- denied). Without this, a directory that fails to scan never gets a
|
|
96
|
+
-- row stored at all, so the moment its *parent* becomes a cache hit,
|
|
97
|
+
-- the error silently stops being reported even though the underlying
|
|
98
|
+
-- problem hasn't gone away. NULL means no error.
|
|
99
|
+
error TEXT,
|
|
100
|
+
UNIQUE(root_id, path)
|
|
101
|
+
);
|
|
102
|
+
CREATE INDEX IF NOT EXISTS idx_directories_root_path ON directories(root_id, path);
|
|
103
|
+
CREATE INDEX IF NOT EXISTS idx_directories_parent ON directories(root_id, parent_path);
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def connect(db_path: Path | None = None) -> sqlite3.Connection:
|
|
108
|
+
if db_path is None:
|
|
109
|
+
db_path = DB_PATH # read at call time so tests can monkeypatch it
|
|
110
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
# check_same_thread=False: the TUI runs scans on a worker thread (to
|
|
112
|
+
# keep the UI responsive) while the connection is created on the main
|
|
113
|
+
# thread. We never use it from two threads *concurrently* -- scans are
|
|
114
|
+
# exclusive/sequenced -- so a single connection without the default's
|
|
115
|
+
# same-thread restriction is safe here.
|
|
116
|
+
conn = sqlite3.connect(db_path, check_same_thread=False)
|
|
117
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
118
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
119
|
+
conn.row_factory = sqlite3.Row
|
|
120
|
+
return conn
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def init_schema(conn: sqlite3.Connection) -> None:
|
|
124
|
+
conn.executescript(SCHEMA)
|
|
125
|
+
conn.commit()
|
|
126
|
+
_migrate(conn)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _migrate(conn: sqlite3.Connection) -> None:
|
|
130
|
+
"""`CREATE TABLE IF NOT EXISTS` doesn't retrofit new columns onto a
|
|
131
|
+
database file created by an older version of diskuh, so do that by
|
|
132
|
+
hand for anything added after the initial schema."""
|
|
133
|
+
cols = {row["name"] for row in conn.execute("PRAGMA table_info(directories)")}
|
|
134
|
+
if "error" not in cols:
|
|
135
|
+
conn.execute("ALTER TABLE directories ADD COLUMN error TEXT")
|
|
136
|
+
conn.commit()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def compute_fingerprint(children_meta: list[tuple[str, str, int, int]]) -> str:
|
|
140
|
+
"""Hash the sorted (name, kind, size, mtime_ns) of a directory's
|
|
141
|
+
immediate children. `kind` is 'f' for file, 'd' for dir."""
|
|
142
|
+
h = hashlib.blake2b(digest_size=16)
|
|
143
|
+
for name, kind, size, mtime_ns in sorted(children_meta):
|
|
144
|
+
h.update(name.encode("utf-8", "surrogateescape"))
|
|
145
|
+
h.update(kind.encode("ascii"))
|
|
146
|
+
h.update(int(size).to_bytes(8, "little", signed=False))
|
|
147
|
+
h.update(int(mtime_ns).to_bytes(8, "little", signed=True))
|
|
148
|
+
return h.hexdigest()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _get_root_id(conn: sqlite3.Connection, root_path: Path) -> int:
|
|
152
|
+
now = time.time()
|
|
153
|
+
row = conn.execute(
|
|
154
|
+
"SELECT id FROM scan_roots WHERE path = ?", (str(root_path),)
|
|
155
|
+
).fetchone()
|
|
156
|
+
if row:
|
|
157
|
+
conn.execute(
|
|
158
|
+
"UPDATE scan_roots SET last_scanned = ? WHERE id = ?", (now, row["id"])
|
|
159
|
+
)
|
|
160
|
+
conn.commit()
|
|
161
|
+
return row["id"]
|
|
162
|
+
cur = conn.execute(
|
|
163
|
+
"INSERT INTO scan_roots (path, last_scanned) VALUES (?, ?)",
|
|
164
|
+
(str(root_path), now),
|
|
165
|
+
)
|
|
166
|
+
conn.commit()
|
|
167
|
+
return cur.lastrowid
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _fetch_row(conn: sqlite3.Connection, root_id: int, path: Path) -> sqlite3.Row | None:
|
|
171
|
+
return conn.execute(
|
|
172
|
+
"SELECT * FROM directories WHERE root_id = ? AND path = ?",
|
|
173
|
+
(root_id, str(path)),
|
|
174
|
+
).fetchone()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _store_row(
|
|
178
|
+
conn: sqlite3.Connection,
|
|
179
|
+
root_id: int,
|
|
180
|
+
path: Path,
|
|
181
|
+
dir_mtime_ns: int,
|
|
182
|
+
entry_count: int,
|
|
183
|
+
fingerprint: str,
|
|
184
|
+
agg_size: int,
|
|
185
|
+
agg_file_count: int,
|
|
186
|
+
error: str | None = None,
|
|
187
|
+
) -> None:
|
|
188
|
+
now = time.time()
|
|
189
|
+
conn.execute(
|
|
190
|
+
"""
|
|
191
|
+
INSERT INTO directories
|
|
192
|
+
(root_id, path, parent_path, dir_mtime_ns, entry_count,
|
|
193
|
+
fingerprint, agg_size, agg_file_count, last_scanned, error)
|
|
194
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
195
|
+
ON CONFLICT(root_id, path) DO UPDATE SET
|
|
196
|
+
parent_path=excluded.parent_path,
|
|
197
|
+
dir_mtime_ns=excluded.dir_mtime_ns,
|
|
198
|
+
entry_count=excluded.entry_count,
|
|
199
|
+
fingerprint=excluded.fingerprint,
|
|
200
|
+
agg_size=excluded.agg_size,
|
|
201
|
+
agg_file_count=excluded.agg_file_count,
|
|
202
|
+
last_scanned=excluded.last_scanned,
|
|
203
|
+
error=excluded.error
|
|
204
|
+
""",
|
|
205
|
+
(
|
|
206
|
+
root_id,
|
|
207
|
+
str(path),
|
|
208
|
+
str(path.parent),
|
|
209
|
+
dir_mtime_ns,
|
|
210
|
+
entry_count,
|
|
211
|
+
fingerprint,
|
|
212
|
+
agg_size,
|
|
213
|
+
agg_file_count,
|
|
214
|
+
now,
|
|
215
|
+
error,
|
|
216
|
+
),
|
|
217
|
+
)
|
|
218
|
+
conn.commit()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _check_dir(
|
|
222
|
+
conn: sqlite3.Connection, root_id: int, path: Path, *, force_rescan: bool
|
|
223
|
+
):
|
|
224
|
+
"""Do the one-directory work needed to decide cache trust: stat the
|
|
225
|
+
directory, look up its cached row, and do a single shallow `scandir` to
|
|
226
|
+
compute a fresh fingerprint. No recursion, no writes -- shared by
|
|
227
|
+
`get_or_refresh`'s traversal and the standalone `is_cached` peek."""
|
|
228
|
+
st = path.lstat() # may raise OSError; caller handles it
|
|
229
|
+
row = None if force_rescan else _fetch_row(conn, root_id, path)
|
|
230
|
+
|
|
231
|
+
children_meta: list[tuple[str, str, int, int]] = []
|
|
232
|
+
live_entries: list[tuple[bool, os.stat_result, Path]] = []
|
|
233
|
+
errors: list[str] = []
|
|
234
|
+
with os.scandir(path) as it: # may raise OSError; caller handles it
|
|
235
|
+
for entry in it:
|
|
236
|
+
try:
|
|
237
|
+
if entry.is_symlink():
|
|
238
|
+
continue
|
|
239
|
+
cst = entry.stat(follow_symlinks=False)
|
|
240
|
+
except OSError as e:
|
|
241
|
+
errors.append(f"{entry.path}: {e}")
|
|
242
|
+
continue
|
|
243
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
244
|
+
kind = "d" if is_dir else "f"
|
|
245
|
+
children_meta.append((entry.name, kind, cst.st_size, cst.st_mtime_ns))
|
|
246
|
+
live_entries.append((is_dir, cst, Path(entry.path)))
|
|
247
|
+
|
|
248
|
+
fingerprint = compute_fingerprint(children_meta)
|
|
249
|
+
entry_count = len(children_meta)
|
|
250
|
+
trusted = (
|
|
251
|
+
row is not None
|
|
252
|
+
and row["fingerprint"] == fingerprint
|
|
253
|
+
and row["dir_mtime_ns"] == st.st_mtime_ns
|
|
254
|
+
and row["entry_count"] == entry_count
|
|
255
|
+
)
|
|
256
|
+
return st, row, live_entries, fingerprint, entry_count, trusted, errors
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def is_cached(conn: sqlite3.Connection, root_path: Path, path: Path) -> bool:
|
|
260
|
+
"""Cheap check: would `get_or_refresh(path)` hit the cache (fast) or
|
|
261
|
+
need a real (re)scan (potentially expensive)? Does one shallow
|
|
262
|
+
`scandir` of `path` itself -- same cost as the real trust check -- but
|
|
263
|
+
never recurses and never writes to the cache. Returns True if `path`
|
|
264
|
+
doesn't exist or can't be read (nothing to scan, so nothing "expensive"
|
|
265
|
+
would happen)."""
|
|
266
|
+
root_path = Path(root_path).resolve()
|
|
267
|
+
path = Path(path).resolve()
|
|
268
|
+
root_id = _get_root_id(conn, root_path)
|
|
269
|
+
try:
|
|
270
|
+
*_, trusted, _errors = _check_dir(conn, root_id, path, force_rescan=False)
|
|
271
|
+
except OSError:
|
|
272
|
+
return True
|
|
273
|
+
return trusted
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def get_or_refresh(
|
|
277
|
+
conn: sqlite3.Connection,
|
|
278
|
+
root_path: Path,
|
|
279
|
+
path: Path,
|
|
280
|
+
*,
|
|
281
|
+
force_rescan: bool = False,
|
|
282
|
+
on_progress: Callable[[Path], None] | None = None,
|
|
283
|
+
) -> DirNode:
|
|
284
|
+
"""Return an aggregated `DirNode` for `path`, using the cache when safe.
|
|
285
|
+
|
|
286
|
+
Directories whose fingerprint still matches the cache are returned with
|
|
287
|
+
their cached aggregate size/file_count and are NOT recursed into further
|
|
288
|
+
(their immediate children are still listed, as lightweight stubs for
|
|
289
|
+
subdirectories, so callers can display or lazily expand one more level).
|
|
290
|
+
Directories that changed (or aren't cached yet, or `force_rescan=True`)
|
|
291
|
+
are fully rescanned, recursing into each child directory independently.
|
|
292
|
+
|
|
293
|
+
`on_progress`, if given, is called with the path of each directory as
|
|
294
|
+
it's visited (whether trusted or rescanned) -- useful for showing live
|
|
295
|
+
progress on a scan that may touch a great many directories.
|
|
296
|
+
"""
|
|
297
|
+
root_path = Path(root_path).resolve()
|
|
298
|
+
path = Path(path).resolve()
|
|
299
|
+
root_id = _get_root_id(conn, root_path)
|
|
300
|
+
|
|
301
|
+
root_node = DirNode(path=path)
|
|
302
|
+
stack: list[DirNode] = [root_node]
|
|
303
|
+
# Nodes that needed a (re)scan, in pop order (parent before its own
|
|
304
|
+
# children, since children are pushed after their parent is popped).
|
|
305
|
+
# Reversing gives child-before-parent, which is what post-order
|
|
306
|
+
# aggregation needs -- same trick as scanner.scan.
|
|
307
|
+
rescanned_order: list[DirNode] = []
|
|
308
|
+
fingerprints: dict[str, str] = {}
|
|
309
|
+
|
|
310
|
+
while stack:
|
|
311
|
+
node = stack.pop()
|
|
312
|
+
if on_progress is not None:
|
|
313
|
+
on_progress(node.path)
|
|
314
|
+
try:
|
|
315
|
+
st, row, live_entries, fingerprint, entry_count, trusted, errors = _check_dir(
|
|
316
|
+
conn, root_id, node.path, force_rescan=force_rescan
|
|
317
|
+
)
|
|
318
|
+
except OSError as e:
|
|
319
|
+
err_msg = f"{node.path}: {e}"
|
|
320
|
+
node.errors.append(err_msg)
|
|
321
|
+
# Persist this, not just report it in-memory this one time --
|
|
322
|
+
# otherwise the moment this node's *parent* becomes a cache
|
|
323
|
+
# hit, the error silently stops being reported even though
|
|
324
|
+
# the underlying problem (e.g. permission denied) hasn't
|
|
325
|
+
# gone away. Best-effort mtime: if even lstat() failed, 0 is
|
|
326
|
+
# fine -- this row exists purely to carry the error forward.
|
|
327
|
+
try:
|
|
328
|
+
mtime_ns = node.path.lstat().st_mtime_ns
|
|
329
|
+
except OSError:
|
|
330
|
+
mtime_ns = 0
|
|
331
|
+
node.mtime_ns = mtime_ns
|
|
332
|
+
_store_row(conn, root_id, node.path, mtime_ns, 0, "", 0, 0, error=err_msg)
|
|
333
|
+
continue
|
|
334
|
+
node.mtime_ns = st.st_mtime_ns
|
|
335
|
+
node.entry_count = entry_count
|
|
336
|
+
node.errors.extend(errors)
|
|
337
|
+
|
|
338
|
+
if trusted:
|
|
339
|
+
node.size = row["agg_size"]
|
|
340
|
+
node.file_count = row["agg_file_count"]
|
|
341
|
+
for is_dir, cst, full_path in live_entries:
|
|
342
|
+
if is_dir:
|
|
343
|
+
# Seed the stub's size/file_count from ITS OWN cached
|
|
344
|
+
# row (a cheap indexed lookup) so it displays correctly
|
|
345
|
+
# even if a caller never expands it further -- e.g.
|
|
346
|
+
# `--depth 1` only ever sees these stubs.
|
|
347
|
+
child_row = _fetch_row(conn, root_id, full_path)
|
|
348
|
+
stub = DirNode(path=full_path, mtime_ns=cst.st_mtime_ns, expanded=False)
|
|
349
|
+
if child_row is not None:
|
|
350
|
+
stub.size = child_row["agg_size"]
|
|
351
|
+
stub.file_count = child_row["agg_file_count"]
|
|
352
|
+
if child_row["error"]:
|
|
353
|
+
# Surface a previously-recorded error even
|
|
354
|
+
# though we're not re-descending into this
|
|
355
|
+
# child (see the "error" column comment in
|
|
356
|
+
# SCHEMA). Known limitation: this only
|
|
357
|
+
# reaches one level up per trusted hop, so an
|
|
358
|
+
# error many levels below several *nested*
|
|
359
|
+
# trusted ancestors may not resurface until
|
|
360
|
+
# something along that chain changes, or
|
|
361
|
+
# --no-cache is used.
|
|
362
|
+
stub.errors.append(child_row["error"])
|
|
363
|
+
node.errors.append(child_row["error"])
|
|
364
|
+
node.children.append(stub)
|
|
365
|
+
else:
|
|
366
|
+
node.children.append(
|
|
367
|
+
FileNode(path=full_path, size=disk_size(cst), mtime_ns=cst.st_mtime_ns)
|
|
368
|
+
)
|
|
369
|
+
continue # cache win: nothing below this directory is touched
|
|
370
|
+
|
|
371
|
+
fingerprints[str(node.path)] = fingerprint
|
|
372
|
+
rescanned_order.append(node)
|
|
373
|
+
for is_dir, cst, full_path in live_entries:
|
|
374
|
+
if is_dir:
|
|
375
|
+
child = DirNode(path=full_path, mtime_ns=cst.st_mtime_ns)
|
|
376
|
+
node.children.append(child)
|
|
377
|
+
stack.append(child)
|
|
378
|
+
else:
|
|
379
|
+
fsize = disk_size(cst)
|
|
380
|
+
fnode = FileNode(path=full_path, size=fsize, mtime_ns=cst.st_mtime_ns)
|
|
381
|
+
node.children.append(fnode)
|
|
382
|
+
node.size += fsize
|
|
383
|
+
node.file_count += 1
|
|
384
|
+
|
|
385
|
+
for node in reversed(rescanned_order):
|
|
386
|
+
for child in node.children:
|
|
387
|
+
if isinstance(child, DirNode):
|
|
388
|
+
node.size += child.size
|
|
389
|
+
node.file_count += child.file_count
|
|
390
|
+
node.errors.extend(child.errors)
|
|
391
|
+
_store_row(
|
|
392
|
+
conn,
|
|
393
|
+
root_id,
|
|
394
|
+
node.path,
|
|
395
|
+
node.mtime_ns,
|
|
396
|
+
node.entry_count,
|
|
397
|
+
fingerprints[str(node.path)],
|
|
398
|
+
node.size,
|
|
399
|
+
node.file_count,
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
return root_node
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def expand(
|
|
406
|
+
conn: sqlite3.Connection,
|
|
407
|
+
root_path: Path,
|
|
408
|
+
node: DirNode,
|
|
409
|
+
*,
|
|
410
|
+
force_rescan: bool = False,
|
|
411
|
+
on_progress: Callable[[Path], None] | None = None,
|
|
412
|
+
) -> None:
|
|
413
|
+
"""Replace a cache-collapsed stub `node` (see `DirNode.expanded`) in
|
|
414
|
+
place with its real children, one level down. A renderer walks the tree
|
|
415
|
+
and calls this on any stub it needs to descend past (e.g. because
|
|
416
|
+
`--depth` asks for rows below it); nodes it never descends into stay
|
|
417
|
+
collapsed and cost nothing further."""
|
|
418
|
+
if node.expanded:
|
|
419
|
+
return
|
|
420
|
+
fresh = get_or_refresh(
|
|
421
|
+
conn, root_path, node.path, force_rescan=force_rescan, on_progress=on_progress
|
|
422
|
+
)
|
|
423
|
+
node.children = fresh.children
|
|
424
|
+
node.entry_count = fresh.entry_count
|
|
425
|
+
node.errors = fresh.errors
|
|
426
|
+
node.size = fresh.size
|
|
427
|
+
node.file_count = fresh.file_count
|
|
428
|
+
node.mtime_ns = fresh.mtime_ns
|
|
429
|
+
node.expanded = True
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def evict_path(conn: sqlite3.Connection, root_path: Path, path: Path) -> None:
|
|
433
|
+
"""Drop cached rows for `path` (and everything under it) plus its
|
|
434
|
+
parent, so the parent's next fingerprint check notices the missing
|
|
435
|
+
entry. Call this after deleting a file/directory from the TUI."""
|
|
436
|
+
root_path = Path(root_path).resolve()
|
|
437
|
+
path = Path(path).resolve()
|
|
438
|
+
root_row = conn.execute(
|
|
439
|
+
"SELECT id FROM scan_roots WHERE path = ?", (str(root_path),)
|
|
440
|
+
).fetchone()
|
|
441
|
+
if root_row is None:
|
|
442
|
+
return
|
|
443
|
+
root_id = root_row["id"]
|
|
444
|
+
conn.execute(
|
|
445
|
+
"DELETE FROM directories WHERE root_id = ? AND (path = ? OR path LIKE ? || '/%')",
|
|
446
|
+
(root_id, str(path), str(path)),
|
|
447
|
+
)
|
|
448
|
+
conn.execute(
|
|
449
|
+
"DELETE FROM directories WHERE root_id = ? AND path = ?",
|
|
450
|
+
(root_id, str(path.parent)),
|
|
451
|
+
)
|
|
452
|
+
conn.commit()
|