rapiDU 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rapidu-0.1.0/.github/workflows/ci.yml +262 -0
- rapidu-0.1.0/.github/workflows/release.yml +50 -0
- rapidu-0.1.0/.gitignore +25 -0
- rapidu-0.1.0/.pre-commit-config.yaml +16 -0
- rapidu-0.1.0/LICENSE +21 -0
- rapidu-0.1.0/MANIFEST.in +6 -0
- rapidu-0.1.0/PKG-INFO +96 -0
- rapidu-0.1.0/README.md +61 -0
- rapidu-0.1.0/pyproject.toml +140 -0
- rapidu-0.1.0/setup.cfg +4 -0
- rapidu-0.1.0/src/rapiDU.egg-info/PKG-INFO +96 -0
- rapidu-0.1.0/src/rapiDU.egg-info/SOURCES.txt +38 -0
- rapidu-0.1.0/src/rapiDU.egg-info/dependency_links.txt +1 -0
- rapidu-0.1.0/src/rapiDU.egg-info/entry_points.txt +3 -0
- rapidu-0.1.0/src/rapiDU.egg-info/requires.txt +9 -0
- rapidu-0.1.0/src/rapiDU.egg-info/scm_file_list.json +42 -0
- rapidu-0.1.0/src/rapiDU.egg-info/scm_version.json +8 -0
- rapidu-0.1.0/src/rapiDU.egg-info/top_level.txt +1 -0
- rapidu-0.1.0/src/rapidu/__init__.py +26 -0
- rapidu-0.1.0/src/rapidu/__main__.py +8 -0
- rapidu-0.1.0/src/rapidu/_version.py +1 -0
- rapidu-0.1.0/src/rapidu/cli.py +492 -0
- rapidu-0.1.0/src/rapidu/deleted.py +170 -0
- rapidu-0.1.0/src/rapidu/fmt.py +68 -0
- rapidu-0.1.0/src/rapidu/py.typed +0 -0
- rapidu-0.1.0/src/rapidu/quota.py +692 -0
- rapidu-0.1.0/src/rapidu/reconcile.py +350 -0
- rapidu-0.1.0/src/rapidu/report.py +1181 -0
- rapidu-0.1.0/src/rapidu/ui.py +676 -0
- rapidu-0.1.0/src/rapidu/walk.py +837 -0
- rapidu-0.1.0/tests/test_allocation.py +181 -0
- rapidu-0.1.0/tests/test_cli.py +271 -0
- rapidu-0.1.0/tests/test_deleted.py +134 -0
- rapidu-0.1.0/tests/test_portability.py +272 -0
- rapidu-0.1.0/tests/test_py36_compat.py +113 -0
- rapidu-0.1.0/tests/test_quota.py +233 -0
- rapidu-0.1.0/tests/test_reconcile.py +250 -0
- rapidu-0.1.0/tests/test_report.py +233 -0
- rapidu-0.1.0/tests/test_ui.py +333 -0
- rapidu-0.1.0/tests/test_walk.py +413 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: ci-${{ github.ref }}
|
|
12
|
+
cancel-in-progress: true
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
# Ruff and mypy produce the same verdict on every interpreter, so running them
|
|
16
|
+
# once is enough; the matrix below is for the code, not the linters.
|
|
17
|
+
lint:
|
|
18
|
+
name: Lint & types
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v5
|
|
22
|
+
with:
|
|
23
|
+
fetch-depth: 0 # setuptools-scm derives the version from tags
|
|
24
|
+
|
|
25
|
+
- uses: actions/setup-python@v6
|
|
26
|
+
with:
|
|
27
|
+
python-version: "3.12"
|
|
28
|
+
|
|
29
|
+
- name: Install package + dev deps
|
|
30
|
+
run: |
|
|
31
|
+
python -m pip install --upgrade pip
|
|
32
|
+
pip install -e ".[dev]"
|
|
33
|
+
|
|
34
|
+
- run: ruff check .
|
|
35
|
+
- run: ruff format --check .
|
|
36
|
+
- run: mypy src/ tests/
|
|
37
|
+
|
|
38
|
+
test:
|
|
39
|
+
name: Test (py${{ matrix.python-version }})
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
strategy:
|
|
42
|
+
fail-fast: false
|
|
43
|
+
matrix:
|
|
44
|
+
# The package declares >=3.6 and is verified by hand on the 3.6.8
|
|
45
|
+
# /usr/bin/python3 of a RHEL8 login node, which is the deployment target
|
|
46
|
+
# that motivates the stdlib-only constraint. tests/test_py36_compat.py
|
|
47
|
+
# guards that floor here; these are the versions GitHub still provides.
|
|
48
|
+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
49
|
+
|
|
50
|
+
steps:
|
|
51
|
+
- uses: actions/checkout@v5
|
|
52
|
+
with:
|
|
53
|
+
fetch-depth: 0
|
|
54
|
+
|
|
55
|
+
- uses: actions/setup-python@v6
|
|
56
|
+
with:
|
|
57
|
+
python-version: ${{ matrix.python-version }}
|
|
58
|
+
allow-prereleases: true
|
|
59
|
+
|
|
60
|
+
- name: Install package + dev deps
|
|
61
|
+
run: |
|
|
62
|
+
python -m pip install --upgrade pip
|
|
63
|
+
pip install -e ".[dev]"
|
|
64
|
+
|
|
65
|
+
# TMPDIR stays on the runner's local disk. The suite handles a networked
|
|
66
|
+
# filesystem -- that is what the settling logic is for -- but the fixture
|
|
67
|
+
# trees would take tens of seconds to settle for no added coverage.
|
|
68
|
+
- name: Test with pytest
|
|
69
|
+
run: python -m pytest --cov=rapidu --cov-report=xml --cov-report=term --cov-fail-under=70
|
|
70
|
+
|
|
71
|
+
- name: Upload coverage to Codecov
|
|
72
|
+
uses: codecov/codecov-action@v5
|
|
73
|
+
with:
|
|
74
|
+
files: ./coverage.xml
|
|
75
|
+
fail_ci_if_error: false
|
|
76
|
+
|
|
77
|
+
# The headline claim, checked rather than asserted: this walker agrees with du
|
|
78
|
+
# byte-for-byte. It is also in the test suite; here it runs against a tree the
|
|
79
|
+
# runner built itself, outside pytest, so a broken fixture cannot hide it.
|
|
80
|
+
agrees-with-du:
|
|
81
|
+
name: Byte-for-byte agreement with du
|
|
82
|
+
runs-on: ubuntu-latest
|
|
83
|
+
steps:
|
|
84
|
+
- uses: actions/checkout@v5
|
|
85
|
+
- uses: actions/setup-python@v6
|
|
86
|
+
with:
|
|
87
|
+
python-version: "3.12"
|
|
88
|
+
|
|
89
|
+
- name: Build a tree with the pathologies that break naive walkers
|
|
90
|
+
run: |
|
|
91
|
+
mkdir -p tree/a/b/c
|
|
92
|
+
for i in $(seq 1 40); do head -c 5000 /dev/urandom > "tree/a/f$i"; done
|
|
93
|
+
for i in $(seq 1 20); do head -c 9000 /dev/urandom > "tree/a/b/g$i"; done
|
|
94
|
+
head -c 3000 /dev/urandom > tree/a/b/c/payload
|
|
95
|
+
for i in $(seq 1 5); do ln tree/a/b/c/payload "tree/link$i"; done # hard links
|
|
96
|
+
truncate -s 1G tree/sparse.bin # sparse
|
|
97
|
+
ln -s /etc/hostname tree/dangling.sym
|
|
98
|
+
sync
|
|
99
|
+
|
|
100
|
+
- name: du and rapidu must return the same integer
|
|
101
|
+
run: |
|
|
102
|
+
cat > walksize.py <<'PY'
|
|
103
|
+
import sys
|
|
104
|
+
from rapidu.walk import walk
|
|
105
|
+
print(walk(sys.argv[1], threads=int(sys.argv[2])).size)
|
|
106
|
+
PY
|
|
107
|
+
# Wait for two consecutive du readings to agree before comparing
|
|
108
|
+
# anything. A runner's ext4 settles instantly, but this is the exact
|
|
109
|
+
# trap the package documents, and a comparison of two measurements
|
|
110
|
+
# taken while the filesystem is still moving would fail for a reason
|
|
111
|
+
# that has nothing to do with the walker.
|
|
112
|
+
prev=-1
|
|
113
|
+
for _ in $(seq 1 20); do
|
|
114
|
+
cur=$(du -s --block-size=1 tree | cut -f1)
|
|
115
|
+
[ "$cur" = "$prev" ] && break
|
|
116
|
+
prev=$cur
|
|
117
|
+
sleep 3
|
|
118
|
+
done
|
|
119
|
+
DU=$(du -s --block-size=1 tree | cut -f1)
|
|
120
|
+
for t in 1 2 4 8 16; do
|
|
121
|
+
SD=$(PYTHONPATH=src python3 walksize.py tree "$t")
|
|
122
|
+
echo "threads=$t du=$DU rapidu=$SD"
|
|
123
|
+
if [ "$DU" != "$SD" ]; then
|
|
124
|
+
echo "::error::rapidu disagreed with du at $t threads ($SD vs $DU)"
|
|
125
|
+
exit 1
|
|
126
|
+
fi
|
|
127
|
+
done
|
|
128
|
+
|
|
129
|
+
# This package exists to be run on a login node during a storage emergency,
|
|
130
|
+
# where nothing is installed and pip may not be able to write anywhere. If it
|
|
131
|
+
# needs an install, or a quota backend, or a TTY, that claim is false.
|
|
132
|
+
zero-install:
|
|
133
|
+
name: Runs with nothing installed
|
|
134
|
+
runs-on: ubuntu-latest
|
|
135
|
+
steps:
|
|
136
|
+
- uses: actions/checkout@v5
|
|
137
|
+
- uses: actions/setup-python@v6
|
|
138
|
+
with:
|
|
139
|
+
python-version: "3.12"
|
|
140
|
+
|
|
141
|
+
- name: No pip, no venv, no dependencies -- just PYTHONPATH
|
|
142
|
+
run: |
|
|
143
|
+
cd /tmp && mkdir -p bare/sub && head -c 100000 /dev/urandom > bare/sub/f
|
|
144
|
+
PYTHONPATH=$GITHUB_WORKSPACE/src python3 -m rapidu bare
|
|
145
|
+
PYTHONPATH=$GITHUB_WORKSPACE/src python3 -m rapidu bare --json | python3 -m json.tool > /dev/null
|
|
146
|
+
|
|
147
|
+
- name: The package imports nothing outside the standard library
|
|
148
|
+
run: |
|
|
149
|
+
python3 - <<'PY'
|
|
150
|
+
import ast, pathlib, sys
|
|
151
|
+
stdlib = set(getattr(sys, "stdlib_module_names", ()))
|
|
152
|
+
assert stdlib, "need Python 3.10+ for stdlib_module_names"
|
|
153
|
+
bad = []
|
|
154
|
+
for path in sorted(pathlib.Path("src/rapidu").glob("*.py")):
|
|
155
|
+
for node in ast.walk(ast.parse(path.read_text())):
|
|
156
|
+
if isinstance(node, ast.Import):
|
|
157
|
+
names = [a.name.split(".")[0] for a in node.names]
|
|
158
|
+
elif isinstance(node, ast.ImportFrom) and node.level == 0:
|
|
159
|
+
names = [(node.module or "").split(".")[0]]
|
|
160
|
+
else:
|
|
161
|
+
continue
|
|
162
|
+
bad += [(path.name, n) for n in names if n and n not in stdlib]
|
|
163
|
+
assert not bad, "non-stdlib imports: {}".format(bad)
|
|
164
|
+
print("stdlib only, confirmed across", len(list(pathlib.Path('src/rapidu').glob('*.py'))), "modules")
|
|
165
|
+
PY
|
|
166
|
+
|
|
167
|
+
# There is no `quota`, no `mmlsquota` and no `lfs` on a GitHub runner, and
|
|
168
|
+
# that is the off-site case: every field must go absent WITH A REASON, and
|
|
169
|
+
# nothing downstream may break. A traceback here is the bug.
|
|
170
|
+
- name: A missing quota backend degrades to n/a, not to zero and not to a crash
|
|
171
|
+
run: |
|
|
172
|
+
pip install -e .
|
|
173
|
+
set +e
|
|
174
|
+
out=$(rdu --quota-only 2>&1); code=$?
|
|
175
|
+
set -e
|
|
176
|
+
echo "$out"
|
|
177
|
+
test $code -eq 1 || { echo "::error::expected exit 1 (attention), got $code"; exit 1; }
|
|
178
|
+
case "$out" in
|
|
179
|
+
*Traceback*) echo "::error::crashed instead of reporting"; exit 1 ;;
|
|
180
|
+
esac
|
|
181
|
+
echo "$out" | grep -q "n/a" || { echo "::error::absent quota did not print n/a"; exit 1; }
|
|
182
|
+
echo "$out" | grep -qi "not on PATH" || { echo "::error::n/a printed without a reason"; exit 1; }
|
|
183
|
+
|
|
184
|
+
- name: Output is clean when redirected -- no escape codes in a support ticket
|
|
185
|
+
run: |
|
|
186
|
+
rdu . > plain.txt
|
|
187
|
+
if grep -qP '\x1b\[' plain.txt; then
|
|
188
|
+
echo "::error::colour escapes leaked into non-TTY output"; exit 1
|
|
189
|
+
fi
|
|
190
|
+
NO_COLOR=1 rdu . --color auto > nocolor.txt
|
|
191
|
+
if grep -qP '\x1b\[' nocolor.txt; then
|
|
192
|
+
echo "::error::NO_COLOR was ignored"; exit 1
|
|
193
|
+
fi
|
|
194
|
+
# --help is painted after argparse has laid it out, which is a second
|
|
195
|
+
# place colour can leak from -- and `rdu --help > usage.txt` is exactly
|
|
196
|
+
# what someone does before pasting it into a ticket.
|
|
197
|
+
rdu --help > help.txt
|
|
198
|
+
if grep -qP '\x1b\[' help.txt; then
|
|
199
|
+
echo "::error::colour escapes leaked into redirected --help"; exit 1
|
|
200
|
+
fi
|
|
201
|
+
|
|
202
|
+
# A -c walk never calls stat, so it has no bytes. Reconciling that against
|
|
203
|
+
# a live quota once fabricated an UNEXPLAINED GAP the size of the whole
|
|
204
|
+
# quota -- the exact failure the reconciler exists to prevent.
|
|
205
|
+
- name: A stat-free walk never manufactures a finding
|
|
206
|
+
run: |
|
|
207
|
+
rdu . -c --json > count.json
|
|
208
|
+
python3 - <<'PY'
|
|
209
|
+
import json
|
|
210
|
+
doc = json.load(open("count.json"))
|
|
211
|
+
for rec in doc.get("reconciliation", []):
|
|
212
|
+
assert rec["verdict"] != "gap", rec
|
|
213
|
+
print("no fabricated gaps from a count-only walk")
|
|
214
|
+
PY
|
|
215
|
+
|
|
216
|
+
# An installed wheel is what a user actually gets. A package that imports from
|
|
217
|
+
# the source tree but not from site-packages is a packaging bug, and the
|
|
218
|
+
# release workflow is the wrong place to discover it.
|
|
219
|
+
wheel:
|
|
220
|
+
name: Built wheel installs and runs
|
|
221
|
+
runs-on: ubuntu-latest
|
|
222
|
+
steps:
|
|
223
|
+
- uses: actions/checkout@v5
|
|
224
|
+
with:
|
|
225
|
+
fetch-depth: 0
|
|
226
|
+
|
|
227
|
+
- uses: actions/setup-python@v6
|
|
228
|
+
with:
|
|
229
|
+
python-version: "3.12"
|
|
230
|
+
|
|
231
|
+
- name: Build sdist and wheel
|
|
232
|
+
run: |
|
|
233
|
+
pip install build twine
|
|
234
|
+
python -m build
|
|
235
|
+
twine check dist/*
|
|
236
|
+
|
|
237
|
+
- name: Install the wheel into a clean venv and use it
|
|
238
|
+
run: |
|
|
239
|
+
python -m venv /tmp/check
|
|
240
|
+
/tmp/check/bin/pip install --quiet dist/*.whl
|
|
241
|
+
/tmp/check/bin/rapidu --version
|
|
242
|
+
/tmp/check/bin/rdu --version # the short alias must be there too
|
|
243
|
+
/tmp/check/bin/rdu /tmp --json | python3 -m json.tool > /dev/null
|
|
244
|
+
/tmp/check/bin/python -c "import rapidu, pathlib; \
|
|
245
|
+
assert (pathlib.Path(rapidu.__file__).parent / 'py.typed').is_file(), \
|
|
246
|
+
'py.typed missing from the wheel'"
|
|
247
|
+
|
|
248
|
+
# The stdlib-only promise is a packaging fact, not just a source fact. The
|
|
249
|
+
# `[dev]` extra legitimately appears here as `; extra == "dev"`, so only
|
|
250
|
+
# unconditional requirements are a failure -- those are what a plain
|
|
251
|
+
# `pip install rapidu` would drag onto a login node.
|
|
252
|
+
- name: The wheel carries no unconditional dependencies
|
|
253
|
+
run: |
|
|
254
|
+
python -m zipfile -e dist/*.whl /tmp/unpacked
|
|
255
|
+
meta=$(find /tmp/unpacked -name METADATA | head -1)
|
|
256
|
+
runtime=$(grep '^Requires-Dist:' "$meta" | grep -v 'extra ==' || true)
|
|
257
|
+
if [ -n "$runtime" ]; then
|
|
258
|
+
echo "::error::the wheel declares runtime dependencies:"
|
|
259
|
+
echo "$runtime"
|
|
260
|
+
exit 1
|
|
261
|
+
fi
|
|
262
|
+
echo "no unconditional Requires-Dist, as intended"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
id-token: write
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build-and-publish:
|
|
14
|
+
name: Build & Publish to PyPI
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v5
|
|
19
|
+
with:
|
|
20
|
+
# setuptools-scm needs full history + tags to derive the version.
|
|
21
|
+
fetch-depth: 0
|
|
22
|
+
|
|
23
|
+
- uses: actions/setup-python@v6
|
|
24
|
+
with:
|
|
25
|
+
python-version: "3.12"
|
|
26
|
+
|
|
27
|
+
- name: Build wheel and sdist
|
|
28
|
+
run: |
|
|
29
|
+
pip install build setuptools-scm
|
|
30
|
+
python -m build
|
|
31
|
+
|
|
32
|
+
- name: Verify tag matches built version
|
|
33
|
+
run: |
|
|
34
|
+
TAG_VERSION="${GITHUB_REF_NAME#v}"
|
|
35
|
+
PKG_VERSION=$(python -m setuptools_scm)
|
|
36
|
+
echo "Tag version: $TAG_VERSION"
|
|
37
|
+
echo "Package version: $PKG_VERSION"
|
|
38
|
+
if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then
|
|
39
|
+
echo "Tag version ($TAG_VERSION) != package version ($PKG_VERSION)"
|
|
40
|
+
exit 1
|
|
41
|
+
fi
|
|
42
|
+
echo "Version match OK"
|
|
43
|
+
|
|
44
|
+
- name: Check artifacts with twine
|
|
45
|
+
run: |
|
|
46
|
+
pip install twine
|
|
47
|
+
twine check dist/*
|
|
48
|
+
|
|
49
|
+
- name: Publish to PyPI (Trusted Publishing)
|
|
50
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
rapidu-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
*.egg-info/
|
|
4
|
+
.eggs/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
*.egg
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
*.log
|
|
12
|
+
*.csv
|
|
13
|
+
.DS_Store
|
|
14
|
+
.coverage
|
|
15
|
+
coverage.xml
|
|
16
|
+
|
|
17
|
+
# Generated by setuptools-scm at build time
|
|
18
|
+
src/rapidu/_version.py
|
|
19
|
+
|
|
20
|
+
# Local (machine-specific) Claude Code config — not shared
|
|
21
|
+
.claude/settings.local.json
|
|
22
|
+
.claude/hooks/
|
|
23
|
+
|
|
24
|
+
# Local audit notes: contains real cluster paths, never publish.
|
|
25
|
+
issues.md
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
# Keep in step with the ruff release CI resolves, or the hook's --fix
|
|
4
|
+
# and CI's check will disagree about import sorting.
|
|
5
|
+
rev: v0.16.1
|
|
6
|
+
hooks:
|
|
7
|
+
- id: ruff
|
|
8
|
+
args: [--fix]
|
|
9
|
+
- id: ruff-format
|
|
10
|
+
|
|
11
|
+
- repo: https://github.com/pre-commit/mirrors-mypy
|
|
12
|
+
rev: v2.3.0
|
|
13
|
+
hooks:
|
|
14
|
+
- id: mypy
|
|
15
|
+
# rapidu has no runtime dependencies, so mypy needs no extras.
|
|
16
|
+
additional_dependencies: []
|
rapidu-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 rapidu contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rapidu-0.1.0/MANIFEST.in
ADDED
rapidu-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rapiDU
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A much faster du that tells you why your quota is full.
|
|
5
|
+
Author-email: Youzhi Yu <yuyouzhi666@icloud.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/PursuitOfDataScience/rapidu
|
|
8
|
+
Project-URL: Repository, https://github.com/PursuitOfDataScience/rapidu.git
|
|
9
|
+
Project-URL: Documentation, https://github.com/PursuitOfDataScience/rapidu#readme
|
|
10
|
+
Project-URL: Issues, https://github.com/PursuitOfDataScience/rapidu/issues
|
|
11
|
+
Keywords: du,disk-usage,quota,inodes,gpfs,lustre,hpc,storage
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Intended Audience :: System Administrators
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering
|
|
20
|
+
Classifier: Topic :: System :: Filesystems
|
|
21
|
+
Classifier: Topic :: System :: Monitoring
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.6
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff<0.17,>=0.15; extra == "dev"
|
|
30
|
+
Requires-Dist: mypy<3,>=1.8; extra == "dev"
|
|
31
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
32
|
+
Requires-Dist: twine>=4.0; extra == "dev"
|
|
33
|
+
Requires-Dist: setuptools-scm>=8; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
<h1 align="center">rapiDU</h1>
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<strong>A much faster <code>du</code> that tells you why your quota is full.</strong>
|
|
40
|
+
</p>
|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
<a href="https://github.com/PursuitOfDataScience/rapidu/actions/workflows/ci.yml"><img src="https://github.com/PursuitOfDataScience/rapidu/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
44
|
+
<img src="https://img.shields.io/badge/python-3.6%2B-blue.svg" alt="Python 3.6+">
|
|
45
|
+
<img src="https://img.shields.io/badge/dependencies-none-brightgreen.svg" alt="No dependencies">
|
|
46
|
+
<img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License">
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
<p align="center">
|
|
50
|
+
<img src="assets/demo.gif" width="900" alt="rapiDU walking a project tree and explaining that it occupies 266.8 MiB to hold 75.5 MiB of data, ranking the same tree by file count, printing a quota table with the age of its snapshot, finding 512 MiB held by a deleted-but-open file descriptor, and catching a freshly written GPFS tree that loses 224 MiB while it settles.">
|
|
51
|
+
</p>
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install rapidu
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Use
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
rdu # this directory: how big, and what is big inside it
|
|
63
|
+
rdu /project/mylab # any other path
|
|
64
|
+
rdu ~/scratch -n 20 # list 20 entries instead of 10
|
|
65
|
+
|
|
66
|
+
rdu -i # rank by file count -- what an inode quota limits
|
|
67
|
+
rdu -c # count files only, no stat: ~8x again on GPFS
|
|
68
|
+
rdu -Q # the quota table, and the age of its figures
|
|
69
|
+
rdu -D # space held by files deleted while still open
|
|
70
|
+
rdu -a # the full audit: quota + /proc scan + reconciliation
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Faster, and the same number
|
|
74
|
+
|
|
75
|
+
<p align="center">
|
|
76
|
+
<picture>
|
|
77
|
+
<source media="(prefers-color-scheme: dark)" srcset="assets/benchmark-dark.png">
|
|
78
|
+
<img src="assets/benchmark-light.png" width="720" alt="Cold GPFS walk: du takes 168.1s against rapiDU's 25.4s on a 792,225-file package cache (6.6x), and 298.5s against 57.4s on a 1,686,589-file project directory (5.2x).">
|
|
79
|
+
</picture>
|
|
80
|
+
</p>
|
|
81
|
+
|
|
82
|
+
Same total as `du`, to the byte. That is checked on every commit.
|
|
83
|
+
|
|
84
|
+
## Reading the table
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
size of tree share files path
|
|
88
|
+
661.5 GiB █████▋░░░░░░░░░░░░ 31.9% 350 checkpoints/
|
|
89
|
+
343.8 GiB ██▊░░░░░░░░░░░░░░░ 16.6% 968 datasets/
|
|
90
|
+
470.9 GiB ▒▒▒▒▒▒▒▒░░░░░░░░░░ 22.9% 4,117 (84 more — use -n 0 for all)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The bar is share of the whole tree, so it always agrees with the number beside
|
|
94
|
+
it. The hatched row is everything not listed. The column you sorted by is the one
|
|
95
|
+
in colour — under `-i`, `files` takes the tone and `size` steps back. Sizes are
|
|
96
|
+
cumulative, so any row agrees with `du -s` on that path.
|
rapidu-0.1.0/README.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
<h1 align="center">rapiDU</h1>
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<strong>A much faster <code>du</code> that tells you why your quota is full.</strong>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<a href="https://github.com/PursuitOfDataScience/rapidu/actions/workflows/ci.yml"><img src="https://github.com/PursuitOfDataScience/rapidu/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
9
|
+
<img src="https://img.shields.io/badge/python-3.6%2B-blue.svg" alt="Python 3.6+">
|
|
10
|
+
<img src="https://img.shields.io/badge/dependencies-none-brightgreen.svg" alt="No dependencies">
|
|
11
|
+
<img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License">
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<p align="center">
|
|
15
|
+
<img src="assets/demo.gif" width="900" alt="rapiDU walking a project tree and explaining that it occupies 266.8 MiB to hold 75.5 MiB of data, ranking the same tree by file count, printing a quota table with the age of its snapshot, finding 512 MiB held by a deleted-but-open file descriptor, and catching a freshly written GPFS tree that loses 224 MiB while it settles.">
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install rapidu
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
rdu # this directory: how big, and what is big inside it
|
|
28
|
+
rdu /project/mylab # any other path
|
|
29
|
+
rdu ~/scratch -n 20 # list 20 entries instead of 10
|
|
30
|
+
|
|
31
|
+
rdu -i # rank by file count -- what an inode quota limits
|
|
32
|
+
rdu -c # count files only, no stat: ~8x again on GPFS
|
|
33
|
+
rdu -Q # the quota table, and the age of its figures
|
|
34
|
+
rdu -D # space held by files deleted while still open
|
|
35
|
+
rdu -a # the full audit: quota + /proc scan + reconciliation
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Faster, and the same number
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<picture>
|
|
42
|
+
<source media="(prefers-color-scheme: dark)" srcset="assets/benchmark-dark.png">
|
|
43
|
+
<img src="assets/benchmark-light.png" width="720" alt="Cold GPFS walk: du takes 168.1s against rapiDU's 25.4s on a 792,225-file package cache (6.6x), and 298.5s against 57.4s on a 1,686,589-file project directory (5.2x).">
|
|
44
|
+
</picture>
|
|
45
|
+
</p>
|
|
46
|
+
|
|
47
|
+
Same total as `du`, to the byte. That is checked on every commit.
|
|
48
|
+
|
|
49
|
+
## Reading the table
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
size of tree share files path
|
|
53
|
+
661.5 GiB █████▋░░░░░░░░░░░░ 31.9% 350 checkpoints/
|
|
54
|
+
343.8 GiB ██▊░░░░░░░░░░░░░░░ 16.6% 968 datasets/
|
|
55
|
+
470.9 GiB ▒▒▒▒▒▒▒▒░░░░░░░░░░ 22.9% 4,117 (84 more — use -n 0 for all)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The bar is share of the whole tree, so it always agrees with the number beside
|
|
59
|
+
it. The hatched row is everything not listed. The column you sorted by is the one
|
|
60
|
+
in colour — under `-i`, `files` takes the tone and `size` steps back. Sizes are
|
|
61
|
+
cumulative, so any row agrees with `du -s` on that path.
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64", "wheel", "setuptools-scm>=8"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
# Display name only. PyPI matches case-insensitively (PEP 503), so this
|
|
7
|
+
# installs as `pip install rapidu` just as readily, and the importable
|
|
8
|
+
# module stays lowercase `rapidu` -- the PyYAML/yaml arrangement.
|
|
9
|
+
name = "rapiDU"
|
|
10
|
+
description = "A much faster du that tells you why your quota is full."
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
license = { text = "MIT" }
|
|
13
|
+
# Stdlib only, and deliberately conservative: the tool is most useful on a bare
|
|
14
|
+
# login node during a quota emergency, where the interpreter is whatever the OS
|
|
15
|
+
# shipped and pip may not be able to write anywhere. Verified running on the
|
|
16
|
+
# 3.6.8 /usr/bin/python3 of a RHEL8 login node; CI covers 3.9-3.13, which is the
|
|
17
|
+
# range GitHub runners still provide.
|
|
18
|
+
requires-python = ">=3.6"
|
|
19
|
+
authors = [{ name = "Youzhi Yu", email = "yuyouzhi666@icloud.com" }]
|
|
20
|
+
# No "slurm" here: this package never calls a scheduler. It reads filesystems,
|
|
21
|
+
# quota backends and /proc, and everything it uniquely reports is a property of
|
|
22
|
+
# a quota'd network filesystem rather than of a batch system.
|
|
23
|
+
keywords = ["du", "disk-usage", "quota", "inodes", "gpfs", "lustre", "hpc", "storage"]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 4 - Beta",
|
|
26
|
+
"Environment :: Console",
|
|
27
|
+
"Intended Audience :: Science/Research",
|
|
28
|
+
"Intended Audience :: System Administrators",
|
|
29
|
+
"License :: OSI Approved :: MIT License",
|
|
30
|
+
"Operating System :: POSIX :: Linux",
|
|
31
|
+
"Programming Language :: Python :: 3",
|
|
32
|
+
"Topic :: Scientific/Engineering",
|
|
33
|
+
"Topic :: System :: Filesystems",
|
|
34
|
+
"Topic :: System :: Monitoring",
|
|
35
|
+
"Typing :: Typed",
|
|
36
|
+
]
|
|
37
|
+
# No runtime dependencies, on purpose. This runs during a storage emergency, on
|
|
38
|
+
# a login node, possibly with a full home directory that pip cannot write to.
|
|
39
|
+
dependencies = []
|
|
40
|
+
dynamic = ["version"]
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/PursuitOfDataScience/rapidu"
|
|
44
|
+
Repository = "https://github.com/PursuitOfDataScience/rapidu.git"
|
|
45
|
+
Documentation = "https://github.com/PursuitOfDataScience/rapidu#readme"
|
|
46
|
+
Issues = "https://github.com/PursuitOfDataScience/rapidu/issues"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
rapidu = "rapidu.cli:main"
|
|
50
|
+
# `rdu` is the short alias, and it is also how the name stays tied to `du`:
|
|
51
|
+
# you type `du`, you type `rdu`.
|
|
52
|
+
#
|
|
53
|
+
# Both names were checked against the registries on 2026-08-02, because
|
|
54
|
+
# Constraint 16 exists. `rapidu` is free on PyPI, npm and crates.io. `rdu` is
|
|
55
|
+
# taken on PyPI by an unrelated chemistry library that ships no console script,
|
|
56
|
+
# and no Debian package installs an `rdu` binary, so the *command* is clear.
|
|
57
|
+
#
|
|
58
|
+
# The previous alias was `sd`, which was wrong: chmln/sd is a widely packaged
|
|
59
|
+
# Rust find-and-replace tool in Debian, Ubuntu, Homebrew and Arch, so on any
|
|
60
|
+
# machine with both installed one of them was silently shadowed depending on
|
|
61
|
+
# PATH order.
|
|
62
|
+
rdu = "rapidu.cli:main"
|
|
63
|
+
|
|
64
|
+
[project.optional-dependencies]
|
|
65
|
+
dev = [
|
|
66
|
+
"pytest>=7.0",
|
|
67
|
+
"pytest-cov>=4.0",
|
|
68
|
+
# Upper-bounded on purpose: a formatter with no ceiling turns `ruff format
|
|
69
|
+
# --check` into a job that fails on a day nobody touched the repo. 0.15 and
|
|
70
|
+
# 0.16 were both verified to leave this tree unchanged.
|
|
71
|
+
"ruff>=0.15,<0.17",
|
|
72
|
+
"mypy>=1.8,<3",
|
|
73
|
+
"build>=1.0",
|
|
74
|
+
"twine>=4.0",
|
|
75
|
+
"setuptools-scm>=8",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
[tool.setuptools.packages.find]
|
|
79
|
+
where = ["src"]
|
|
80
|
+
include = ["rapidu*"]
|
|
81
|
+
|
|
82
|
+
[tool.setuptools.package-data]
|
|
83
|
+
rapidu = ["py.typed"]
|
|
84
|
+
|
|
85
|
+
[tool.setuptools_scm]
|
|
86
|
+
# Written at build/install time so the version needs no runtime metadata
|
|
87
|
+
# lookup -- importlib.metadata does not exist on the 3.6 login-node python.
|
|
88
|
+
version_file = "src/rapidu/_version.py"
|
|
89
|
+
# setuptools-scm's stock template opens with `from __future__ import
|
|
90
|
+
# annotations` and annotates with PEP 604 unions, which is a SyntaxError on
|
|
91
|
+
# 3.6 and would break the login-node path this package exists to serve.
|
|
92
|
+
# Emit the one line actually needed instead. Guarded by
|
|
93
|
+
# tests/test_py36_compat.py.
|
|
94
|
+
# Must be a TOML *basic* string: a literal ('...') one would not expand \n.
|
|
95
|
+
version_file_template = "__version__ = \"{version}\"\n"
|
|
96
|
+
|
|
97
|
+
[tool.ruff]
|
|
98
|
+
target-version = "py38"
|
|
99
|
+
line-length = 100
|
|
100
|
+
|
|
101
|
+
[tool.ruff.lint]
|
|
102
|
+
select = ["E", "F", "W", "I", "B", "C4", "SIM"]
|
|
103
|
+
|
|
104
|
+
[tool.ruff.lint.per-file-ignores]
|
|
105
|
+
"src/rapidu/cli.py" = ["T201"]
|
|
106
|
+
"src/rapidu/report.py" = ["T201"]
|
|
107
|
+
# The deleted-fd tests must hold descriptors open across statements -- that is
|
|
108
|
+
# precisely the condition under test -- so a context manager is not available.
|
|
109
|
+
"tests/test_deleted.py" = ["SIM115"]
|
|
110
|
+
# Same for the demo renderer: its unlinked-but-open scene only exists while the
|
|
111
|
+
# descriptor is still held, which a `with` block would close.
|
|
112
|
+
"assets/render_demo.py" = ["SIM115"]
|
|
113
|
+
|
|
114
|
+
[tool.ruff.format]
|
|
115
|
+
quote-style = "double"
|
|
116
|
+
|
|
117
|
+
[tool.mypy]
|
|
118
|
+
python_version = "3.10" # mypy's own floor; the runtime floor is 3.6, guarded by tests/test_py36_compat.py
|
|
119
|
+
ignore_missing_imports = true
|
|
120
|
+
warn_unused_ignores = false
|
|
121
|
+
exclude = "src/rapidu/_version.py"
|
|
122
|
+
|
|
123
|
+
[tool.pytest.ini_options]
|
|
124
|
+
minversion = "7.0"
|
|
125
|
+
testpaths = ["tests"]
|
|
126
|
+
addopts = "-q"
|
|
127
|
+
|
|
128
|
+
[tool.coverage.run]
|
|
129
|
+
source = ["rapidu"]
|
|
130
|
+
omit = ["*/tests/*", "*/_version.py"]
|
|
131
|
+
|
|
132
|
+
[tool.coverage.report]
|
|
133
|
+
exclude_lines = [
|
|
134
|
+
"pragma: no cover",
|
|
135
|
+
"def __repr__",
|
|
136
|
+
"raise AssertionError",
|
|
137
|
+
"raise NotImplementedError",
|
|
138
|
+
"if __name__ == .__main__.:",
|
|
139
|
+
"if TYPE_CHECKING:",
|
|
140
|
+
]
|