findupe 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- findupe/__init__.py +25 -0
- findupe/cache.py +135 -0
- findupe/cli.py +333 -0
- findupe/clones.py +116 -0
- findupe/dashboard.py +160 -0
- findupe/discover.py +265 -0
- findupe/grouping.py +293 -0
- findupe/hashing.py +130 -0
- findupe/imaging.py +174 -0
- findupe/ledger.py +159 -0
- findupe/models.py +138 -0
- findupe/paths.py +38 -0
- findupe/report.py +379 -0
- findupe/stats.py +114 -0
- findupe/trash.py +409 -0
- findupe-0.4.0.dist-info/METADATA +168 -0
- findupe-0.4.0.dist-info/RECORD +20 -0
- findupe-0.4.0.dist-info/WHEEL +4 -0
- findupe-0.4.0.dist-info/entry_points.txt +2 -0
- findupe-0.4.0.dist-info/licenses/LICENSE +21 -0
findupe/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""findupe — safe duplicate finder & reviewer for macOS."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _read_version() -> str:
|
|
9
|
+
try:
|
|
10
|
+
return version("findupe")
|
|
11
|
+
except PackageNotFoundError:
|
|
12
|
+
# No installed dist-info (e.g. running straight from a checkout without
|
|
13
|
+
# `uv sync`/`pip install -e .` first) — fall back to pyproject.toml so
|
|
14
|
+
# there's still a single source of truth for the version.
|
|
15
|
+
import tomllib
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
pyproject = Path(__file__).resolve().parent.parent.parent / "pyproject.toml"
|
|
19
|
+
try:
|
|
20
|
+
return tomllib.loads(pyproject.read_text())["project"]["version"]
|
|
21
|
+
except (OSError, KeyError):
|
|
22
|
+
return "0.0.0+unknown"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__version__ = _read_version()
|
findupe/cache.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Persistent SQLite index so re-scans only hash new or changed files.
|
|
2
|
+
|
|
3
|
+
A cache row is valid only if (path, size, mtime_ns, volume) all match — any
|
|
4
|
+
change forces recompute. WAL mode + batched transactions keep the index
|
|
5
|
+
consistent if a scan is interrupted; resuming is just re-running the scan.
|
|
6
|
+
Written from the main thread only (workers return results; caller stores).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sqlite3
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .models import FileRecord, norm_path
|
|
17
|
+
from .paths import resolve_data_home
|
|
18
|
+
|
|
19
|
+
_SCHEMA_VERSION = 2 # v2: volume_uuid keying + capture_subsec
|
|
20
|
+
|
|
21
|
+
_SCHEMA = """
|
|
22
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
23
|
+
path TEXT PRIMARY KEY,
|
|
24
|
+
size INTEGER NOT NULL,
|
|
25
|
+
mtime_ns INTEGER NOT NULL,
|
|
26
|
+
volume_uuid TEXT NOT NULL,
|
|
27
|
+
exact_hash TEXT,
|
|
28
|
+
phash INTEGER,
|
|
29
|
+
dhash INTEGER,
|
|
30
|
+
width INTEGER,
|
|
31
|
+
height INTEGER,
|
|
32
|
+
capture_key TEXT,
|
|
33
|
+
capture_subsec TEXT,
|
|
34
|
+
last_seen TEXT NOT NULL
|
|
35
|
+
);
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _to_signed(v: int | None) -> int | None:
|
|
40
|
+
"""SQLite INTEGER is signed 64-bit; perceptual hashes are unsigned 64-bit."""
|
|
41
|
+
if v is None:
|
|
42
|
+
return None
|
|
43
|
+
return v - (1 << 64) if v >= (1 << 63) else v
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _to_unsigned(v: int | None) -> int | None:
|
|
47
|
+
if v is None:
|
|
48
|
+
return None
|
|
49
|
+
return v + (1 << 64) if v < 0 else v
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class CachedInfo:
|
|
54
|
+
exact_hash: str | None
|
|
55
|
+
phash: int | None
|
|
56
|
+
dhash: int | None
|
|
57
|
+
width: int | None
|
|
58
|
+
height: int | None
|
|
59
|
+
capture_key: str | None
|
|
60
|
+
capture_subsec: str | None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Cache:
|
|
64
|
+
def __init__(self, db_path: Path | None = None) -> None:
|
|
65
|
+
db_path = db_path if db_path is not None else resolve_data_home() / "index.db"
|
|
66
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
self.db_path = db_path
|
|
68
|
+
self.conn = sqlite3.connect(db_path)
|
|
69
|
+
self.conn.execute("PRAGMA journal_mode=WAL")
|
|
70
|
+
version = self.conn.execute("PRAGMA user_version").fetchone()[0]
|
|
71
|
+
if version != _SCHEMA_VERSION:
|
|
72
|
+
# cheap-to-rebuild cache: recreate rather than migrate
|
|
73
|
+
self.conn.execute("DROP TABLE IF EXISTS files")
|
|
74
|
+
self.conn.execute(f"PRAGMA user_version={_SCHEMA_VERSION}")
|
|
75
|
+
self.conn.execute(_SCHEMA)
|
|
76
|
+
self.conn.commit()
|
|
77
|
+
self.hits = 0
|
|
78
|
+
self.misses = 0
|
|
79
|
+
|
|
80
|
+
def lookup(self, rec: FileRecord) -> CachedInfo | None:
|
|
81
|
+
row = self.conn.execute(
|
|
82
|
+
"SELECT exact_hash, phash, dhash, width, height, capture_key, capture_subsec "
|
|
83
|
+
"FROM files WHERE path=? AND size=? AND mtime_ns=? AND volume_uuid=?",
|
|
84
|
+
(norm_path(rec.path), rec.size, rec.mtime_ns, rec.volume_uuid),
|
|
85
|
+
).fetchone()
|
|
86
|
+
if row is None:
|
|
87
|
+
self.misses += 1
|
|
88
|
+
return None
|
|
89
|
+
self.hits += 1
|
|
90
|
+
exact_hash, phash, dhash, width, height, capture_key, capture_subsec = row
|
|
91
|
+
return CachedInfo(
|
|
92
|
+
exact_hash, _to_unsigned(phash), _to_unsigned(dhash),
|
|
93
|
+
width, height, capture_key, capture_subsec,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def store(self, records: list[FileRecord]) -> None:
|
|
97
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
98
|
+
self.conn.executemany(
|
|
99
|
+
# COALESCE merges pipeline stages (exact pass, then perceptual pass) for the
|
|
100
|
+
# SAME file version. If size/mtime changed, the old values are stale and must
|
|
101
|
+
# be replaced outright — hence the CASE guard on every merged column.
|
|
102
|
+
"INSERT INTO files (path, size, mtime_ns, volume_uuid, exact_hash, phash, dhash,"
|
|
103
|
+
" width, height, capture_key, capture_subsec, last_seen) VALUES (?,?,?,?,?,?,?,?,?,?,?,?) "
|
|
104
|
+
"ON CONFLICT(path) DO UPDATE SET "
|
|
105
|
+
+ ", ".join(
|
|
106
|
+
f"{col} = CASE WHEN files.size = excluded.size AND files.mtime_ns = excluded.mtime_ns"
|
|
107
|
+
f" THEN COALESCE(excluded.{col}, files.{col}) ELSE excluded.{col} END"
|
|
108
|
+
for col in ("exact_hash", "phash", "dhash", "width", "height",
|
|
109
|
+
"capture_key", "capture_subsec")
|
|
110
|
+
)
|
|
111
|
+
+ ", size=excluded.size, mtime_ns=excluded.mtime_ns,"
|
|
112
|
+
" volume_uuid=excluded.volume_uuid, last_seen=excluded.last_seen",
|
|
113
|
+
[
|
|
114
|
+
(
|
|
115
|
+
norm_path(r.path), r.size, r.mtime_ns, r.volume_uuid, r.exact_hash,
|
|
116
|
+
_to_signed(r.phash), _to_signed(r.dhash),
|
|
117
|
+
r.width, r.height, r.capture_key, r.capture_subsec, now,
|
|
118
|
+
)
|
|
119
|
+
for r in records
|
|
120
|
+
],
|
|
121
|
+
)
|
|
122
|
+
self.conn.commit()
|
|
123
|
+
|
|
124
|
+
def clear(self) -> None:
|
|
125
|
+
self.conn.execute("DELETE FROM files")
|
|
126
|
+
self.conn.commit()
|
|
127
|
+
|
|
128
|
+
def close(self) -> None:
|
|
129
|
+
self.conn.close()
|
|
130
|
+
|
|
131
|
+
def __enter__(self) -> Cache:
|
|
132
|
+
return self
|
|
133
|
+
|
|
134
|
+
def __exit__(self, *exc) -> None:
|
|
135
|
+
self.close()
|
findupe/cli.py
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
"""findupe CLI: scan / apply / undo / cache clear.
|
|
2
|
+
|
|
3
|
+
scan never deletes; apply only acts on a reviewed selection JSON and asks for a
|
|
4
|
+
typed confirmation; undo restores from the Trash. See README for the workflow.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import grouping
|
|
16
|
+
from .cache import Cache
|
|
17
|
+
from .discover import discover
|
|
18
|
+
from .grouping import build_families
|
|
19
|
+
from .hashing import ensure_hashes, group_exact
|
|
20
|
+
from .imaging import compute_perceptual
|
|
21
|
+
from .dashboard import render_dashboard_html
|
|
22
|
+
from .ledger import list_scans, load_scan, record_scan
|
|
23
|
+
from .models import FileRecord, ScanResult
|
|
24
|
+
from .report import _is_image_family, generate_reports
|
|
25
|
+
from .stats import (
|
|
26
|
+
aggregate_undo_totals,
|
|
27
|
+
applied_scan_ids,
|
|
28
|
+
duplicates_timeline,
|
|
29
|
+
reclaimed_timeline,
|
|
30
|
+
render_stats_text,
|
|
31
|
+
)
|
|
32
|
+
from .trash import FakeTrasher, FinderTrasher, apply_selection, list_manifests, undo
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _fmt_bytes(n: float) -> str:
|
|
36
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
37
|
+
if n < 1024 or unit == "TB":
|
|
38
|
+
return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B"
|
|
39
|
+
n /= 1024
|
|
40
|
+
return f"{n} B"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _collect_hash_errors(
|
|
44
|
+
records: list[FileRecord], companions: list[FileRecord]
|
|
45
|
+
) -> list[tuple[Path, str]]:
|
|
46
|
+
"""Dedup by path: discover._attach_companions appends one shared sidecar
|
|
47
|
+
(e.g. an XMP) to EVERY primary in its stem-group, so a companion with the
|
|
48
|
+
same path can appear more than once in `companions` — one real problem
|
|
49
|
+
file must not be counted or reported more than once."""
|
|
50
|
+
seen: dict[Path, str] = {}
|
|
51
|
+
for rec in records + companions:
|
|
52
|
+
if rec.hash_error and rec.path not in seen:
|
|
53
|
+
seen[rec.path] = rec.hash_error
|
|
54
|
+
return list(seen.items())
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def cmd_scan(args: argparse.Namespace) -> int:
|
|
58
|
+
roots = [Path(p).expanduser().resolve() for p in args.paths]
|
|
59
|
+
bad_roots = [r for r in roots if not r.is_dir()]
|
|
60
|
+
if bad_roots:
|
|
61
|
+
for r in bad_roots:
|
|
62
|
+
print(f"error: {r}: not a directory or not mounted", file=sys.stderr)
|
|
63
|
+
return 2
|
|
64
|
+
print(f"discovering files under {len(roots)} root(s)…")
|
|
65
|
+
disc = discover(roots, exclude_globs=args.exclude, materialize=args.materialize)
|
|
66
|
+
print(f" {len(disc.records)} files · {len(disc.skipped_stubs)} cloud stubs skipped · "
|
|
67
|
+
f"{len(disc.skipped_managed)} managed libraries refused · {len(disc.hardlink_notes)} hardlinks · "
|
|
68
|
+
f"{len(disc.zero_byte)} zero-byte · {len(disc.errors)} read errors")
|
|
69
|
+
if disc.skipped_managed:
|
|
70
|
+
print(" refused (managed libraries):")
|
|
71
|
+
for p in disc.skipped_managed:
|
|
72
|
+
print(f" {p}")
|
|
73
|
+
if disc.errors:
|
|
74
|
+
print(f" {len(disc.errors)} read errors — see report notes for details")
|
|
75
|
+
|
|
76
|
+
with Cache(args.db) as cache:
|
|
77
|
+
print("exact pass (BLAKE2b funnel)…")
|
|
78
|
+
exact = group_exact(disc.records, cache=cache)
|
|
79
|
+
print(f" {len(exact)} exact-duplicate groups")
|
|
80
|
+
|
|
81
|
+
print("perceptual pass (images)…")
|
|
82
|
+
compute_perceptual(disc.records, cache=cache, workers=args.workers)
|
|
83
|
+
|
|
84
|
+
families, possible = build_families(
|
|
85
|
+
disc.records, exact, threshold_possible=args.threshold
|
|
86
|
+
)
|
|
87
|
+
members = [r for f in families for p in f.partitions for r in p.files]
|
|
88
|
+
companions = [c for r in members for c in r.companions]
|
|
89
|
+
ensure_hashes(members + companions, cache=cache)
|
|
90
|
+
|
|
91
|
+
hash_errors = _collect_hash_errors(disc.records, companions)
|
|
92
|
+
pointer = " — see report notes for details" if hash_errors else ""
|
|
93
|
+
print(f" {len(hash_errors)} decode/hash errors{pointer}")
|
|
94
|
+
|
|
95
|
+
scan_id = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
96
|
+
scan = ScanResult(
|
|
97
|
+
scan_id=scan_id, roots=roots, families=families,
|
|
98
|
+
skipped_stubs=disc.skipped_stubs, skipped_managed=disc.skipped_managed,
|
|
99
|
+
errors=disc.errors, hardlink_notes=disc.hardlink_notes, zero_byte=disc.zero_byte,
|
|
100
|
+
hash_errors=hash_errors,
|
|
101
|
+
)
|
|
102
|
+
img_path, other_path = generate_reports(scan, possible, Path(args.output))
|
|
103
|
+
|
|
104
|
+
img_families = [f for f in families if _is_image_family(f)]
|
|
105
|
+
other_families = [f for f in families if not _is_image_family(f)]
|
|
106
|
+
|
|
107
|
+
def _cat_summary(label: str, fams: list, path: Path) -> None:
|
|
108
|
+
surplus = sum(f.surplus_count for f in fams)
|
|
109
|
+
reclaimable = sum(f.surplus_bytes for f in fams)
|
|
110
|
+
print(f" {label}: {len(fams)} families · {surplus} surplus · "
|
|
111
|
+
f"{_fmt_bytes(reclaimable)} reclaimable — {path.resolve()}")
|
|
112
|
+
|
|
113
|
+
print(f"\n{len(families)} duplicate families · {len(possible)} possible matches (review-only)")
|
|
114
|
+
_cat_summary("images", img_families, img_path)
|
|
115
|
+
_cat_summary("other ", other_families, other_path)
|
|
116
|
+
print("next: open each report, review, Export selection, then run apply once per\n"
|
|
117
|
+
" exported file, e.g.:\n"
|
|
118
|
+
f" findupe apply findupe-selection-{scan_id}-images.json\n"
|
|
119
|
+
f" findupe apply findupe-selection-{scan_id}-other.json")
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
record_scan(scan, possible, img_families, other_families,
|
|
123
|
+
(img_path, other_path), scans_dir=args.scans_dir)
|
|
124
|
+
except Exception as e: # archival is a convenience on an already-succeeded scan
|
|
125
|
+
print(f"warning: could not archive this scan to history: {e}", file=sys.stderr)
|
|
126
|
+
return 0
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _make_trasher(args: argparse.Namespace):
|
|
130
|
+
return FakeTrasher(Path(args.trash_dir)) if args.trash_dir else FinderTrasher()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def cmd_apply(args: argparse.Namespace) -> int:
|
|
134
|
+
try:
|
|
135
|
+
selection = json.loads(Path(args.selection).read_text())
|
|
136
|
+
except OSError as e:
|
|
137
|
+
print(f"cannot read selection file: {e}", file=sys.stderr)
|
|
138
|
+
return 2
|
|
139
|
+
except json.JSONDecodeError as e:
|
|
140
|
+
print(f"selection file is not valid JSON: {e}", file=sys.stderr)
|
|
141
|
+
return 2
|
|
142
|
+
trasher = _make_trasher(args)
|
|
143
|
+
|
|
144
|
+
plan, manifest_path = apply_selection(
|
|
145
|
+
selection, trasher, dry_run=True, undo_dir=args.undo_dir
|
|
146
|
+
)
|
|
147
|
+
if plan.fatal:
|
|
148
|
+
print(f"REFUSED: {plan.fatal}", file=sys.stderr)
|
|
149
|
+
return 2
|
|
150
|
+
for fam, reason in plan.rejected_families.items():
|
|
151
|
+
print(f" rejected {fam}: {reason}", file=sys.stderr)
|
|
152
|
+
for path, reason in plan.skipped:
|
|
153
|
+
print(f" skipped {path}: {reason}", file=sys.stderr)
|
|
154
|
+
if not plan.to_trash:
|
|
155
|
+
print("nothing to do (all entries were skipped or rejected)")
|
|
156
|
+
return 1
|
|
157
|
+
|
|
158
|
+
comps = f" + {len(plan.companions)} companion file(s)" if plan.companions else ""
|
|
159
|
+
print(f"will move {len(plan.to_trash)} file(s){comps} "
|
|
160
|
+
f"({_fmt_bytes(plan.bytes_to_trash)}) to the Trash")
|
|
161
|
+
if args.dry_run:
|
|
162
|
+
for e in plan.to_trash:
|
|
163
|
+
print(f" would trash: {e['path']}")
|
|
164
|
+
for c in plan.companions:
|
|
165
|
+
print(f" would trash: {c['path']} (companion)")
|
|
166
|
+
return 0
|
|
167
|
+
|
|
168
|
+
answer = input("type 'trash' to confirm (anything else aborts): ")
|
|
169
|
+
if answer.strip().lower() != "trash":
|
|
170
|
+
print("aborted — nothing was moved")
|
|
171
|
+
return 1
|
|
172
|
+
|
|
173
|
+
plan, manifest_path = apply_selection(
|
|
174
|
+
selection, trasher, dry_run=False, undo_dir=args.undo_dir
|
|
175
|
+
)
|
|
176
|
+
print(f"trashed {len(plan.to_trash)} file(s); "
|
|
177
|
+
f"{len(plan.skipped)} skipped (see above)")
|
|
178
|
+
if manifest_path:
|
|
179
|
+
print(f"undo manifest: {manifest_path}")
|
|
180
|
+
print(f"to restore: findupe undo {manifest_path.name}")
|
|
181
|
+
return 0
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def cmd_undo(args: argparse.Namespace) -> int:
|
|
185
|
+
manifests = list_manifests(args.undo_dir)
|
|
186
|
+
if not args.manifest:
|
|
187
|
+
if not manifests:
|
|
188
|
+
print("no undo manifests")
|
|
189
|
+
return 0
|
|
190
|
+
for m in manifests:
|
|
191
|
+
print(m.name)
|
|
192
|
+
return 0
|
|
193
|
+
match = next(
|
|
194
|
+
(m for m in manifests if m.name == args.manifest or m.stem == args.manifest
|
|
195
|
+
or m.name.startswith(args.manifest)),
|
|
196
|
+
None,
|
|
197
|
+
)
|
|
198
|
+
if match is None:
|
|
199
|
+
candidate = Path(args.manifest)
|
|
200
|
+
match = candidate if candidate.is_file() else None
|
|
201
|
+
if match is None:
|
|
202
|
+
print(f"no manifest matching {args.manifest!r}", file=sys.stderr)
|
|
203
|
+
return 2
|
|
204
|
+
results = undo(match, trasher=_make_trasher(args), undo_dir=args.undo_dir)
|
|
205
|
+
for path, outcome in results:
|
|
206
|
+
print(f" {outcome}: {path}")
|
|
207
|
+
restored = sum(1 for _, o in results if o == "restored")
|
|
208
|
+
print(f"restored {restored}/{len(results)}")
|
|
209
|
+
return 0
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def cmd_stats(args: argparse.Namespace) -> int:
|
|
213
|
+
records = list_scans(args.scans_dir)
|
|
214
|
+
totals = aggregate_undo_totals(args.undo_dir, args.scans_dir)
|
|
215
|
+
applied = applied_scan_ids(args.undo_dir)
|
|
216
|
+
print(render_stats_text(records, totals, applied))
|
|
217
|
+
if args.html is not None:
|
|
218
|
+
html_doc = render_dashboard_html(
|
|
219
|
+
records, totals, applied,
|
|
220
|
+
reclaimed_timeline(args.undo_dir), duplicates_timeline(records),
|
|
221
|
+
)
|
|
222
|
+
args.html.write_text(html_doc, encoding="utf-8")
|
|
223
|
+
print(f"dashboard: {args.html.resolve()}")
|
|
224
|
+
return 0
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _find_scan(scan_id: str, scans_dir: Path):
|
|
228
|
+
"""Prefix-tolerant lookup, mirroring cmd_undo's manifest matching."""
|
|
229
|
+
exact = load_scan(scan_id, scans_dir)
|
|
230
|
+
if exact is not None:
|
|
231
|
+
return exact
|
|
232
|
+
return next((r for r in list_scans(scans_dir) if r.scan_id.startswith(scan_id)), None)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def cmd_history(args: argparse.Namespace) -> int:
|
|
236
|
+
if not args.scan_id:
|
|
237
|
+
records = list_scans(args.scans_dir)
|
|
238
|
+
if not records:
|
|
239
|
+
print("no archived scans")
|
|
240
|
+
return 0
|
|
241
|
+
applied = applied_scan_ids(args.undo_dir)
|
|
242
|
+
for r in records:
|
|
243
|
+
tag = "applied" if r.scan_id in applied else "not applied"
|
|
244
|
+
print(f"{r.scan_id} {r.duplicate_families} families "
|
|
245
|
+
f"{_fmt_bytes(r.surplus_bytes)} reclaimable — [{tag}]")
|
|
246
|
+
return 0
|
|
247
|
+
rec = _find_scan(args.scan_id, args.scans_dir)
|
|
248
|
+
if rec is None:
|
|
249
|
+
print(f"no archived scan matching {args.scan_id!r}", file=sys.stderr)
|
|
250
|
+
return 2
|
|
251
|
+
applied = rec.scan_id in applied_scan_ids(args.undo_dir)
|
|
252
|
+
print(f"{rec.scan_id} ({'applied' if applied else 'not applied'})")
|
|
253
|
+
print(f" {rec.duplicate_families} duplicate families · {rec.possible_matches} possible matches")
|
|
254
|
+
for category, path in rec.report_paths.items():
|
|
255
|
+
print(f" {category}: {path.resolve() if path else '(report copy missing)'}")
|
|
256
|
+
return 0
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def cmd_cache_clear(args: argparse.Namespace) -> int:
|
|
260
|
+
with Cache(args.db) as cache:
|
|
261
|
+
cache.clear()
|
|
262
|
+
db_path = cache.db_path
|
|
263
|
+
print(f"cache cleared: {db_path}")
|
|
264
|
+
return 0
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def main(argv: list[str] | None = None) -> int:
|
|
268
|
+
if sys.platform != "darwin":
|
|
269
|
+
print("findupe requires macOS — it relies on Finder/AppleScript for Trash "
|
|
270
|
+
"integration and APFS-specific behavior.", file=sys.stderr)
|
|
271
|
+
return 2
|
|
272
|
+
parser = argparse.ArgumentParser(
|
|
273
|
+
prog="findupe",
|
|
274
|
+
description="Safe duplicate finder: scan -> review HTML report -> apply -> (undo)",
|
|
275
|
+
)
|
|
276
|
+
# default=None, not a pre-resolved path: each is resolved lazily inside the
|
|
277
|
+
# function that actually needs it, only when still None, so passing an
|
|
278
|
+
# explicit flag here genuinely bypasses the one-time
|
|
279
|
+
# ~/.dupefinder -> ~/.findupe migration check — it never fires just
|
|
280
|
+
# because argparse filled in a default.
|
|
281
|
+
parser.add_argument("--db", type=Path, default=None, help=argparse.SUPPRESS)
|
|
282
|
+
parser.add_argument("--undo-dir", type=Path, default=None, help=argparse.SUPPRESS)
|
|
283
|
+
parser.add_argument("--scans-dir", type=Path, default=None, help=argparse.SUPPRESS)
|
|
284
|
+
parser.add_argument("--trash-dir", help="use a plain directory instead of the macOS Trash")
|
|
285
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
286
|
+
|
|
287
|
+
p_scan = sub.add_parser("scan", help="find duplicates and write the review report")
|
|
288
|
+
p_scan.add_argument("paths", nargs="+")
|
|
289
|
+
p_scan.add_argument("--exclude", action="append", default=[], metavar="GLOB")
|
|
290
|
+
p_scan.add_argument("--materialize", action="store_true",
|
|
291
|
+
help="download iCloud stubs instead of skipping them")
|
|
292
|
+
p_scan.add_argument("--threshold", type=int, default=grouping.THRESHOLD_POSSIBLE,
|
|
293
|
+
help="max pHash distance for the review-only 'possible' tier")
|
|
294
|
+
p_scan.add_argument("-o", "--output", default="report.html",
|
|
295
|
+
help="base report path; writes <name>-images.html and <name>-other.html")
|
|
296
|
+
p_scan.add_argument("--workers", type=int, default=4, help=argparse.SUPPRESS)
|
|
297
|
+
p_scan.set_defaults(func=cmd_scan)
|
|
298
|
+
|
|
299
|
+
p_apply = sub.add_parser("apply", help="move a reviewed selection to the Trash")
|
|
300
|
+
p_apply.add_argument("selection")
|
|
301
|
+
p_apply.add_argument("--dry-run", action="store_true")
|
|
302
|
+
p_apply.set_defaults(func=cmd_apply)
|
|
303
|
+
|
|
304
|
+
p_undo = sub.add_parser("undo", help="restore a previous apply (no arg: list manifests)")
|
|
305
|
+
p_undo.add_argument("manifest", nargs="?")
|
|
306
|
+
p_undo.set_defaults(func=cmd_undo)
|
|
307
|
+
|
|
308
|
+
p_cache = sub.add_parser("cache", help="cache maintenance")
|
|
309
|
+
cache_sub = p_cache.add_subparsers(dest="cache_command", required=True)
|
|
310
|
+
p_clear = cache_sub.add_parser("clear")
|
|
311
|
+
p_clear.set_defaults(func=cmd_cache_clear)
|
|
312
|
+
|
|
313
|
+
p_stats = sub.add_parser("stats", help="all-time totals across every scan and apply")
|
|
314
|
+
p_stats.add_argument("--html", nargs="?", const=Path("findupe-dashboard.html"),
|
|
315
|
+
default=None, type=Path, metavar="PATH",
|
|
316
|
+
help="also write an HTML dashboard (optional output path)")
|
|
317
|
+
p_stats.set_defaults(func=cmd_stats)
|
|
318
|
+
|
|
319
|
+
p_history = sub.add_parser("history", help="list archived scans, or show one by id")
|
|
320
|
+
p_history.add_argument("scan_id", nargs="?")
|
|
321
|
+
p_history.set_defaults(func=cmd_history)
|
|
322
|
+
|
|
323
|
+
args = parser.parse_args(argv)
|
|
324
|
+
try:
|
|
325
|
+
return args.func(args)
|
|
326
|
+
except KeyboardInterrupt:
|
|
327
|
+
print("\ninterrupted — nothing partial was deleted; cache keeps completed work",
|
|
328
|
+
file=sys.stderr)
|
|
329
|
+
return 130
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
if __name__ == "__main__":
|
|
333
|
+
sys.exit(main())
|
findupe/clones.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""APFS clone detection via physical-extent comparison (F_LOG2PHYS_EXT).
|
|
2
|
+
|
|
3
|
+
Spiked and verified on real hardware (2026-07-15): a `cp -c` clone reports
|
|
4
|
+
the exact same physical device offset as its original; an independent copy
|
|
5
|
+
does not; two independently-written files with identical content don't
|
|
6
|
+
overlap either (ruling out any content-based-dedup false positive). A
|
|
7
|
+
partially-edited clone still shares extents for its unedited remainder —
|
|
8
|
+
detecting that correctly needs interval overlap, not exact-offset equality,
|
|
9
|
+
which is what `shares_physical_extents` does.
|
|
10
|
+
|
|
11
|
+
Only ever called within an already-confirmed exact-hash duplicate cluster
|
|
12
|
+
(the only files clones can ever be) — never scan-wide, since the fcntl call
|
|
13
|
+
has real per-file cost.
|
|
14
|
+
|
|
15
|
+
Fails safe in BOTH directions, and the two directions are not symmetric:
|
|
16
|
+
- Any error (non-APFS volume, permission issue, file vanished) -> treat as
|
|
17
|
+
"not a clone". A detection failure must never HIDE reclaimable space that
|
|
18
|
+
genuinely exists.
|
|
19
|
+
- A *successful* call that reports device offset 0 for a file's first
|
|
20
|
+
extent -> also treated as "not a clone" and the whole probe for that file
|
|
21
|
+
is discarded. This is the more dangerous direction: a filesystem that
|
|
22
|
+
returns a "successful" but meaningless placeholder offset (rather than a
|
|
23
|
+
real error) for an unsupported case could make two genuinely-independent
|
|
24
|
+
files look identical, causing this code to falsely claim trashing one
|
|
25
|
+
frees no space — undercounting, the opposite of the error case, and worse
|
|
26
|
+
than not having this feature at all. Empirically verified on this real
|
|
27
|
+
machine (2026-07-15): FAT32 and exFAT volumes both raise a clean
|
|
28
|
+
OSError(ENOTSUP) rather than returning a placeholder — this guard is
|
|
29
|
+
deliberate defense-in-depth for filesystems/drivers not tested here, not
|
|
30
|
+
a response to an observed failure.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import os
|
|
36
|
+
import struct
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
|
|
39
|
+
_F_LOG2PHYS_EXT = 65
|
|
40
|
+
_STRUCT_FMT = "<Iqq" # struct log2phys, #pragma pack(4): uint, off_t, off_t
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _extent_map(path: Path) -> list[tuple[int, int]] | None:
|
|
44
|
+
"""[(device_offset, length), ...] covering the whole file. None on any
|
|
45
|
+
error, or if the kernel reports a suspicious 0 device offset."""
|
|
46
|
+
import fcntl # Unix-only; imported lazily so this module stays importable
|
|
47
|
+
# on Windows — cli.py's sys.platform guard runs before any code path that
|
|
48
|
+
# would actually reach this function, but module-level imports elsewhere
|
|
49
|
+
# in the chain (cli -> grouping -> clones) happen before that guard does.
|
|
50
|
+
try:
|
|
51
|
+
fd = os.open(path, os.O_RDONLY)
|
|
52
|
+
except OSError:
|
|
53
|
+
return None
|
|
54
|
+
try:
|
|
55
|
+
try:
|
|
56
|
+
size = os.fstat(fd).st_size
|
|
57
|
+
except OSError:
|
|
58
|
+
return None
|
|
59
|
+
if size == 0:
|
|
60
|
+
return None
|
|
61
|
+
extents: list[tuple[int, int]] = []
|
|
62
|
+
file_offset = 0
|
|
63
|
+
while file_offset < size:
|
|
64
|
+
query = struct.pack(_STRUCT_FMT, 0, size - file_offset, file_offset)
|
|
65
|
+
try:
|
|
66
|
+
result = fcntl.fcntl(fd, _F_LOG2PHYS_EXT, query)
|
|
67
|
+
except OSError:
|
|
68
|
+
return None # unsupported (non-APFS, network volume, etc.)
|
|
69
|
+
_flags, contig, dev_off = struct.unpack(_STRUCT_FMT, result)
|
|
70
|
+
if dev_off <= 0:
|
|
71
|
+
# a real device offset is never 0 (reserved/boot area) — treat
|
|
72
|
+
# as a placeholder/unsupported response, not real data
|
|
73
|
+
return None
|
|
74
|
+
if contig <= 0:
|
|
75
|
+
break # avoid an infinite loop if the kernel reports no progress
|
|
76
|
+
extents.append((dev_off, contig))
|
|
77
|
+
file_offset += contig
|
|
78
|
+
return extents
|
|
79
|
+
finally:
|
|
80
|
+
os.close(fd)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _overlaps(a: list[tuple[int, int]], b: list[tuple[int, int]]) -> bool:
|
|
84
|
+
return any(
|
|
85
|
+
da < db + lb and db < da + la
|
|
86
|
+
for da, la in a
|
|
87
|
+
for db, lb in b
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class KeeperExtents:
|
|
92
|
+
"""One file's extent map, computed once and checked against many others —
|
|
93
|
+
for a cluster with N surplus files, the keeper would otherwise get
|
|
94
|
+
re-probed N times for no reason."""
|
|
95
|
+
|
|
96
|
+
def __init__(self, path: Path) -> None:
|
|
97
|
+
self._extents = _extent_map(path)
|
|
98
|
+
|
|
99
|
+
def shares_with(self, other: Path) -> bool:
|
|
100
|
+
"""True if `other` shares ANY physical storage with this extent map —
|
|
101
|
+
an APFS clone relationship, even if `other` has since been partially
|
|
102
|
+
edited (the unedited remainder still overlaps). False (never raises)
|
|
103
|
+
if either side's extent map can't be trusted (unreadable, unsupported
|
|
104
|
+
volume, or a suspicious response) or if they genuinely share nothing."""
|
|
105
|
+
if not self._extents:
|
|
106
|
+
return False
|
|
107
|
+
other_extents = _extent_map(other)
|
|
108
|
+
if not other_extents:
|
|
109
|
+
return False
|
|
110
|
+
return _overlaps(self._extents, other_extents)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def shares_physical_extents(a: Path, b: Path) -> bool:
|
|
114
|
+
"""Convenience wrapper for a single one-off comparison. See KeeperExtents
|
|
115
|
+
for reusing one side's extent map across multiple comparisons."""
|
|
116
|
+
return KeeperExtents(a).shares_with(b)
|