findupe 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- findupe-0.4.0/.github/workflows/release.yml +97 -0
- findupe-0.4.0/.gitignore +11 -0
- findupe-0.4.0/.python-version +1 -0
- findupe-0.4.0/CHANGELOG.md +58 -0
- findupe-0.4.0/LICENSE +21 -0
- findupe-0.4.0/PKG-INFO +168 -0
- findupe-0.4.0/README.md +142 -0
- findupe-0.4.0/docs/superpowers/specs/2026-07-09-dupefinder-design.md +165 -0
- findupe-0.4.0/docs/superpowers/specs/2026-07-11-report-split-and-release-automation.md +103 -0
- findupe-0.4.0/docs/superpowers/specs/2026-07-12-make-problems-loud.md +114 -0
- findupe-0.4.0/docs/superpowers/specs/2026-07-12-scan-history-and-dashboard.md +158 -0
- findupe-0.4.0/pyproject.toml +58 -0
- findupe-0.4.0/src/findupe/__init__.py +25 -0
- findupe-0.4.0/src/findupe/cache.py +135 -0
- findupe-0.4.0/src/findupe/cli.py +333 -0
- findupe-0.4.0/src/findupe/clones.py +116 -0
- findupe-0.4.0/src/findupe/dashboard.py +160 -0
- findupe-0.4.0/src/findupe/discover.py +265 -0
- findupe-0.4.0/src/findupe/grouping.py +293 -0
- findupe-0.4.0/src/findupe/hashing.py +130 -0
- findupe-0.4.0/src/findupe/imaging.py +174 -0
- findupe-0.4.0/src/findupe/ledger.py +159 -0
- findupe-0.4.0/src/findupe/models.py +138 -0
- findupe-0.4.0/src/findupe/paths.py +38 -0
- findupe-0.4.0/src/findupe/report.py +379 -0
- findupe-0.4.0/src/findupe/stats.py +114 -0
- findupe-0.4.0/src/findupe/trash.py +409 -0
- findupe-0.4.0/tests/conftest.py +67 -0
- findupe-0.4.0/tests/test_cli.py +20 -0
- findupe-0.4.0/tests/test_clones.py +105 -0
- findupe-0.4.0/tests/test_dashboard.py +83 -0
- findupe-0.4.0/tests/test_discover.py +153 -0
- findupe-0.4.0/tests/test_e2e.py +513 -0
- findupe-0.4.0/tests/test_grouping.py +278 -0
- findupe-0.4.0/tests/test_hashing_cache.py +136 -0
- findupe-0.4.0/tests/test_imaging.py +108 -0
- findupe-0.4.0/tests/test_ledger.py +177 -0
- findupe-0.4.0/tests/test_paths.py +107 -0
- findupe-0.4.0/tests/test_report.py +283 -0
- findupe-0.4.0/tests/test_stats.py +157 -0
- findupe-0.4.0/tests/test_trash_apply.py +262 -0
- findupe-0.4.0/uv.lock +1006 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: write
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
release:
|
|
12
|
+
# skip the tool's own bump commit — avoids re-running on the push it makes
|
|
13
|
+
if: ${{ !startsWith(github.event.head_commit.message, 'bump:') }}
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
outputs:
|
|
16
|
+
version: ${{ steps.cz.outputs.version }}
|
|
17
|
+
previous_version: ${{ steps.cz.outputs.previous_version }}
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
|
20
|
+
with:
|
|
21
|
+
fetch-depth: 0 # commitizen needs full history since the last tag
|
|
22
|
+
token: ${{ secrets.GITHUB_TOKEN }}
|
|
23
|
+
# NOT persist-credentials: false here (unlike `build` below) —
|
|
24
|
+
# commitizen-action's push: true default relies on the credential
|
|
25
|
+
# actions/checkout leaves in .git/config to push the bump commit + tag.
|
|
26
|
+
|
|
27
|
+
- id: cz
|
|
28
|
+
uses: commitizen-tools/commitizen-action@338bbd841b75aaee6bf5340e1fa12f6ab58ff9ff # 0.27.1
|
|
29
|
+
with:
|
|
30
|
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
31
|
+
changelog_increment_filename: _release_notes.md
|
|
32
|
+
|
|
33
|
+
- name: Create GitHub Release
|
|
34
|
+
# commitizen-action always sets `version` (= `cz version --project`,
|
|
35
|
+
# the static pyproject version) even on a no-bump run, since a
|
|
36
|
+
# no-increment `cz bump` exits via the action's default no_raise=21.
|
|
37
|
+
# Comparing against `previous_version` is the real bump signal.
|
|
38
|
+
if: ${{ steps.cz.outputs.version != steps.cz.outputs.previous_version }}
|
|
39
|
+
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
|
|
40
|
+
with:
|
|
41
|
+
tag_name: v${{ steps.cz.outputs.version }}
|
|
42
|
+
body_path: _release_notes.md
|
|
43
|
+
env:
|
|
44
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
45
|
+
|
|
46
|
+
# Build and publish are separate jobs, and publish gets NO checkout — per
|
|
47
|
+
# the publish action's own README: separating build from publish means
|
|
48
|
+
# anything maliciously injected into the build environment can't ride
|
|
49
|
+
# along into the job that holds PyPI's OIDC trust.
|
|
50
|
+
build:
|
|
51
|
+
needs: release
|
|
52
|
+
if: ${{ needs.release.outputs.version != needs.release.outputs.previous_version }}
|
|
53
|
+
runs-on: ubuntu-latest
|
|
54
|
+
permissions:
|
|
55
|
+
contents: read # only checks out code + builds; no git writes, unlike `release`
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
|
58
|
+
with:
|
|
59
|
+
# commitizen-action already pushed the bump commit + tag to main
|
|
60
|
+
# inside the `release` job (push: true is its default) — a plain
|
|
61
|
+
# checkout here would still resolve to the pre-bump commit that
|
|
62
|
+
# triggered this run, not the bumped pyproject.toml `uv build`
|
|
63
|
+
# needs to read. Check out the tag the release job just created.
|
|
64
|
+
ref: v${{ needs.release.outputs.version }}
|
|
65
|
+
# this job only reads + builds, never pushes — no reason for uv
|
|
66
|
+
# build or its dependencies to have a retained git credential.
|
|
67
|
+
persist-credentials: false
|
|
68
|
+
|
|
69
|
+
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
|
70
|
+
|
|
71
|
+
- run: uv build
|
|
72
|
+
|
|
73
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
74
|
+
with:
|
|
75
|
+
name: dist
|
|
76
|
+
path: dist/
|
|
77
|
+
|
|
78
|
+
publish:
|
|
79
|
+
needs: [release, build]
|
|
80
|
+
if: ${{ needs.release.outputs.version != needs.release.outputs.previous_version }}
|
|
81
|
+
runs-on: ubuntu-latest
|
|
82
|
+
environment:
|
|
83
|
+
name: pypi
|
|
84
|
+
url: https://pypi.org/p/findupe
|
|
85
|
+
permissions:
|
|
86
|
+
id-token: write # mandatory for PyPI Trusted Publishing (OIDC); scoped to this job only
|
|
87
|
+
steps:
|
|
88
|
+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
89
|
+
with:
|
|
90
|
+
name: dist
|
|
91
|
+
path: dist/
|
|
92
|
+
|
|
93
|
+
- name: Publish to PyPI
|
|
94
|
+
# Pinned to the commit tagged v1.14.0 (not the release/v1 branch
|
|
95
|
+
# pointer, which moves) — this job carries Trusted Publishing
|
|
96
|
+
# authority, so it gets stricter pinning than the rest of this file.
|
|
97
|
+
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
findupe-0.4.0/.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. Versions follow
|
|
4
|
+
[Semantic Versioning](https://semver.org/). New entries are generated
|
|
5
|
+
automatically by [Commitizen](https://commitizen-tools.github.io/commitizen/)
|
|
6
|
+
from [Conventional Commits](https://www.conventionalcommits.org/) on every
|
|
7
|
+
qualifying push to `main` — the heading style below matches what it emits.
|
|
8
|
+
|
|
9
|
+
## v0.4.0 (2026-07-16)
|
|
10
|
+
|
|
11
|
+
### BREAKING CHANGE
|
|
12
|
+
|
|
13
|
+
- the CLI command changes from `dupefinder` to `findupe`,
|
|
14
|
+
and the on-disk data directory moves from `~/.dupefinder/` to
|
|
15
|
+
`~/.findupe/` (auto-migrated in place on first run). Any scripts,
|
|
16
|
+
aliases, or cron jobs invoking `dupefinder` directly, or reading
|
|
17
|
+
`~/.dupefinder/` paths directly instead of through the CLI, need updating.
|
|
18
|
+
|
|
19
|
+
### Feat
|
|
20
|
+
|
|
21
|
+
- PyPI-publish CI (OIDC), and APFS clone detection
|
|
22
|
+
- rename project dupefinder -> findupe, honest space labels, PyPI-ready metadata
|
|
23
|
+
|
|
24
|
+
### Fix
|
|
25
|
+
|
|
26
|
+
- address PR review findings (Codex + CodeRabbit)
|
|
27
|
+
|
|
28
|
+
## v0.3.0 (2026-07-13)
|
|
29
|
+
|
|
30
|
+
### Feat
|
|
31
|
+
|
|
32
|
+
- persistent scan history, stats totals, and an HTML dashboard
|
|
33
|
+
|
|
34
|
+
## v0.2.1 (2026-07-12)
|
|
35
|
+
|
|
36
|
+
### Fix
|
|
37
|
+
|
|
38
|
+
- fail fast on bad scan roots, surface hash/decode errors loudly
|
|
39
|
+
|
|
40
|
+
## v0.2.0 (2026-07-11)
|
|
41
|
+
|
|
42
|
+
### Feat
|
|
43
|
+
|
|
44
|
+
- split HTML report into images/other categories
|
|
45
|
+
|
|
46
|
+
## v0.1.0 (2026-07-09)
|
|
47
|
+
|
|
48
|
+
### Feat
|
|
49
|
+
|
|
50
|
+
- exact-duplicate detection for any file type via size → 64KB-edge → full BLAKE2b funnel
|
|
51
|
+
- same-photo-across-formats detection (HEIC/JPEG/RAW, re-encodes, resized exports) via orientation-normalized pHash + dHash; keeping `X.CR3` + `X.jpg` side by side is never flagged
|
|
52
|
+
- self-contained offline HTML review report: side-by-side thumbnails, per-keeper pre-checked candidates, live reclaim counter, one-click selection export
|
|
53
|
+
- review-first `apply` to the real macOS Trash (all volumes), typed confirmation, full re-verification at apply time, last-copy-per-partition protection
|
|
54
|
+
- `undo` restores from an atomically-written manifest, immune to Finder's collision renames
|
|
55
|
+
- companions (Live Photo `.MOV`, `.XMP`/`.AAE` sidecars) ride along with their primary on trash and undo
|
|
56
|
+
- safety guards: Photos/Lightroom denylist, iCloud/Dropbox stub handling (`--materialize`), hardlink/symlink/zero-byte exclusion, burst/low-entropy flagging, cloud-sync badges
|
|
57
|
+
- persistent SQLite hash cache; re-scans only hash new/changed files
|
|
58
|
+
- validated end-to-end against a real 6,177-photo library; two rounds of real-data tuning plus a 20-agent adversarial code review (12 defects found and fixed) drove false positives to zero
|
findupe-0.4.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 jyshnkr
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
findupe-0.4.0/PKG-INFO
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: findupe
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Safe duplicate finder & reviewer for macOS — exact + perceptual image matching, review-first deletion to Trash
|
|
5
|
+
Project-URL: Homepage, https://github.com/jyshnkr/findupe
|
|
6
|
+
Project-URL: Repository, https://github.com/jyshnkr/findupe
|
|
7
|
+
Project-URL: Changelog, https://github.com/jyshnkr/findupe/blob/main/CHANGELOG.md
|
|
8
|
+
Author-email: JayaShankar Mangina <jyshnkr.ai@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: cli,deduplication,duplicates,images,macos,perceptual-hash,photos
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
18
|
+
Classifier: Topic :: Utilities
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: imagehash>=4.3.2
|
|
21
|
+
Requires-Dist: pillow-heif>=1.4
|
|
22
|
+
Requires-Dist: pillow>=12.0
|
|
23
|
+
Requires-Dist: pybktree>=1.1
|
|
24
|
+
Requires-Dist: rawpy>=0.27
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# findupe
|
|
28
|
+
|
|
29
|
+
Safe duplicate finder & reviewer for macOS. Finds **exact duplicates** (any file type,
|
|
30
|
+
zero false positives) and **same-photo-different-format duplicates** (HEIC / JPEG / RAW,
|
|
31
|
+
re-encodes, resized exports), then lets *you* review everything visually in an HTML
|
|
32
|
+
report before anything moves — to the real macOS **Trash**, never deleted outright.
|
|
33
|
+
|
|
34
|
+
Built for photographers: keeping `X.CR3` + `X.jpg` side by side is intentional and is
|
|
35
|
+
**never** flagged. Only surplus copies *within* a format (`X copy.jpg`, `X_2.jpg`,
|
|
36
|
+
a re-imported CR3) become deletion candidates.
|
|
37
|
+
|
|
38
|
+
## Workflow
|
|
39
|
+
|
|
40
|
+
`scan` writes two reports — one for images (exact + perceptual matching), one
|
|
41
|
+
for everything else (PDFs, text, archives, ... — exact-hash matches only,
|
|
42
|
+
since perceptual matching never applies to non-images). Review and apply each
|
|
43
|
+
independently.
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
1. uv run findupe scan ~/Pictures/inbox "/Volumes/Extreme SSD/photos"
|
|
47
|
+
2. open report-images.html # photo/image duplicates — thumbnails, adjust checkboxes
|
|
48
|
+
open report-other.html # everything else — plain checkbox + path + size rows
|
|
49
|
+
3. # click "Export selection" on each ->
|
|
50
|
+
# findupe-selection-<id>-images.json
|
|
51
|
+
# findupe-selection-<id>-other.json
|
|
52
|
+
4. uv run findupe apply findupe-selection-<id>-images.json --dry-run # preview
|
|
53
|
+
uv run findupe apply findupe-selection-<id>-other.json --dry-run # preview
|
|
54
|
+
5. uv run findupe apply findupe-selection-<id>-images.json # typed confirmation
|
|
55
|
+
uv run findupe apply findupe-selection-<id>-other.json # typed confirmation
|
|
56
|
+
6. uv run findupe undo # list restore points
|
|
57
|
+
uv run findupe undo <manifest> # put everything back
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## How it decides two files are "the same"
|
|
61
|
+
|
|
62
|
+
| Tier | Test | Shown as |
|
|
63
|
+
|---|---|---|
|
|
64
|
+
| Exact | size → 64KB-edges hash → full **BLAKE2b** | Exact duplicates (pre-checked per keeper rule) |
|
|
65
|
+
| Strong visual | **pHash ≤ 2 AND dHash ≤ 2** after EXIF-orientation normalization | Same image, multiple versions |
|
|
66
|
+
| Possible | pHash 3–8 | Review-only — no checkboxes, the tool will not touch these |
|
|
67
|
+
|
|
68
|
+
Thresholds were calibrated on real files: a HEIC→JPEG export measures distance **0**
|
|
69
|
+
(even resized); different photos measure ≥ 28. Burst frames are the treacherous case —
|
|
70
|
+
static-scene frames shot in the same second can hash **identically** — so three extra
|
|
71
|
+
guards demote them to review-only, verified against a real 6,177-photo library:
|
|
72
|
+
|
|
73
|
+
- capture metadata (time + exposure) must match for any strong match to form;
|
|
74
|
+
- **SubSecTimeOriginal** must match too — it differs between burst frames shot within
|
|
75
|
+
the same second (`'75'` vs `'97'` on consecutive EOS R6 II frames);
|
|
76
|
+
- **RAW↔RAW pairs are never perceptually strong.** Every real-world RAW duplicate is a
|
|
77
|
+
byte-identical copy (nobody re-encodes a CR3), so RAW deletion candidates come only
|
|
78
|
+
from the exact tier — burst frames whose previews collide land in review-only.
|
|
79
|
+
|
|
80
|
+
RAW files are fingerprinted via their embedded JPEG preview (rawpy; exiftool fallback).
|
|
81
|
+
And "surplus" is computed only within *directly-matched* same-format clusters — a file
|
|
82
|
+
that merely shares a family through a chain of cross-format links renders as an
|
|
83
|
+
informational "sibling", never as a deletion candidate.
|
|
84
|
+
|
|
85
|
+
## Safety model
|
|
86
|
+
|
|
87
|
+
- **`scan` has no delete authority.** Deletion happens only through `apply`, which takes
|
|
88
|
+
the selection file you exported from the report after human review.
|
|
89
|
+
- **Everything is re-verified at apply time** — every keeper and every candidate is
|
|
90
|
+
re-checked (existence, size, full BLAKE2b). A file that changed since the scan is
|
|
91
|
+
skipped; a keeper that changed rejects its whole partition; a selection that lists a
|
|
92
|
+
keeper for deletion is rejected outright.
|
|
93
|
+
- **The last copy always survives**: at most `n-1` files of a (family, format) partition
|
|
94
|
+
can be trashed, enforced independently of the report UI.
|
|
95
|
+
- **Real Trash, all volumes**: batched Finder AppleScript, so "Put Back" works — external
|
|
96
|
+
drives use their own `.Trashes` (pre-flight checked; a volume without a working Trash is
|
|
97
|
+
refused, never silently permanent-deleted).
|
|
98
|
+
- **Undo manifest written before anything moves** (atomic write), and `undo` re-locates
|
|
99
|
+
files in the Trash by size + hash — immune to Finder's collision renames.
|
|
100
|
+
- **Never touched at all**: hardlinks (deleting one reclaims nothing — informational),
|
|
101
|
+
Photos/Lightroom library internals (hard denylist), symlinks, iCloud dataless stubs
|
|
102
|
+
(skipped and listed; `--materialize` downloads them on purpose), zero-byte files.
|
|
103
|
+
- **Companions ride along**: Live Photo `.MOV`s and `XMP`/`AAE` sidecars are trashed with
|
|
104
|
+
their primary and restored with it on undo.
|
|
105
|
+
- **Flagged families are never pre-checked**: >3 visually-matched same-format files
|
|
106
|
+
("possible-burst") or near-uniform images ("low-entropy") require deliberate clicks.
|
|
107
|
+
- Files in iCloud/Dropbox-synced folders carry a ☁ badge — deleting them propagates to
|
|
108
|
+
your other devices.
|
|
109
|
+
- **APFS clones are detected where possible** (`⧉ clone — 0 B freed` badge, via physical
|
|
110
|
+
extent comparison — `F_LOG2PHYS_EXT`) and excluded from the reclaimable total, since
|
|
111
|
+
trashing one frees nothing while its keeper survives. Detection isn't foolproof (some
|
|
112
|
+
volumes/setups can't be probed, and only clone-of-keeper is checked, not
|
|
113
|
+
clone-of-another-candidate) — an undetected clone still reclaims no space when
|
|
114
|
+
trashed, same as before this existed (the report footer explains this too).
|
|
115
|
+
|
|
116
|
+
## Install
|
|
117
|
+
|
|
118
|
+
Requires macOS.
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
pipx install findupe
|
|
122
|
+
# or
|
|
123
|
+
uv tool install findupe
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Then `findupe --help`. Running on a non-macOS platform refuses immediately with a
|
|
127
|
+
clear error — the Trash integration (Finder/AppleScript) and clone-detection notes
|
|
128
|
+
are macOS/APFS-specific.
|
|
129
|
+
|
|
130
|
+
## Dev setup
|
|
131
|
+
|
|
132
|
+
Requires macOS + [uv](https://docs.astral.sh/uv/). Python 3.11+ and all dependencies
|
|
133
|
+
(Pillow, pillow-heif, imagehash, rawpy, pybktree) are resolved automatically.
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
uv sync
|
|
137
|
+
uv run pytest # 145 tests
|
|
138
|
+
uv run findupe --help
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
First `apply` may trigger a one-time macOS permission prompt ("Terminal wants to control
|
|
142
|
+
Finder") — that's the Trash integration. If you deny it, apply aborts safely.
|
|
143
|
+
|
|
144
|
+
All state lives under `~/.findupe/`: the hash cache (`index.db`, re-scans only hash
|
|
145
|
+
new/changed files), scan history (`scans/`), and undo manifests (`undo/`).
|
|
146
|
+
`findupe cache clear` resets the hash cache only. If you have an existing
|
|
147
|
+
`~/.dupefinder/` from before the `findupe` rename, it's moved into place
|
|
148
|
+
automatically, once, the first time you run any command that doesn't override
|
|
149
|
+
`--db`/`--undo-dir`/`--scans-dir`.
|
|
150
|
+
|
|
151
|
+
## Commit conventions & releases
|
|
152
|
+
|
|
153
|
+
Commits to `main` follow [Conventional Commits](https://www.conventionalcommits.org/):
|
|
154
|
+
`feat:` for user-facing additions, `fix:` for bug fixes, `chore:`/`docs:`/`test:`
|
|
155
|
+
for everything with no release impact. A qualifying push is picked up automatically by
|
|
156
|
+
[Commitizen](https://commitizen-tools.github.io/commitizen/) — it computes the next
|
|
157
|
+
[semantic version](https://semver.org/) (`feat` → minor, `fix`/`perf`/`refactor` → patch,
|
|
158
|
+
`feat!`/`BREAKING CHANGE` → major), updates `CHANGELOG.md`, tags the release, and a GitHub
|
|
159
|
+
Action turns that tag into a [GitHub Release](https://github.com/jyshnkr/findupe/releases)
|
|
160
|
+
with no manual step. See `.github/workflows/release.yml`.
|
|
161
|
+
|
|
162
|
+
## Deliberately out of scope (v1)
|
|
163
|
+
|
|
164
|
+
Scanning inside Photos/Lightroom libraries · OCR screenshot discrimination · config
|
|
165
|
+
file · GUI · scheduling. (APFS clone detection shipped — see Safety model above.) See
|
|
166
|
+
`docs/superpowers/specs/2026-07-09-dupefinder-design.md` for the full original design +
|
|
167
|
+
rationale (written before the project was renamed from `dupefinder` to `findupe`, and
|
|
168
|
+
before clone detection existed).
|
findupe-0.4.0/README.md
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# findupe
|
|
2
|
+
|
|
3
|
+
Safe duplicate finder & reviewer for macOS. Finds **exact duplicates** (any file type,
|
|
4
|
+
zero false positives) and **same-photo-different-format duplicates** (HEIC / JPEG / RAW,
|
|
5
|
+
re-encodes, resized exports), then lets *you* review everything visually in an HTML
|
|
6
|
+
report before anything moves — to the real macOS **Trash**, never deleted outright.
|
|
7
|
+
|
|
8
|
+
Built for photographers: keeping `X.CR3` + `X.jpg` side by side is intentional and is
|
|
9
|
+
**never** flagged. Only surplus copies *within* a format (`X copy.jpg`, `X_2.jpg`,
|
|
10
|
+
a re-imported CR3) become deletion candidates.
|
|
11
|
+
|
|
12
|
+
## Workflow
|
|
13
|
+
|
|
14
|
+
`scan` writes two reports — one for images (exact + perceptual matching), one
|
|
15
|
+
for everything else (PDFs, text, archives, ... — exact-hash matches only,
|
|
16
|
+
since perceptual matching never applies to non-images). Review and apply each
|
|
17
|
+
independently.
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
1. uv run findupe scan ~/Pictures/inbox "/Volumes/Extreme SSD/photos"
|
|
21
|
+
2. open report-images.html # photo/image duplicates — thumbnails, adjust checkboxes
|
|
22
|
+
open report-other.html # everything else — plain checkbox + path + size rows
|
|
23
|
+
3. # click "Export selection" on each ->
|
|
24
|
+
# findupe-selection-<id>-images.json
|
|
25
|
+
# findupe-selection-<id>-other.json
|
|
26
|
+
4. uv run findupe apply findupe-selection-<id>-images.json --dry-run # preview
|
|
27
|
+
uv run findupe apply findupe-selection-<id>-other.json --dry-run # preview
|
|
28
|
+
5. uv run findupe apply findupe-selection-<id>-images.json # typed confirmation
|
|
29
|
+
uv run findupe apply findupe-selection-<id>-other.json # typed confirmation
|
|
30
|
+
6. uv run findupe undo # list restore points
|
|
31
|
+
uv run findupe undo <manifest> # put everything back
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## How it decides two files are "the same"
|
|
35
|
+
|
|
36
|
+
| Tier | Test | Shown as |
|
|
37
|
+
|---|---|---|
|
|
38
|
+
| Exact | size → 64KB-edges hash → full **BLAKE2b** | Exact duplicates (pre-checked per keeper rule) |
|
|
39
|
+
| Strong visual | **pHash ≤ 2 AND dHash ≤ 2** after EXIF-orientation normalization | Same image, multiple versions |
|
|
40
|
+
| Possible | pHash 3–8 | Review-only — no checkboxes, the tool will not touch these |
|
|
41
|
+
|
|
42
|
+
Thresholds were calibrated on real files: a HEIC→JPEG export measures distance **0**
|
|
43
|
+
(even resized); different photos measure ≥ 28. Burst frames are the treacherous case —
|
|
44
|
+
static-scene frames shot in the same second can hash **identically** — so three extra
|
|
45
|
+
guards demote them to review-only, verified against a real 6,177-photo library:
|
|
46
|
+
|
|
47
|
+
- capture metadata (time + exposure) must match for any strong match to form;
|
|
48
|
+
- **SubSecTimeOriginal** must match too — it differs between burst frames shot within
|
|
49
|
+
the same second (`'75'` vs `'97'` on consecutive EOS R6 II frames);
|
|
50
|
+
- **RAW↔RAW pairs are never perceptually strong.** Every real-world RAW duplicate is a
|
|
51
|
+
byte-identical copy (nobody re-encodes a CR3), so RAW deletion candidates come only
|
|
52
|
+
from the exact tier — burst frames whose previews collide land in review-only.
|
|
53
|
+
|
|
54
|
+
RAW files are fingerprinted via their embedded JPEG preview (rawpy; exiftool fallback).
|
|
55
|
+
And "surplus" is computed only within *directly-matched* same-format clusters — a file
|
|
56
|
+
that merely shares a family through a chain of cross-format links renders as an
|
|
57
|
+
informational "sibling", never as a deletion candidate.
|
|
58
|
+
|
|
59
|
+
## Safety model
|
|
60
|
+
|
|
61
|
+
- **`scan` has no delete authority.** Deletion happens only through `apply`, which takes
|
|
62
|
+
the selection file you exported from the report after human review.
|
|
63
|
+
- **Everything is re-verified at apply time** — every keeper and every candidate is
|
|
64
|
+
re-checked (existence, size, full BLAKE2b). A file that changed since the scan is
|
|
65
|
+
skipped; a keeper that changed rejects its whole partition; a selection that lists a
|
|
66
|
+
keeper for deletion is rejected outright.
|
|
67
|
+
- **The last copy always survives**: at most `n-1` files of a (family, format) partition
|
|
68
|
+
can be trashed, enforced independently of the report UI.
|
|
69
|
+
- **Real Trash, all volumes**: batched Finder AppleScript, so "Put Back" works — external
|
|
70
|
+
drives use their own `.Trashes` (pre-flight checked; a volume without a working Trash is
|
|
71
|
+
refused, never silently permanent-deleted).
|
|
72
|
+
- **Undo manifest written before anything moves** (atomic write), and `undo` re-locates
|
|
73
|
+
files in the Trash by size + hash — immune to Finder's collision renames.
|
|
74
|
+
- **Never touched at all**: hardlinks (deleting one reclaims nothing — informational),
|
|
75
|
+
Photos/Lightroom library internals (hard denylist), symlinks, iCloud dataless stubs
|
|
76
|
+
(skipped and listed; `--materialize` downloads them on purpose), zero-byte files.
|
|
77
|
+
- **Companions ride along**: Live Photo `.MOV`s and `XMP`/`AAE` sidecars are trashed with
|
|
78
|
+
their primary and restored with it on undo.
|
|
79
|
+
- **Flagged families are never pre-checked**: >3 visually-matched same-format files
|
|
80
|
+
("possible-burst") or near-uniform images ("low-entropy") require deliberate clicks.
|
|
81
|
+
- Files in iCloud/Dropbox-synced folders carry a ☁ badge — deleting them propagates to
|
|
82
|
+
your other devices.
|
|
83
|
+
- **APFS clones are detected where possible** (`⧉ clone — 0 B freed` badge, via physical
|
|
84
|
+
extent comparison — `F_LOG2PHYS_EXT`) and excluded from the reclaimable total, since
|
|
85
|
+
trashing one frees nothing while its keeper survives. Detection isn't foolproof (some
|
|
86
|
+
volumes/setups can't be probed, and only clone-of-keeper is checked, not
|
|
87
|
+
clone-of-another-candidate) — an undetected clone still reclaims no space when
|
|
88
|
+
trashed, same as before this existed (the report footer explains this too).
|
|
89
|
+
|
|
90
|
+
## Install
|
|
91
|
+
|
|
92
|
+
Requires macOS.
|
|
93
|
+
|
|
94
|
+
```sh
|
|
95
|
+
pipx install findupe
|
|
96
|
+
# or
|
|
97
|
+
uv tool install findupe
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Then `findupe --help`. Running on a non-macOS platform refuses immediately with a
|
|
101
|
+
clear error — the Trash integration (Finder/AppleScript) and clone-detection notes
|
|
102
|
+
are macOS/APFS-specific.
|
|
103
|
+
|
|
104
|
+
## Dev setup
|
|
105
|
+
|
|
106
|
+
Requires macOS + [uv](https://docs.astral.sh/uv/). Python 3.11+ and all dependencies
|
|
107
|
+
(Pillow, pillow-heif, imagehash, rawpy, pybktree) are resolved automatically.
|
|
108
|
+
|
|
109
|
+
```sh
|
|
110
|
+
uv sync
|
|
111
|
+
uv run pytest # 145 tests
|
|
112
|
+
uv run findupe --help
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
First `apply` may trigger a one-time macOS permission prompt ("Terminal wants to control
|
|
116
|
+
Finder") — that's the Trash integration. If you deny it, apply aborts safely.
|
|
117
|
+
|
|
118
|
+
All state lives under `~/.findupe/`: the hash cache (`index.db`, re-scans only hash
|
|
119
|
+
new/changed files), scan history (`scans/`), and undo manifests (`undo/`).
|
|
120
|
+
`findupe cache clear` resets the hash cache only. If you have an existing
|
|
121
|
+
`~/.dupefinder/` from before the `findupe` rename, it's moved into place
|
|
122
|
+
automatically, once, the first time you run any command that doesn't override
|
|
123
|
+
`--db`/`--undo-dir`/`--scans-dir`.
|
|
124
|
+
|
|
125
|
+
## Commit conventions & releases
|
|
126
|
+
|
|
127
|
+
Commits to `main` follow [Conventional Commits](https://www.conventionalcommits.org/):
|
|
128
|
+
`feat:` for user-facing additions, `fix:` for bug fixes, `chore:`/`docs:`/`test:`
|
|
129
|
+
for everything with no release impact. A qualifying push is picked up automatically by
|
|
130
|
+
[Commitizen](https://commitizen-tools.github.io/commitizen/) — it computes the next
|
|
131
|
+
[semantic version](https://semver.org/) (`feat` → minor, `fix`/`perf`/`refactor` → patch,
|
|
132
|
+
`feat!`/`BREAKING CHANGE` → major), updates `CHANGELOG.md`, tags the release, and a GitHub
|
|
133
|
+
Action turns that tag into a [GitHub Release](https://github.com/jyshnkr/findupe/releases)
|
|
134
|
+
with no manual step. See `.github/workflows/release.yml`.
|
|
135
|
+
|
|
136
|
+
## Deliberately out of scope (v1)
|
|
137
|
+
|
|
138
|
+
Scanning inside Photos/Lightroom libraries · OCR screenshot discrimination · config
|
|
139
|
+
file · GUI · scheduling. (APFS clone detection shipped — see Safety model above.) See
|
|
140
|
+
`docs/superpowers/specs/2026-07-09-dupefinder-design.md` for the full original design +
|
|
141
|
+
rationale (written before the project was renamed from `dupefinder` to `findupe`, and
|
|
142
|
+
before clone detection existed).
|