media-workflow 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- media_workflow/__init__.py +27 -0
- media_workflow/__main__.py +4 -0
- media_workflow/cli.py +232 -0
- media_workflow/duplicates.py +339 -0
- media_workflow/fix_times.py +176 -0
- media_workflow/importer.py +730 -0
- media_workflow-0.2.0.dist-info/METADATA +254 -0
- media_workflow-0.2.0.dist-info/RECORD +12 -0
- media_workflow-0.2.0.dist-info/WHEEL +5 -0
- media_workflow-0.2.0.dist-info/entry_points.txt +2 -0
- media_workflow-0.2.0.dist-info/licenses/LICENSE +201 -0
- media_workflow-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Media workflow: import, organize, and de-duplicate photo collections."""
|
|
2
|
+
|
|
3
|
+
from .importer import (
|
|
4
|
+
ExifMetadata,
|
|
5
|
+
ImportResult,
|
|
6
|
+
MetadataError,
|
|
7
|
+
NotAnImageError,
|
|
8
|
+
build_filename,
|
|
9
|
+
build_relative_path,
|
|
10
|
+
import_images,
|
|
11
|
+
read_metadata,
|
|
12
|
+
sanitize_component,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"ExifMetadata",
|
|
17
|
+
"ImportResult",
|
|
18
|
+
"MetadataError",
|
|
19
|
+
"NotAnImageError",
|
|
20
|
+
"build_filename",
|
|
21
|
+
"build_relative_path",
|
|
22
|
+
"import_images",
|
|
23
|
+
"read_metadata",
|
|
24
|
+
"sanitize_component",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
__version__ = "0.1.2"
|
media_workflow/cli.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""Command-line interface for the media-workflow package."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from . import duplicates, fix_times
|
|
10
|
+
from .importer import ImportResult, import_images
|
|
11
|
+
|
|
12
|
+
SOURCE_ENV = "MEDIA_WORKFLOW_SOURCE"
|
|
13
|
+
ROOT_ENV = "MEDIA_WORKFLOW_ROOT"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _env_default(name: str) -> str | None:
|
|
17
|
+
"""Return a non-empty environment value, or ``None`` when unset/empty."""
|
|
18
|
+
|
|
19
|
+
return os.environ.get(name) or None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _print_import_results(results: list[ImportResult], *, quiet: bool = False) -> None:
|
|
23
|
+
for result in results:
|
|
24
|
+
if result.status == "moved":
|
|
25
|
+
print(f"moved: {result.source} -> {result.destination}")
|
|
26
|
+
elif result.status == "would_move":
|
|
27
|
+
print(f"[dry run] would move: {result.source} -> {result.destination}")
|
|
28
|
+
elif result.status == "unchanged":
|
|
29
|
+
print(f"unchanged: {result.source}")
|
|
30
|
+
elif result.status == "failed":
|
|
31
|
+
print(f"error: {result.source}: {result.reason}", file=sys.stderr)
|
|
32
|
+
elif result.status == "skipped" and not quiet:
|
|
33
|
+
print(f"skipped: {result.source} ({result.reason})")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _cmd_import(args: argparse.Namespace) -> int:
|
|
37
|
+
try:
|
|
38
|
+
results = import_images(
|
|
39
|
+
args.source,
|
|
40
|
+
args.destination,
|
|
41
|
+
recursive=args.recursive,
|
|
42
|
+
dry_run=args.dry_run,
|
|
43
|
+
include_subseconds=not args.no_subseconds,
|
|
44
|
+
)
|
|
45
|
+
except FileNotFoundError as exc:
|
|
46
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
47
|
+
return 1
|
|
48
|
+
|
|
49
|
+
_print_import_results(results, quiet=args.quiet)
|
|
50
|
+
|
|
51
|
+
moved = sum(r.status in ("moved", "would_move") for r in results)
|
|
52
|
+
skipped = sum(r.status in ("skipped", "failed") for r in results)
|
|
53
|
+
if not args.quiet:
|
|
54
|
+
print(
|
|
55
|
+
f"{moved} file(s) {'would be ' if args.dry_run else ''}moved, {skipped} skipped"
|
|
56
|
+
)
|
|
57
|
+
return 1 if any(r.status == "failed" for r in results) else 0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
61
|
+
parser = argparse.ArgumentParser(
|
|
62
|
+
prog="media-workflow",
|
|
63
|
+
description="Import, organize, and de-duplicate photo and video collections.",
|
|
64
|
+
)
|
|
65
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
66
|
+
|
|
67
|
+
importer = subparsers.add_parser(
|
|
68
|
+
"import",
|
|
69
|
+
help="rename camera photos and videos into a YYYY/MM archive tree",
|
|
70
|
+
description=(
|
|
71
|
+
"Rename photos and videos into "
|
|
72
|
+
"YYYY/MM/YYYYMMDD_HHMMSS[.subsec]_Make_Model.ext."
|
|
73
|
+
),
|
|
74
|
+
)
|
|
75
|
+
importer.add_argument(
|
|
76
|
+
"source",
|
|
77
|
+
nargs="?",
|
|
78
|
+
default=_env_default(SOURCE_ENV),
|
|
79
|
+
help=f"directory (or one file) to import (or set {SOURCE_ENV})",
|
|
80
|
+
)
|
|
81
|
+
importer.add_argument(
|
|
82
|
+
"--destination",
|
|
83
|
+
"-d",
|
|
84
|
+
default=_env_default(ROOT_ENV) or ".",
|
|
85
|
+
help=f"archive root for the YYYY/MM tree (default: ${ROOT_ENV} or current directory)",
|
|
86
|
+
)
|
|
87
|
+
importer.add_argument(
|
|
88
|
+
"--recursive",
|
|
89
|
+
"-r",
|
|
90
|
+
action="store_true",
|
|
91
|
+
help="also import files in subdirectories of SOURCE",
|
|
92
|
+
)
|
|
93
|
+
importer.add_argument(
|
|
94
|
+
"--dry-run",
|
|
95
|
+
"-n",
|
|
96
|
+
action="store_true",
|
|
97
|
+
help="show what would happen without moving files",
|
|
98
|
+
)
|
|
99
|
+
importer.add_argument(
|
|
100
|
+
"--no-subseconds",
|
|
101
|
+
action="store_true",
|
|
102
|
+
help="omit SubSecTimeOriginal from filenames",
|
|
103
|
+
)
|
|
104
|
+
importer.add_argument(
|
|
105
|
+
"--quiet",
|
|
106
|
+
"-q",
|
|
107
|
+
action="store_true",
|
|
108
|
+
help="do not report skipped files or the summary",
|
|
109
|
+
)
|
|
110
|
+
importer.set_defaults(func=_cmd_import)
|
|
111
|
+
|
|
112
|
+
duplicate_parser = subparsers.add_parser(
|
|
113
|
+
"duplicates",
|
|
114
|
+
help="find/remove duplicate copies such as beach_1.jpg",
|
|
115
|
+
)
|
|
116
|
+
duplicate_parser.add_argument(
|
|
117
|
+
"root",
|
|
118
|
+
nargs="?",
|
|
119
|
+
default=_env_default(ROOT_ENV) or ".",
|
|
120
|
+
help=f"directory to scan (default: ${ROOT_ENV} or current directory)",
|
|
121
|
+
)
|
|
122
|
+
duplicate_parser.add_argument(
|
|
123
|
+
"--apply",
|
|
124
|
+
action="store_true",
|
|
125
|
+
help="actually delete/move files (default is dry run)",
|
|
126
|
+
)
|
|
127
|
+
duplicate_parser.add_argument(
|
|
128
|
+
"--trash",
|
|
129
|
+
action="store_true",
|
|
130
|
+
help="move duplicates into .dtrash instead of deleting",
|
|
131
|
+
)
|
|
132
|
+
duplicate_parser.add_argument(
|
|
133
|
+
"--fuzzy",
|
|
134
|
+
action="store_true",
|
|
135
|
+
help="match images that differ in resolution or JPEG quality",
|
|
136
|
+
)
|
|
137
|
+
duplicate_parser.add_argument(
|
|
138
|
+
"--threshold",
|
|
139
|
+
type=int,
|
|
140
|
+
default=6,
|
|
141
|
+
metavar="BITS",
|
|
142
|
+
help="Hamming distance for --fuzzy matches (default: 6 of 64 bits)",
|
|
143
|
+
)
|
|
144
|
+
duplicate_parser.add_argument(
|
|
145
|
+
"--verbose",
|
|
146
|
+
action="store_true",
|
|
147
|
+
help="log fuzzy pairs that were considered but not flagged",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def _cmd_duplicates(args: argparse.Namespace) -> int:
|
|
151
|
+
forwarded = [args.root]
|
|
152
|
+
for flag in ("apply", "trash", "fuzzy", "verbose"):
|
|
153
|
+
if getattr(args, flag):
|
|
154
|
+
forwarded.append("--" + flag)
|
|
155
|
+
if args.threshold != 6:
|
|
156
|
+
forwarded.extend(["--threshold", str(args.threshold)])
|
|
157
|
+
return duplicates.main(forwarded)
|
|
158
|
+
|
|
159
|
+
duplicate_parser.set_defaults(func=_cmd_duplicates)
|
|
160
|
+
|
|
161
|
+
fix_parser = subparsers.add_parser(
|
|
162
|
+
"fix-times",
|
|
163
|
+
help="shift EXIF dates and re-import JPEGs",
|
|
164
|
+
)
|
|
165
|
+
fix_parser.add_argument(
|
|
166
|
+
"root",
|
|
167
|
+
nargs="?",
|
|
168
|
+
default=_env_default(ROOT_ENV),
|
|
169
|
+
help=f"directory containing JPEGs to fix (or set {ROOT_ENV})",
|
|
170
|
+
)
|
|
171
|
+
fix_parser.add_argument(
|
|
172
|
+
"--destination",
|
|
173
|
+
"-d",
|
|
174
|
+
default=_env_default(ROOT_ENV) or ".",
|
|
175
|
+
help=f"archive root for renamed files (default: ${ROOT_ENV} or current directory)",
|
|
176
|
+
)
|
|
177
|
+
fix_parser.add_argument(
|
|
178
|
+
"--hours",
|
|
179
|
+
type=float,
|
|
180
|
+
default=1.0,
|
|
181
|
+
help="hours to add to all EXIF dates (default: 1)",
|
|
182
|
+
)
|
|
183
|
+
fix_parser.add_argument(
|
|
184
|
+
"--no-recursive",
|
|
185
|
+
action="store_true",
|
|
186
|
+
help="only process JPEGs directly inside ROOT",
|
|
187
|
+
)
|
|
188
|
+
fix_parser.add_argument(
|
|
189
|
+
"--dry-run",
|
|
190
|
+
"-n",
|
|
191
|
+
action="store_true",
|
|
192
|
+
help="report what would happen without shifting or moving files",
|
|
193
|
+
)
|
|
194
|
+
fix_parser.add_argument(
|
|
195
|
+
"--exiftool",
|
|
196
|
+
default="exiftool",
|
|
197
|
+
help="exiftool executable name/path (default: exiftool)",
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
def _cmd_fix_times(args: argparse.Namespace) -> int:
|
|
201
|
+
forwarded = [
|
|
202
|
+
args.root,
|
|
203
|
+
"--destination",
|
|
204
|
+
args.destination,
|
|
205
|
+
"--hours",
|
|
206
|
+
str(args.hours),
|
|
207
|
+
"--exiftool",
|
|
208
|
+
args.exiftool,
|
|
209
|
+
]
|
|
210
|
+
if args.no_recursive:
|
|
211
|
+
forwarded.append("--no-recursive")
|
|
212
|
+
if args.dry_run:
|
|
213
|
+
forwarded.append("--dry-run")
|
|
214
|
+
return fix_times.main(forwarded)
|
|
215
|
+
|
|
216
|
+
fix_parser.set_defaults(func=_cmd_fix_times)
|
|
217
|
+
|
|
218
|
+
return parser
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def main(argv=None) -> int:
|
|
222
|
+
parser = build_parser()
|
|
223
|
+
args = parser.parse_args(argv)
|
|
224
|
+
if args.command == "import" and args.source is None:
|
|
225
|
+
parser.error(f"source is required: pass SOURCE or set {SOURCE_ENV}")
|
|
226
|
+
if args.command == "fix-times" and args.root is None:
|
|
227
|
+
parser.error(f"root is required: pass ROOT or set {ROOT_ENV}")
|
|
228
|
+
return args.func(args)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
if __name__ == "__main__":
|
|
232
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""Find and remove duplicate copies of photos and videos.
|
|
2
|
+
|
|
3
|
+
This is the Python package port of the original ``duplicates.py`` script.
|
|
4
|
+
Behaviour is preserved:
|
|
5
|
+
|
|
6
|
+
* dry-run by default; pass ``--apply`` to change files;
|
|
7
|
+
* ``--trash`` moves duplicates into ``.dtrash`` instead of deleting them;
|
|
8
|
+
* byte-identical JPEGs and videos (same SHA-256) are duplicates by default;
|
|
9
|
+
* ``--fuzzy`` adds a perceptual dHash gate and a pixel-MAE verification for
|
|
10
|
+
photos; videos stay exact-hash only;
|
|
11
|
+
* differing extended-XMP digests veto a fuzzy match.
|
|
12
|
+
|
|
13
|
+
Run through the package CLI::
|
|
14
|
+
|
|
15
|
+
media-workflow duplicates [ROOT] --apply --trash
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import hashlib
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import shutil
|
|
25
|
+
import sys
|
|
26
|
+
|
|
27
|
+
from PIL import Image
|
|
28
|
+
|
|
29
|
+
TRASH_DIR = ".dtrash"
|
|
30
|
+
|
|
31
|
+
# Second-stage fuzzy verification: mean absolute pixel difference (0-255
|
|
32
|
+
# scale) after normalizing both images to 64x64 grayscale. Re-exports of the
|
|
33
|
+
# same image measure well under 1; distinct photos of the same scene measure
|
|
34
|
+
# far above 3.
|
|
35
|
+
MAE_THRESHOLD = 3
|
|
36
|
+
|
|
37
|
+
IMAGE_SUFFIXES = frozenset({".jpg", ".jpeg"})
|
|
38
|
+
VIDEO_SUFFIXES = frozenset({".mp4", ".mov", ".m4v", ".3gp"})
|
|
39
|
+
|
|
40
|
+
# A filename like "beach_1.jpg", "Beach_02.jpeg", or "clip_1.mp4" triggers
|
|
41
|
+
# the duplicate check.
|
|
42
|
+
COPY_RE = re.compile(
|
|
43
|
+
r"^(?P<base>.+?)_?\d+(\.(?:jpe?g|mp4|mov|m4v|3gp))$", re.IGNORECASE
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _extension_pattern(suffix):
|
|
48
|
+
suffix = suffix.lower()
|
|
49
|
+
if suffix in IMAGE_SUFFIXES:
|
|
50
|
+
return r"(?:jpe?g)"
|
|
51
|
+
return re.escape(suffix.lstrip("."))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def candidates(files, base, suffix):
|
|
55
|
+
"""Files in one directory that could be the original or copies of *base*.
|
|
56
|
+
|
|
57
|
+
Anchored to exactly ``base.jpg``, ``base.jpeg``, ``base_N.jpg/jpeg``, or
|
|
58
|
+
the video equivalents (case-insensitive) so unrelated files that merely
|
|
59
|
+
share a prefix are never touched. Photos treat ``.jpg`` and ``.jpeg`` as
|
|
60
|
+
the same extension; videos require the exact extension.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
pattern = re.compile(
|
|
64
|
+
r"^" + re.escape(base) + r"(_\d+)?\." + _extension_pattern(suffix) + r"$",
|
|
65
|
+
re.IGNORECASE,
|
|
66
|
+
)
|
|
67
|
+
return [f for f in files if pattern.match(f)]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _is_image_name(name):
|
|
71
|
+
return os.path.splitext(name)[1].lower() in IMAGE_SUFFIXES
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def sha256sum(filename):
|
|
75
|
+
try:
|
|
76
|
+
with open(filename, "rb", buffering=0) as f:
|
|
77
|
+
return hashlib.file_digest(f, "sha256").hexdigest()
|
|
78
|
+
except OSError as exc:
|
|
79
|
+
print(f"warning: cannot read {filename}: {exc}", file=sys.stderr)
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def dhash(filename, size=8):
|
|
84
|
+
"""Perceptual hash: robust against rescaling and JPEG recompression."""
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
with Image.open(filename) as img:
|
|
88
|
+
img = img.convert("L").resize((size + 1, size), Image.Resampling.LANCZOS)
|
|
89
|
+
pixels = img.tobytes()
|
|
90
|
+
except Exception as exc:
|
|
91
|
+
print(f"warning: cannot read image {filename}: {exc}", file=sys.stderr)
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
bits = 0
|
|
95
|
+
for row in range(size):
|
|
96
|
+
for col in range(size):
|
|
97
|
+
left = pixels[row * (size + 1) + col]
|
|
98
|
+
right = pixels[row * (size + 1) + col + 1]
|
|
99
|
+
bits = (bits << 1) | (left > right)
|
|
100
|
+
return bits
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def hamming(a, b):
|
|
104
|
+
return (a ^ b).bit_count()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def normalized_mae(path1, path2, size=64):
|
|
108
|
+
"""Mean absolute pixel difference (0-255) between two normalized images.
|
|
109
|
+
|
|
110
|
+
Robust against rescaling and JPEG recompression: re-exports of the same
|
|
111
|
+
image score near 0, distinct images score much higher.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
pixels = []
|
|
115
|
+
for path in (path1, path2):
|
|
116
|
+
with Image.open(path) as img:
|
|
117
|
+
pixels.append(
|
|
118
|
+
img.convert("L")
|
|
119
|
+
.resize((size, size), Image.Resampling.LANCZOS)
|
|
120
|
+
.tobytes()
|
|
121
|
+
)
|
|
122
|
+
total = sum(abs(a - b) for a, b in zip(*pixels))
|
|
123
|
+
return total / (size * size)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# xmpNote:HasExtendedXMP holds an MD5 over the extended XMP payload. Camera
|
|
127
|
+
# apps (e.g. Google HDR+) store depth maps / gain maps there, so differing
|
|
128
|
+
# digests mean the files carry different auxiliary data: distinct photos.
|
|
129
|
+
XMP_DIGEST_RE = re.compile(r'xmpNote:HasExtendedXMP="([^"]+)"')
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def extended_xmp_digest(path):
|
|
133
|
+
"""Extended-XMP payload digest, or ``None`` if absent/unreadable."""
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
with Image.open(path) as img:
|
|
137
|
+
xmp = img.info.get("xmp")
|
|
138
|
+
except Exception as exc:
|
|
139
|
+
print(f"warning: cannot read metadata {path}: {exc}", file=sys.stderr)
|
|
140
|
+
return None
|
|
141
|
+
if not xmp:
|
|
142
|
+
return None
|
|
143
|
+
match = XMP_DIGEST_RE.search(xmp.decode("utf-8", "replace"))
|
|
144
|
+
return match.group(1) if match else None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def discard(path, use_trash, trash_dir):
|
|
148
|
+
if use_trash:
|
|
149
|
+
# Mirror the subdirectory layout so files can be restored to their
|
|
150
|
+
# original location unambiguously.
|
|
151
|
+
rel = os.path.relpath(path, os.path.dirname(trash_dir))
|
|
152
|
+
dest = os.path.join(trash_dir, rel)
|
|
153
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
154
|
+
n = 1
|
|
155
|
+
while os.path.exists(dest):
|
|
156
|
+
stem, ext = os.path.splitext(dest)
|
|
157
|
+
dest = f"{stem}_{n}{ext}"
|
|
158
|
+
n += 1
|
|
159
|
+
shutil.move(path, dest)
|
|
160
|
+
return f"moved to {dest}"
|
|
161
|
+
os.remove(path)
|
|
162
|
+
return "deleted"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def process_dir(
|
|
166
|
+
root,
|
|
167
|
+
files,
|
|
168
|
+
apply_changes,
|
|
169
|
+
use_trash,
|
|
170
|
+
trash_dir,
|
|
171
|
+
fuzzy,
|
|
172
|
+
threshold,
|
|
173
|
+
top,
|
|
174
|
+
verbose,
|
|
175
|
+
):
|
|
176
|
+
def display(path):
|
|
177
|
+
# Skip the part of the path already given by the scan root.
|
|
178
|
+
return os.path.relpath(path, top)
|
|
179
|
+
|
|
180
|
+
copies = [f for f in files if COPY_RE.match(f)]
|
|
181
|
+
if not copies:
|
|
182
|
+
return
|
|
183
|
+
|
|
184
|
+
seen_bases = set()
|
|
185
|
+
for copy in copies:
|
|
186
|
+
base = COPY_RE.match(copy).group("base")
|
|
187
|
+
if base in seen_bases:
|
|
188
|
+
continue
|
|
189
|
+
seen_bases.add(base)
|
|
190
|
+
|
|
191
|
+
matches = candidates(files, base, os.path.splitext(copy)[1])
|
|
192
|
+
if len(matches) < 2:
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
# Highest quality survives: largest file first; ties broken by
|
|
196
|
+
# preferring the original name, then alphabetical.
|
|
197
|
+
def size_of(name):
|
|
198
|
+
try:
|
|
199
|
+
return os.path.getsize(os.path.join(root, name))
|
|
200
|
+
except OSError:
|
|
201
|
+
return -1
|
|
202
|
+
|
|
203
|
+
# Successive stable sorts: last key (size) dominates, earlier keys
|
|
204
|
+
# only break ties.
|
|
205
|
+
matches.sort(key=lambda f: f.lower())
|
|
206
|
+
matches.sort(key=lambda f: COPY_RE.match(f) is not None)
|
|
207
|
+
matches.sort(key=size_of, reverse=True)
|
|
208
|
+
|
|
209
|
+
kept = [] # (path, sha256, dhash)
|
|
210
|
+
dupes = [] # (path, kept_path, reason)
|
|
211
|
+
for name in matches:
|
|
212
|
+
path = os.path.join(root, name)
|
|
213
|
+
file_hash = sha256sum(path)
|
|
214
|
+
if file_hash is None:
|
|
215
|
+
continue
|
|
216
|
+
phash = dhash(path) if fuzzy and _is_image_name(name) else None
|
|
217
|
+
dup_of = None
|
|
218
|
+
for kept_path, kept_hash, kept_phash in kept:
|
|
219
|
+
if file_hash == kept_hash:
|
|
220
|
+
dup_of = (kept_path, "same hash (identical files)")
|
|
221
|
+
break
|
|
222
|
+
if phash is not None and kept_phash is not None:
|
|
223
|
+
dist = hamming(phash, kept_phash)
|
|
224
|
+
mae = normalized_mae(path, kept_path)
|
|
225
|
+
if dist <= threshold and mae <= MAE_THRESHOLD:
|
|
226
|
+
xa = extended_xmp_digest(path)
|
|
227
|
+
xb = extended_xmp_digest(kept_path)
|
|
228
|
+
if xa is not None and xb is not None and xa != xb:
|
|
229
|
+
if verbose:
|
|
230
|
+
kept_suffix = os.path.basename(kept_path)[len(base) :]
|
|
231
|
+
print(
|
|
232
|
+
f"[no match] {display(path)} vs "
|
|
233
|
+
f"{kept_suffix}: extended XMP differs "
|
|
234
|
+
f"({xa[:8]}\u2026 vs {xb[:8]}\u2026), pixel "
|
|
235
|
+
f"difference {mae:.1f}/255"
|
|
236
|
+
)
|
|
237
|
+
else:
|
|
238
|
+
dup_of = (
|
|
239
|
+
kept_path,
|
|
240
|
+
f"perceptual match (dHash distance "
|
|
241
|
+
f"{dist}/{64} bits, pixel difference "
|
|
242
|
+
f"{mae:.1f}/255)",
|
|
243
|
+
)
|
|
244
|
+
break
|
|
245
|
+
elif verbose:
|
|
246
|
+
kept_suffix = os.path.basename(kept_path)[len(base) :]
|
|
247
|
+
print(
|
|
248
|
+
f"[no match] {display(path)} vs {kept_suffix}: "
|
|
249
|
+
f"dHash distance {dist}/{64} bits "
|
|
250
|
+
f"(threshold {threshold}), pixel difference "
|
|
251
|
+
f"{mae:.1f}/255 (limit {MAE_THRESHOLD:.1f})"
|
|
252
|
+
)
|
|
253
|
+
if dup_of:
|
|
254
|
+
dupes.append((path, *dup_of))
|
|
255
|
+
else:
|
|
256
|
+
kept.append((path, file_hash, phash))
|
|
257
|
+
|
|
258
|
+
for path, dup_of, reason in dupes:
|
|
259
|
+
# Both names share the directory and the base; show only the
|
|
260
|
+
# survivor's differing suffix (e.g. "_1.jpg" or plain ".jpg").
|
|
261
|
+
dup_suffix = os.path.basename(dup_of)[len(base) :]
|
|
262
|
+
if apply_changes:
|
|
263
|
+
try:
|
|
264
|
+
result = discard(path, use_trash, trash_dir)
|
|
265
|
+
print(
|
|
266
|
+
f"removed: {display(path)} (duplicate of {dup_suffix}, "
|
|
267
|
+
f"{reason}; {result})"
|
|
268
|
+
)
|
|
269
|
+
except OSError as exc:
|
|
270
|
+
print(f"error: failed to remove {path}: {exc}", file=sys.stderr)
|
|
271
|
+
else:
|
|
272
|
+
print(
|
|
273
|
+
f"[dry run] would delete: {display(path)} (duplicate of "
|
|
274
|
+
f"{dup_suffix}, {reason})"
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def main(argv=None):
|
|
279
|
+
parser = argparse.ArgumentParser(
|
|
280
|
+
prog="media-workflow duplicates",
|
|
281
|
+
description=__doc__,
|
|
282
|
+
)
|
|
283
|
+
parser.add_argument("root", nargs="?", default=".", help="directory to scan")
|
|
284
|
+
parser.add_argument(
|
|
285
|
+
"--apply",
|
|
286
|
+
action="store_true",
|
|
287
|
+
help="actually delete/move files (default is dry run)",
|
|
288
|
+
)
|
|
289
|
+
parser.add_argument(
|
|
290
|
+
"--trash",
|
|
291
|
+
action="store_true",
|
|
292
|
+
help="move duplicates into .dtrash instead of deleting",
|
|
293
|
+
)
|
|
294
|
+
parser.add_argument(
|
|
295
|
+
"--fuzzy",
|
|
296
|
+
action="store_true",
|
|
297
|
+
help="also match images that differ in resolution or JPEG quality "
|
|
298
|
+
"(perceptual hash)",
|
|
299
|
+
)
|
|
300
|
+
parser.add_argument(
|
|
301
|
+
"--threshold",
|
|
302
|
+
type=int,
|
|
303
|
+
default=6,
|
|
304
|
+
metavar="BITS",
|
|
305
|
+
help="Hamming distance for --fuzzy matches (default: 6 of 64 bits)",
|
|
306
|
+
)
|
|
307
|
+
parser.add_argument(
|
|
308
|
+
"--verbose",
|
|
309
|
+
action="store_true",
|
|
310
|
+
help="log fuzzy candidate pairs that were considered but not flagged, "
|
|
311
|
+
"with their similarity values",
|
|
312
|
+
)
|
|
313
|
+
args = parser.parse_args(argv)
|
|
314
|
+
|
|
315
|
+
if not args.fuzzy and args.threshold != 6:
|
|
316
|
+
print("warning: --threshold ignored without --fuzzy", file=sys.stderr)
|
|
317
|
+
|
|
318
|
+
top = os.path.abspath(args.root)
|
|
319
|
+
trash_dir = os.path.join(top, TRASH_DIR)
|
|
320
|
+
|
|
321
|
+
for root, dirs, files in os.walk(top):
|
|
322
|
+
dirs[:] = [d for d in dirs if d != TRASH_DIR]
|
|
323
|
+
process_dir(
|
|
324
|
+
root,
|
|
325
|
+
files,
|
|
326
|
+
args.apply,
|
|
327
|
+
args.trash,
|
|
328
|
+
trash_dir,
|
|
329
|
+
args.fuzzy,
|
|
330
|
+
args.threshold,
|
|
331
|
+
top,
|
|
332
|
+
args.verbose,
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
return 0
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
if __name__ == "__main__":
|
|
339
|
+
raise SystemExit(main())
|