dupehunt 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dupehunt-0.0.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 lqxnjk
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,108 @@
1
+ Metadata-Version: 2.5
2
+ Name: dupehunt
3
+ Version: 0.0.1
4
+ Summary: Fast, dependency-free duplicate file finder with a friendly CLI.
5
+ Project-URL: Homepage, https://github.com/lqxnjk/dupehunt
6
+ Project-URL: Source, https://github.com/lqxnjk/dupehunt
7
+ Author: lqxnjk
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: cleanup,cli,dedupe,disk-usage,duplicate,duplicates,files,hash
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: End Users/Desktop
14
+ Classifier: Intended Audience :: System Administrators
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Topic :: System :: Filesystems
20
+ Classifier: Topic :: Utilities
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+
24
+ # dupehunt
25
+
26
+ Find duplicate files by content hash. Zero dependencies, one command, works on
27
+ Windows, macOS and Linux.
28
+
29
+ ```console
30
+ $ dupehunt ~/Downloads
31
+ 3 group(s), 9 file(s), 412.6 MB reclaimable
32
+
33
+ [1] 180.0 MB x 3 (360.0 MB wasted)
34
+ /home/me/Downloads/ubuntu.iso
35
+ /home/me/Downloads/ubuntu (1).iso
36
+ /home/me/Downloads/backup/ubuntu.iso
37
+
38
+ [2] 26.3 MB x 2 (26.3 MB wasted)
39
+ ...
40
+ ```
41
+
42
+ ## Install
43
+
44
+ ```console
45
+ pip install dupehunt
46
+ ```
47
+
48
+ ## Usage
49
+
50
+ ```console
51
+ dupehunt [paths ...] [options]
52
+ ```
53
+
54
+ | Option | Meaning |
55
+ | --- | --- |
56
+ | `paths` | Files or directories to scan. Default: current directory |
57
+ | `-m, --min-size` | Skip files smaller than this. `1K`, `4M`, `1G` all accepted |
58
+ | `-a, --algo` | `blake2b` (default), `sha256`, `sha1`, `md5` |
59
+ | `-n, --top` | Show only the N largest groups |
60
+ | `--json` | Machine readable output |
61
+ | `--delete` | Delete extra copies, keeping the first path of each group |
62
+ | `--yes` | Required together with `--delete` to actually delete |
63
+
64
+ ### Examples
65
+
66
+ ```console
67
+ # biggest wins only, files above 10 MB
68
+ dupehunt ~/Pictures -m 10M -n 20
69
+
70
+ # feed the result to another tool
71
+ dupehunt ~/data --json > dupes.json
72
+
73
+ # preview what a cleanup would remove, without touching anything
74
+ dupehunt ~/data --delete -m 1M
75
+
76
+ # really delete the extra copies
77
+ dupehunt ~/data --delete --yes -m 1M
78
+ ```
79
+
80
+ `--delete` without `--yes` is always a dry run: it prints the plan and removes
81
+ nothing. Nothing is ever deleted unless both flags are present.
82
+
83
+ ## How it works
84
+
85
+ Three passes, cheapest first, so a full disk scan stays fast:
86
+
87
+ 1. Group files by exact size. Unique sizes drop out immediately.
88
+ 2. Hash the first 8 KiB of each remaining candidate.
89
+ 3. Hash the whole file only for candidates that still collide.
90
+
91
+ Symlinks are never followed, and hard links to an already seen inode are counted
92
+ once, so the same physical data is not reported as duplicates of itself.
93
+
94
+ ## Use as a library
95
+
96
+ ```python
97
+ from pathlib import Path
98
+ from dupehunt import find_duplicates, total_wasted
99
+
100
+ groups = find_duplicates([Path(".")], n_min_size=1024 * 1024)
101
+ for group in groups:
102
+ print(group.size, group.wasted, [str(p) for p in group.paths])
103
+ print(total_wasted(groups))
104
+ ```
105
+
106
+ ## License
107
+
108
+ MIT
@@ -0,0 +1,85 @@
1
+ # dupehunt
2
+
3
+ Find duplicate files by content hash. Zero dependencies, one command, works on
4
+ Windows, macOS and Linux.
5
+
6
+ ```console
7
+ $ dupehunt ~/Downloads
8
+ 3 group(s), 9 file(s), 412.6 MB reclaimable
9
+
10
+ [1] 180.0 MB x 3 (360.0 MB wasted)
11
+ /home/me/Downloads/ubuntu.iso
12
+ /home/me/Downloads/ubuntu (1).iso
13
+ /home/me/Downloads/backup/ubuntu.iso
14
+
15
+ [2] 26.3 MB x 2 (26.3 MB wasted)
16
+ ...
17
+ ```
18
+
19
+ ## Install
20
+
21
+ ```console
22
+ pip install dupehunt
23
+ ```
24
+
25
+ ## Usage
26
+
27
+ ```console
28
+ dupehunt [paths ...] [options]
29
+ ```
30
+
31
+ | Option | Meaning |
32
+ | --- | --- |
33
+ | `paths` | Files or directories to scan. Default: current directory |
34
+ | `-m, --min-size` | Skip files smaller than this. `1K`, `4M`, `1G` all accepted |
35
+ | `-a, --algo` | `blake2b` (default), `sha256`, `sha1`, `md5` |
36
+ | `-n, --top` | Show only the N largest groups |
37
+ | `--json` | Machine readable output |
38
+ | `--delete` | Delete extra copies, keeping the first path of each group |
39
+ | `--yes` | Required together with `--delete` to actually delete |
40
+
41
+ ### Examples
42
+
43
+ ```console
44
+ # biggest wins only, files above 10 MB
45
+ dupehunt ~/Pictures -m 10M -n 20
46
+
47
+ # feed the result to another tool
48
+ dupehunt ~/data --json > dupes.json
49
+
50
+ # preview what a cleanup would remove, without touching anything
51
+ dupehunt ~/data --delete -m 1M
52
+
53
+ # really delete the extra copies
54
+ dupehunt ~/data --delete --yes -m 1M
55
+ ```
56
+
57
+ `--delete` without `--yes` is always a dry run: it prints the plan and removes
58
+ nothing. Nothing is ever deleted unless both flags are present.
59
+
60
+ ## How it works
61
+
62
+ Three passes, cheapest first, so a full disk scan stays fast:
63
+
64
+ 1. Group files by exact size. Unique sizes drop out immediately.
65
+ 2. Hash the first 8 KiB of each remaining candidate.
66
+ 3. Hash the whole file only for candidates that still collide.
67
+
68
+ Symlinks are never followed, and hard links to an already seen inode are counted
69
+ once, so the same physical data is not reported as duplicates of itself.
70
+
71
+ ## Use as a library
72
+
73
+ ```python
74
+ from pathlib import Path
75
+ from dupehunt import find_duplicates, total_wasted
76
+
77
+ groups = find_duplicates([Path(".")], n_min_size=1024 * 1024)
78
+ for group in groups:
79
+ print(group.size, group.wasted, [str(p) for p in group.paths])
80
+ print(total_wasted(groups))
81
+ ```
82
+
83
+ ## License
84
+
85
+ MIT
@@ -0,0 +1,57 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "dupehunt"
7
+ version = "0.0.1"
8
+ description = "Fast, dependency-free duplicate file finder with a friendly CLI."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "lqxnjk" }]
13
+ keywords = [
14
+ "duplicate",
15
+ "duplicates",
16
+ "dedupe",
17
+ "files",
18
+ "hash",
19
+ "disk-usage",
20
+ "cleanup",
21
+ "cli",
22
+ ]
23
+ classifiers = [
24
+ "Development Status :: 3 - Alpha",
25
+ "Environment :: Console",
26
+ "Intended Audience :: End Users/Desktop",
27
+ "Intended Audience :: System Administrators",
28
+ "License :: OSI Approved :: MIT License",
29
+ "Operating System :: OS Independent",
30
+ "Programming Language :: Python :: 3",
31
+ "Programming Language :: Python :: 3 :: Only",
32
+ "Topic :: System :: Filesystems",
33
+ "Topic :: Utilities",
34
+ ]
35
+ dependencies = []
36
+
37
+ [project.scripts]
38
+ dupehunt = "dupehunt.cli:main"
39
+
40
+ [project.urls]
41
+ Homepage = "https://github.com/lqxnjk/dupehunt"
42
+ Source = "https://github.com/lqxnjk/dupehunt"
43
+
44
+ [tool.pytest.ini_options]
45
+ pythonpath = ["src"]
46
+ testpaths = ["tests"]
47
+
48
+ [tool.hatch.build.targets.wheel]
49
+ packages = ["src/dupehunt"]
50
+
51
+ [tool.hatch.build.targets.sdist]
52
+ include = [
53
+ "src/dupehunt",
54
+ "tests",
55
+ "README.md",
56
+ "LICENSE",
57
+ ]
@@ -0,0 +1,31 @@
1
+ """
2
+ Author: lqxnjk <lqxnjk@qq.com>
3
+ Created: 2026-10-23 10:06
4
+ Modified: 2026-10-23 10:06
5
+ Python Version: 3.10
6
+ """
7
+
8
+ from .cli import __version__, main
9
+ from .core import (
10
+ SUPPORTED_ALGOS,
11
+ DuplicateGroup,
12
+ ScanError,
13
+ find_duplicates,
14
+ human_size,
15
+ iter_files,
16
+ parse_size,
17
+ total_wasted,
18
+ )
19
+
20
+ __all__ = [
21
+ "SUPPORTED_ALGOS",
22
+ "DuplicateGroup",
23
+ "ScanError",
24
+ "__version__",
25
+ "find_duplicates",
26
+ "human_size",
27
+ "iter_files",
28
+ "main",
29
+ "parse_size",
30
+ "total_wasted",
31
+ ]
@@ -0,0 +1,13 @@
1
+ """
2
+ Author: lqxnjk <lqxnjk@qq.com>
3
+ Created: 2026-10-23 10:06
4
+ Modified: 2026-10-23 10:06
5
+ Python Version: 3.10
6
+ """
7
+
8
+ import sys
9
+
10
+ from .cli import main
11
+
12
+ if __name__ == "__main__":
13
+ sys.exit(main())
@@ -0,0 +1,174 @@
1
+ """
2
+ Author: lqxnjk <lqxnjk@qq.com>
3
+ Created: 2026-10-23 10:06
4
+ Modified: 2026-10-23 10:06
5
+ Python Version: 3.10
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import logging
13
+ import os
14
+ import sys
15
+ from pathlib import Path
16
+ from typing import Sequence
17
+
18
+ from .core import (
19
+ SUPPORTED_ALGOS,
20
+ DuplicateGroup,
21
+ find_duplicates,
22
+ human_size,
23
+ parse_size,
24
+ total_wasted,
25
+ )
26
+
27
+ __version__ = "0.0.1"
28
+
29
+
30
+ def _build_parser() -> argparse.ArgumentParser:
31
+ parser = argparse.ArgumentParser(
32
+ prog="dupehunt",
33
+ description="Find duplicate files by content hash.",
34
+ )
35
+ parser.add_argument(
36
+ "paths",
37
+ nargs="*",
38
+ default=["."],
39
+ help="files or directories to scan (default: current directory)",
40
+ )
41
+ parser.add_argument(
42
+ "-m",
43
+ "--min-size",
44
+ default="1",
45
+ help="ignore files smaller than this, e.g. 1K, 4M, 1G (default: 1)",
46
+ )
47
+ parser.add_argument(
48
+ "-a",
49
+ "--algo",
50
+ default="blake2b",
51
+ choices=list(SUPPORTED_ALGOS),
52
+ help="hash algorithm (default: blake2b, fastest)",
53
+ )
54
+ parser.add_argument(
55
+ "-n",
56
+ "--top",
57
+ type=int,
58
+ default=0,
59
+ help="show only the N largest groups (0 = all)",
60
+ )
61
+ parser.add_argument("--json", action="store_true", help="machine readable output")
62
+ parser.add_argument(
63
+ "--delete",
64
+ action="store_true",
65
+ help="delete extra copies, keeping the first path in each group (dry run unless --yes)",
66
+ )
67
+ parser.add_argument(
68
+ "--yes",
69
+ action="store_true",
70
+ help="confirm deletion; only meaningful together with --delete",
71
+ )
72
+ parser.add_argument("--version", action="version", version=f"dupehunt {__version__}")
73
+ return parser
74
+
75
+
76
+ def _render_text(m_groups: Sequence[DuplicateGroup], m_roots: Sequence[str]) -> str:
77
+ if not m_groups:
78
+ return "no duplicates found"
79
+
80
+ lines = [
81
+ f"{len(m_groups)} group(s), "
82
+ f"{sum(len(group.paths) for group in m_groups)} file(s), "
83
+ f"{human_size(total_wasted(m_groups))} reclaimable",
84
+ "",
85
+ ]
86
+ for index, group in enumerate(m_groups, start=1):
87
+ lines.append(
88
+ f"[{index}] {human_size(group.size)} x {len(group.paths)}"
89
+ f" ({human_size(group.wasted)} wasted)"
90
+ )
91
+ lines.extend(f" {path}" for path in group.paths)
92
+ lines.append("")
93
+ return "\n".join(lines).rstrip()
94
+
95
+
96
+ def _render_json(m_groups: Sequence[DuplicateGroup], m_roots: Sequence[str]) -> str:
97
+ payload = {
98
+ "roots": list(m_roots),
99
+ "groups": [
100
+ {
101
+ "size": group.size,
102
+ "digest": group.digest,
103
+ "wasted": group.wasted,
104
+ "paths": [str(path) for path in group.paths],
105
+ }
106
+ for group in m_groups
107
+ ],
108
+ "total_wasted": total_wasted(m_groups),
109
+ }
110
+ return json.dumps(payload, ensure_ascii=False, indent=2)
111
+
112
+
113
+ def _delete_extras(m_groups: Sequence[DuplicateGroup], n_confirm: bool) -> int:
114
+ """Delete every path but the first in each group. Returns the number removed."""
115
+ targets = [path for group in m_groups for path in group.paths[1:]]
116
+ if not targets:
117
+ return 0
118
+ if not n_confirm:
119
+ print(
120
+ f"dry run: {len(targets)} file(s) would be deleted "
121
+ "(re-run with --delete --yes to apply)",
122
+ file=sys.stderr,
123
+ )
124
+ for path in targets:
125
+ print(f" would delete {path}", file=sys.stderr)
126
+ return 0
127
+
128
+ removed = 0
129
+ for path in targets:
130
+ try:
131
+ os.remove(path)
132
+ removed += 1
133
+ except OSError as exc:
134
+ logging.getLogger(__name__).error("failed to delete %s: %s", path, exc)
135
+ return removed
136
+
137
+
138
+ def main(m_argv: Sequence[str] | None = None) -> int:
139
+ parser = _build_parser()
140
+ args = parser.parse_args(m_argv)
141
+
142
+ try:
143
+ min_size = parse_size(args.min_size)
144
+ except ValueError as exc:
145
+ parser.error(str(exc))
146
+ return 2
147
+
148
+ roots = [Path(item) for item in args.paths]
149
+ missing = [root for root in roots if not root.exists()]
150
+ if missing:
151
+ for root in missing:
152
+ print(f"path does not exist: {root}", file=sys.stderr)
153
+ return 2
154
+
155
+ try:
156
+ groups = find_duplicates(roots, n_min_size=min_size, n_algo=args.algo)
157
+ except OSError as exc:
158
+ logging.getLogger(__name__).error("scan failed: %s", exc)
159
+ return 1
160
+
161
+ if args.top > 0:
162
+ groups = groups[: args.top]
163
+
164
+ if args.json:
165
+ print(_render_json(groups, args.paths))
166
+ else:
167
+ print(_render_text(groups, args.paths))
168
+
169
+ if args.delete:
170
+ removed = _delete_extras(groups, n_confirm=args.yes)
171
+ if removed:
172
+ print(f"deleted {removed} file(s)", file=sys.stderr)
173
+
174
+ return 0
@@ -0,0 +1,189 @@
1
+ """
2
+ Author: lqxnjk <lqxnjk@qq.com>
3
+ Created: 2026-10-23 10:06
4
+ Modified: 2026-10-23 10:06
5
+ Python Version: 3.10
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import logging
12
+ import os
13
+ from collections import defaultdict
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Iterable, Iterator, Sequence
17
+
18
+ LOGGER = logging.getLogger(__name__)
19
+
20
+ CHUNK_SIZE = 1 << 20
21
+ QUICK_SIZE = 8192
22
+ SUPPORTED_ALGOS = ("blake2b", "sha256", "sha1", "md5")
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class DuplicateGroup:
27
+ """A set of two or more files with identical content."""
28
+
29
+ size: int
30
+ digest: str
31
+ paths: list[Path]
32
+
33
+ @property
34
+ def wasted(self) -> int:
35
+ """Bytes that would be reclaimed by keeping a single copy."""
36
+ return self.size * (len(self.paths) - 1)
37
+
38
+
39
+ class ScanError(OSError):
40
+ """Raised when a scan root cannot be read."""
41
+
42
+
43
+ def human_size(m_num: int) -> str:
44
+ """Format a byte count as a short human readable string."""
45
+ value = float(m_num)
46
+ for unit in ("B", "KB", "MB", "GB", "TB"):
47
+ if value < 1024 or unit == "TB":
48
+ if unit == "B":
49
+ return f"{int(value)} {unit}"
50
+ return f"{value:.1f} {unit}"
51
+ value /= 1024
52
+ return f"{value:.1f} PB"
53
+
54
+
55
+ def parse_size(m_text: str) -> int:
56
+ """Parse ``"1024"``, ``"10K"``, ``"1.5M"``, ``"2G"`` into bytes."""
57
+ raw = m_text.strip().upper().rstrip("B")
58
+ multiplier = 1
59
+ if raw and raw[-1] in "KMGTP":
60
+ multiplier = 1024 ** ("KMGTP".index(raw[-1]) + 1)
61
+ raw = raw[:-1]
62
+ try:
63
+ return int(float(raw) * multiplier)
64
+ except ValueError as exc:
65
+ raise ValueError(f"invalid size: {m_text!r}") from exc
66
+
67
+
68
+ def _hash_file(m_path: Path, m_algo: str, n_limit: int | None = None) -> str:
69
+ """Hash a file, optionally reading at most ``n_limit`` bytes."""
70
+ digest = hashlib.new(m_algo)
71
+ remaining = n_limit
72
+ with m_path.open("rb") as handle:
73
+ while True:
74
+ want = CHUNK_SIZE if remaining is None else min(CHUNK_SIZE, remaining)
75
+ if want <= 0:
76
+ break
77
+ block = handle.read(want)
78
+ if not block:
79
+ break
80
+ digest.update(block)
81
+ if remaining is not None:
82
+ remaining -= len(block)
83
+ return digest.hexdigest()
84
+
85
+
86
+ def iter_files(m_roots: Sequence[Path], n_min_size: int = 1) -> Iterator[tuple[Path, int]]:
87
+ """Yield ``(path, size)`` for regular files under ``m_roots``.
88
+
89
+ Symlinks are never followed and hard links to an already seen inode are
90
+ skipped, so the same physical data is not reported twice. Unreadable
91
+ entries are logged and skipped instead of aborting the whole scan.
92
+ """
93
+ seen_inodes: set[tuple[int, int]] = set()
94
+ for root in m_roots:
95
+ if root.is_file():
96
+ try:
97
+ stat = root.stat()
98
+ except OSError as exc:
99
+ LOGGER.warning("skip unreadable file %s: %s", root, exc)
100
+ continue
101
+ if stat.st_size >= n_min_size:
102
+ yield root, stat.st_size
103
+ continue
104
+ if not root.is_dir():
105
+ raise ScanError(f"not a directory: {root}")
106
+ stack = [root]
107
+ while stack:
108
+ current = stack.pop()
109
+ try:
110
+ entries = list(os.scandir(current))
111
+ except OSError as exc:
112
+ LOGGER.warning("skip unreadable directory %s: %s", current, exc)
113
+ continue
114
+ for entry in entries:
115
+ try:
116
+ if entry.is_dir(follow_symlinks=False):
117
+ stack.append(Path(entry.path))
118
+ continue
119
+ if not entry.is_file(follow_symlinks=False):
120
+ continue
121
+ stat = entry.stat(follow_symlinks=False)
122
+ except OSError as exc:
123
+ LOGGER.warning("skip unreadable entry %s: %s", entry.path, exc)
124
+ continue
125
+ inode = (stat.st_dev, stat.st_ino)
126
+ if inode != (0, 0):
127
+ if inode in seen_inodes:
128
+ LOGGER.debug("skip hard link %s", entry.path)
129
+ continue
130
+ seen_inodes.add(inode)
131
+ if stat.st_size >= n_min_size:
132
+ yield Path(entry.path), stat.st_size
133
+
134
+
135
+ def find_duplicates(
136
+ m_roots: Sequence[Path],
137
+ n_min_size: int = 1,
138
+ n_algo: str = "blake2b",
139
+ ) -> list[DuplicateGroup]:
140
+ """Return duplicate groups sorted by reclaimable space, largest first.
141
+
142
+ Three passes keep large scans cheap: group by size, then hash the first
143
+ few KiB, then hash the whole file only for surviving candidates.
144
+ """
145
+ if n_algo not in SUPPORTED_ALGOS:
146
+ raise ValueError(f"unsupported algorithm: {n_algo!r}")
147
+
148
+ by_size: dict[int, list[Path]] = defaultdict(list)
149
+ for path, size in iter_files(m_roots, n_min_size):
150
+ by_size[size].append(path)
151
+
152
+ candidates = [(size, path) for size, paths in by_size.items() if len(paths) > 1 for path in paths]
153
+ if not candidates:
154
+ return []
155
+
156
+ by_quick: dict[tuple[int, str], list[Path]] = defaultdict(list)
157
+ for size, path in candidates:
158
+ try:
159
+ quick = _hash_file(path, n_algo, QUICK_SIZE)
160
+ except OSError as exc:
161
+ LOGGER.warning("skip unreadable candidate %s: %s", path, exc)
162
+ continue
163
+ by_quick[(size, quick)].append(path)
164
+
165
+ groups: list[DuplicateGroup] = []
166
+ for (size, quick), paths in by_quick.items():
167
+ if len(paths) < 2:
168
+ continue
169
+ if size <= QUICK_SIZE:
170
+ groups.append(DuplicateGroup(size=size, digest=quick, paths=paths))
171
+ continue
172
+ by_full: dict[str, list[Path]] = defaultdict(list)
173
+ for path in paths:
174
+ try:
175
+ by_full[_hash_file(path, n_algo)].append(path)
176
+ except OSError as exc:
177
+ LOGGER.warning("skip unreadable candidate %s: %s", path, exc)
178
+ continue
179
+ for digest, same in by_full.items():
180
+ if len(same) > 1:
181
+ groups.append(DuplicateGroup(size=size, digest=digest, paths=same))
182
+
183
+ groups.sort(key=lambda group: group.wasted, reverse=True)
184
+ return groups
185
+
186
+
187
+ def total_wasted(m_groups: Iterable[DuplicateGroup]) -> int:
188
+ """Sum reclaimable bytes across groups."""
189
+ return sum(group.wasted for group in m_groups)
@@ -0,0 +1,106 @@
1
+ """
2
+ Author: lqxnjk <lqxnjk@qq.com>
3
+ Created: 2026-10-23 10:06
4
+ Modified: 2026-10-23 10:06
5
+ Python Version: 3.10
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from pathlib import Path
11
+
12
+ import pytest
13
+
14
+ from dupehunt import find_duplicates, human_size, parse_size, total_wasted
15
+
16
+
17
+ def _write(path: Path, content: bytes) -> Path:
18
+ path.parent.mkdir(parents=True, exist_ok=True)
19
+ path.write_bytes(content)
20
+ return path
21
+
22
+
23
+ def test_detects_identical_files(tmp_path: Path) -> None:
24
+ payload = b"same content" * 100
25
+ _write(tmp_path / "a.bin", payload)
26
+ _write(tmp_path / "sub" / "b.bin", payload)
27
+ _write(tmp_path / "c.bin", b"different")
28
+ _write(tmp_path / "d.bin", b"also different, longer payload")
29
+
30
+ groups = find_duplicates([tmp_path])
31
+
32
+ assert len(groups) == 1
33
+ assert groups[0].size == len(payload)
34
+ assert {p.name for p in groups[0].paths} == {"a.bin", "b.bin"}
35
+ assert groups[0].wasted == len(payload)
36
+
37
+
38
+ def test_ignores_unique_files(tmp_path: Path) -> None:
39
+ _write(tmp_path / "a.txt", b"alpha")
40
+ _write(tmp_path / "b.txt", b"beta!")
41
+ assert find_duplicates([tmp_path]) == []
42
+
43
+
44
+ def test_respects_min_size(tmp_path: Path) -> None:
45
+ _write(tmp_path / "a.txt", b"tiny")
46
+ _write(tmp_path / "b.txt", b"tiny")
47
+ assert find_duplicates([tmp_path], n_min_size=10) == []
48
+ assert len(find_duplicates([tmp_path], n_min_size=1)) == 1
49
+
50
+
51
+ def test_large_file_uses_full_hash(tmp_path: Path) -> None:
52
+ head = b"x" * 8192
53
+ _write(tmp_path / "a.bin", head + b"tail-one")
54
+ _write(tmp_path / "b.bin", head + b"tail-two")
55
+
56
+ assert find_duplicates([tmp_path]) == []
57
+
58
+
59
+ def test_groups_sorted_by_wasted(tmp_path: Path) -> None:
60
+ big = b"b" * 5000
61
+ small = b"s" * 100
62
+ _write(tmp_path / "big1", big)
63
+ _write(tmp_path / "big2", big)
64
+ _write(tmp_path / "small1", small)
65
+ _write(tmp_path / "small2", small)
66
+
67
+ groups = find_duplicates([tmp_path])
68
+
69
+ assert [g.size for g in groups] == [5000, 100]
70
+ assert total_wasted(groups) == 5100
71
+
72
+
73
+ def test_accepts_single_file_root(tmp_path: Path) -> None:
74
+ target = _write(tmp_path / "one.bin", b"payload")
75
+ assert find_duplicates([target]) == []
76
+
77
+
78
+ def test_missing_root_raises(tmp_path: Path) -> None:
79
+ with pytest.raises(OSError):
80
+ find_duplicates([tmp_path / "nope"])
81
+
82
+
83
+ @pytest.mark.parametrize(
84
+ ("text", "expected"),
85
+ [("1", 1), ("1K", 1024), ("1k", 1024), ("1.5M", 1572864), ("2G", 2 * 1024**3)],
86
+ )
87
+ def test_parse_size(text: str, expected: int) -> None:
88
+ assert parse_size(text) == expected
89
+
90
+
91
+ def test_parse_size_rejects_garbage() -> None:
92
+ with pytest.raises(ValueError):
93
+ parse_size("abc")
94
+
95
+
96
+ @pytest.mark.parametrize(
97
+ ("num", "expected"),
98
+ [(0, "0 B"), (1023, "1023 B"), (1024, "1.0 KB"), (1536, "1.5 KB")],
99
+ )
100
+ def test_human_size(num: int, expected: str) -> None:
101
+ assert human_size(num) == expected
102
+
103
+
104
+ def test_unsupported_algo(tmp_path: Path) -> None:
105
+ with pytest.raises(ValueError):
106
+ find_duplicates([tmp_path], n_algo="crc99")