dupehunt 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dupehunt-0.0.1/LICENSE +21 -0
- dupehunt-0.0.1/PKG-INFO +108 -0
- dupehunt-0.0.1/README.md +85 -0
- dupehunt-0.0.1/pyproject.toml +57 -0
- dupehunt-0.0.1/src/dupehunt/__init__.py +31 -0
- dupehunt-0.0.1/src/dupehunt/__main__.py +13 -0
- dupehunt-0.0.1/src/dupehunt/cli.py +174 -0
- dupehunt-0.0.1/src/dupehunt/core.py +189 -0
- dupehunt-0.0.1/tests/test_core.py +106 -0
dupehunt-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 lqxnjk
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dupehunt-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dupehunt
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Fast, dependency-free duplicate file finder with a friendly CLI.
|
|
5
|
+
Project-URL: Homepage, https://github.com/lqxnjk/dupehunt
|
|
6
|
+
Project-URL: Source, https://github.com/lqxnjk/dupehunt
|
|
7
|
+
Author: lqxnjk
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: cleanup,cli,dedupe,disk-usage,duplicate,duplicates,files,hash
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
14
|
+
Classifier: Intended Audience :: System Administrators
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: System :: Filesystems
|
|
20
|
+
Classifier: Topic :: Utilities
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# dupehunt
|
|
25
|
+
|
|
26
|
+
Find duplicate files by content hash. Zero dependencies, one command, works on
|
|
27
|
+
Windows, macOS and Linux.
|
|
28
|
+
|
|
29
|
+
```console
|
|
30
|
+
$ dupehunt ~/Downloads
|
|
31
|
+
3 group(s), 9 file(s), 412.6 MB reclaimable
|
|
32
|
+
|
|
33
|
+
[1] 180.0 MB x 3 (360.0 MB wasted)
|
|
34
|
+
/home/me/Downloads/ubuntu.iso
|
|
35
|
+
/home/me/Downloads/ubuntu (1).iso
|
|
36
|
+
/home/me/Downloads/backup/ubuntu.iso
|
|
37
|
+
|
|
38
|
+
[2] 26.3 MB x 2 (26.3 MB wasted)
|
|
39
|
+
...
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```console
|
|
45
|
+
pip install dupehunt
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Usage
|
|
49
|
+
|
|
50
|
+
```console
|
|
51
|
+
dupehunt [paths ...] [options]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
| Option | Meaning |
|
|
55
|
+
| --- | --- |
|
|
56
|
+
| `paths` | Files or directories to scan. Default: current directory |
|
|
57
|
+
| `-m, --min-size` | Skip files smaller than this. `1K`, `4M`, `1G` all accepted |
|
|
58
|
+
| `-a, --algo` | `blake2b` (default), `sha256`, `sha1`, `md5` |
|
|
59
|
+
| `-n, --top` | Show only the N largest groups |
|
|
60
|
+
| `--json` | Machine readable output |
|
|
61
|
+
| `--delete` | Delete extra copies, keeping the first path of each group |
|
|
62
|
+
| `--yes` | Required together with `--delete` to actually delete |
|
|
63
|
+
|
|
64
|
+
### Examples
|
|
65
|
+
|
|
66
|
+
```console
|
|
67
|
+
# biggest wins only, files above 10 MB
|
|
68
|
+
dupehunt ~/Pictures -m 10M -n 20
|
|
69
|
+
|
|
70
|
+
# feed the result to another tool
|
|
71
|
+
dupehunt ~/data --json > dupes.json
|
|
72
|
+
|
|
73
|
+
# preview what a cleanup would remove, without touching anything
|
|
74
|
+
dupehunt ~/data --delete -m 1M
|
|
75
|
+
|
|
76
|
+
# really delete the extra copies
|
|
77
|
+
dupehunt ~/data --delete --yes -m 1M
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`--delete` without `--yes` is always a dry run: it prints the plan and removes
|
|
81
|
+
nothing. Nothing is ever deleted unless both flags are present.
|
|
82
|
+
|
|
83
|
+
## How it works
|
|
84
|
+
|
|
85
|
+
Three passes, cheapest first, so a full disk scan stays fast:
|
|
86
|
+
|
|
87
|
+
1. Group files by exact size. Unique sizes drop out immediately.
|
|
88
|
+
2. Hash the first 8 KiB of each remaining candidate.
|
|
89
|
+
3. Hash the whole file only for candidates that still collide.
|
|
90
|
+
|
|
91
|
+
Symlinks are never followed, and hard links to an already seen inode are counted
|
|
92
|
+
once, so the same physical data is not reported as duplicates of itself.
|
|
93
|
+
|
|
94
|
+
## Use as a library
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from pathlib import Path
|
|
98
|
+
from dupehunt import find_duplicates, total_wasted
|
|
99
|
+
|
|
100
|
+
groups = find_duplicates([Path(".")], n_min_size=1024 * 1024)
|
|
101
|
+
for group in groups:
|
|
102
|
+
print(group.size, group.wasted, [str(p) for p in group.paths])
|
|
103
|
+
print(total_wasted(groups))
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## License
|
|
107
|
+
|
|
108
|
+
MIT
|
dupehunt-0.0.1/README.md
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# dupehunt
|
|
2
|
+
|
|
3
|
+
Find duplicate files by content hash. Zero dependencies, one command, works on
|
|
4
|
+
Windows, macOS and Linux.
|
|
5
|
+
|
|
6
|
+
```console
|
|
7
|
+
$ dupehunt ~/Downloads
|
|
8
|
+
3 group(s), 9 file(s), 412.6 MB reclaimable
|
|
9
|
+
|
|
10
|
+
[1] 180.0 MB x 3 (360.0 MB wasted)
|
|
11
|
+
/home/me/Downloads/ubuntu.iso
|
|
12
|
+
/home/me/Downloads/ubuntu (1).iso
|
|
13
|
+
/home/me/Downloads/backup/ubuntu.iso
|
|
14
|
+
|
|
15
|
+
[2] 26.3 MB x 2 (26.3 MB wasted)
|
|
16
|
+
...
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
```console
|
|
22
|
+
pip install dupehunt
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
```console
|
|
28
|
+
dupehunt [paths ...] [options]
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
| Option | Meaning |
|
|
32
|
+
| --- | --- |
|
|
33
|
+
| `paths` | Files or directories to scan. Default: current directory |
|
|
34
|
+
| `-m, --min-size` | Skip files smaller than this. `1K`, `4M`, `1G` all accepted |
|
|
35
|
+
| `-a, --algo` | `blake2b` (default), `sha256`, `sha1`, `md5` |
|
|
36
|
+
| `-n, --top` | Show only the N largest groups |
|
|
37
|
+
| `--json` | Machine readable output |
|
|
38
|
+
| `--delete` | Delete extra copies, keeping the first path of each group |
|
|
39
|
+
| `--yes` | Required together with `--delete` to actually delete |
|
|
40
|
+
|
|
41
|
+
### Examples
|
|
42
|
+
|
|
43
|
+
```console
|
|
44
|
+
# biggest wins only, files above 10 MB
|
|
45
|
+
dupehunt ~/Pictures -m 10M -n 20
|
|
46
|
+
|
|
47
|
+
# feed the result to another tool
|
|
48
|
+
dupehunt ~/data --json > dupes.json
|
|
49
|
+
|
|
50
|
+
# preview what a cleanup would remove, without touching anything
|
|
51
|
+
dupehunt ~/data --delete -m 1M
|
|
52
|
+
|
|
53
|
+
# really delete the extra copies
|
|
54
|
+
dupehunt ~/data --delete --yes -m 1M
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
`--delete` without `--yes` is always a dry run: it prints the plan and removes
|
|
58
|
+
nothing. Nothing is ever deleted unless both flags are present.
|
|
59
|
+
|
|
60
|
+
## How it works
|
|
61
|
+
|
|
62
|
+
Three passes, cheapest first, so a full disk scan stays fast:
|
|
63
|
+
|
|
64
|
+
1. Group files by exact size. Unique sizes drop out immediately.
|
|
65
|
+
2. Hash the first 8 KiB of each remaining candidate.
|
|
66
|
+
3. Hash the whole file only for candidates that still collide.
|
|
67
|
+
|
|
68
|
+
Symlinks are never followed, and hard links to an already seen inode are counted
|
|
69
|
+
once, so the same physical data is not reported as duplicates of itself.
|
|
70
|
+
|
|
71
|
+
## Use as a library
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from pathlib import Path
|
|
75
|
+
from dupehunt import find_duplicates, total_wasted
|
|
76
|
+
|
|
77
|
+
groups = find_duplicates([Path(".")], n_min_size=1024 * 1024)
|
|
78
|
+
for group in groups:
|
|
79
|
+
print(group.size, group.wasted, [str(p) for p in group.paths])
|
|
80
|
+
print(total_wasted(groups))
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## License
|
|
84
|
+
|
|
85
|
+
MIT
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dupehunt"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
description = "Fast, dependency-free duplicate file finder with a friendly CLI."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "lqxnjk" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"duplicate",
|
|
15
|
+
"duplicates",
|
|
16
|
+
"dedupe",
|
|
17
|
+
"files",
|
|
18
|
+
"hash",
|
|
19
|
+
"disk-usage",
|
|
20
|
+
"cleanup",
|
|
21
|
+
"cli",
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 3 - Alpha",
|
|
25
|
+
"Environment :: Console",
|
|
26
|
+
"Intended Audience :: End Users/Desktop",
|
|
27
|
+
"Intended Audience :: System Administrators",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
32
|
+
"Topic :: System :: Filesystems",
|
|
33
|
+
"Topic :: Utilities",
|
|
34
|
+
]
|
|
35
|
+
dependencies = []
|
|
36
|
+
|
|
37
|
+
[project.scripts]
|
|
38
|
+
dupehunt = "dupehunt.cli:main"
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/lqxnjk/dupehunt"
|
|
42
|
+
Source = "https://github.com/lqxnjk/dupehunt"
|
|
43
|
+
|
|
44
|
+
[tool.pytest.ini_options]
|
|
45
|
+
pythonpath = ["src"]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.wheel]
|
|
49
|
+
packages = ["src/dupehunt"]
|
|
50
|
+
|
|
51
|
+
[tool.hatch.build.targets.sdist]
|
|
52
|
+
include = [
|
|
53
|
+
"src/dupehunt",
|
|
54
|
+
"tests",
|
|
55
|
+
"README.md",
|
|
56
|
+
"LICENSE",
|
|
57
|
+
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Author: lqxnjk <lqxnjk@qq.com>
|
|
3
|
+
Created: 2026-10-23 10:06
|
|
4
|
+
Modified: 2026-10-23 10:06
|
|
5
|
+
Python Version: 3.10
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .cli import __version__, main
|
|
9
|
+
from .core import (
|
|
10
|
+
SUPPORTED_ALGOS,
|
|
11
|
+
DuplicateGroup,
|
|
12
|
+
ScanError,
|
|
13
|
+
find_duplicates,
|
|
14
|
+
human_size,
|
|
15
|
+
iter_files,
|
|
16
|
+
parse_size,
|
|
17
|
+
total_wasted,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"SUPPORTED_ALGOS",
|
|
22
|
+
"DuplicateGroup",
|
|
23
|
+
"ScanError",
|
|
24
|
+
"__version__",
|
|
25
|
+
"find_duplicates",
|
|
26
|
+
"human_size",
|
|
27
|
+
"iter_files",
|
|
28
|
+
"main",
|
|
29
|
+
"parse_size",
|
|
30
|
+
"total_wasted",
|
|
31
|
+
]
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Author: lqxnjk <lqxnjk@qq.com>
|
|
3
|
+
Created: 2026-10-23 10:06
|
|
4
|
+
Modified: 2026-10-23 10:06
|
|
5
|
+
Python Version: 3.10
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Sequence
|
|
17
|
+
|
|
18
|
+
from .core import (
|
|
19
|
+
SUPPORTED_ALGOS,
|
|
20
|
+
DuplicateGroup,
|
|
21
|
+
find_duplicates,
|
|
22
|
+
human_size,
|
|
23
|
+
parse_size,
|
|
24
|
+
total_wasted,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
__version__ = "0.0.1"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
31
|
+
parser = argparse.ArgumentParser(
|
|
32
|
+
prog="dupehunt",
|
|
33
|
+
description="Find duplicate files by content hash.",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"paths",
|
|
37
|
+
nargs="*",
|
|
38
|
+
default=["."],
|
|
39
|
+
help="files or directories to scan (default: current directory)",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"-m",
|
|
43
|
+
"--min-size",
|
|
44
|
+
default="1",
|
|
45
|
+
help="ignore files smaller than this, e.g. 1K, 4M, 1G (default: 1)",
|
|
46
|
+
)
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"-a",
|
|
49
|
+
"--algo",
|
|
50
|
+
default="blake2b",
|
|
51
|
+
choices=list(SUPPORTED_ALGOS),
|
|
52
|
+
help="hash algorithm (default: blake2b, fastest)",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"-n",
|
|
56
|
+
"--top",
|
|
57
|
+
type=int,
|
|
58
|
+
default=0,
|
|
59
|
+
help="show only the N largest groups (0 = all)",
|
|
60
|
+
)
|
|
61
|
+
parser.add_argument("--json", action="store_true", help="machine readable output")
|
|
62
|
+
parser.add_argument(
|
|
63
|
+
"--delete",
|
|
64
|
+
action="store_true",
|
|
65
|
+
help="delete extra copies, keeping the first path in each group (dry run unless --yes)",
|
|
66
|
+
)
|
|
67
|
+
parser.add_argument(
|
|
68
|
+
"--yes",
|
|
69
|
+
action="store_true",
|
|
70
|
+
help="confirm deletion; only meaningful together with --delete",
|
|
71
|
+
)
|
|
72
|
+
parser.add_argument("--version", action="version", version=f"dupehunt {__version__}")
|
|
73
|
+
return parser
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _render_text(m_groups: Sequence[DuplicateGroup], m_roots: Sequence[str]) -> str:
|
|
77
|
+
if not m_groups:
|
|
78
|
+
return "no duplicates found"
|
|
79
|
+
|
|
80
|
+
lines = [
|
|
81
|
+
f"{len(m_groups)} group(s), "
|
|
82
|
+
f"{sum(len(group.paths) for group in m_groups)} file(s), "
|
|
83
|
+
f"{human_size(total_wasted(m_groups))} reclaimable",
|
|
84
|
+
"",
|
|
85
|
+
]
|
|
86
|
+
for index, group in enumerate(m_groups, start=1):
|
|
87
|
+
lines.append(
|
|
88
|
+
f"[{index}] {human_size(group.size)} x {len(group.paths)}"
|
|
89
|
+
f" ({human_size(group.wasted)} wasted)"
|
|
90
|
+
)
|
|
91
|
+
lines.extend(f" {path}" for path in group.paths)
|
|
92
|
+
lines.append("")
|
|
93
|
+
return "\n".join(lines).rstrip()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _render_json(m_groups: Sequence[DuplicateGroup], m_roots: Sequence[str]) -> str:
|
|
97
|
+
payload = {
|
|
98
|
+
"roots": list(m_roots),
|
|
99
|
+
"groups": [
|
|
100
|
+
{
|
|
101
|
+
"size": group.size,
|
|
102
|
+
"digest": group.digest,
|
|
103
|
+
"wasted": group.wasted,
|
|
104
|
+
"paths": [str(path) for path in group.paths],
|
|
105
|
+
}
|
|
106
|
+
for group in m_groups
|
|
107
|
+
],
|
|
108
|
+
"total_wasted": total_wasted(m_groups),
|
|
109
|
+
}
|
|
110
|
+
return json.dumps(payload, ensure_ascii=False, indent=2)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _delete_extras(m_groups: Sequence[DuplicateGroup], n_confirm: bool) -> int:
|
|
114
|
+
"""Delete every path but the first in each group. Returns the number removed."""
|
|
115
|
+
targets = [path for group in m_groups for path in group.paths[1:]]
|
|
116
|
+
if not targets:
|
|
117
|
+
return 0
|
|
118
|
+
if not n_confirm:
|
|
119
|
+
print(
|
|
120
|
+
f"dry run: {len(targets)} file(s) would be deleted "
|
|
121
|
+
"(re-run with --delete --yes to apply)",
|
|
122
|
+
file=sys.stderr,
|
|
123
|
+
)
|
|
124
|
+
for path in targets:
|
|
125
|
+
print(f" would delete {path}", file=sys.stderr)
|
|
126
|
+
return 0
|
|
127
|
+
|
|
128
|
+
removed = 0
|
|
129
|
+
for path in targets:
|
|
130
|
+
try:
|
|
131
|
+
os.remove(path)
|
|
132
|
+
removed += 1
|
|
133
|
+
except OSError as exc:
|
|
134
|
+
logging.getLogger(__name__).error("failed to delete %s: %s", path, exc)
|
|
135
|
+
return removed
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def main(m_argv: Sequence[str] | None = None) -> int:
|
|
139
|
+
parser = _build_parser()
|
|
140
|
+
args = parser.parse_args(m_argv)
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
min_size = parse_size(args.min_size)
|
|
144
|
+
except ValueError as exc:
|
|
145
|
+
parser.error(str(exc))
|
|
146
|
+
return 2
|
|
147
|
+
|
|
148
|
+
roots = [Path(item) for item in args.paths]
|
|
149
|
+
missing = [root for root in roots if not root.exists()]
|
|
150
|
+
if missing:
|
|
151
|
+
for root in missing:
|
|
152
|
+
print(f"path does not exist: {root}", file=sys.stderr)
|
|
153
|
+
return 2
|
|
154
|
+
|
|
155
|
+
try:
|
|
156
|
+
groups = find_duplicates(roots, n_min_size=min_size, n_algo=args.algo)
|
|
157
|
+
except OSError as exc:
|
|
158
|
+
logging.getLogger(__name__).error("scan failed: %s", exc)
|
|
159
|
+
return 1
|
|
160
|
+
|
|
161
|
+
if args.top > 0:
|
|
162
|
+
groups = groups[: args.top]
|
|
163
|
+
|
|
164
|
+
if args.json:
|
|
165
|
+
print(_render_json(groups, args.paths))
|
|
166
|
+
else:
|
|
167
|
+
print(_render_text(groups, args.paths))
|
|
168
|
+
|
|
169
|
+
if args.delete:
|
|
170
|
+
removed = _delete_extras(groups, n_confirm=args.yes)
|
|
171
|
+
if removed:
|
|
172
|
+
print(f"deleted {removed} file(s)", file=sys.stderr)
|
|
173
|
+
|
|
174
|
+
return 0
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Author: lqxnjk <lqxnjk@qq.com>
|
|
3
|
+
Created: 2026-10-23 10:06
|
|
4
|
+
Modified: 2026-10-23 10:06
|
|
5
|
+
Python Version: 3.10
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
from collections import defaultdict
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Iterable, Iterator, Sequence
|
|
17
|
+
|
|
18
|
+
LOGGER = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
CHUNK_SIZE = 1 << 20
|
|
21
|
+
QUICK_SIZE = 8192
|
|
22
|
+
SUPPORTED_ALGOS = ("blake2b", "sha256", "sha1", "md5")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class DuplicateGroup:
|
|
27
|
+
"""A set of two or more files with identical content."""
|
|
28
|
+
|
|
29
|
+
size: int
|
|
30
|
+
digest: str
|
|
31
|
+
paths: list[Path]
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def wasted(self) -> int:
|
|
35
|
+
"""Bytes that would be reclaimed by keeping a single copy."""
|
|
36
|
+
return self.size * (len(self.paths) - 1)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ScanError(OSError):
|
|
40
|
+
"""Raised when a scan root cannot be read."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def human_size(m_num: int) -> str:
|
|
44
|
+
"""Format a byte count as a short human readable string."""
|
|
45
|
+
value = float(m_num)
|
|
46
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
47
|
+
if value < 1024 or unit == "TB":
|
|
48
|
+
if unit == "B":
|
|
49
|
+
return f"{int(value)} {unit}"
|
|
50
|
+
return f"{value:.1f} {unit}"
|
|
51
|
+
value /= 1024
|
|
52
|
+
return f"{value:.1f} PB"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_size(m_text: str) -> int:
|
|
56
|
+
"""Parse ``"1024"``, ``"10K"``, ``"1.5M"``, ``"2G"`` into bytes."""
|
|
57
|
+
raw = m_text.strip().upper().rstrip("B")
|
|
58
|
+
multiplier = 1
|
|
59
|
+
if raw and raw[-1] in "KMGTP":
|
|
60
|
+
multiplier = 1024 ** ("KMGTP".index(raw[-1]) + 1)
|
|
61
|
+
raw = raw[:-1]
|
|
62
|
+
try:
|
|
63
|
+
return int(float(raw) * multiplier)
|
|
64
|
+
except ValueError as exc:
|
|
65
|
+
raise ValueError(f"invalid size: {m_text!r}") from exc
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _hash_file(m_path: Path, m_algo: str, n_limit: int | None = None) -> str:
|
|
69
|
+
"""Hash a file, optionally reading at most ``n_limit`` bytes."""
|
|
70
|
+
digest = hashlib.new(m_algo)
|
|
71
|
+
remaining = n_limit
|
|
72
|
+
with m_path.open("rb") as handle:
|
|
73
|
+
while True:
|
|
74
|
+
want = CHUNK_SIZE if remaining is None else min(CHUNK_SIZE, remaining)
|
|
75
|
+
if want <= 0:
|
|
76
|
+
break
|
|
77
|
+
block = handle.read(want)
|
|
78
|
+
if not block:
|
|
79
|
+
break
|
|
80
|
+
digest.update(block)
|
|
81
|
+
if remaining is not None:
|
|
82
|
+
remaining -= len(block)
|
|
83
|
+
return digest.hexdigest()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def iter_files(m_roots: Sequence[Path], n_min_size: int = 1) -> Iterator[tuple[Path, int]]:
|
|
87
|
+
"""Yield ``(path, size)`` for regular files under ``m_roots``.
|
|
88
|
+
|
|
89
|
+
Symlinks are never followed and hard links to an already seen inode are
|
|
90
|
+
skipped, so the same physical data is not reported twice. Unreadable
|
|
91
|
+
entries are logged and skipped instead of aborting the whole scan.
|
|
92
|
+
"""
|
|
93
|
+
seen_inodes: set[tuple[int, int]] = set()
|
|
94
|
+
for root in m_roots:
|
|
95
|
+
if root.is_file():
|
|
96
|
+
try:
|
|
97
|
+
stat = root.stat()
|
|
98
|
+
except OSError as exc:
|
|
99
|
+
LOGGER.warning("skip unreadable file %s: %s", root, exc)
|
|
100
|
+
continue
|
|
101
|
+
if stat.st_size >= n_min_size:
|
|
102
|
+
yield root, stat.st_size
|
|
103
|
+
continue
|
|
104
|
+
if not root.is_dir():
|
|
105
|
+
raise ScanError(f"not a directory: {root}")
|
|
106
|
+
stack = [root]
|
|
107
|
+
while stack:
|
|
108
|
+
current = stack.pop()
|
|
109
|
+
try:
|
|
110
|
+
entries = list(os.scandir(current))
|
|
111
|
+
except OSError as exc:
|
|
112
|
+
LOGGER.warning("skip unreadable directory %s: %s", current, exc)
|
|
113
|
+
continue
|
|
114
|
+
for entry in entries:
|
|
115
|
+
try:
|
|
116
|
+
if entry.is_dir(follow_symlinks=False):
|
|
117
|
+
stack.append(Path(entry.path))
|
|
118
|
+
continue
|
|
119
|
+
if not entry.is_file(follow_symlinks=False):
|
|
120
|
+
continue
|
|
121
|
+
stat = entry.stat(follow_symlinks=False)
|
|
122
|
+
except OSError as exc:
|
|
123
|
+
LOGGER.warning("skip unreadable entry %s: %s", entry.path, exc)
|
|
124
|
+
continue
|
|
125
|
+
inode = (stat.st_dev, stat.st_ino)
|
|
126
|
+
if inode != (0, 0):
|
|
127
|
+
if inode in seen_inodes:
|
|
128
|
+
LOGGER.debug("skip hard link %s", entry.path)
|
|
129
|
+
continue
|
|
130
|
+
seen_inodes.add(inode)
|
|
131
|
+
if stat.st_size >= n_min_size:
|
|
132
|
+
yield Path(entry.path), stat.st_size
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def find_duplicates(
|
|
136
|
+
m_roots: Sequence[Path],
|
|
137
|
+
n_min_size: int = 1,
|
|
138
|
+
n_algo: str = "blake2b",
|
|
139
|
+
) -> list[DuplicateGroup]:
|
|
140
|
+
"""Return duplicate groups sorted by reclaimable space, largest first.
|
|
141
|
+
|
|
142
|
+
Three passes keep large scans cheap: group by size, then hash the first
|
|
143
|
+
few KiB, then hash the whole file only for surviving candidates.
|
|
144
|
+
"""
|
|
145
|
+
if n_algo not in SUPPORTED_ALGOS:
|
|
146
|
+
raise ValueError(f"unsupported algorithm: {n_algo!r}")
|
|
147
|
+
|
|
148
|
+
by_size: dict[int, list[Path]] = defaultdict(list)
|
|
149
|
+
for path, size in iter_files(m_roots, n_min_size):
|
|
150
|
+
by_size[size].append(path)
|
|
151
|
+
|
|
152
|
+
candidates = [(size, path) for size, paths in by_size.items() if len(paths) > 1 for path in paths]
|
|
153
|
+
if not candidates:
|
|
154
|
+
return []
|
|
155
|
+
|
|
156
|
+
by_quick: dict[tuple[int, str], list[Path]] = defaultdict(list)
|
|
157
|
+
for size, path in candidates:
|
|
158
|
+
try:
|
|
159
|
+
quick = _hash_file(path, n_algo, QUICK_SIZE)
|
|
160
|
+
except OSError as exc:
|
|
161
|
+
LOGGER.warning("skip unreadable candidate %s: %s", path, exc)
|
|
162
|
+
continue
|
|
163
|
+
by_quick[(size, quick)].append(path)
|
|
164
|
+
|
|
165
|
+
groups: list[DuplicateGroup] = []
|
|
166
|
+
for (size, quick), paths in by_quick.items():
|
|
167
|
+
if len(paths) < 2:
|
|
168
|
+
continue
|
|
169
|
+
if size <= QUICK_SIZE:
|
|
170
|
+
groups.append(DuplicateGroup(size=size, digest=quick, paths=paths))
|
|
171
|
+
continue
|
|
172
|
+
by_full: dict[str, list[Path]] = defaultdict(list)
|
|
173
|
+
for path in paths:
|
|
174
|
+
try:
|
|
175
|
+
by_full[_hash_file(path, n_algo)].append(path)
|
|
176
|
+
except OSError as exc:
|
|
177
|
+
LOGGER.warning("skip unreadable candidate %s: %s", path, exc)
|
|
178
|
+
continue
|
|
179
|
+
for digest, same in by_full.items():
|
|
180
|
+
if len(same) > 1:
|
|
181
|
+
groups.append(DuplicateGroup(size=size, digest=digest, paths=same))
|
|
182
|
+
|
|
183
|
+
groups.sort(key=lambda group: group.wasted, reverse=True)
|
|
184
|
+
return groups
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def total_wasted(m_groups: Iterable[DuplicateGroup]) -> int:
|
|
188
|
+
"""Sum reclaimable bytes across groups."""
|
|
189
|
+
return sum(group.wasted for group in m_groups)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Author: lqxnjk <lqxnjk@qq.com>
|
|
3
|
+
Created: 2026-10-23 10:06
|
|
4
|
+
Modified: 2026-10-23 10:06
|
|
5
|
+
Python Version: 3.10
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
from dupehunt import find_duplicates, human_size, parse_size, total_wasted
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _write(path: Path, content: bytes) -> Path:
|
|
18
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
19
|
+
path.write_bytes(content)
|
|
20
|
+
return path
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_detects_identical_files(tmp_path: Path) -> None:
|
|
24
|
+
payload = b"same content" * 100
|
|
25
|
+
_write(tmp_path / "a.bin", payload)
|
|
26
|
+
_write(tmp_path / "sub" / "b.bin", payload)
|
|
27
|
+
_write(tmp_path / "c.bin", b"different")
|
|
28
|
+
_write(tmp_path / "d.bin", b"also different, longer payload")
|
|
29
|
+
|
|
30
|
+
groups = find_duplicates([tmp_path])
|
|
31
|
+
|
|
32
|
+
assert len(groups) == 1
|
|
33
|
+
assert groups[0].size == len(payload)
|
|
34
|
+
assert {p.name for p in groups[0].paths} == {"a.bin", "b.bin"}
|
|
35
|
+
assert groups[0].wasted == len(payload)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_ignores_unique_files(tmp_path: Path) -> None:
|
|
39
|
+
_write(tmp_path / "a.txt", b"alpha")
|
|
40
|
+
_write(tmp_path / "b.txt", b"beta!")
|
|
41
|
+
assert find_duplicates([tmp_path]) == []
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_respects_min_size(tmp_path: Path) -> None:
|
|
45
|
+
_write(tmp_path / "a.txt", b"tiny")
|
|
46
|
+
_write(tmp_path / "b.txt", b"tiny")
|
|
47
|
+
assert find_duplicates([tmp_path], n_min_size=10) == []
|
|
48
|
+
assert len(find_duplicates([tmp_path], n_min_size=1)) == 1
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_large_file_uses_full_hash(tmp_path: Path) -> None:
|
|
52
|
+
head = b"x" * 8192
|
|
53
|
+
_write(tmp_path / "a.bin", head + b"tail-one")
|
|
54
|
+
_write(tmp_path / "b.bin", head + b"tail-two")
|
|
55
|
+
|
|
56
|
+
assert find_duplicates([tmp_path]) == []
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_groups_sorted_by_wasted(tmp_path: Path) -> None:
|
|
60
|
+
big = b"b" * 5000
|
|
61
|
+
small = b"s" * 100
|
|
62
|
+
_write(tmp_path / "big1", big)
|
|
63
|
+
_write(tmp_path / "big2", big)
|
|
64
|
+
_write(tmp_path / "small1", small)
|
|
65
|
+
_write(tmp_path / "small2", small)
|
|
66
|
+
|
|
67
|
+
groups = find_duplicates([tmp_path])
|
|
68
|
+
|
|
69
|
+
assert [g.size for g in groups] == [5000, 100]
|
|
70
|
+
assert total_wasted(groups) == 5100
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_accepts_single_file_root(tmp_path: Path) -> None:
|
|
74
|
+
target = _write(tmp_path / "one.bin", b"payload")
|
|
75
|
+
assert find_duplicates([target]) == []
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_missing_root_raises(tmp_path: Path) -> None:
|
|
79
|
+
with pytest.raises(OSError):
|
|
80
|
+
find_duplicates([tmp_path / "nope"])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@pytest.mark.parametrize(
|
|
84
|
+
("text", "expected"),
|
|
85
|
+
[("1", 1), ("1K", 1024), ("1k", 1024), ("1.5M", 1572864), ("2G", 2 * 1024**3)],
|
|
86
|
+
)
|
|
87
|
+
def test_parse_size(text: str, expected: int) -> None:
|
|
88
|
+
assert parse_size(text) == expected
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_parse_size_rejects_garbage() -> None:
|
|
92
|
+
with pytest.raises(ValueError):
|
|
93
|
+
parse_size("abc")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@pytest.mark.parametrize(
|
|
97
|
+
("num", "expected"),
|
|
98
|
+
[(0, "0 B"), (1023, "1023 B"), (1024, "1.0 KB"), (1536, "1.5 KB")],
|
|
99
|
+
)
|
|
100
|
+
def test_human_size(num: int, expected: str) -> None:
|
|
101
|
+
assert human_size(num) == expected
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_unsupported_algo(tmp_path: Path) -> None:
|
|
105
|
+
with pytest.raises(ValueError):
|
|
106
|
+
find_duplicates([tmp_path], n_algo="crc99")
|