Immaculate 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. immaculate-0.3.0/.github/workflows/release-onnx.yml +47 -0
  2. {immaculate-0.2.0 → immaculate-0.3.0}/.github/workflows/release.yml +22 -30
  3. {immaculate-0.2.0 → immaculate-0.3.0}/Cargo.lock +1 -1
  4. {immaculate-0.2.0 → immaculate-0.3.0}/Cargo.toml +3 -3
  5. {immaculate-0.2.0 → immaculate-0.3.0}/PKG-INFO +5 -2
  6. {immaculate-0.2.0 → immaculate-0.3.0}/README.md +1 -0
  7. {immaculate-0.2.0 → immaculate-0.3.0}/pyproject.toml +23 -3
  8. {immaculate-0.2.0 → immaculate-0.3.0}/rust/lib.rs +67 -12
  9. {immaculate-0.2.0 → immaculate-0.3.0}/src/immaculate/__init__.py +1 -1
  10. {immaculate-0.2.0 → immaculate-0.3.0}/src/immaculate/_duplicates.py +25 -20
  11. {immaculate-0.2.0 → immaculate-0.3.0}/tests/test_immaculate.py +38 -5
  12. {immaculate-0.2.0 → immaculate-0.3.0}/uv.lock +102 -8
  13. immaculate-0.2.0/src/immaculate/models/SSCD_LICENSE +0 -21
  14. immaculate-0.2.0/src/immaculate/models/sscd.onnx +0 -0
  15. immaculate-0.2.0/src/immaculate/models/sscd_distilled.onnx +0 -0
  16. {immaculate-0.2.0 → immaculate-0.3.0}/.github/workflows/ci.yml +0 -0
  17. {immaculate-0.2.0 → immaculate-0.3.0}/.gitignore +0 -0
  18. {immaculate-0.2.0 → immaculate-0.3.0}/.pre-commit-config.yaml +0 -0
  19. {immaculate-0.2.0 → immaculate-0.3.0}/.python-version +0 -0
  20. {immaculate-0.2.0 → immaculate-0.3.0}/LICENSE +0 -0
  21. {immaculate-0.2.0 → immaculate-0.3.0}/clippy.toml +0 -0
  22. {immaculate-0.2.0 → immaculate-0.3.0}/docs/api.md +0 -0
  23. {immaculate-0.2.0 → immaculate-0.3.0}/docs/index.md +0 -0
  24. {immaculate-0.2.0 → immaculate-0.3.0}/mkdocs.yml +0 -0
  25. {immaculate-0.2.0 → immaculate-0.3.0}/rust/multi_index.rs +0 -0
  26. {immaculate-0.2.0 → immaculate-0.3.0}/rust/pdq.rs +0 -0
  27. {immaculate-0.2.0 → immaculate-0.3.0}/rust/perceptual_hash.rs +0 -0
  28. {immaculate-0.2.0 → immaculate-0.3.0}/rust/phash.rs +0 -0
  29. {immaculate-0.2.0 → immaculate-0.3.0}/rust/side_cache.rs +0 -0
  30. {immaculate-0.2.0 → immaculate-0.3.0}/rust/sscd.rs +0 -0
  31. {immaculate-0.2.0 → immaculate-0.3.0}/rust/test_util.rs +0 -0
  32. {immaculate-0.2.0 → immaculate-0.3.0}/src/immaculate/__main__.py +0 -0
  33. {immaculate-0.2.0 → immaculate-0.3.0}/src/immaculate/_core.pyi +0 -0
  34. {immaculate-0.2.0 → immaculate-0.3.0}/src/immaculate/py.typed +0 -0
@@ -0,0 +1,47 @@
1
+ name: Release immaculate-onnx
2
+
3
+ # Builds the immaculate-onnx package and, on an immaculate-onnx-v<version> tag, publishes it to
4
+ # PyPI; see DEPLOYMENT.md. It is a workflow of its own, not a job in release.yml, so its trusted
5
+ # publisher differs from immaculate's: PyPI would otherwise give the upload immaculate's token,
6
+ # which can't create or upload to another project
7
+ on:
8
+ push:
9
+ tags: ["immaculate-onnx-v*"]
10
+ workflow_dispatch:
11
+
12
+ permissions:
13
+ contents: read
14
+
15
+ jobs:
16
+ # Pure Python, so one wheel serves every platform
17
+ build:
18
+ runs-on: ubuntu-latest
19
+ steps:
20
+ - uses: actions/checkout@v5
21
+ - uses: astral-sh/setup-uv@v6
22
+ - run: uv build --package immaculate-onnx --out-dir dist
23
+ - uses: actions/upload-artifact@v4
24
+ with:
25
+ name: immaculate-onnx
26
+ path: dist/*
27
+
28
+ # Uploads use trusted publishing, so no tokens are stored; PyPI trusts this workflow in the
29
+ # pypi environment, which needs approval
30
+ publish-pypi:
31
+ if: startsWith(github.ref, 'refs/tags/immaculate-onnx-v')
32
+ needs: build
33
+ runs-on: ubuntu-latest
34
+ environment:
35
+ name: pypi
36
+ url: https://pypi.org/project/immaculate-onnx/
37
+ permissions:
38
+ id-token: write
39
+ steps:
40
+ - uses: actions/download-artifact@v4
41
+ with:
42
+ name: immaculate-onnx
43
+ path: dist
44
+ # PyPI never accepts a version twice, so a test tag mustn't upload the real version
45
+ - name: Check the tag matches the version
46
+ run: test -f "dist/immaculate_onnx-${GITHUB_REF_NAME#immaculate-onnx-v}.tar.gz"
47
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -1,8 +1,9 @@
1
1
  name: Release
2
2
 
3
- # Builds the sdist and a wheel for each platform ONNX Runtime has wheels for, tests each
4
- # wheel on its own platform, and on a version tag publishes them to TestPyPI, then PyPI; see
5
- # DEPLOYMENT.md
3
+ # Builds the sdist and a wheel for each platform ONNX Runtime has wheels for, so the sscd extra
4
+ # installs wherever Immaculate does, tests each wheel on its own platform, and on a version tag
5
+ # publishes them to PyPI; see DEPLOYMENT.md. The immaculate-onnx package has its own workflow,
6
+ # release-onnx.yml
6
7
  on:
7
8
  push:
8
9
  tags: ["v*"]
@@ -41,8 +42,9 @@ jobs:
41
42
  # ONNX Runtime's own Linux wheels need glibc 2.28
42
43
  manylinux: "2_28"
43
44
  - uses: astral-sh/setup-uv@v6
44
- # Tests the built wheel, not the source tree: the dev group has no project, so syncing
45
- # it doesn't build Immaculate
45
+ # Tests the built wheel, not the source tree: the dev group doesn't include Immaculate,
46
+ # so syncing it doesn't build it. It does install immaculate-onnx from the source tree,
47
+ # for the tests of the SSCD methods
46
48
  - name: Test the wheel
47
49
  shell: bash
48
50
  # Only uv sync is given the Python: with UV_PYTHON set, uv pip would install into that
@@ -55,7 +57,7 @@ jobs:
55
57
  uv run --no-sync pytest
56
58
  - uses: actions/upload-artifact@v4
57
59
  with:
58
- name: wheel-${{ matrix.os }}
60
+ name: immaculate-wheel-${{ matrix.os }}
59
61
  path: dist/*.whl
60
62
 
61
63
  sdist:
@@ -69,44 +71,34 @@ jobs:
69
71
  args: --out dist
70
72
  - uses: actions/upload-artifact@v4
71
73
  with:
72
- name: sdist
74
+ name: immaculate-sdist
73
75
  path: dist/*.tar.gz
74
76
 
75
- # Uploads use trusted publishing, so no tokens are stored; each index trusts this workflow
76
- # in the environment named below
77
- publish-testpypi:
77
+ # Uploads use trusted publishing, so no tokens are stored; PyPI trusts this workflow in the
78
+ # pypi environment, which needs approval
79
+ publish-pypi:
78
80
  if: startsWith(github.ref, 'refs/tags/v')
79
81
  needs: [wheels, sdist]
80
82
  runs-on: ubuntu-latest
81
83
  environment:
82
- name: testpypi
83
- url: https://test.pypi.org/project/immaculate/
84
+ name: pypi
85
+ url: https://pypi.org/project/immaculate/
84
86
  permissions:
85
87
  id-token: write
86
88
  steps:
87
89
  - uses: actions/download-artifact@v4
88
90
  with:
91
+ pattern: immaculate-*
89
92
  path: dist
90
93
  merge-multiple: true
91
94
  # PyPI never accepts a version twice, so a tag like v0.1.0-test mustn't upload 0.1.0
92
95
  - name: Check the tag matches the version
93
96
  run: test -f "dist/immaculate-${GITHUB_REF_NAME#v}.tar.gz"
94
- - uses: pypa/gh-action-pypi-publish@release/v1
95
- with:
96
- repository-url: https://test.pypi.org/legacy/
97
-
98
- # The pypi environment needs approval, so there's time to check the TestPyPI release first
99
- publish-pypi:
100
- needs: publish-testpypi
101
- runs-on: ubuntu-latest
102
- environment:
103
- name: pypi
104
- url: https://pypi.org/project/immaculate/
105
- permissions:
106
- id-token: write
107
- steps:
108
- - uses: actions/download-artifact@v4
109
- with:
110
- path: dist
111
- merge-multiple: true
97
+ # Otherwise installing the sscd extra would fail; release-onnx.yml publishes it
98
+ - name: Check the pinned immaculate-onnx is on PyPI
99
+ run: |
100
+ version=$(tar -xzOf dist/immaculate-*.tar.gz --wildcards '*/PKG-INFO' \
101
+ | sed -n 's/^Requires-Dist: immaculate-onnx==\([^ ;]*\).*/\1/p')
102
+ test -n "$version"
103
+ curl -fsS -o /dev/null "https://pypi.org/pypi/immaculate-onnx/$version/json"
112
104
  - uses: pypa/gh-action-pypi-publish@release/v1
@@ -174,7 +174,7 @@ dependencies = [
174
174
 
175
175
  [[package]]
176
176
  name = "immaculate"
177
- version = "0.2.0"
177
+ version = "0.3.0"
178
178
  dependencies = [
179
179
  "image",
180
180
  "ort",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "immaculate"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  edition = "2024"
5
5
  license-file = "LICENSE"
6
6
  publish = false
@@ -13,8 +13,8 @@ crate-type = ["cdylib"]
13
13
 
14
14
  [dependencies]
15
15
  image = { version = "0.25.10", default-features = false, features = ["bmp", "gif", "jpeg", "png", "tiff", "webp"] }
16
- # ONNX Runtime is loaded at runtime from the onnxruntime Python package, so it isn't built into
17
- # the wheel. Without the api-* features ort needs only ONNX Runtime 1.17 or later.
16
+ # ONNX Runtime is loaded at runtime from the onnxruntime Python package, which the sscd extra
17
+ # installs, so it isn't built into the wheel. Without the api-* features ort needs only ONNX Runtime 1.17 or later.
18
18
  ort = { version = "=2.0.0-rc.14", default-features = false, features = ["load-dynamic", "std"] }
19
19
  pyo3 = { version = "0.30.1", features = ["abi3-py312"] }
20
20
  rayon = "1.12.0"
@@ -1,8 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: Immaculate
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Classifier: Programming Language :: Python :: 3
5
- Requires-Dist: onnxruntime>=1.17
5
+ Requires-Dist: immaculate-onnx==0.1.0 ; extra == 'sscd'
6
+ Requires-Dist: onnxruntime>=1.17 ; extra == 'sscd'
7
+ Provides-Extra: sscd
6
8
  License-File: LICENSE
7
9
  Summary: Find exact and near-duplicate images in a folder, fast.
8
10
  Author: Peter Batchelor
@@ -21,6 +23,7 @@ Find exact and near-duplicate images in a folder, fast.
21
23
 
22
24
  ```bash
23
25
  pip install immaculate
26
+ pip install "immaculate[sscd]" # adds ONNX Runtime and the models for the sscd and sscd_distilled methods
24
27
  ```
25
28
 
26
29
  ## Usage
@@ -6,6 +6,7 @@ Find exact and near-duplicate images in a folder, fast.
6
6
 
7
7
  ```bash
8
8
  pip install immaculate
9
+ pip install "immaculate[sscd]" # adds ONNX Runtime and the models for the sscd and sscd_distilled methods
9
10
  ```
10
11
 
11
12
  ## Usage
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "Immaculate"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  description = "Find exact and near-duplicate images in a folder, fast."
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Peter Batchelor" }]
@@ -10,8 +10,14 @@ classifiers = [
10
10
  "Programming Language :: Python :: 3",
11
11
  ]
12
12
  requires-python = ">=3.12"
13
- dependencies = [
14
- # Runs the "sscd" method's model; the Rust bindings need 1.17 or later
13
+ dependencies = []
14
+
15
+ [project.optional-dependencies]
16
+ sscd = [
17
+ # The "sscd" and "sscd_distilled" methods' models, in their own package because they would
18
+ # be nearly all of the size of Immaculate's wheels
19
+ "immaculate-onnx==0.1.0",
20
+ # Runs the models; the Rust bindings need 1.17 or later
15
21
  "onnxruntime>=1.17",
16
22
  ]
17
23
 
@@ -32,6 +38,12 @@ benchmark = [
32
38
  "pyarrow==26.0.0",
33
39
  ]
34
40
  dev = [
41
+ # The tests run the SSCD methods, and the release workflow syncs this group without the
42
+ # project, so its sscd extra isn't installed
43
+ "immaculate-onnx",
44
+ # ONNX Runtime's last Intel Mac wheels are 1.23.2
45
+ "onnxruntime==1.31.0; sys_platform != 'darwin' or platform_machine != 'x86_64'",
46
+ "onnxruntime==1.23.2; sys_platform == 'darwin' and platform_machine == 'x86_64'",
35
47
  "pillow==13.0.0",
36
48
  "pre-commit==4.6.1",
37
49
  "pytest==9.1.1",
@@ -58,6 +70,8 @@ features = ["pyo3/extension-module"]
58
70
  exclude = [
59
71
  { path = "benchmark/**/*", format = "sdist" },
60
72
  { path = "DEPLOYMENT.md", format = "sdist" },
73
+ # Published as its own package, immaculate-onnx
74
+ { path = "immaculate-onnx/**/*", format = "sdist" },
61
75
  ]
62
76
 
63
77
  [tool.uv]
@@ -65,6 +79,12 @@ exclude = [
65
79
  cache-keys = [{ file = "pyproject.toml" }, { file = "Cargo.toml" }, { file = "rust/**/*.rs" }]
66
80
  default-groups = ["dev", "benchmark"]
67
81
 
82
+ [tool.uv.workspace]
83
+ members = ["immaculate-onnx"]
84
+
85
+ [tool.uv.sources]
86
+ immaculate-onnx = { workspace = true }
87
+
68
88
  [tool.pytest.ini_options]
69
89
  addopts = "-q"
70
90
  testpaths = ["tests"]
@@ -33,7 +33,7 @@ use pyo3::prelude::*;
33
33
  use pyo3::types::PyTuple;
34
34
  use rayon::prelude::*;
35
35
  use walkdir::WalkDir;
36
- use xxhash_rust::xxh3::Xxh3Default;
36
+ use xxhash_rust::xxh3::{Xxh3Default, xxh3_128};
37
37
 
38
38
  use crate::pdq::Pdq;
39
39
  use crate::perceptual_hash::PerceptualHash;
@@ -167,6 +167,53 @@ fn pixel_hash(image: &DynamicImage) -> u128 {
167
167
  hasher.digest128()
168
168
  }
169
169
 
170
+ /// For each image, the index of the first image whose file has the same contents, which is
171
+ /// its own index if no earlier file does.
172
+ ///
173
+ /// Only files whose size another file shares are read, and their contents compared by 128-bit
174
+ /// xxh3, so a folder without byte-identical copies costs little more than a `stat` per file.
175
+ /// A file that can't be read or measured is treated as unique, to be skipped when it fails to
176
+ /// decode. Stops early if `cancelled` is set.
177
+ fn first_identical_files(
178
+ images: &[PathBuf],
179
+ cancelled: &AtomicBool,
180
+ ) -> Result<Vec<usize>, DedupError> {
181
+ let sizes: Vec<Option<u64>> = images
182
+ .par_iter()
183
+ .map(|path| fs::metadata(path).ok().map(|metadata| metadata.len()))
184
+ .collect();
185
+ let mut files_by_size: HashMap<u64, usize> = HashMap::new();
186
+ for &size in sizes.iter().flatten() {
187
+ *files_by_size.entry(size).or_default() += 1;
188
+ }
189
+ let contents: Vec<Option<(usize, u128)>> = images
190
+ .par_iter()
191
+ .zip(&sizes)
192
+ .map(|(path, size)| {
193
+ if cancelled.load(Ordering::Relaxed) {
194
+ return Err(DedupError::Cancelled);
195
+ }
196
+ let shared = size.is_some_and(|size| files_by_size[&size] > 1);
197
+ // The length is read again in case the file changed since it was measured
198
+ Ok(shared
199
+ .then(|| fs::read(path).ok())
200
+ .flatten()
201
+ .map(|bytes| (bytes.len(), xxh3_128(&bytes))))
202
+ })
203
+ .collect::<Result<_, DedupError>>()?;
204
+
205
+ let mut first_with_contents: HashMap<(usize, u128), usize> = HashMap::new();
206
+ Ok(contents
207
+ .into_iter()
208
+ .enumerate()
209
+ .map(|(index, key)| {
210
+ key.map_or(index, |key| {
211
+ *first_with_contents.entry(key).or_insert(index)
212
+ })
213
+ })
214
+ .collect())
215
+ }
216
+
170
217
  struct ImageHashes<H> {
171
218
  pixels: u128,
172
219
  perceptual: H,
@@ -239,24 +286,32 @@ fn find_duplicate_images<P: PerceptualHash>(
239
286
  let images = find_images(input_dir, &progress.cancelled)?;
240
287
  // Only this function sets it, once
241
288
  let _ = progress.total.set(images.len());
289
+ // Byte-identical files are decoded and hashed once, through the first of them
290
+ let firsts = first_identical_files(&images, &progress.cancelled)?;
291
+ let mut copies = vec![0; images.len()];
292
+ for &first in &firsts {
293
+ copies[first] += 1;
294
+ }
295
+ // Only set for the first of each set of byte-identical files
242
296
  let hashes: Vec<Option<ImageHashes<P::Hash>>> = images
243
297
  .par_iter()
244
- .map(|path| {
298
+ .zip(&copies)
299
+ .map(|(path, &copies)| {
300
+ if copies == 0 {
301
+ return Ok(None);
302
+ }
245
303
  if progress.cancelled.load(Ordering::Relaxed) {
246
304
  return Err(DedupError::Cancelled);
247
305
  }
248
306
  let hashes = hash_image(path, method);
249
- progress.done.fetch_add(1, Ordering::Relaxed);
307
+ progress.done.fetch_add(copies, Ordering::Relaxed);
250
308
  hashes
251
309
  })
252
310
  .collect::<Result<_, DedupError>>()?;
311
+ let image_hashes = |index: usize| hashes[firsts[index]].as_ref();
253
312
 
254
- let loaded = || {
255
- hashes
256
- .iter()
257
- .enumerate()
258
- .filter_map(|(index, hashes)| hashes.as_ref().map(|hashes| (index, hashes)))
259
- };
313
+ let loaded =
314
+ || (0..images.len()).filter_map(|index| image_hashes(index).map(|hashes| (index, hashes)));
260
315
  let exact = group_by_key(loaded().map(|(index, hashes)| (index, hashes.pixels)));
261
316
  let perceptual: Vec<(usize, &P::Hash)> = loaded()
262
317
  .map(|(index, hashes)| (index, &hashes.perceptual))
@@ -276,10 +331,10 @@ fn find_duplicate_images<P: PerceptualHash>(
276
331
  })
277
332
  .collect()
278
333
  };
279
- let skipped = hashes
334
+ let skipped = images
280
335
  .iter()
281
- .zip(&images)
282
- .filter(|(hashes, _)| hashes.is_none())
336
+ .enumerate()
337
+ .filter(|&(index, _)| image_hashes(index).is_none())
283
338
  .map(|(_, path)| path.clone())
284
339
  .collect();
285
340
  Ok(Duplicates {
@@ -3,4 +3,4 @@
3
3
  from immaculate._duplicates import Duplicates, Method, find_duplicates
4
4
 
5
5
  __all__ = ["Duplicates", "Method", "find_duplicates"]
6
- __version__ = "0.2.0"
6
+ __version__ = "0.3.0"
@@ -9,15 +9,8 @@ from immaculate import _core
9
9
  type Method = Literal["pdq", "phash", "sscd", "sscd_distilled"]
10
10
  """How near duplicates are found; see `find_duplicates`."""
11
11
 
12
- SSCD_MODEL = Path(__file__).parent / "models" / "sscd.onnx"
13
- """SSCD's `sscd_disc_mixup` model quantised to int8 for ONNX Runtime. Meta's weights are MIT
14
- licensed; see `SSCD_LICENSE` in the same folder."""
15
- SSCD_DISTILLED_MODEL = Path(__file__).parent / "models" / "sscd_distilled.onnx"
16
- """A MobileNetV4 Conv-S student trained to reproduce SSCD's embeddings, with fp16 weights and
17
- fp32 maths, made by `dev/export_mobilenetv4_onnx.py`. Its trunk starts from timm's
18
- ImageNet-trained weights (Apache 2.0), and it was distilled from SSCD (`SSCD_LICENSE`) on
19
- Conceptual Captions 3M images."""
20
- SSCD_MODELS: dict[str, Path] = {"sscd": SSCD_MODEL, "sscd_distilled": SSCD_DISTILLED_MODEL}
12
+ SSCD_METHODS = ("sscd", "sscd_distilled")
13
+ """The methods that need the `sscd` extra, which installs ONNX Runtime and their models."""
21
14
 
22
15
 
23
16
  class Duplicates(NamedTuple):
@@ -52,8 +45,9 @@ def find_duplicates(
52
45
  library. Catches resized and recompressed copies.
53
46
  - `"sscd"`: images whose embeddings from Meta's SSCD copy detection model
54
47
  have cosine similarity of at least 0.75. Catches heavier edits than hashes, such as
55
- crops and overlays, but is much slower. The model ships with Immaculate, quantised to
56
- int8, and is run with ONNX Runtime. Matches chain, as with `"pdq"`.
48
+ crops and overlays, but is much slower. The model, quantised to int8, and ONNX Runtime,
49
+ which runs it, come with the `sscd` extra: `pip install "immaculate[sscd]"`. Matches
50
+ chain, as with `"pdq"`.
57
51
  - `"sscd_distilled"`: as `"sscd"`, with the same 0.75 threshold, but using a small
58
52
  MobileNetV4 model trained to reproduce SSCD's embeddings. About 4x faster than
59
53
  `"sscd"`, at some cost in recall.
@@ -65,16 +59,15 @@ def find_duplicates(
65
59
 
66
60
  Raises `FileNotFoundError` or `NotADirectoryError` if `input_dir` is not a folder,
67
61
  `OSError` if part of it cannot be read, `ValueError` for an unknown `method`,
68
- `ModuleNotFoundError` if `"sscd"` or `"sscd_distilled"` is used but ONNX Runtime, one of
69
- Immaculate's dependencies, is missing, and `RuntimeError` if ONNX Runtime can't load the
70
- model.
62
+ `ModuleNotFoundError` if `"sscd"` or `"sscd_distilled"` is used without the `sscd` extra,
63
+ and `RuntimeError` if ONNX Runtime can't load the model.
71
64
  """
72
- if method in SSCD_MODELS:
65
+ if method in SSCD_METHODS:
73
66
  return Duplicates(
74
67
  *_core.find_duplicates(
75
68
  input_dir,
76
69
  method,
77
- SSCD_MODELS[method],
70
+ _sscd_model(method),
78
71
  _onnxruntime_library(method),
79
72
  progress=progress,
80
73
  )
@@ -82,6 +75,15 @@ def find_duplicates(
82
75
  return Duplicates(*_core.find_duplicates(input_dir, method, progress=progress))
83
76
 
84
77
 
78
+ def _sscd_model(method: str) -> Path:
79
+ """The model `method` runs, from the `immaculate-onnx` package."""
80
+ try:
81
+ import immaculate_onnx # noqa: PLC0415 - only installed with the sscd extra
82
+ except ModuleNotFoundError:
83
+ raise ModuleNotFoundError(_missing_sscd_extra(method)) from None
84
+ return immaculate_onnx.MODELS[method]
85
+
86
+
85
87
  def _onnxruntime_library(method: str = "sscd") -> Path:
86
88
  """The C library in the `onnxruntime` package, found without importing the package."""
87
89
  spec = importlib.util.find_spec("onnxruntime")
@@ -95,8 +97,11 @@ def _onnxruntime_library(method: str = "sscd") -> Path:
95
97
  )
96
98
  if libraries:
97
99
  return libraries[0]
98
- msg = (
99
- f'method "{method}" needs ONNX Runtime, which is missing; install it with: '
100
- "pip install onnxruntime"
100
+ raise ModuleNotFoundError(_missing_sscd_extra(method))
101
+
102
+
103
+ def _missing_sscd_extra(method: str) -> str:
104
+ return (
105
+ f'method "{method}" needs the sscd extra, which is missing; install it with: '
106
+ 'pip install "immaculate[sscd]"'
101
107
  )
102
- raise ModuleNotFoundError(msg)
@@ -2,6 +2,7 @@ import importlib.metadata
2
2
  import importlib.util
3
3
  import json
4
4
  import os
5
+ import re
5
6
  import subprocess
6
7
  import sys
7
8
  from collections.abc import Callable
@@ -9,13 +10,14 @@ from pathlib import Path
9
10
  from typing import get_args
10
11
 
11
12
  import pytest
13
+ from immaculate_onnx import SSCD_DISTILLED_MODEL, SSCD_MODEL
12
14
  from PIL import Image, ImageDraw, ImageFilter, ImageOps
13
15
 
14
16
  import immaculate
15
17
  from immaculate import Duplicates, Method, _core, find_duplicates
16
18
  from immaculate.__main__ import cli
17
19
  from immaculate.__main__ import main as cli_main
18
- from immaculate._duplicates import SSCD_DISTILLED_MODEL, SSCD_MODEL, _onnxruntime_library
20
+ from immaculate._duplicates import _onnxruntime_library
19
21
 
20
22
  METHODS: tuple[Method, ...] = get_args(Method.__value__)
21
23
  HASH_METHODS = tuple(method for method in METHODS if method not in {"sscd", "sscd_distilled"})
@@ -121,7 +123,7 @@ def test_hashes_are_unchanged(tmp_path: Path, name: str, phash: int, pdq: str) -
121
123
 
122
124
 
123
125
  def test_version() -> None:
124
- assert immaculate.__version__ == "0.2.0"
126
+ assert immaculate.__version__ == "0.3.0"
125
127
 
126
128
 
127
129
  def test_exact_duplicates_match_on_pixels_not_file_bytes(tmp_path: Path) -> None:
@@ -135,6 +137,26 @@ def test_exact_duplicates_match_on_pixels_not_file_bytes(tmp_path: Path) -> None
135
137
  assert find_duplicates(tmp_path) == Duplicates(exact=[[png, bmp, nested]], near=[], skipped=[])
136
138
 
137
139
 
140
+ @pytest.mark.parametrize("method", METHODS)
141
+ def test_byte_identical_copies_share_results(tmp_path: Path, method: Method) -> None:
142
+ image = make_image(1)
143
+ original = save(image, tmp_path / "a.png")
144
+ copy = tmp_path / "b.png"
145
+ copy.write_bytes(original.read_bytes())
146
+ pixel_copy = save(image, tmp_path / "c.png", compress_level=1)
147
+ recompressed = save(image, tmp_path / "d.jpg", quality=90)
148
+ corrupt = tmp_path / "e.png"
149
+ corrupt.write_bytes(b"not an image")
150
+ corrupt_copy = tmp_path / "f.png"
151
+ corrupt_copy.write_bytes(b"not an image")
152
+
153
+ result = find_duplicates(tmp_path, method)
154
+
155
+ assert result.exact == [[original, copy, pixel_copy]]
156
+ assert result.near == [[original, copy, pixel_copy, recompressed]]
157
+ assert result.skipped == [corrupt, corrupt_copy]
158
+
159
+
138
160
  @pytest.mark.parametrize("method", HASH_METHODS)
139
161
  def test_near_duplicates(tmp_path: Path, method: Method) -> None:
140
162
  image = make_image(1)
@@ -211,7 +233,7 @@ def test_sscd_matches_crops(tmp_path: Path) -> None:
211
233
  assert find_duplicates(tmp_path).near == []
212
234
 
213
235
 
214
- def test_sscd_models_are_bundled() -> None:
236
+ def test_sscd_models_are_packaged() -> None:
215
237
  assert SSCD_MODEL.is_file()
216
238
  assert SSCD_DISTILLED_MODEL.is_file()
217
239
  assert (SSCD_MODEL.parent / "SSCD_LICENSE").is_file()
@@ -222,7 +244,17 @@ def test_sscd_needs_onnxruntime(
222
244
  tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: Method
223
245
  ) -> None:
224
246
  monkeypatch.setattr(importlib.util, "find_spec", lambda _: None)
225
- with pytest.raises(ModuleNotFoundError, match="pip install onnxruntime"):
247
+ with pytest.raises(ModuleNotFoundError, match=re.escape('pip install "immaculate[sscd]"')):
248
+ find_duplicates(tmp_path, method)
249
+
250
+
251
+ @pytest.mark.parametrize("method", ["sscd", "sscd_distilled"])
252
+ def test_sscd_needs_its_models(
253
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: Method
254
+ ) -> None:
255
+ # None in sys.modules makes importing the package raise ModuleNotFoundError
256
+ monkeypatch.setitem(sys.modules, "immaculate_onnx", None)
257
+ with pytest.raises(ModuleNotFoundError, match=re.escape('pip install "immaculate[sscd]"')):
226
258
  find_duplicates(tmp_path, method)
227
259
 
228
260
 
@@ -280,7 +312,8 @@ def test_progress_ends_with_every_image_done(tmp_path: Path, method: Method) ->
280
312
  for seed in range(3):
281
313
  save(make_image(seed), tmp_path / f"{seed}.png")
282
314
  (tmp_path / "corrupt.png").write_bytes(b"not an image")
283
- images = 4
315
+ (tmp_path / "copy.png").write_bytes((tmp_path / "0.png").read_bytes())
316
+ images = 5
284
317
  calls: list[tuple[int, int]] = []
285
318
 
286
319
  find_duplicates(tmp_path, method, progress=lambda done, total: calls.append((done, total)))
@@ -1,6 +1,16 @@
1
1
  version = 1
2
2
  revision = 3
3
3
  requires-python = ">=3.12"
4
+ resolution-markers = [
5
+ "platform_machine != 'x86_64' or sys_platform != 'darwin'",
6
+ "platform_machine == 'x86_64' and sys_platform == 'darwin'",
7
+ ]
8
+
9
+ [manifest]
10
+ members = [
11
+ "immaculate",
12
+ "immaculate-onnx",
13
+ ]
4
14
 
5
15
  [[package]]
6
16
  name = "babel"
@@ -192,6 +202,18 @@ wheels = [
192
202
  { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
193
203
  ]
194
204
 
205
+ [[package]]
206
+ name = "coloredlogs"
207
+ version = "15.0.1"
208
+ source = { registry = "https://pypi.org/simple" }
209
+ dependencies = [
210
+ { name = "humanfriendly", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
211
+ ]
212
+ sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" }
213
+ wheels = [
214
+ { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018, upload-time = "2021-06-11T10:22:42.561Z" },
215
+ ]
216
+
195
217
  [[package]]
196
218
  name = "distlib"
197
219
  version = "0.4.4"
@@ -239,6 +261,15 @@ wheels = [
239
261
  { url = "https://files.pythonhosted.org/packages/7e/28/bb989a55ac56dabf30765d2e7ea0710c032d9b67d8a7599f91fdcb2e00b1/griffelib-2.4.0-py3-none-any.whl", hash = "sha256:0a7db47b0bdf5123a6ceca2f71aa996742a4f04d9048046778f4a9006772d27f", size = 172791, upload-time = "2026-10-19T16:26:49.511Z" },
240
262
  ]
241
263
 
264
+ [[package]]
265
+ name = "humanfriendly"
266
+ version = "10.0"
267
+ source = { registry = "https://pypi.org/simple" }
268
+ sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" }
269
+ wheels = [
270
+ { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" },
271
+ ]
272
+
242
273
  [[package]]
243
274
  name = "identify"
244
275
  version = "2.6.20"
@@ -259,10 +290,14 @@ wheels = [
259
290
 
260
291
  [[package]]
261
292
  name = "immaculate"
262
- version = "0.2.0"
293
+ version = "0.3.0"
263
294
  source = { editable = "." }
264
- dependencies = [
265
- { name = "onnxruntime" },
295
+
296
+ [package.optional-dependencies]
297
+ sscd = [
298
+ { name = "immaculate-onnx" },
299
+ { name = "onnxruntime", version = "1.23.2", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
300
+ { name = "onnxruntime", version = "1.31.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
266
301
  ]
267
302
 
268
303
  [package.dev-dependencies]
@@ -272,6 +307,9 @@ benchmark = [
272
307
  { name = "pyarrow" },
273
308
  ]
274
309
  dev = [
310
+ { name = "immaculate-onnx" },
311
+ { name = "onnxruntime", version = "1.23.2", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
312
+ { name = "onnxruntime", version = "1.31.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
275
313
  { name = "pillow" },
276
314
  { name = "pre-commit" },
277
315
  { name = "pytest" },
@@ -287,7 +325,11 @@ docs = [
287
325
  ]
288
326
 
289
327
  [package.metadata]
290
- requires-dist = [{ name = "onnxruntime", specifier = ">=1.17" }]
328
+ requires-dist = [
329
+ { name = "immaculate-onnx", marker = "extra == 'sscd'", editable = "immaculate-onnx" },
330
+ { name = "onnxruntime", marker = "extra == 'sscd'", specifier = ">=1.17" },
331
+ ]
332
+ provides-extras = ["sscd"]
291
333
 
292
334
  [package.metadata.requires-dev]
293
335
  benchmark = [
@@ -296,6 +338,9 @@ benchmark = [
296
338
  { name = "pyarrow", specifier = "==26.0.0" },
297
339
  ]
298
340
  dev = [
341
+ { name = "immaculate-onnx", editable = "immaculate-onnx" },
342
+ { name = "onnxruntime", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'", specifier = "==1.31.0" },
343
+ { name = "onnxruntime", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'", specifier = "==1.23.2" },
299
344
  { name = "pillow", specifier = "==13.0.0" },
300
345
  { name = "pre-commit", specifier = "==4.6.1" },
301
346
  { name = "pytest", specifier = "==9.1.1" },
@@ -310,6 +355,11 @@ docs = [
310
355
  { name = "ruff", specifier = "==0.16.1" },
311
356
  ]
312
357
 
358
+ [[package]]
359
+ name = "immaculate-onnx"
360
+ version = "0.1.0"
361
+ source = { editable = "immaculate-onnx" }
362
+
313
363
  [[package]]
314
364
  name = "iniconfig"
315
365
  version = "2.3.1"
@@ -589,6 +639,15 @@ wheels = [
589
639
  { url = "https://files.pythonhosted.org/packages/62/0f/f7f926cfa06317bbdbd3cf0ff82f9e3a91767352eb01519c81499fddf575/mkdocstrings_python-2.0.9-py3-none-any.whl", hash = "sha256:c0233eff3f84d78110df50541918e7ec2bcff8e1614faddea34290a90321a416", size = 105698, upload-time = "2026-09-22T13:43:26.345Z" },
590
640
  ]
591
641
 
642
+ [[package]]
643
+ name = "mpmath"
644
+ version = "1.3.0"
645
+ source = { registry = "https://pypi.org/simple" }
646
+ sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" }
647
+ wheels = [
648
+ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" },
649
+ ]
650
+
592
651
  [[package]]
593
652
  name = "nodeenv"
594
653
  version = "1.11.0"
@@ -671,15 +730,38 @@ wheels = [
671
730
  { url = "https://files.pythonhosted.org/packages/48/7f/c2d1b436b6e7cfebac140c2579a298344b85f2991a2ce5c3615cefb29400/numpy-2.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:7a14a461d9340f1b46b8648578aed9cdb8b3b018a8fac6c1dde2c9192a01a87f", size = 10883718, upload-time = "2026-10-10T20:05:28.547Z" },
672
731
  ]
673
732
 
733
+ [[package]]
734
+ name = "onnxruntime"
735
+ version = "1.23.2"
736
+ source = { registry = "https://pypi.org/simple" }
737
+ resolution-markers = [
738
+ "platform_machine == 'x86_64' and sys_platform == 'darwin'",
739
+ ]
740
+ dependencies = [
741
+ { name = "coloredlogs", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
742
+ { name = "flatbuffers", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
743
+ { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
744
+ { name = "packaging", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
745
+ { name = "protobuf", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
746
+ { name = "sympy", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
747
+ ]
748
+ wheels = [
749
+ { url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705, upload-time = "2025-10-22T03:46:40.415Z" },
750
+ { url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691, upload-time = "2025-10-22T03:46:43.518Z" },
751
+ ]
752
+
674
753
  [[package]]
675
754
  name = "onnxruntime"
676
755
  version = "1.31.0"
677
756
  source = { registry = "https://pypi.org/simple" }
757
+ resolution-markers = [
758
+ "platform_machine != 'x86_64' or sys_platform != 'darwin'",
759
+ ]
678
760
  dependencies = [
679
- { name = "flatbuffers" },
680
- { name = "numpy" },
681
- { name = "packaging" },
682
- { name = "protobuf" },
761
+ { name = "flatbuffers", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
762
+ { name = "numpy", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
763
+ { name = "packaging", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
764
+ { name = "protobuf", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
683
765
  ]
684
766
  wheels = [
685
767
  { url = "https://files.pythonhosted.org/packages/b3/bd/2ac094311163b803e3626c3937461d6900934bd56cca7601f6150ff860c3/onnxruntime-1.31.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:aaab9b3af536b06ca27ab5e35e3d429c97457ce76cf298af103f687e8b9975c0", size = 20882054, upload-time = "2026-10-09T04:18:18.811Z" },
@@ -1074,6 +1156,18 @@ wheels = [
1074
1156
  { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
1075
1157
  ]
1076
1158
 
1159
+ [[package]]
1160
+ name = "sympy"
1161
+ version = "1.14.0"
1162
+ source = { registry = "https://pypi.org/simple" }
1163
+ dependencies = [
1164
+ { name = "mpmath", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" },
1165
+ ]
1166
+ sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" }
1167
+ wheels = [
1168
+ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
1169
+ ]
1170
+
1077
1171
  [[package]]
1078
1172
  name = "ty"
1079
1173
  version = "0.0.65"
@@ -1,21 +0,0 @@
1
- MIT License
2
-
3
- Copyright (c) Meta Platforms, Inc. and affiliates.
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE.
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes