image-optimizer-webui 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,362 @@
1
+ """Shared helpers for reusing previously produced output files.
2
+
3
+ `settings_fingerprint` and `is_valid_reusable_output` started life in main.py
4
+ for the GUI's skip_existing feature; they live here so the CLI content-hash
5
+ cache can use the exact same definitions (one source of truth for "which
6
+ settings change the pixels" and "is this file safe to hand back").
7
+ main.py re-imports them under their old private names.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ import json
13
+ import os
14
+ import re
15
+ import shutil
16
+ import stat
17
+ import time
18
+ import uuid
19
+ from pathlib import Path
20
+ from typing import Iterable, NamedTuple, Optional
21
+
22
+ import PIL
23
+ from PIL import Image
24
+
25
+
26
+ def settings_fingerprint(
27
+ quality: str, compression_mode: str, max_width: int,
28
+ protected_colors: Optional[list], dithering: bool, keep_exif: bool, output_format: str,
29
+ ) -> str:
30
+ """Fingerprints the effective per-file compression settings that
31
+ produced (or would produce) a given output. Used by skip_existing to
32
+ tell "the source hasn't changed AND these are the same settings that
33
+ made this file last time" apart from "the source happens to be
34
+ unchanged but the user asked for something different this run" - the
35
+ two look identical if you only check the source's mtime, which is all
36
+ the reuse check used to do.
37
+
38
+ Order-independent on protected_colors (case/whitespace-normalized) so
39
+ ["#FF0000", "#00ff00"] and ["#00FF00", "#ff0000"] fingerprint the same,
40
+ matching how the color-map step itself treats them."""
41
+ colors = sorted((c or "").strip().lower() for c in (protected_colors or []))
42
+ payload = json.dumps({
43
+ "quality": quality,
44
+ "compression_mode": compression_mode,
45
+ "max_width": max_width,
46
+ "protected_colors": colors,
47
+ "dithering": bool(dithering),
48
+ "keep_exif": bool(keep_exif),
49
+ "output_format": output_format,
50
+ }, sort_keys=True)
51
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
52
+
53
+
54
+ def is_valid_reusable_output(path: Path, expected_format: str) -> bool:
55
+ """Best-effort corruption check for a skip_existing reuse candidate.
56
+
57
+ A non-empty file isn't strong enough evidence that it's safe to reuse:
58
+ if pngquant/oxipng was killed mid-write on a previous run (crash, OOM
59
+ kill, power loss), it can leave behind a file that's non-empty but
60
+ truncated or structurally broken. Reusing that file would silently
61
+ hand the user a corrupt "optimized" image instead of recompressing it.
62
+
63
+ Uses Pillow's verify() (structural/header check, doesn't decode full
64
+ pixel data - cheap) and confirms the format actually matches what
65
+ this run expects, so e.g. a stray same-named file from a different
66
+ output_format never gets reused by mistake."""
67
+ try:
68
+ with Image.open(path) as img:
69
+ img.verify()
70
+ fmt = (img.format or "").upper()
71
+ except Exception:
72
+ return False
73
+ expected = {
74
+ "png": "PNG",
75
+ "webp": "WEBP",
76
+ "jpg": "JPEG",
77
+ "jpeg": "JPEG",
78
+ "avif": "AVIF",
79
+ }.get(expected_format, "PNG")
80
+ return fmt == expected
81
+
82
+
83
+ # Bump when pipeline logic changes in a way that alters output bytes for the
84
+ # same input + settings + encoders. Folded into every cache key, so a bump
85
+ # orphans (and LRU eventually reclaims) all older entries.
86
+ CACHE_SCHEMA_VERSION = 1
87
+
88
+ # Output formats the CLI cache may serve. tests/test_determinism.py is the
89
+ # gate: a format whose real encoder is found non-deterministic there must be
90
+ # removed from this set (and noted in the README).
91
+ CACHEABLE_FORMATS = frozenset({"png", "jpg", "webp", "avif"})
92
+
93
+ # CLI default for --cache-max-size-mb (0 = unlimited). Shared with argparse so
94
+ # the "flag without --cache-dir" hint and the post-run warning cannot drift
95
+ # from the flag's own default.
96
+ DEFAULT_CACHE_MAX_MB = 1024
97
+
98
+ # Which external encoder binaries (Optimizer attribute names) feed each
99
+ # output format. WebP is Pillow-only, so it is covered by the Pillow version
100
+ # in the encoder fingerprint instead.
101
+ _FORMAT_ENCODERS = {
102
+ "png": ("pngquant_path", "oxipng_path"),
103
+ "jpg": ("cjpeg_path",),
104
+ "webp": (),
105
+ "avif": ("avifenc_path",),
106
+ }
107
+
108
+ # Stale temp files from a crashed writer are swept at the end of every CLI
109
+ # run after this long (ContentCache.sweep_stale_tmp), independent of the
110
+ # size cap.
111
+ _STALE_TMP_SECONDS = 3600
112
+
113
+ # `--cache-dir` is user-supplied, so it may point at an existing folder full
114
+ # of unrelated files. Only paths shaped exactly like the ones this module
115
+ # writes are ever counted, swept or evicted: a shard is a two-hex-digit
116
+ # directory, an entry is "<64 hex key>.<format>", and a temp file is an entry
117
+ # name plus ".tmp-<pid>-<8 hex>". Everything else is left alone. The format
118
+ # list comes from CACHEABLE_FORMATS so adding a cacheable format cannot leave
119
+ # its entries unevictable or unsweepable.
120
+ _CACHE_EXTENSIONS = "|".join(sorted(CACHEABLE_FORMATS))
121
+ _SHARD_RE = re.compile(r"^[0-9a-f]{2}$")
122
+ _ENTRY_RE = re.compile(rf"^[0-9a-f]{{64}}\.(?:{_CACHE_EXTENSIONS})$")
123
+ _TMP_RE = re.compile(
124
+ rf"^[0-9a-f]{{64}}\.(?:{_CACHE_EXTENSIONS})\.tmp-\d+-[0-9a-f]{{8}}$"
125
+ )
126
+
127
+ # Retries for os.replace() on Windows, where replacing a destination that is
128
+ # momentarily open for reading (probe() validating or copying an entry) or
129
+ # being replaced by another writer fails with PermissionError instead of
130
+ # waiting. Backoff is short: every retry is already an unusual collision.
131
+ _REPLACE_ATTEMPTS = 6
132
+ _REPLACE_BACKOFF_SECONDS = 0.005
133
+
134
+
135
+ def _publish(tmp: Path, entry: Path, output_format: str) -> None:
136
+ """Atomically move a finished temp file into place.
137
+
138
+ os.replace() is atomic on every platform, but on Windows it can fail
139
+ with PermissionError while the destination is open elsewhere: a reader
140
+ (probe) has the entry open, or another writer is replacing the same key
141
+ at the same moment. Retry briefly; when the final attempt still fails
142
+ but the entry now exists AND passes the same structural check a hit
143
+ would, another writer has already published a complete file for this
144
+ key, which is just as valid a result as ours, so there is nothing left
145
+ to do. An entry that exists but fails the check (a corrupt leftover that
146
+ probe() just treated as a miss) is not a result: the PermissionError is
147
+ re-raised so the caller reports the failed write instead of silently
148
+ leaving the bad entry in place to miss again on every run."""
149
+ for attempt in range(_REPLACE_ATTEMPTS):
150
+ try:
151
+ os.replace(tmp, entry)
152
+ return
153
+ except PermissionError:
154
+ if attempt == _REPLACE_ATTEMPTS - 1:
155
+ if entry.is_file() and is_valid_reusable_output(entry, output_format):
156
+ return
157
+ raise
158
+ time.sleep(_REPLACE_BACKOFF_SECONDS * (2 ** attempt))
159
+
160
+
161
+ def file_sha256(path: Path) -> str:
162
+ h = hashlib.sha256()
163
+ with open(path, "rb") as f:
164
+ for chunk in iter(lambda: f.read(1024 * 1024), b""):
165
+ h.update(chunk)
166
+ return h.hexdigest()
167
+
168
+
169
+ def encoder_fingerprint(optimizer, output_format: str) -> str:
170
+ """Identifies the encoders that will produce this run's output: the
171
+ content hash of each external binary the format uses, plus the Pillow
172
+ version (it does the WebP encode and every resize/convert step). Swapping
173
+ a bundled or PATH binary for another build therefore invalidates the cache
174
+ on its own. Computed once per run by the caller."""
175
+ parts = [f"pillow={PIL.__version__}"]
176
+ for attr in _FORMAT_ENCODERS.get(output_format, ()):
177
+ path = getattr(optimizer, attr, None)
178
+ if path is None:
179
+ parts.append(f"{attr}=none")
180
+ continue
181
+ try:
182
+ parts.append(f"{attr}={file_sha256(Path(path))}")
183
+ except OSError:
184
+ # Unreadable binary: a unique value per run means nothing is
185
+ # ever served from or trusted against a fingerprint we could
186
+ # not compute.
187
+ parts.append(f"{attr}=unreadable-{uuid.uuid4().hex}")
188
+ return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
189
+
190
+
191
+ class CacheProbe(NamedTuple):
192
+ key: str
193
+ original_size: int
194
+ # Size of the file copied to the output path, or None on a miss.
195
+ hit_size: Optional[int]
196
+
197
+
198
+ class ContentCache:
199
+ """Content-addressed store for finished compression results, independent
200
+ of any output directory (so a caller that uses a fresh temp output dir on
201
+ every run still gets hits).
202
+
203
+ Layout: <cache_dir>/<key[:2]>/<key>.<ext>
204
+ Key: sha256(input bytes sha256 + settings fingerprint + encoder
205
+ fingerprint + CACHE_SCHEMA_VERSION)
206
+
207
+ All methods are blocking file I/O; call them via asyncio.to_thread.
208
+ Concurrent writers to the same key are fine: entries are written to a
209
+ temp file and moved into place with os.replace, so a reader never
210
+ observes a partial file. On Windows a replace can transiently fail while
211
+ the destination is open (a reader copying it, or another writer); store()
212
+ retries and treats "another writer already published an entry" as
213
+ success, so last write wins whenever both succeed and any complete entry
214
+ is a valid result for the key.
215
+ """
216
+
217
+ def __init__(self, cache_dir: Path, output_format: str, settings_fp: str, encoder_fp: str):
218
+ self.root = Path(cache_dir)
219
+ self.output_format = output_format
220
+ self._ext = "." + output_format
221
+ self._settings_fp = settings_fp
222
+ self._encoder_fp = encoder_fp
223
+
224
+ def key_for_digest(self, input_sha256: str) -> str:
225
+ payload = "|".join(
226
+ [input_sha256, self._settings_fp, self._encoder_fp, str(CACHE_SCHEMA_VERSION)]
227
+ )
228
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
229
+
230
+ def entry_path(self, key: str) -> Path:
231
+ return self.root / key[:2] / (key + self._ext)
232
+
233
+ def probe(self, input_path: Path, out_path: Path) -> CacheProbe:
234
+ """Hash the input, and on a valid cache entry copy it to out_path.
235
+ An entry that fails structural validation (truncated, wrong format)
236
+ counts as a miss; the caller recompresses and store() overwrites it."""
237
+ original_size = os.path.getsize(input_path)
238
+ key = self.key_for_digest(file_sha256(input_path))
239
+ entry = self.entry_path(key)
240
+ if not entry.is_file() or not is_valid_reusable_output(entry, self.output_format):
241
+ return CacheProbe(key, original_size, None)
242
+ shutil.copy2(entry, out_path)
243
+ try:
244
+ # Recency for LRU eviction.
245
+ os.utime(entry, None)
246
+ except OSError:
247
+ pass
248
+ return CacheProbe(key, original_size, os.path.getsize(out_path))
249
+
250
+ def store(self, key: str, produced: Path) -> Path:
251
+ """Atomically publish a successful result under `key`. Never call
252
+ for a failed result. Returns the entry path, so the caller can hand
253
+ this run's fresh entries to evict(protect=...) instead of having
254
+ them deleted in the very run that wrote them (the returned path also
255
+ exists when another writer published the entry first)."""
256
+ entry = self.entry_path(key)
257
+ entry.parent.mkdir(parents=True, exist_ok=True)
258
+ tmp = entry.with_name(f"{entry.name}.tmp-{os.getpid()}-{uuid.uuid4().hex[:8]}")
259
+ try:
260
+ shutil.copyfile(produced, tmp)
261
+ _publish(tmp, entry, self.output_format)
262
+ finally:
263
+ try:
264
+ tmp.unlink()
265
+ except FileNotFoundError:
266
+ pass
267
+ return entry
268
+
269
+ def sweep_stale_tmp(self) -> int:
270
+ """Delete .tmp-* leftovers from crashed writers older than
271
+ _STALE_TMP_SECONDS. Called at the end of every CLI run regardless of
272
+ the size cap: with --cache-max-size-mb 0 evict() never runs, and
273
+ without this sweep those files would accumulate forever. Returns the
274
+ number of files removed.
275
+
276
+ Only names this module writes are considered (see _TMP_RE): the cache
277
+ directory is user-supplied and may contain unrelated files."""
278
+ if not self.root.is_dir():
279
+ return 0
280
+ now = time.time()
281
+ removed = 0
282
+ for shard in self._shards():
283
+ for f in shard.iterdir():
284
+ if not _TMP_RE.match(f.name) or not f.name.startswith(shard.name):
285
+ continue
286
+ try:
287
+ st = f.stat()
288
+ except OSError:
289
+ continue
290
+ if now - st.st_mtime > _STALE_TMP_SECONDS:
291
+ try:
292
+ f.unlink()
293
+ removed += 1
294
+ except OSError:
295
+ pass
296
+ return removed
297
+
298
+ def evict(
299
+ self,
300
+ max_bytes: int,
301
+ protect: Optional[set] = None,
302
+ stats: Optional[dict] = None,
303
+ ) -> int:
304
+ """Delete least-recently-used entries (oldest mtime first) until the
305
+ total is at or under max_bytes. max_bytes <= 0 means unlimited.
306
+ Returns the number of entries removed. Entries whose source image is
307
+ gone are never touched by hits, so they age out here naturally.
308
+
309
+ `protect` lists entry paths that must not be chosen as victims: the
310
+ entries written during the current run. Without this, a cap smaller
311
+ than one entry (or already-full cache plus a fresh write) would
312
+ delete the just-written entry in the very run that produced it, so
313
+ the cache could never serve a single hit; the cap may transiently be
314
+ exceeded because of them. They are ordinary candidates again on the
315
+ next run, where LRU order decides (they are the newest, so they go
316
+ last).
317
+
318
+ `stats`, when given a dict, receives {"total_after": bytes} so the
319
+ caller can warn when the cap is still exceeded (e.g. a single entry
320
+ larger than the cap, or everything left protected)."""
321
+ if max_bytes <= 0 or not self.root.is_dir():
322
+ return 0
323
+ protected = protect or set()
324
+ entries: list[tuple[float, int, Path]] = []
325
+ total = 0
326
+ for shard in self._shards():
327
+ for f in shard.iterdir():
328
+ # Only real cache entries count and only they may be
329
+ # deleted: the directory is user-supplied and may hold
330
+ # unrelated files. Temp files are sweep_stale_tmp()'s job;
331
+ # they are never counted toward the cap nor chosen as
332
+ # victims.
333
+ if not _ENTRY_RE.match(f.name) or not f.name.startswith(shard.name):
334
+ continue
335
+ try:
336
+ st = f.stat()
337
+ except OSError:
338
+ continue
339
+ if not stat.S_ISREG(st.st_mode):
340
+ continue
341
+ entries.append((st.st_mtime, st.st_size, f))
342
+ total += st.st_size
343
+ removed = 0
344
+ for _mtime, size, f in sorted(entries, key=lambda e: e[0]):
345
+ if total <= max_bytes:
346
+ break
347
+ if f in protected:
348
+ continue
349
+ try:
350
+ f.unlink()
351
+ except OSError:
352
+ continue
353
+ total -= size
354
+ removed += 1
355
+ if stats is not None:
356
+ stats["total_after"] = total
357
+ return removed
358
+
359
+ def _shards(self) -> Iterable[Path]:
360
+ for d in self.root.iterdir():
361
+ if _SHARD_RE.match(d.name) and d.is_dir():
362
+ yield d