image-optimizer-webui 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- image_optimizer/__init__.py +3 -0
- image_optimizer/__main__.py +2 -0
- image_optimizer/binaries.py +430 -0
- image_optimizer/cache.py +362 -0
- image_optimizer/main.py +3929 -0
- image_optimizer/models.py +147 -0
- image_optimizer/optimizer.py +1378 -0
- image_optimizer/templates/favicon.ico +0 -0
- image_optimizer/templates/index.html +3066 -0
- image_optimizer/watcher.py +180 -0
- image_optimizer_webui-1.1.0.dist-info/METADATA +388 -0
- image_optimizer_webui-1.1.0.dist-info/RECORD +16 -0
- image_optimizer_webui-1.1.0.dist-info/WHEEL +5 -0
- image_optimizer_webui-1.1.0.dist-info/entry_points.txt +2 -0
- image_optimizer_webui-1.1.0.dist-info/licenses/LICENSE +21 -0
- image_optimizer_webui-1.1.0.dist-info/top_level.txt +1 -0
image_optimizer/cache.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""Shared helpers for reusing previously produced output files.
|
|
2
|
+
|
|
3
|
+
`settings_fingerprint` and `is_valid_reusable_output` started life in main.py
|
|
4
|
+
for the GUI's skip_existing feature; they live here so the CLI content-hash
|
|
5
|
+
cache can use the exact same definitions (one source of truth for "which
|
|
6
|
+
settings change the pixels" and "is this file safe to hand back").
|
|
7
|
+
main.py re-imports them under their old private names.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import shutil
|
|
16
|
+
import stat
|
|
17
|
+
import time
|
|
18
|
+
import uuid
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Iterable, NamedTuple, Optional
|
|
21
|
+
|
|
22
|
+
import PIL
|
|
23
|
+
from PIL import Image
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def settings_fingerprint(
|
|
27
|
+
quality: str, compression_mode: str, max_width: int,
|
|
28
|
+
protected_colors: Optional[list], dithering: bool, keep_exif: bool, output_format: str,
|
|
29
|
+
) -> str:
|
|
30
|
+
"""Fingerprints the effective per-file compression settings that
|
|
31
|
+
produced (or would produce) a given output. Used by skip_existing to
|
|
32
|
+
tell "the source hasn't changed AND these are the same settings that
|
|
33
|
+
made this file last time" apart from "the source happens to be
|
|
34
|
+
unchanged but the user asked for something different this run" - the
|
|
35
|
+
two look identical if you only check the source's mtime, which is all
|
|
36
|
+
the reuse check used to do.
|
|
37
|
+
|
|
38
|
+
Order-independent on protected_colors (case/whitespace-normalized) so
|
|
39
|
+
["#FF0000", "#00ff00"] and ["#00FF00", "#ff0000"] fingerprint the same,
|
|
40
|
+
matching how the color-map step itself treats them."""
|
|
41
|
+
colors = sorted((c or "").strip().lower() for c in (protected_colors or []))
|
|
42
|
+
payload = json.dumps({
|
|
43
|
+
"quality": quality,
|
|
44
|
+
"compression_mode": compression_mode,
|
|
45
|
+
"max_width": max_width,
|
|
46
|
+
"protected_colors": colors,
|
|
47
|
+
"dithering": bool(dithering),
|
|
48
|
+
"keep_exif": bool(keep_exif),
|
|
49
|
+
"output_format": output_format,
|
|
50
|
+
}, sort_keys=True)
|
|
51
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def is_valid_reusable_output(path: Path, expected_format: str) -> bool:
|
|
55
|
+
"""Best-effort corruption check for a skip_existing reuse candidate.
|
|
56
|
+
|
|
57
|
+
A non-empty file isn't strong enough evidence that it's safe to reuse:
|
|
58
|
+
if pngquant/oxipng was killed mid-write on a previous run (crash, OOM
|
|
59
|
+
kill, power loss), it can leave behind a file that's non-empty but
|
|
60
|
+
truncated or structurally broken. Reusing that file would silently
|
|
61
|
+
hand the user a corrupt "optimized" image instead of recompressing it.
|
|
62
|
+
|
|
63
|
+
Uses Pillow's verify() (structural/header check, doesn't decode full
|
|
64
|
+
pixel data - cheap) and confirms the format actually matches what
|
|
65
|
+
this run expects, so e.g. a stray same-named file from a different
|
|
66
|
+
output_format never gets reused by mistake."""
|
|
67
|
+
try:
|
|
68
|
+
with Image.open(path) as img:
|
|
69
|
+
img.verify()
|
|
70
|
+
fmt = (img.format or "").upper()
|
|
71
|
+
except Exception:
|
|
72
|
+
return False
|
|
73
|
+
expected = {
|
|
74
|
+
"png": "PNG",
|
|
75
|
+
"webp": "WEBP",
|
|
76
|
+
"jpg": "JPEG",
|
|
77
|
+
"jpeg": "JPEG",
|
|
78
|
+
"avif": "AVIF",
|
|
79
|
+
}.get(expected_format, "PNG")
|
|
80
|
+
return fmt == expected
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# Bump when pipeline logic changes in a way that alters output bytes for the
|
|
84
|
+
# same input + settings + encoders. Folded into every cache key, so a bump
|
|
85
|
+
# orphans (and LRU eventually reclaims) all older entries.
|
|
86
|
+
CACHE_SCHEMA_VERSION = 1
|
|
87
|
+
|
|
88
|
+
# Output formats the CLI cache may serve. tests/test_determinism.py is the
|
|
89
|
+
# gate: a format whose real encoder is found non-deterministic there must be
|
|
90
|
+
# removed from this set (and noted in the README).
|
|
91
|
+
CACHEABLE_FORMATS = frozenset({"png", "jpg", "webp", "avif"})
|
|
92
|
+
|
|
93
|
+
# CLI default for --cache-max-size-mb (0 = unlimited). Shared with argparse so
|
|
94
|
+
# the "flag without --cache-dir" hint and the post-run warning cannot drift
|
|
95
|
+
# from the flag's own default.
|
|
96
|
+
DEFAULT_CACHE_MAX_MB = 1024
|
|
97
|
+
|
|
98
|
+
# Which external encoder binaries (Optimizer attribute names) feed each
|
|
99
|
+
# output format. WebP is Pillow-only, so it is covered by the Pillow version
|
|
100
|
+
# in the encoder fingerprint instead.
|
|
101
|
+
_FORMAT_ENCODERS = {
|
|
102
|
+
"png": ("pngquant_path", "oxipng_path"),
|
|
103
|
+
"jpg": ("cjpeg_path",),
|
|
104
|
+
"webp": (),
|
|
105
|
+
"avif": ("avifenc_path",),
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
# Stale temp files from a crashed writer are swept at the end of every CLI
|
|
109
|
+
# run after this long (ContentCache.sweep_stale_tmp), independent of the
|
|
110
|
+
# size cap.
|
|
111
|
+
_STALE_TMP_SECONDS = 3600
|
|
112
|
+
|
|
113
|
+
# `--cache-dir` is user-supplied, so it may point at an existing folder full
|
|
114
|
+
# of unrelated files. Only paths shaped exactly like the ones this module
|
|
115
|
+
# writes are ever counted, swept or evicted: a shard is a two-hex-digit
|
|
116
|
+
# directory, an entry is "<64 hex key>.<format>", and a temp file is an entry
|
|
117
|
+
# name plus ".tmp-<pid>-<8 hex>". Everything else is left alone. The format
|
|
118
|
+
# list comes from CACHEABLE_FORMATS so adding a cacheable format cannot leave
|
|
119
|
+
# its entries unevictable or unsweepable.
|
|
120
|
+
_CACHE_EXTENSIONS = "|".join(sorted(CACHEABLE_FORMATS))
|
|
121
|
+
_SHARD_RE = re.compile(r"^[0-9a-f]{2}$")
|
|
122
|
+
_ENTRY_RE = re.compile(rf"^[0-9a-f]{{64}}\.(?:{_CACHE_EXTENSIONS})$")
|
|
123
|
+
_TMP_RE = re.compile(
|
|
124
|
+
rf"^[0-9a-f]{{64}}\.(?:{_CACHE_EXTENSIONS})\.tmp-\d+-[0-9a-f]{{8}}$"
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
# Retries for os.replace() on Windows, where replacing a destination that is
|
|
128
|
+
# momentarily open for reading (probe() validating or copying an entry) or
|
|
129
|
+
# being replaced by another writer fails with PermissionError instead of
|
|
130
|
+
# waiting. Backoff is short: every retry is already an unusual collision.
|
|
131
|
+
_REPLACE_ATTEMPTS = 6
|
|
132
|
+
_REPLACE_BACKOFF_SECONDS = 0.005
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _publish(tmp: Path, entry: Path, output_format: str) -> None:
|
|
136
|
+
"""Atomically move a finished temp file into place.
|
|
137
|
+
|
|
138
|
+
os.replace() is atomic on every platform, but on Windows it can fail
|
|
139
|
+
with PermissionError while the destination is open elsewhere: a reader
|
|
140
|
+
(probe) has the entry open, or another writer is replacing the same key
|
|
141
|
+
at the same moment. Retry briefly; when the final attempt still fails
|
|
142
|
+
but the entry now exists AND passes the same structural check a hit
|
|
143
|
+
would, another writer has already published a complete file for this
|
|
144
|
+
key, which is just as valid a result as ours, so there is nothing left
|
|
145
|
+
to do. An entry that exists but fails the check (a corrupt leftover that
|
|
146
|
+
probe() just treated as a miss) is not a result: the PermissionError is
|
|
147
|
+
re-raised so the caller reports the failed write instead of silently
|
|
148
|
+
leaving the bad entry in place to miss again on every run."""
|
|
149
|
+
for attempt in range(_REPLACE_ATTEMPTS):
|
|
150
|
+
try:
|
|
151
|
+
os.replace(tmp, entry)
|
|
152
|
+
return
|
|
153
|
+
except PermissionError:
|
|
154
|
+
if attempt == _REPLACE_ATTEMPTS - 1:
|
|
155
|
+
if entry.is_file() and is_valid_reusable_output(entry, output_format):
|
|
156
|
+
return
|
|
157
|
+
raise
|
|
158
|
+
time.sleep(_REPLACE_BACKOFF_SECONDS * (2 ** attempt))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def file_sha256(path: Path) -> str:
|
|
162
|
+
h = hashlib.sha256()
|
|
163
|
+
with open(path, "rb") as f:
|
|
164
|
+
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
|
165
|
+
h.update(chunk)
|
|
166
|
+
return h.hexdigest()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def encoder_fingerprint(optimizer, output_format: str) -> str:
|
|
170
|
+
"""Identifies the encoders that will produce this run's output: the
|
|
171
|
+
content hash of each external binary the format uses, plus the Pillow
|
|
172
|
+
version (it does the WebP encode and every resize/convert step). Swapping
|
|
173
|
+
a bundled or PATH binary for another build therefore invalidates the cache
|
|
174
|
+
on its own. Computed once per run by the caller."""
|
|
175
|
+
parts = [f"pillow={PIL.__version__}"]
|
|
176
|
+
for attr in _FORMAT_ENCODERS.get(output_format, ()):
|
|
177
|
+
path = getattr(optimizer, attr, None)
|
|
178
|
+
if path is None:
|
|
179
|
+
parts.append(f"{attr}=none")
|
|
180
|
+
continue
|
|
181
|
+
try:
|
|
182
|
+
parts.append(f"{attr}={file_sha256(Path(path))}")
|
|
183
|
+
except OSError:
|
|
184
|
+
# Unreadable binary: a unique value per run means nothing is
|
|
185
|
+
# ever served from or trusted against a fingerprint we could
|
|
186
|
+
# not compute.
|
|
187
|
+
parts.append(f"{attr}=unreadable-{uuid.uuid4().hex}")
|
|
188
|
+
return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class CacheProbe(NamedTuple):
|
|
192
|
+
key: str
|
|
193
|
+
original_size: int
|
|
194
|
+
# Size of the file copied to the output path, or None on a miss.
|
|
195
|
+
hit_size: Optional[int]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class ContentCache:
|
|
199
|
+
"""Content-addressed store for finished compression results, independent
|
|
200
|
+
of any output directory (so a caller that uses a fresh temp output dir on
|
|
201
|
+
every run still gets hits).
|
|
202
|
+
|
|
203
|
+
Layout: <cache_dir>/<key[:2]>/<key>.<ext>
|
|
204
|
+
Key: sha256(input bytes sha256 + settings fingerprint + encoder
|
|
205
|
+
fingerprint + CACHE_SCHEMA_VERSION)
|
|
206
|
+
|
|
207
|
+
All methods are blocking file I/O; call them via asyncio.to_thread.
|
|
208
|
+
Concurrent writers to the same key are fine: entries are written to a
|
|
209
|
+
temp file and moved into place with os.replace, so a reader never
|
|
210
|
+
observes a partial file. On Windows a replace can transiently fail while
|
|
211
|
+
the destination is open (a reader copying it, or another writer); store()
|
|
212
|
+
retries and treats "another writer already published an entry" as
|
|
213
|
+
success, so last write wins whenever both succeed and any complete entry
|
|
214
|
+
is a valid result for the key.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
def __init__(self, cache_dir: Path, output_format: str, settings_fp: str, encoder_fp: str):
|
|
218
|
+
self.root = Path(cache_dir)
|
|
219
|
+
self.output_format = output_format
|
|
220
|
+
self._ext = "." + output_format
|
|
221
|
+
self._settings_fp = settings_fp
|
|
222
|
+
self._encoder_fp = encoder_fp
|
|
223
|
+
|
|
224
|
+
def key_for_digest(self, input_sha256: str) -> str:
|
|
225
|
+
payload = "|".join(
|
|
226
|
+
[input_sha256, self._settings_fp, self._encoder_fp, str(CACHE_SCHEMA_VERSION)]
|
|
227
|
+
)
|
|
228
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
229
|
+
|
|
230
|
+
def entry_path(self, key: str) -> Path:
|
|
231
|
+
return self.root / key[:2] / (key + self._ext)
|
|
232
|
+
|
|
233
|
+
def probe(self, input_path: Path, out_path: Path) -> CacheProbe:
|
|
234
|
+
"""Hash the input, and on a valid cache entry copy it to out_path.
|
|
235
|
+
An entry that fails structural validation (truncated, wrong format)
|
|
236
|
+
counts as a miss; the caller recompresses and store() overwrites it."""
|
|
237
|
+
original_size = os.path.getsize(input_path)
|
|
238
|
+
key = self.key_for_digest(file_sha256(input_path))
|
|
239
|
+
entry = self.entry_path(key)
|
|
240
|
+
if not entry.is_file() or not is_valid_reusable_output(entry, self.output_format):
|
|
241
|
+
return CacheProbe(key, original_size, None)
|
|
242
|
+
shutil.copy2(entry, out_path)
|
|
243
|
+
try:
|
|
244
|
+
# Recency for LRU eviction.
|
|
245
|
+
os.utime(entry, None)
|
|
246
|
+
except OSError:
|
|
247
|
+
pass
|
|
248
|
+
return CacheProbe(key, original_size, os.path.getsize(out_path))
|
|
249
|
+
|
|
250
|
+
def store(self, key: str, produced: Path) -> Path:
|
|
251
|
+
"""Atomically publish a successful result under `key`. Never call
|
|
252
|
+
for a failed result. Returns the entry path, so the caller can hand
|
|
253
|
+
this run's fresh entries to evict(protect=...) instead of having
|
|
254
|
+
them deleted in the very run that wrote them (the returned path also
|
|
255
|
+
exists when another writer published the entry first)."""
|
|
256
|
+
entry = self.entry_path(key)
|
|
257
|
+
entry.parent.mkdir(parents=True, exist_ok=True)
|
|
258
|
+
tmp = entry.with_name(f"{entry.name}.tmp-{os.getpid()}-{uuid.uuid4().hex[:8]}")
|
|
259
|
+
try:
|
|
260
|
+
shutil.copyfile(produced, tmp)
|
|
261
|
+
_publish(tmp, entry, self.output_format)
|
|
262
|
+
finally:
|
|
263
|
+
try:
|
|
264
|
+
tmp.unlink()
|
|
265
|
+
except FileNotFoundError:
|
|
266
|
+
pass
|
|
267
|
+
return entry
|
|
268
|
+
|
|
269
|
+
def sweep_stale_tmp(self) -> int:
|
|
270
|
+
"""Delete .tmp-* leftovers from crashed writers older than
|
|
271
|
+
_STALE_TMP_SECONDS. Called at the end of every CLI run regardless of
|
|
272
|
+
the size cap: with --cache-max-size-mb 0 evict() never runs, and
|
|
273
|
+
without this sweep those files would accumulate forever. Returns the
|
|
274
|
+
number of files removed.
|
|
275
|
+
|
|
276
|
+
Only names this module writes are considered (see _TMP_RE): the cache
|
|
277
|
+
directory is user-supplied and may contain unrelated files."""
|
|
278
|
+
if not self.root.is_dir():
|
|
279
|
+
return 0
|
|
280
|
+
now = time.time()
|
|
281
|
+
removed = 0
|
|
282
|
+
for shard in self._shards():
|
|
283
|
+
for f in shard.iterdir():
|
|
284
|
+
if not _TMP_RE.match(f.name) or not f.name.startswith(shard.name):
|
|
285
|
+
continue
|
|
286
|
+
try:
|
|
287
|
+
st = f.stat()
|
|
288
|
+
except OSError:
|
|
289
|
+
continue
|
|
290
|
+
if now - st.st_mtime > _STALE_TMP_SECONDS:
|
|
291
|
+
try:
|
|
292
|
+
f.unlink()
|
|
293
|
+
removed += 1
|
|
294
|
+
except OSError:
|
|
295
|
+
pass
|
|
296
|
+
return removed
|
|
297
|
+
|
|
298
|
+
def evict(
|
|
299
|
+
self,
|
|
300
|
+
max_bytes: int,
|
|
301
|
+
protect: Optional[set] = None,
|
|
302
|
+
stats: Optional[dict] = None,
|
|
303
|
+
) -> int:
|
|
304
|
+
"""Delete least-recently-used entries (oldest mtime first) until the
|
|
305
|
+
total is at or under max_bytes. max_bytes <= 0 means unlimited.
|
|
306
|
+
Returns the number of entries removed. Entries whose source image is
|
|
307
|
+
gone are never touched by hits, so they age out here naturally.
|
|
308
|
+
|
|
309
|
+
`protect` lists entry paths that must not be chosen as victims: the
|
|
310
|
+
entries written during the current run. Without this, a cap smaller
|
|
311
|
+
than one entry (or already-full cache plus a fresh write) would
|
|
312
|
+
delete the just-written entry in the very run that produced it, so
|
|
313
|
+
the cache could never serve a single hit; the cap may transiently be
|
|
314
|
+
exceeded because of them. They are ordinary candidates again on the
|
|
315
|
+
next run, where LRU order decides (they are the newest, so they go
|
|
316
|
+
last).
|
|
317
|
+
|
|
318
|
+
`stats`, when given a dict, receives {"total_after": bytes} so the
|
|
319
|
+
caller can warn when the cap is still exceeded (e.g. a single entry
|
|
320
|
+
larger than the cap, or everything left protected)."""
|
|
321
|
+
if max_bytes <= 0 or not self.root.is_dir():
|
|
322
|
+
return 0
|
|
323
|
+
protected = protect or set()
|
|
324
|
+
entries: list[tuple[float, int, Path]] = []
|
|
325
|
+
total = 0
|
|
326
|
+
for shard in self._shards():
|
|
327
|
+
for f in shard.iterdir():
|
|
328
|
+
# Only real cache entries count and only they may be
|
|
329
|
+
# deleted: the directory is user-supplied and may hold
|
|
330
|
+
# unrelated files. Temp files are sweep_stale_tmp()'s job;
|
|
331
|
+
# they are never counted toward the cap nor chosen as
|
|
332
|
+
# victims.
|
|
333
|
+
if not _ENTRY_RE.match(f.name) or not f.name.startswith(shard.name):
|
|
334
|
+
continue
|
|
335
|
+
try:
|
|
336
|
+
st = f.stat()
|
|
337
|
+
except OSError:
|
|
338
|
+
continue
|
|
339
|
+
if not stat.S_ISREG(st.st_mode):
|
|
340
|
+
continue
|
|
341
|
+
entries.append((st.st_mtime, st.st_size, f))
|
|
342
|
+
total += st.st_size
|
|
343
|
+
removed = 0
|
|
344
|
+
for _mtime, size, f in sorted(entries, key=lambda e: e[0]):
|
|
345
|
+
if total <= max_bytes:
|
|
346
|
+
break
|
|
347
|
+
if f in protected:
|
|
348
|
+
continue
|
|
349
|
+
try:
|
|
350
|
+
f.unlink()
|
|
351
|
+
except OSError:
|
|
352
|
+
continue
|
|
353
|
+
total -= size
|
|
354
|
+
removed += 1
|
|
355
|
+
if stats is not None:
|
|
356
|
+
stats["total_after"] = total
|
|
357
|
+
return removed
|
|
358
|
+
|
|
359
|
+
def _shards(self) -> Iterable[Path]:
|
|
360
|
+
for d in self.root.iterdir():
|
|
361
|
+
if _SHARD_RE.match(d.name) and d.is_dir():
|
|
362
|
+
yield d
|