sillybos 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sillybos/__init__.py +49 -0
- sillybos/_version.py +24 -0
- sillybos/check.py +206 -0
- sillybos/cli.py +119 -0
- sillybos/headers.py +236 -0
- sillybos/pdf.py +126 -0
- sillybos/pillow.py +49 -0
- sillybos/policy.py +91 -0
- sillybos/py.typed +0 -0
- sillybos/sniff.py +120 -0
- sillybos-0.1.0.dist-info/METADATA +245 -0
- sillybos-0.1.0.dist-info/RECORD +15 -0
- sillybos-0.1.0.dist-info/WHEEL +4 -0
- sillybos-0.1.0.dist-info/entry_points.txt +2 -0
- sillybos-0.1.0.dist-info/licenses/LICENSE +21 -0
sillybos/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""sillybos: decide before decoding.
|
|
2
|
+
|
|
3
|
+
A σίλλυβος (*sillybos*, Latin *sittybus*) was the little parchment tag that
|
|
4
|
+
hung from a rolled scroll in an ancient library, giving its title and author
|
|
5
|
+
so that a reader could tell what a scroll was without unrolling it. This
|
|
6
|
+
library is that tag for the files a scanning or OCR pipeline is handed. It
|
|
7
|
+
answers one narrow question before any image library or PDF renderer opens a
|
|
8
|
+
file: *is this one of the formats I mean to read, within the limits I
|
|
9
|
+
expect, and if not, why?*
|
|
10
|
+
|
|
11
|
+
It reads the first bytes to learn what the file really is, whatever its name
|
|
12
|
+
says; reads the header of a JPEG, PNG or TIFF with code of its own, never a
|
|
13
|
+
decoder's; asks poppler's ``pdfinfo``, in a separate process, how many pages
|
|
14
|
+
a PDF has and how large they are; and refuses, with a reason in plain words,
|
|
15
|
+
anything outside a :class:`Policy`.
|
|
16
|
+
|
|
17
|
+
>>> from sillybos import check
|
|
18
|
+
>>> verdict = check("scan.png") # doctest: +SKIP
|
|
19
|
+
>>> verdict.ok, verdict.reasons # doctest: +SKIP
|
|
20
|
+
(False, ('says PNG, is PostScript',))
|
|
21
|
+
|
|
22
|
+
The whole surface is :func:`check`, which returns a :class:`Verdict`;
|
|
23
|
+
:func:`require`, which raises :class:`Refused` instead; :class:`Policy`,
|
|
24
|
+
with the presets :data:`DEFAULT` and :data:`STRICT`; and, with the
|
|
25
|
+
``[pillow]`` extra, :func:`sillybos.pillow.open_checked`.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from sillybos.check import Facts, Refused, Verdict, check, require
|
|
31
|
+
from sillybos.policy import DEFAULT, FORMATS, STRICT, Policy
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
from sillybos._version import __version__
|
|
35
|
+
except ImportError: # pragma: no cover - only in a checkout that has not been built
|
|
36
|
+
__version__ = "0.0.0"
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"DEFAULT",
|
|
40
|
+
"FORMATS",
|
|
41
|
+
"STRICT",
|
|
42
|
+
"Facts",
|
|
43
|
+
"Policy",
|
|
44
|
+
"Refused",
|
|
45
|
+
"Verdict",
|
|
46
|
+
"__version__",
|
|
47
|
+
"check",
|
|
48
|
+
"require",
|
|
49
|
+
]
|
sillybos/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
sillybos/check.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""The check itself: one function, one verdict.
|
|
2
|
+
|
|
3
|
+
:func:`check` runs the steps in the order that keeps the cheapest and safest
|
|
4
|
+
first, and stops handing the file to anything further once it has a reason
|
|
5
|
+
to refuse it:
|
|
6
|
+
|
|
7
|
+
1. The file's size, from the file system. Over the limit, nothing reads it
|
|
8
|
+
but the sniff.
|
|
9
|
+
2. Its first bytes, to name the format it really is. A format the policy
|
|
10
|
+
does not allow is refused by name, and no reader of ours runs on it.
|
|
11
|
+
3. Whether its name agrees with its contents.
|
|
12
|
+
4. Its header, with the reader for its format (ours for JPEG, PNG and TIFF;
|
|
13
|
+
``pdfinfo`` in a separate process for PDF): dimensions, frames, pages.
|
|
14
|
+
5. The policy's ceilings against what the header said.
|
|
15
|
+
|
|
16
|
+
Every reason is a sentence an operator can act on, and the verdict carries
|
|
17
|
+
all of them, not just the first.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from sillybos import headers, sniff
|
|
27
|
+
from sillybos.pdf import pdf as read_pdf
|
|
28
|
+
from sillybos.policy import DEFAULT, FORMATS, Policy
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class Facts:
|
|
33
|
+
"""What was read from the file before any decoder saw it.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
bytes: The file's size.
|
|
37
|
+
width: Pixels across the first page or frame; for a PDF, at the
|
|
38
|
+
policy's render dpi.
|
|
39
|
+
height: Pixels down, likewise.
|
|
40
|
+
pages: Pages in a PDF, frames in a TIFF or animated PNG, 1 otherwise.
|
|
41
|
+
For a file with more than the policy allows, the walk may have
|
|
42
|
+
stopped early: the number is then "more than" the limit.
|
|
43
|
+
max_pixels: The most pixels any one page or frame would decode to,
|
|
44
|
+
or ``None`` if the page count was over the limit and the sizes
|
|
45
|
+
were not read.
|
|
46
|
+
encrypted: Whether a PDF is encrypted. Always ``False`` for an image.
|
|
47
|
+
pages_exact: ``False`` when the frame walk stopped at the limit, so
|
|
48
|
+
``pages`` is a floor, not a count.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
bytes: int
|
|
52
|
+
width: int
|
|
53
|
+
height: int
|
|
54
|
+
pages: int
|
|
55
|
+
max_pixels: int | None
|
|
56
|
+
encrypted: bool = False
|
|
57
|
+
pages_exact: bool = True
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class Verdict:
|
|
62
|
+
"""The answer to "may a decoder open this file?".
|
|
63
|
+
|
|
64
|
+
Attributes:
|
|
65
|
+
path: The file.
|
|
66
|
+
ok: ``True`` if nothing was found against it.
|
|
67
|
+
format: What the file really is, from its contents: a name from
|
|
68
|
+
:data:`sillybos.FORMATS`, another name the library knows only
|
|
69
|
+
to refuse (``"postscript"``), or ``None`` for an unknown format.
|
|
70
|
+
claimed: What its extension says it is, or ``None`` if the extension
|
|
71
|
+
is not one the library maps.
|
|
72
|
+
facts: What the header said, or ``None`` if it was never read.
|
|
73
|
+
reasons: Why the file was refused, one sentence each. Empty when
|
|
74
|
+
``ok``.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
path: Path
|
|
78
|
+
ok: bool
|
|
79
|
+
format: str | None
|
|
80
|
+
claimed: str | None
|
|
81
|
+
facts: Facts | None
|
|
82
|
+
reasons: tuple[str, ...]
|
|
83
|
+
|
|
84
|
+
def __str__(self) -> str:
|
|
85
|
+
if self.ok:
|
|
86
|
+
return f"{self.path.name}: ok ({self.summary()})"
|
|
87
|
+
return f"{self.path.name}: refused. " + " ".join(r + "." for r in self.reasons)
|
|
88
|
+
|
|
89
|
+
def summary(self) -> str:
|
|
90
|
+
"""One line of facts: ``"JPEG, 4000x6000, 1 page, 2.1 MB"``."""
|
|
91
|
+
parts = [sniff.describe(self.format)]
|
|
92
|
+
if self.facts is not None:
|
|
93
|
+
parts.append(f"{self.facts.width}x{self.facts.height}")
|
|
94
|
+
parts.append(f"{self.facts.pages} page{'s' if self.facts.pages != 1 else ''}")
|
|
95
|
+
parts.append(f"{self.facts.bytes / 1_000_000:.1f} MB")
|
|
96
|
+
return ", ".join(parts)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Refused(Exception):
|
|
100
|
+
"""A file did not pass. Raised by :func:`require`; carries the :class:`Verdict`."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, verdict: Verdict) -> None:
|
|
103
|
+
self.verdict = verdict
|
|
104
|
+
super().__init__(str(verdict))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def check(path: str | os.PathLike[str], policy: Policy = DEFAULT) -> Verdict:
|
|
108
|
+
"""Decide whether a decoder may open ``path``, without opening it with one.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
path: The file to check. It must exist; a missing file is the
|
|
112
|
+
caller's error, not a verdict, and raises :class:`FileNotFoundError`.
|
|
113
|
+
policy: The limits to apply. :data:`sillybos.DEFAULT` if not given.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
A :class:`Verdict`. Never raises for anything found *in* the file:
|
|
117
|
+
a malformed header, a lying name and a page too large are all
|
|
118
|
+
reasons, not exceptions.
|
|
119
|
+
|
|
120
|
+
Raises:
|
|
121
|
+
FileNotFoundError: If the file does not exist.
|
|
122
|
+
sillybos.pdf.PdfInfoMissing: If the file is a PDF and ``pdfinfo``
|
|
123
|
+
is not installed.
|
|
124
|
+
"""
|
|
125
|
+
path = Path(path)
|
|
126
|
+
size = path.stat().st_size
|
|
127
|
+
reasons: list[str] = []
|
|
128
|
+
claimed = sniff.claimed_by(path.suffix)
|
|
129
|
+
|
|
130
|
+
if size > policy.max_bytes:
|
|
131
|
+
reasons.append(f"{size:,} bytes; the limit is {policy.max_bytes:,}")
|
|
132
|
+
|
|
133
|
+
with path.open("rb") as f:
|
|
134
|
+
head = f.read(sniff.HEAD_BYTES)
|
|
135
|
+
if not head:
|
|
136
|
+
return Verdict(path, False, None, claimed, None, ("the file is empty",))
|
|
137
|
+
fmt = sniff.sniff(head)
|
|
138
|
+
|
|
139
|
+
if fmt is None or not policy.allows(fmt):
|
|
140
|
+
reasons.append(_wrong_format(fmt, claimed, policy))
|
|
141
|
+
return Verdict(path, False, fmt, claimed, None, tuple(reasons))
|
|
142
|
+
|
|
143
|
+
if policy.name_must_match and claimed is not None and claimed != fmt:
|
|
144
|
+
reasons.append(f"says {sniff.describe(claimed)}, is {sniff.describe(fmt)}")
|
|
145
|
+
return Verdict(path, False, fmt, claimed, None, tuple(reasons))
|
|
146
|
+
|
|
147
|
+
if reasons:
|
|
148
|
+
# Over the byte limit: do not read on, and in particular do not
|
|
149
|
+
# start pdfinfo on it.
|
|
150
|
+
return Verdict(path, False, fmt, claimed, None, tuple(reasons))
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
facts = _facts(path, fmt, f, size, policy)
|
|
154
|
+
except headers.HeaderError as exc:
|
|
155
|
+
reasons.append(f"the header could not be read: {exc}")
|
|
156
|
+
return Verdict(path, False, fmt, claimed, None, tuple(reasons))
|
|
157
|
+
|
|
158
|
+
reasons.extend(_against_limits(facts, policy))
|
|
159
|
+
return Verdict(path, not reasons, fmt, claimed, facts, tuple(reasons))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def require(path: str | os.PathLike[str], policy: Policy = DEFAULT) -> Verdict:
|
|
163
|
+
"""As :func:`check`, but raise :class:`Refused` instead of returning a refusal."""
|
|
164
|
+
verdict = check(path, policy)
|
|
165
|
+
if not verdict.ok:
|
|
166
|
+
raise Refused(verdict)
|
|
167
|
+
return verdict
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _wrong_format(fmt: str | None, claimed: str | None, policy: Policy) -> str:
|
|
171
|
+
real = sniff.describe(fmt)
|
|
172
|
+
allowed = ", ".join(sniff.describe(name) for name in sorted(policy.formats))
|
|
173
|
+
if claimed is not None and claimed != fmt:
|
|
174
|
+
return f"says {sniff.describe(claimed)}, is {real}"
|
|
175
|
+
if fmt in FORMATS:
|
|
176
|
+
return f"is {real}, which this policy does not allow (allowed: {allowed})"
|
|
177
|
+
return f"is {real}, not one of the formats a scan can be (allowed: {allowed})"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _facts(path: Path, fmt: str, f: headers.Reader, size: int, policy: Policy) -> Facts:
|
|
181
|
+
if fmt == "pdf":
|
|
182
|
+
info = read_pdf(path, policy)
|
|
183
|
+
return Facts(size, info.width, info.height, info.pages, info.max_pixels, info.encrypted)
|
|
184
|
+
header = headers.READERS[fmt](f, size, policy.max_pages)
|
|
185
|
+
return Facts(
|
|
186
|
+
size,
|
|
187
|
+
header.width,
|
|
188
|
+
header.height,
|
|
189
|
+
header.pages,
|
|
190
|
+
header.max_pixels if header.complete else None,
|
|
191
|
+
pages_exact=header.complete,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _against_limits(facts: Facts, policy: Policy) -> list[str]:
|
|
196
|
+
reasons = []
|
|
197
|
+
if facts.pages > policy.max_pages:
|
|
198
|
+
if facts.pages_exact:
|
|
199
|
+
reasons.append(f"{facts.pages:,} pages; the limit is {policy.max_pages:,}")
|
|
200
|
+
else:
|
|
201
|
+
reasons.append(f"more than {policy.max_pages:,} pages, which is the limit")
|
|
202
|
+
if facts.max_pixels is not None and facts.max_pixels > policy.max_pixels:
|
|
203
|
+
reasons.append(f"a page of {facts.max_pixels:,} pixels; the limit is {policy.max_pixels:,}")
|
|
204
|
+
if facts.encrypted and not policy.allow_encrypted_pdf:
|
|
205
|
+
reasons.append("the PDF is encrypted")
|
|
206
|
+
return reasons
|
sillybos/cli.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""The ``sillybos`` command: check files, print verdicts, exit non-zero on a refusal.
|
|
2
|
+
|
|
3
|
+
For a shell pipeline or a Dockerfile::
|
|
4
|
+
|
|
5
|
+
sillybos scans/*.tif && run-the-ocr scans/
|
|
6
|
+
|
|
7
|
+
Exit status 0 when every file passed, 1 when any was refused, 2 when the
|
|
8
|
+
command could not do its job (a missing file, no ``pdfinfo`` for a PDF).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import dataclasses
|
|
15
|
+
import json
|
|
16
|
+
import sys
|
|
17
|
+
from collections.abc import Sequence
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from sillybos import __version__, check
|
|
21
|
+
from sillybos.pdf import PdfInfoMissing
|
|
22
|
+
from sillybos.policy import DEFAULT, STRICT, Policy
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _policy(args: argparse.Namespace) -> Policy:
|
|
26
|
+
base = STRICT if args.strict else DEFAULT
|
|
27
|
+
changes = {
|
|
28
|
+
name: value
|
|
29
|
+
for name, value in (
|
|
30
|
+
("max_bytes", args.max_bytes),
|
|
31
|
+
("max_pixels", args.max_pixels),
|
|
32
|
+
("max_pages", args.max_pages),
|
|
33
|
+
)
|
|
34
|
+
if value is not None
|
|
35
|
+
}
|
|
36
|
+
if args.formats:
|
|
37
|
+
changes["formats"] = frozenset(args.formats.split(","))
|
|
38
|
+
if args.any_name:
|
|
39
|
+
changes["name_must_match"] = False
|
|
40
|
+
return dataclasses.replace(base, **changes) if changes else base
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
44
|
+
parser = argparse.ArgumentParser(
|
|
45
|
+
prog="sillybos",
|
|
46
|
+
description=(
|
|
47
|
+
"Decide before decoding: say whether each file is one of the formats a scan "
|
|
48
|
+
"can be, within the limits, before any image library or PDF renderer opens it."
|
|
49
|
+
),
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument("files", nargs="+", type=Path, metavar="FILE")
|
|
52
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--strict", action="store_true", help="the strict preset, for files from outside"
|
|
55
|
+
)
|
|
56
|
+
parser.add_argument(
|
|
57
|
+
"--formats",
|
|
58
|
+
metavar="LIST",
|
|
59
|
+
help="comma-separated formats to allow, from jpeg,png,tiff,pdf (default: all four)",
|
|
60
|
+
)
|
|
61
|
+
parser.add_argument("--max-bytes", type=int, metavar="N", help="largest file, in bytes")
|
|
62
|
+
parser.add_argument(
|
|
63
|
+
"--max-pixels", type=int, metavar="N", help="most pixels in any one page or frame"
|
|
64
|
+
)
|
|
65
|
+
parser.add_argument("--max-pages", type=int, metavar="N", help="most pages or frames")
|
|
66
|
+
parser.add_argument(
|
|
67
|
+
"--any-name",
|
|
68
|
+
action="store_true",
|
|
69
|
+
help="do not refuse a file whose extension disagrees with its contents",
|
|
70
|
+
)
|
|
71
|
+
parser.add_argument("--json", action="store_true", help="one JSON object per line")
|
|
72
|
+
parser.add_argument(
|
|
73
|
+
"-q", "--quiet", action="store_true", help="print only the files that were refused"
|
|
74
|
+
)
|
|
75
|
+
return parser
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _line(verdict: object, as_json: bool) -> str:
|
|
79
|
+
from sillybos.check import Verdict
|
|
80
|
+
|
|
81
|
+
if not isinstance(verdict, Verdict): # pragma: no cover - typing guard
|
|
82
|
+
raise TypeError(type(verdict))
|
|
83
|
+
if as_json:
|
|
84
|
+
record = dataclasses.asdict(verdict)
|
|
85
|
+
record["path"] = str(verdict.path)
|
|
86
|
+
return json.dumps(record, ensure_ascii=False)
|
|
87
|
+
if verdict.ok:
|
|
88
|
+
return f"ok {verdict.path} ({verdict.summary()})"
|
|
89
|
+
return f"refused {verdict.path} {'; '.join(verdict.reasons)}"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
93
|
+
args = build_parser().parse_args(argv)
|
|
94
|
+
try:
|
|
95
|
+
policy = _policy(args)
|
|
96
|
+
except ValueError as exc:
|
|
97
|
+
print(f"sillybos: {exc}", file=sys.stderr)
|
|
98
|
+
return 2
|
|
99
|
+
status = 0
|
|
100
|
+
for path in args.files:
|
|
101
|
+
try:
|
|
102
|
+
verdict = check(path, policy)
|
|
103
|
+
except FileNotFoundError:
|
|
104
|
+
print(f"sillybos: {path}: no such file", file=sys.stderr)
|
|
105
|
+
status = 2
|
|
106
|
+
continue
|
|
107
|
+
except PdfInfoMissing as exc:
|
|
108
|
+
print(f"sillybos: {exc}", file=sys.stderr)
|
|
109
|
+
return 2
|
|
110
|
+
if not verdict.ok:
|
|
111
|
+
status = max(status, 1)
|
|
112
|
+
if verdict.ok and args.quiet:
|
|
113
|
+
continue
|
|
114
|
+
print(_line(verdict, args.json))
|
|
115
|
+
return status
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__": # pragma: no cover
|
|
119
|
+
sys.exit(main())
|
sillybos/headers.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Width, height and page count from a JPEG, PNG or TIFF header, without a decoder.
|
|
2
|
+
|
|
3
|
+
Each reader here is a few dozen lines that understand just enough of its
|
|
4
|
+
format to find the dimensions and count the frames. None of them decodes a
|
|
5
|
+
pixel, and none of them is a decoder's own header code: the point of the
|
|
6
|
+
library is that a file is judged before any of that runs. Every read is
|
|
7
|
+
bounded, every offset is checked against the file's size, and a TIFF's
|
|
8
|
+
chain of frames is walked with a limit and a memory of where it has been,
|
|
9
|
+
so that a crafted file cannot send the reader round in a circle or off the
|
|
10
|
+
end of the file.
|
|
11
|
+
|
|
12
|
+
A header the reader cannot make sense of raises :class:`HeaderError` with a
|
|
13
|
+
reason; the check turns that into a refusal. "Cannot make sense of" is
|
|
14
|
+
deliberately broad: a file that is not shaped the way its format says is not
|
|
15
|
+
one to hand to a decoder, whatever a more forgiving reader would do with it.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import struct
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import IO
|
|
23
|
+
|
|
24
|
+
Reader = IO[bytes]
|
|
25
|
+
|
|
26
|
+
MAX_SEGMENTS = 4096
|
|
27
|
+
"""How many JPEG segments or PNG chunks to walk before giving up: a bound
|
|
28
|
+
on the time a crafted file can cost. A real scan has tens."""
|
|
29
|
+
|
|
30
|
+
MAX_DIMENSION = 1 << 31
|
|
31
|
+
"""A width or height at or above this is not a dimension; it is a bug or
|
|
32
|
+
an attack, and refused before the multiplication that would overflow a C
|
|
33
|
+
decoder."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class HeaderError(ValueError):
|
|
37
|
+
"""The header is not shaped the way its format says."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Header:
|
|
42
|
+
"""What a header reader found.
|
|
43
|
+
|
|
44
|
+
Attributes:
|
|
45
|
+
width: Pixels across, for the first frame.
|
|
46
|
+
height: Pixels down, for the first frame.
|
|
47
|
+
pages: Frames in the file (1 for an ordinary image). For a TIFF with
|
|
48
|
+
more frames than ``limit``, ``limit + 1``: the walk stopped.
|
|
49
|
+
max_pixels: The most pixels in any one frame the reader looked at.
|
|
50
|
+
complete: ``False`` if the walk stopped at the limit, so ``pages``
|
|
51
|
+
and ``max_pixels`` describe only the frames it reached.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
width: int
|
|
55
|
+
height: int
|
|
56
|
+
pages: int
|
|
57
|
+
max_pixels: int
|
|
58
|
+
complete: bool = True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _read_exactly(f: Reader, n: int, what: str) -> bytes:
|
|
62
|
+
data = f.read(n)
|
|
63
|
+
if len(data) != n:
|
|
64
|
+
raise HeaderError(f"the file ends inside its {what}")
|
|
65
|
+
return data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _dimensions(width: int, height: int, what: str) -> None:
|
|
69
|
+
if width < 1 or height < 1:
|
|
70
|
+
raise HeaderError(f"the {what} gives a zero width or height")
|
|
71
|
+
if width >= MAX_DIMENSION or height >= MAX_DIMENSION:
|
|
72
|
+
raise HeaderError(f"the {what} gives a width or height of {max(width, height):,}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# --- JPEG -------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
# Start-of-frame markers, which carry the dimensions. C4 (DHT), C8 (JPG
|
|
78
|
+
# extension) and CC (DAC) share the range but are not frames.
|
|
79
|
+
_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
|
80
|
+
# Markers with no length field.
|
|
81
|
+
_STANDALONE = {0x01, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def jpeg(f: Reader, size: int, limit: int) -> Header:
|
|
85
|
+
"""Read a JPEG's frame header: walk the segments to the first SOF marker."""
|
|
86
|
+
del limit # a JPEG has one frame
|
|
87
|
+
f.seek(0)
|
|
88
|
+
if _read_exactly(f, 2, "header") != b"\xff\xd8":
|
|
89
|
+
raise HeaderError("no JPEG start-of-image marker")
|
|
90
|
+
for _ in range(MAX_SEGMENTS):
|
|
91
|
+
byte = _read_exactly(f, 1, "segment marker")
|
|
92
|
+
if byte != b"\xff":
|
|
93
|
+
raise HeaderError("a segment does not start with a marker")
|
|
94
|
+
marker = _read_exactly(f, 1, "segment marker")[0]
|
|
95
|
+
while marker == 0xFF: # fill bytes before a marker are allowed
|
|
96
|
+
marker = _read_exactly(f, 1, "segment marker")[0]
|
|
97
|
+
if marker in _STANDALONE:
|
|
98
|
+
continue
|
|
99
|
+
if marker in (0xD9, 0xDA):
|
|
100
|
+
raise HeaderError("the image data starts before any frame header")
|
|
101
|
+
(length,) = struct.unpack(">H", _read_exactly(f, 2, "segment length"))
|
|
102
|
+
if length < 2:
|
|
103
|
+
raise HeaderError("a segment claims a length shorter than its own length field")
|
|
104
|
+
if marker in _SOF:
|
|
105
|
+
if length < 7:
|
|
106
|
+
raise HeaderError("the frame header is too short to hold the dimensions")
|
|
107
|
+
_precision, height, width = struct.unpack(">BHH", _read_exactly(f, 5, "frame header"))
|
|
108
|
+
if height == 0:
|
|
109
|
+
# Legal (a DNL marker supplies it later) and never seen in a
|
|
110
|
+
# scan; refusing it keeps every decoder's rarest path closed.
|
|
111
|
+
raise HeaderError("the height is deferred to a DNL marker")
|
|
112
|
+
_dimensions(width, height, "frame header")
|
|
113
|
+
return Header(width, height, 1, width * height)
|
|
114
|
+
next_offset = f.tell() + length - 2
|
|
115
|
+
if next_offset > size:
|
|
116
|
+
raise HeaderError("a segment runs past the end of the file")
|
|
117
|
+
f.seek(next_offset)
|
|
118
|
+
raise HeaderError(f"no frame header in the first {MAX_SEGMENTS} segments")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# --- PNG --------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def png(f: Reader, size: int, limit: int) -> Header:
|
|
125
|
+
"""Read a PNG's IHDR, and count the frames of an animated one (acTL)."""
|
|
126
|
+
del limit # the frame count is one field; no walk to bound
|
|
127
|
+
f.seek(8) # the signature was checked by sniff
|
|
128
|
+
length, kind = struct.unpack(">I4s", _read_exactly(f, 8, "first chunk"))
|
|
129
|
+
if kind != b"IHDR" or length != 13:
|
|
130
|
+
raise HeaderError("the first chunk is not a 13-byte IHDR")
|
|
131
|
+
width, height = struct.unpack(">II", _read_exactly(f, 8, "IHDR"))
|
|
132
|
+
_dimensions(width, height, "IHDR")
|
|
133
|
+
f.seek(13 - 8 + 4, 1) # the rest of IHDR, and its CRC
|
|
134
|
+
pages = 1
|
|
135
|
+
for _ in range(MAX_SEGMENTS):
|
|
136
|
+
head = f.read(8)
|
|
137
|
+
if len(head) < 8:
|
|
138
|
+
break
|
|
139
|
+
length, kind = struct.unpack(">I4s", head)
|
|
140
|
+
if kind in (b"IDAT", b"IEND"):
|
|
141
|
+
break
|
|
142
|
+
if kind == b"acTL" and length >= 4:
|
|
143
|
+
(pages,) = struct.unpack(">I", _read_exactly(f, 4, "acTL"))
|
|
144
|
+
if pages < 1:
|
|
145
|
+
raise HeaderError("acTL gives zero frames")
|
|
146
|
+
break
|
|
147
|
+
next_offset = f.tell() + length + 4
|
|
148
|
+
if next_offset > size:
|
|
149
|
+
raise HeaderError("a chunk runs past the end of the file")
|
|
150
|
+
f.seek(next_offset)
|
|
151
|
+
return Header(width, height, pages, width * height)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# --- TIFF -------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
_TIFF_SHORT = 3
|
|
157
|
+
_TIFF_LONG = 4
|
|
158
|
+
_IMAGE_WIDTH = 256
|
|
159
|
+
_IMAGE_LENGTH = 257
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def tiff(f: Reader, size: int, limit: int) -> Header:
|
|
163
|
+
"""Walk a classic TIFF's IFD chain: dimensions per frame, frames counted.
|
|
164
|
+
|
|
165
|
+
Stops after ``limit + 1`` frames and reports that many, so that a file
|
|
166
|
+
with a million frames costs the time of ``limit`` reads, not a million.
|
|
167
|
+
An IFD that points at itself, at an earlier one, or past the end of the
|
|
168
|
+
file is an error, not a loop.
|
|
169
|
+
"""
|
|
170
|
+
f.seek(0)
|
|
171
|
+
order = _read_exactly(f, 2, "header")
|
|
172
|
+
endian = {b"II": "<", b"MM": ">"}.get(order)
|
|
173
|
+
if endian is None:
|
|
174
|
+
raise HeaderError("no TIFF byte-order mark")
|
|
175
|
+
(magic,) = struct.unpack(endian + "H", _read_exactly(f, 2, "header"))
|
|
176
|
+
if magic != 42:
|
|
177
|
+
raise HeaderError("not a classic TIFF (BigTIFF is refused by name)")
|
|
178
|
+
(offset,) = struct.unpack(endian + "I", _read_exactly(f, 4, "header"))
|
|
179
|
+
seen: set[int] = set()
|
|
180
|
+
first: tuple[int, int] | None = None
|
|
181
|
+
max_pixels = 0
|
|
182
|
+
pages = 0
|
|
183
|
+
while offset != 0:
|
|
184
|
+
if pages > limit:
|
|
185
|
+
if first is None:
|
|
186
|
+
return _no_frames()
|
|
187
|
+
return Header(first[0], first[1], pages, max_pixels, complete=False)
|
|
188
|
+
if offset in seen:
|
|
189
|
+
raise HeaderError("the frame chain loops back on itself")
|
|
190
|
+
if offset + 2 > size or offset < 8:
|
|
191
|
+
raise HeaderError("a frame directory lies outside the file")
|
|
192
|
+
seen.add(offset)
|
|
193
|
+
f.seek(offset)
|
|
194
|
+
(count,) = struct.unpack(endian + "H", _read_exactly(f, 2, "frame directory"))
|
|
195
|
+
if offset + 2 + count * 12 + 4 > size:
|
|
196
|
+
raise HeaderError("a frame directory runs past the end of the file")
|
|
197
|
+
width = height = None
|
|
198
|
+
for _ in range(count):
|
|
199
|
+
tag, kind, n, raw = struct.unpack(
|
|
200
|
+
endian + "HHI4s", _read_exactly(f, 12, "frame directory")
|
|
201
|
+
)
|
|
202
|
+
if tag not in (_IMAGE_WIDTH, _IMAGE_LENGTH):
|
|
203
|
+
continue
|
|
204
|
+
if n != 1:
|
|
205
|
+
raise HeaderError("a dimension tag holds more than one value")
|
|
206
|
+
if kind == _TIFF_SHORT:
|
|
207
|
+
# A SHORT sits in the first two bytes of the field in either
|
|
208
|
+
# byte order: the value is left-justified, not right-aligned.
|
|
209
|
+
(value,) = struct.unpack(endian + "H", raw[:2])
|
|
210
|
+
elif kind == _TIFF_LONG:
|
|
211
|
+
(value,) = struct.unpack(endian + "I", raw)
|
|
212
|
+
else:
|
|
213
|
+
raise HeaderError("a dimension tag is neither SHORT nor LONG")
|
|
214
|
+
if tag == _IMAGE_WIDTH:
|
|
215
|
+
width = value
|
|
216
|
+
else:
|
|
217
|
+
height = value
|
|
218
|
+
if width is None or height is None:
|
|
219
|
+
raise HeaderError(f"frame {pages + 1} has no width or height")
|
|
220
|
+
_dimensions(width, height, f"frame {pages + 1}")
|
|
221
|
+
pages += 1
|
|
222
|
+
max_pixels = max(max_pixels, width * height)
|
|
223
|
+
if first is None:
|
|
224
|
+
first = (width, height)
|
|
225
|
+
(offset,) = struct.unpack(endian + "I", _read_exactly(f, 4, "frame directory"))
|
|
226
|
+
if first is None:
|
|
227
|
+
return _no_frames()
|
|
228
|
+
return Header(first[0], first[1], pages, max_pixels)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _no_frames() -> Header:
|
|
232
|
+
raise HeaderError("the file has no frames")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
READERS = {"jpeg": jpeg, "png": png, "tiff": tiff}
|
|
236
|
+
"""The header reader for each raster format in :data:`sillybos.FORMATS`."""
|
sillybos/pdf.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Page count and page sizes of a PDF, from poppler's ``pdfinfo``.
|
|
2
|
+
|
|
3
|
+
A PDF's structure cannot be read from a fixed header the way an image's can:
|
|
4
|
+
the page count lives in an object tree that may be compressed, and finding
|
|
5
|
+
it means parsing. Rather than carry a PDF parser (which would be exactly the
|
|
6
|
+
kind of code the library exists to keep away from untrusted files), this
|
|
7
|
+
asks ``pdfinfo``, which parses the structure without rendering a page, in a
|
|
8
|
+
separate process with a time limit. A hang or a crash in ``pdfinfo`` is then
|
|
9
|
+
one file's refusal, not the caller's.
|
|
10
|
+
|
|
11
|
+
``pdfinfo`` comes with poppler (``brew install poppler``, ``apt install
|
|
12
|
+
poppler-utils``), which anything that renders PDFs in Python already needs.
|
|
13
|
+
Without it, a PDF cannot be checked, and :class:`PdfInfoMissing` says so;
|
|
14
|
+
that is a configuration error, not a verdict on the file.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import subprocess
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from sillybos.headers import HeaderError
|
|
26
|
+
from sillybos.policy import Policy
|
|
27
|
+
|
|
28
|
+
_PAGES = re.compile(r"^Pages:\s+(\d+)\s*$", re.MULTILINE)
|
|
29
|
+
_ENCRYPTED = re.compile(r"^Encrypted:\s+yes", re.MULTILINE)
|
|
30
|
+
_PAGE_SIZE = re.compile(r"^Page(?:\s+(\d+))? size:\s+([\d.]+) x ([\d.]+) pts", re.MULTILINE)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class PdfInfoMissing(RuntimeError):
|
|
34
|
+
"""``pdfinfo`` is not on the PATH, so no PDF can be checked."""
|
|
35
|
+
|
|
36
|
+
def __init__(self) -> None:
|
|
37
|
+
super().__init__(
|
|
38
|
+
"pdfinfo (from poppler) is needed to check a PDF and is not on the PATH: "
|
|
39
|
+
"`brew install poppler` on macOS, `apt install poppler-utils` on Debian"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class PdfHeader:
|
|
45
|
+
"""What ``pdfinfo`` reported.
|
|
46
|
+
|
|
47
|
+
Attributes:
|
|
48
|
+
pages: The page count.
|
|
49
|
+
encrypted: Whether the file is encrypted.
|
|
50
|
+
width: The first page's width in pixels at the policy's render dpi.
|
|
51
|
+
height: The first page's height, likewise.
|
|
52
|
+
max_pixels: The most pixels any page would render to. ``None`` when
|
|
53
|
+
the page count was over the limit and the sizes were not asked
|
|
54
|
+
for.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
pages: int
|
|
58
|
+
encrypted: bool
|
|
59
|
+
width: int
|
|
60
|
+
height: int
|
|
61
|
+
max_pixels: int | None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _pdfinfo(args: list[str], path: Path, timeout: float) -> str:
|
|
65
|
+
executable = shutil.which("pdfinfo")
|
|
66
|
+
if executable is None:
|
|
67
|
+
raise PdfInfoMissing
|
|
68
|
+
try:
|
|
69
|
+
# The arguments are a fixed list plus the path being checked: nothing
|
|
70
|
+
# here is a shell, and the file's contents go to pdfinfo's parser,
|
|
71
|
+
# which is the one process this library is willing to expose to them.
|
|
72
|
+
done = subprocess.run( # noqa: S603
|
|
73
|
+
[executable, *args, str(path)],
|
|
74
|
+
capture_output=True,
|
|
75
|
+
text=True,
|
|
76
|
+
errors="replace",
|
|
77
|
+
timeout=timeout,
|
|
78
|
+
check=False,
|
|
79
|
+
)
|
|
80
|
+
except subprocess.TimeoutExpired:
|
|
81
|
+
raise HeaderError(f"pdfinfo did not finish within {timeout:g} seconds") from None
|
|
82
|
+
if done.returncode != 0:
|
|
83
|
+
detail = done.stderr.strip().splitlines()
|
|
84
|
+
reason = detail[-1] if detail else f"exit status {done.returncode}"
|
|
85
|
+
raise HeaderError(f"pdfinfo could not read it ({reason})")
|
|
86
|
+
return done.stdout
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _pixels(points: str, dpi: int) -> float:
|
|
90
|
+
return float(points) / 72 * dpi
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def pdf(path: Path, policy: Policy) -> PdfHeader:
|
|
94
|
+
"""Ask ``pdfinfo`` for the page count, encryption and, within the limit, every page's size.
|
|
95
|
+
|
|
96
|
+
Page sizes are only asked for when the count is within ``policy.max_pages``:
|
|
97
|
+
a listing of a hundred thousand pages is work the file should not be able
|
|
98
|
+
to cause.
|
|
99
|
+
|
|
100
|
+
Raises:
|
|
101
|
+
HeaderError: If ``pdfinfo`` refuses the file or runs out of time.
|
|
102
|
+
PdfInfoMissing: If ``pdfinfo`` is not installed.
|
|
103
|
+
"""
|
|
104
|
+
summary = _pdfinfo([], path, policy.pdfinfo_timeout)
|
|
105
|
+
pages_match = _PAGES.search(summary)
|
|
106
|
+
if pages_match is None:
|
|
107
|
+
raise HeaderError("pdfinfo reported no page count")
|
|
108
|
+
pages = int(pages_match.group(1))
|
|
109
|
+
if pages < 1:
|
|
110
|
+
raise HeaderError("the PDF has no pages")
|
|
111
|
+
encrypted = _ENCRYPTED.search(summary) is not None
|
|
112
|
+
first = _PAGE_SIZE.search(summary)
|
|
113
|
+
if first is None:
|
|
114
|
+
raise HeaderError("pdfinfo reported no page size")
|
|
115
|
+
width = round(_pixels(first.group(2), policy.render_dpi))
|
|
116
|
+
height = round(_pixels(first.group(3), policy.render_dpi))
|
|
117
|
+
if pages > policy.max_pages:
|
|
118
|
+
return PdfHeader(pages, encrypted, width, height, None)
|
|
119
|
+
listing = _pdfinfo(["-f", "1", "-l", str(pages)], path, policy.pdfinfo_timeout)
|
|
120
|
+
sizes = [
|
|
121
|
+
_pixels(w, policy.render_dpi) * _pixels(h, policy.render_dpi)
|
|
122
|
+
for _page, w, h in _PAGE_SIZE.findall(listing)
|
|
123
|
+
]
|
|
124
|
+
if not sizes:
|
|
125
|
+
raise HeaderError("pdfinfo reported no page sizes")
|
|
126
|
+
return PdfHeader(pages, encrypted, width, height, round(max(sizes)))
|
sillybos/pillow.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Check a file, then open it with Pillow using only the readers the policy allows.
|
|
2
|
+
|
|
3
|
+
The one-line fix that October 2026's Pillow advisories made everyone want,
|
|
4
|
+
packaged: ``Image.open`` takes a ``formats`` list, and a file that is not one
|
|
5
|
+
of them is never handed to the other readers, whatever it claims to be.
|
|
6
|
+
:func:`open_checked` runs :func:`sillybos.require` first, so a file that is
|
|
7
|
+
too large, lies about its format or has a page beyond the limit is refused
|
|
8
|
+
before Pillow sees it, and then opens it with ``formats`` set from the
|
|
9
|
+
policy.
|
|
10
|
+
|
|
11
|
+
Needs the ``[pillow]`` extra: ``pip install 'sillybos[pillow]'``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
from typing import TYPE_CHECKING
|
|
18
|
+
|
|
19
|
+
from sillybos.check import require
|
|
20
|
+
from sillybos.policy import DEFAULT, Policy
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from PIL import Image
|
|
24
|
+
|
|
25
|
+
_PILLOW_NAMES = {"jpeg": "JPEG", "png": "PNG", "tiff": "TIFF"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def pillow_formats(policy: Policy = DEFAULT) -> list[str]:
|
|
29
|
+
"""The ``formats`` argument for ``Image.open``, from a policy: ``["JPEG", "PNG", "TIFF"]``."""
|
|
30
|
+
return [_PILLOW_NAMES[name] for name in sorted(policy.formats) if name in _PILLOW_NAMES]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def open_checked(path: str | os.PathLike[str], policy: Policy = DEFAULT) -> Image.Image:
|
|
34
|
+
"""Check ``path`` against ``policy``, then open it with only the allowed Pillow readers.
|
|
35
|
+
|
|
36
|
+
A PDF passes the check but Pillow does not read PDFs; opening one raises
|
|
37
|
+
``PIL.UnidentifiedImageError``, as it would for any file outside
|
|
38
|
+
``formats``.
|
|
39
|
+
|
|
40
|
+
Raises:
|
|
41
|
+
sillybos.Refused: If the file does not pass the policy.
|
|
42
|
+
ImportError: If Pillow is not installed.
|
|
43
|
+
"""
|
|
44
|
+
try:
|
|
45
|
+
from PIL import Image
|
|
46
|
+
except ImportError:
|
|
47
|
+
raise ImportError("open_checked needs Pillow: pip install 'sillybos[pillow]'") from None
|
|
48
|
+
require(path, policy)
|
|
49
|
+
return Image.open(path, formats=pillow_formats(policy))
|
sillybos/policy.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""What a file is allowed to be.
|
|
2
|
+
|
|
3
|
+
A :class:`Policy` is a plain settings object: which formats may pass, and the
|
|
4
|
+
ceilings on size, pixels and pages. Two presets cover most uses. The numbers
|
|
5
|
+
in :data:`DEFAULT` come from a real corpus of archival scans (the largest
|
|
6
|
+
page in it is 39 million pixels at 600 dpi; the longest book is 166 pages)
|
|
7
|
+
with room to spare; :data:`STRICT` is for files from further away.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
FORMATS: tuple[str, ...] = ("jpeg", "png", "tiff", "pdf")
|
|
15
|
+
"""The only formats the library will ever pass: the ones a scan can be.
|
|
16
|
+
Everything else is refused by name. Adding a format is adding attack surface,
|
|
17
|
+
so one is added only when someone shows a scan in it."""
|
|
18
|
+
|
|
19
|
+
MIB = 1024 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Policy:
|
|
24
|
+
"""The limits a file must sit inside to pass :func:`sillybos.check`.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
formats: Which of :data:`FORMATS` may pass. A file of any other
|
|
28
|
+
format is refused whatever its name says.
|
|
29
|
+
max_bytes: The largest file, in bytes. Checked before anything is
|
|
30
|
+
read, so an enormous file is never handed to ``pdfinfo``.
|
|
31
|
+
max_pixels: The most pixels in any one page or frame once decoded.
|
|
32
|
+
For a PDF page, the pixels it would render to at ``render_dpi``.
|
|
33
|
+
The default is the threshold at which Pillow warns of a
|
|
34
|
+
decompression bomb, enforced here rather than left to a global
|
|
35
|
+
that another library may have raised.
|
|
36
|
+
max_pages: The most pages in a PDF, or frames in a multi-frame TIFF
|
|
37
|
+
or animated PNG. Bounds the time and the memory a run takes,
|
|
38
|
+
and stops the header walk itself on a file with a million
|
|
39
|
+
empty frames.
|
|
40
|
+
render_dpi: The resolution the PDF renderer will be run at, so a
|
|
41
|
+
page's size in points can be turned into pixels before the
|
|
42
|
+
renderer allocates them.
|
|
43
|
+
name_must_match: Refuse a file whose extension says one allowed
|
|
44
|
+
format and whose contents are another (a JPEG called
|
|
45
|
+
``.png``). The readers would cope, but a file whose name lies
|
|
46
|
+
is not one to trust, and the mismatch is the first thing an
|
|
47
|
+
operator wants to know.
|
|
48
|
+
allow_encrypted_pdf: Pass a PDF that ``pdfinfo`` reports as
|
|
49
|
+
encrypted. A scanned book never is, so the default refuses.
|
|
50
|
+
pdfinfo_timeout: Seconds to give ``pdfinfo`` before giving up on a
|
|
51
|
+
PDF, which is then refused. A crafted file can make a parser
|
|
52
|
+
loop; a limit turns that into one file's failure.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
formats: frozenset[str] = field(default_factory=lambda: frozenset(FORMATS))
|
|
56
|
+
max_bytes: int = 512 * MIB
|
|
57
|
+
max_pixels: int = 89_478_485
|
|
58
|
+
max_pages: int = 500
|
|
59
|
+
render_dpi: int = 200
|
|
60
|
+
name_must_match: bool = True
|
|
61
|
+
allow_encrypted_pdf: bool = False
|
|
62
|
+
pdfinfo_timeout: float = 30.0
|
|
63
|
+
|
|
64
|
+
def __post_init__(self) -> None:
|
|
65
|
+
unknown = set(self.formats) - set(FORMATS)
|
|
66
|
+
if unknown:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"unknown format(s) {sorted(unknown)}; the formats are {', '.join(FORMATS)}"
|
|
69
|
+
)
|
|
70
|
+
if not self.formats:
|
|
71
|
+
raise ValueError("a policy must allow at least one format")
|
|
72
|
+
for name in ("max_bytes", "max_pixels", "max_pages", "render_dpi"):
|
|
73
|
+
if getattr(self, name) < 1:
|
|
74
|
+
raise ValueError(f"{name} must be at least 1")
|
|
75
|
+
if self.pdfinfo_timeout <= 0:
|
|
76
|
+
raise ValueError("pdfinfo_timeout must be positive")
|
|
77
|
+
|
|
78
|
+
def allows(self, fmt: str) -> bool:
|
|
79
|
+
"""Whether ``fmt`` (a name from :data:`FORMATS`) may pass."""
|
|
80
|
+
return fmt in self.formats
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
DEFAULT = Policy()
|
|
84
|
+
"""For files from a trusted laptop or your own scanner."""
|
|
85
|
+
|
|
86
|
+
STRICT = Policy(
|
|
87
|
+
max_bytes=128 * MIB,
|
|
88
|
+
max_pixels=50_000_000,
|
|
89
|
+
max_pages=100,
|
|
90
|
+
)
|
|
91
|
+
"""For files from outside: smaller, shorter, and no more lenient on format."""
|
sillybos/py.typed
ADDED
|
File without changes
|
sillybos/sniff.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""What a file really is, from its first bytes.
|
|
2
|
+
|
|
3
|
+
An image library decides a file's format from its contents and then runs the
|
|
4
|
+
reader for that format, so a file's extension protects nothing: a PostScript
|
|
5
|
+
file named ``page.png`` reaches the PostScript reader. This module looks at
|
|
6
|
+
the same bytes first, and names the format, so that the check can refuse by
|
|
7
|
+
name before any reader runs.
|
|
8
|
+
|
|
9
|
+
The table names more formats than the library will ever pass. The extra
|
|
10
|
+
names exist for the reason text: "says PNG, is PostScript" tells an operator
|
|
11
|
+
more than "says PNG, is something else".
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
HEAD_BYTES = 1024
|
|
17
|
+
"""How much of a file :func:`sniff` needs. The longest signature here is
|
|
18
|
+
twelve bytes; the rest is slack for formats that put their mark later."""
|
|
19
|
+
|
|
20
|
+
# (name, offset, signature); checked in order, first match wins.
|
|
21
|
+
_SIGNATURES: tuple[tuple[str, int, bytes], ...] = (
|
|
22
|
+
("jpeg", 0, b"\xff\xd8\xff"),
|
|
23
|
+
("png", 0, b"\x89PNG\r\n\x1a\n"),
|
|
24
|
+
("tiff", 0, b"II*\x00"),
|
|
25
|
+
("tiff", 0, b"MM\x00*"),
|
|
26
|
+
("bigtiff", 0, b"II+\x00"),
|
|
27
|
+
("bigtiff", 0, b"MM\x00+"),
|
|
28
|
+
("pdf", 0, b"%PDF-"),
|
|
29
|
+
("postscript", 0, b"%!PS"),
|
|
30
|
+
("postscript", 0, b"\xc5\xd0\xd3\xc6"), # DOS EPS binary header
|
|
31
|
+
("gif", 0, b"GIF87a"),
|
|
32
|
+
("gif", 0, b"GIF89a"),
|
|
33
|
+
("bmp", 0, b"BM"),
|
|
34
|
+
("ole", 0, b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"), # FlashPix, old Office
|
|
35
|
+
("psd", 0, b"8BPS"),
|
|
36
|
+
("jpeg2000", 0, b"\x00\x00\x00\x0cjP \r\n\x87\n"),
|
|
37
|
+
("jpeg2000", 0, b"\xff\x4f\xff\x51"),
|
|
38
|
+
("zip", 0, b"PK\x03\x04"),
|
|
39
|
+
("gzip", 0, b"\x1f\x8b"),
|
|
40
|
+
("ico", 0, b"\x00\x00\x01\x00"),
|
|
41
|
+
("xml", 0, b"<?xml"),
|
|
42
|
+
("xml", 0, b"<svg"),
|
|
43
|
+
("pcx", 0, b"\x0a"),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
_DESCRIPTIONS: dict[str, str] = {
|
|
47
|
+
"jpeg": "JPEG",
|
|
48
|
+
"png": "PNG",
|
|
49
|
+
"tiff": "TIFF",
|
|
50
|
+
"bigtiff": "BigTIFF",
|
|
51
|
+
"pdf": "PDF",
|
|
52
|
+
"postscript": "PostScript",
|
|
53
|
+
"gif": "GIF",
|
|
54
|
+
"bmp": "BMP",
|
|
55
|
+
"ole": "an OLE compound file (FlashPix, or an old Office document)",
|
|
56
|
+
"psd": "a Photoshop document",
|
|
57
|
+
"jpeg2000": "JPEG 2000",
|
|
58
|
+
"webp": "WebP",
|
|
59
|
+
"heif": "HEIF or AVIF",
|
|
60
|
+
"zip": "a zip archive",
|
|
61
|
+
"gzip": "a gzip file",
|
|
62
|
+
"ico": "a Windows icon",
|
|
63
|
+
"xml": "XML (or SVG)",
|
|
64
|
+
"pcx": "PCX",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
_CLAIMS: dict[str, str] = {
|
|
68
|
+
".jpg": "jpeg",
|
|
69
|
+
".jpeg": "jpeg",
|
|
70
|
+
".jpe": "jpeg",
|
|
71
|
+
".jfif": "jpeg",
|
|
72
|
+
".png": "png",
|
|
73
|
+
".tif": "tiff",
|
|
74
|
+
".tiff": "tiff",
|
|
75
|
+
".pdf": "pdf",
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def sniff(head: bytes) -> str | None:
|
|
80
|
+
"""Name the format of a file from its first bytes, or ``None`` if unknown.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
head: The first :data:`HEAD_BYTES` of the file (fewer, if the file is
|
|
84
|
+
shorter).
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
A lower-case name: one of :data:`sillybos.FORMATS`, one of the other
|
|
88
|
+
names the library knows only to refuse, or ``None``.
|
|
89
|
+
"""
|
|
90
|
+
for name, offset, signature in _SIGNATURES:
|
|
91
|
+
if head[offset : offset + len(signature)] == signature:
|
|
92
|
+
return name
|
|
93
|
+
# RIFF containers and ISO base media files carry their type a few bytes in.
|
|
94
|
+
if head[:4] == b"RIFF" and head[8:12] == b"WEBP":
|
|
95
|
+
return "webp"
|
|
96
|
+
if head[4:8] == b"ftyp":
|
|
97
|
+
brand = head[8:12]
|
|
98
|
+
if brand in {b"heic", b"heix", b"hevc", b"hevx", b"mif1", b"msf1", b"avif", b"avis"}:
|
|
99
|
+
return "heif"
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def describe(name: str | None) -> str:
|
|
104
|
+
"""The format's name for a reason sentence: ``"PostScript"``, ``"a zip archive"``."""
|
|
105
|
+
if name is None:
|
|
106
|
+
return "an unknown format"
|
|
107
|
+
return _DESCRIPTIONS.get(name, name)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def claimed_by(suffix: str) -> str | None:
|
|
111
|
+
"""What a file's extension says it is, as a name from :data:`sillybos.FORMATS`.
|
|
112
|
+
|
|
113
|
+
``None`` for an extension the library does not map, including no
|
|
114
|
+
extension at all: such a file makes no claim, so it cannot lie.
|
|
115
|
+
"""
|
|
116
|
+
return _CLAIMS.get(suffix.lower())
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
FORMATS_KNOWN = frozenset(_DESCRIPTIONS)
|
|
120
|
+
"""Every name sniff() can return, for telling "not allowed" from "unknown"."""
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sillybos
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Decide before decoding: check that a file is a JPEG, PNG, TIFF or PDF within your limits before any image library or PDF renderer opens it
|
|
5
|
+
Project-URL: Homepage, https://github.com/scattercode/sillybos
|
|
6
|
+
Project-URL: Source, https://github.com/scattercode/sillybos
|
|
7
|
+
Project-URL: Changelog, https://github.com/scattercode/sillybos/blob/main/CHANGELOG.md
|
|
8
|
+
Project-URL: Issues, https://github.com/scattercode/sillybos/issues
|
|
9
|
+
Project-URL: Tetrak, https://tetrak.dev/
|
|
10
|
+
Author: Stephen Masters, Yvette Mankerian
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: archives,decompression-bomb,file-validation,image,libraries,ocr,pdf,pillow,poppler,scans,security
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Information Technology
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
25
|
+
Classifier: Topic :: Security
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.11
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
30
|
+
Requires-Dist: pillow>=13.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
34
|
+
Provides-Extra: pillow
|
|
35
|
+
Requires-Dist: pillow>=13.0; extra == 'pillow'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# sillybos
|
|
39
|
+
|
|
40
|
+
<img src="https://raw.githubusercontent.com/scattercode/sillybos/main/docs/logo.png" alt="A rolled scroll with a small parchment tag hanging from it, reading: metadata, caps and format vetted" width="280" align="right">
|
|
41
|
+
|
|
42
|
+
**Decide before decoding.** A σίλλυβος (*sillybos*; Latin *sittybus*) was
|
|
43
|
+
the little parchment tag that hung from a rolled scroll in an ancient
|
|
44
|
+
library, carrying its title and author, so that a reader could tell what a
|
|
45
|
+
scroll was without unrolling it. Cicero had them made for his books. This
|
|
46
|
+
library is that tag for the files a scanning or OCR pipeline is handed: it
|
|
47
|
+
reads the tag, and says whether the scroll should be unrolled at all.
|
|
48
|
+
|
|
49
|
+
It answers one narrow question, before any image library or PDF renderer
|
|
50
|
+
opens a file: *is this one of the formats I mean to read, within the limits
|
|
51
|
+
I expect, and if not, why?* It names the format from the file's first bytes,
|
|
52
|
+
whatever the file is called; reads the header of a JPEG, PNG or TIFF with a
|
|
53
|
+
few dozen lines of its own, never a decoder's; asks poppler's `pdfinfo`, in
|
|
54
|
+
a separate process with a time limit, how many pages a PDF has and how large
|
|
55
|
+
they are; and refuses, in a plain sentence, anything outside the policy you
|
|
56
|
+
give it.
|
|
57
|
+
|
|
58
|
+
> **Status: alpha.** The API is small and may still move. It is in use in
|
|
59
|
+
> [Tetrak OCR](https://tetrak.dev/), which is where it came from.
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install sillybos # the check; no dependencies
|
|
63
|
+
pip install 'sillybos[pillow]' # plus open_checked(), which opens the file with Pillow
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
A PDF needs `pdfinfo` from poppler on the PATH (`brew install poppler`,
|
|
67
|
+
`apt install poppler-utils`). Anything that renders PDFs in Python already
|
|
68
|
+
has it.
|
|
69
|
+
|
|
70
|
+
## Why
|
|
71
|
+
|
|
72
|
+
Every program that reads scans hands each file to a decoder: Pillow for
|
|
73
|
+
images, poppler for PDFs, then an OCR engine. Decoders are large, old
|
|
74
|
+
programs that read complicated formats, and they have flaws. In October
|
|
75
|
+
2026, Pillow 13.0.0 fixed four serious ones at once. One of them
|
|
76
|
+
(CVE-2026-112900) ran commands on the computer when an EPS file was opened
|
|
77
|
+
and Ghostscript was installed; and a file's extension was no protection,
|
|
78
|
+
because Pillow decides what a file is from its contents, so a PostScript
|
|
79
|
+
file named `page.png` went straight to the PostScript reader.
|
|
80
|
+
|
|
81
|
+
The fixes to Pillow arrived; the lesson stays. A pipeline that opens files
|
|
82
|
+
it did not make should decide, from the file's own bytes and a policy of
|
|
83
|
+
its own, whether a decoder may have it, and should do that deciding with
|
|
84
|
+
code that is not itself a decoder.
|
|
85
|
+
|
|
86
|
+
## Use
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
>>> from sillybos import check
|
|
90
|
+
>>> verdict = check("scans/page-012.png")
|
|
91
|
+
>>> verdict.ok
|
|
92
|
+
False
|
|
93
|
+
>>> verdict.reasons
|
|
94
|
+
('says PNG, is PostScript',)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
>>> verdict = check("scans/register.tif")
|
|
99
|
+
>>> verdict.ok, verdict.format, verdict.facts
|
|
100
|
+
(True, 'tiff', Facts(bytes=48213990, width=4960, height=7016, pages=32, max_pixels=34799360, encrypted=False))
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`check` never raises for anything found *in* the file. A lying name, a
|
|
104
|
+
malformed header, a page too large and a thousand-page PDF are all reasons,
|
|
105
|
+
not exceptions, and a verdict carries every reason it found. `require` is
|
|
106
|
+
the same check for callers who would rather catch one exception:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from sillybos import require, Refused
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
require(path)
|
|
113
|
+
except Refused as refusal:
|
|
114
|
+
quarantine(path, refusal.verdict.reasons)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### The policy
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from sillybos import Policy, DEFAULT, STRICT, check
|
|
121
|
+
|
|
122
|
+
check(path) # DEFAULT: a trusted laptop, your own scanner
|
|
123
|
+
check(path, STRICT) # files from outside: smaller and shorter
|
|
124
|
+
check(path, Policy(formats={"tiff", "pdf"}, max_pages=50))
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
| Setting | `DEFAULT` | `STRICT` | What it bounds |
|
|
128
|
+
|---|---|---|---|
|
|
129
|
+
| `formats` | jpeg, png, tiff, pdf | the same | What may pass at all. Nothing else is ever allowed, whatever the policy says |
|
|
130
|
+
| `max_bytes` | 512 MiB | 128 MiB | The file. Checked before anything reads it |
|
|
131
|
+
| `max_pixels` | 89,478,485 | 50,000,000 | Any one page or frame, decoded. For a PDF page, at `render_dpi`. The default is Pillow's own decompression-bomb threshold, enforced here rather than left to a global |
|
|
132
|
+
| `max_pages` | 500 | 100 | Pages of a PDF, frames of a TIFF or animated PNG. Also bounds the header walk itself |
|
|
133
|
+
| `render_dpi` | 200 | 200 | What you will render PDF pages at, so a page's pixels are known before the renderer allocates them |
|
|
134
|
+
| `name_must_match` | yes | yes | Refuse a JPEG called `.png`. The readers would cope; the lie is what you want to know about |
|
|
135
|
+
| `allow_encrypted_pdf` | no | no | A scanned book is never encrypted |
|
|
136
|
+
| `pdfinfo_timeout` | 30 s | 30 s | How long `pdfinfo` may take before the file is refused |
|
|
137
|
+
|
|
138
|
+
The defaults were set from a real corpus of archival scans (largest page 39
|
|
139
|
+
million pixels at 600 dpi; longest book 166 pages) with room to spare. Set
|
|
140
|
+
yours the same way: from the largest real file you have, times two or three.
|
|
141
|
+
|
|
142
|
+
### With Pillow
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from sillybos.pillow import open_checked
|
|
146
|
+
|
|
147
|
+
with open_checked(
|
|
148
|
+
path
|
|
149
|
+
) as image: # checked first; then Image.open(path, formats=["JPEG", "PNG", "TIFF"])
|
|
150
|
+
...
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`Image.open` takes a `formats` list, and a file outside it is never handed
|
|
154
|
+
to the other readers. `open_checked` runs the check, then opens with that
|
|
155
|
+
list set from the policy. That one argument is the whole of the October fix
|
|
156
|
+
for images, and it costs nothing; the check in front of it is what makes
|
|
157
|
+
the limits bite before Pillow allocates a pixel.
|
|
158
|
+
|
|
159
|
+
### The command
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
$ sillybos scans/*
|
|
163
|
+
ok scans/page-001.tif (TIFF, 4960x7016, 1 page, 1.6 MB)
|
|
164
|
+
refused scans/page-002.png says PNG, is PostScript
|
|
165
|
+
refused scans/poster.jpg a page of 1,200,000,000 pixels; the limit is 89,478,485
|
|
166
|
+
refused scans/report.pdf 1,412 pages; the limit is 500
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Exit status 0 when every file passed, 1 when any was refused, 2 when the
|
|
170
|
+
command could not do its job. `--strict`, `--formats`, `--max-pages`,
|
|
171
|
+
`--max-pixels`, `--max-bytes` and `--any-name` adjust the policy; `--json`
|
|
172
|
+
prints one object per line for a pipeline; `-q` prints only refusals.
|
|
173
|
+
|
|
174
|
+
## What it does, exactly
|
|
175
|
+
|
|
176
|
+
1. **Size**, from the file system. Over the limit, nothing reads the file
|
|
177
|
+
but the next step.
|
|
178
|
+
2. **Format, from the first bytes.** JPEG, PNG, TIFF and PDF are the four
|
|
179
|
+
it can pass. It also knows PostScript, GIF, BMP, WebP, HEIF, JPEG 2000,
|
|
180
|
+
BigTIFF, Photoshop, OLE (FlashPix, old Office), zip, gzip, icons and
|
|
181
|
+
XML, so that a refusal can say *what* the file is. Anything else is "an
|
|
182
|
+
unknown format". Refused by name, and no reader of ours runs on it.
|
|
183
|
+
3. **Name against contents.** `says PNG, is JPEG`.
|
|
184
|
+
4. **The header**, with a reader of its own for each raster format: the
|
|
185
|
+
dimensions from a JPEG's frame header, a PNG's `IHDR` (and `acTL` frame
|
|
186
|
+
count), and every directory in a TIFF's chain, which is walked with a
|
|
187
|
+
limit and a memory of where it has been, so that a chain that loops or
|
|
188
|
+
points off the end of the file is a refusal, not a hang. Every read is
|
|
189
|
+
bounded and every offset is checked against the file's size. For a PDF,
|
|
190
|
+
`pdfinfo` reports the page count, whether it is encrypted and, if the
|
|
191
|
+
count is within the limit, every page's size.
|
|
192
|
+
5. **The limits**, against what the header said.
|
|
193
|
+
|
|
194
|
+
## What it does not do
|
|
195
|
+
|
|
196
|
+
Say this plainly, because a security library carries an implied promise.
|
|
197
|
+
|
|
198
|
+
- **It does not protect against a flaw in the readers you *do* allow.** A
|
|
199
|
+
crafted JPEG that exploits a bug in your JPEG decoder passes this check,
|
|
200
|
+
because it is a JPEG of a sensible size. What this library removes is the
|
|
201
|
+
attack surface of every format you did not mean to read, and the whole
|
|
202
|
+
class of files that are too big to be safe. For the readers you keep:
|
|
203
|
+
update them, and run them in a boxed process with memory and time
|
|
204
|
+
limits (a recipe for that is planned; it is deliberately not a
|
|
205
|
+
dependency).
|
|
206
|
+
- **It does not scan for malware.** ClamAV does that, for malware that has
|
|
207
|
+
been seen before.
|
|
208
|
+
- **It does not repair, re-encode or "sanitise" a file.** That needs a
|
|
209
|
+
decoder, and the moment this library decoded anything it would become
|
|
210
|
+
the kind of code it exists to keep away from untrusted files.
|
|
211
|
+
- **It does not look inside a PDF beyond what `pdfinfo` reports.** A PDF's
|
|
212
|
+
structure cannot be read from a fixed header; it has to be parsed, and
|
|
213
|
+
the parser this library trusts is poppler's, in its own process, with a
|
|
214
|
+
timeout. Checks for JavaScript, launch actions, embedded files and forms
|
|
215
|
+
(things a scanned book never has) are the next step.
|
|
216
|
+
- **It does not validate uploads.** File names, path traversal and zip
|
|
217
|
+
bombs belong to a web framework's upload handling. This library starts
|
|
218
|
+
after the file is on disk and asks only whether a decoder may open it.
|
|
219
|
+
- **It does not read formats beyond the four.** WebP, JPEG 2000, HEIC and
|
|
220
|
+
the rest are refused by name until someone shows a scan in one. Each
|
|
221
|
+
format added is attack surface added.
|
|
222
|
+
|
|
223
|
+
## Development
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
uv venv .venv && source .venv/bin/activate
|
|
227
|
+
uv pip install -e '.[dev]'
|
|
228
|
+
ruff check . && ruff format --check . && mypy && pytest --cov
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
The tests build the hostile corpus by hand at test time (`tests/corpus.py`):
|
|
232
|
+
renamed PostScript, a 100-byte PNG that claims ten billion pixels, a TIFF
|
|
233
|
+
with ten thousand frames, a TIFF whose frame chain loops, a PDF of a
|
|
234
|
+
thousand pages, a PDF with a page 200 inches wide. Nothing hostile is
|
|
235
|
+
checked in. The readers are also checked against Pillow's reading of real
|
|
236
|
+
files, so that they agree with the decoder they stand in front of.
|
|
237
|
+
|
|
238
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the checks and the commit style,
|
|
239
|
+
and [SECURITY.md](SECURITY.md) for how to report a flaw privately.
|
|
240
|
+
|
|
241
|
+
## Credits
|
|
242
|
+
|
|
243
|
+
Made by [Stephen Masters](https://scattercode.dev) and Yvette Mankerian as
|
|
244
|
+
part of [Tetrak](https://tetrak.dev/), an OCR and catalogue-search toolkit
|
|
245
|
+
for archives, where the check sits in front of every scan. MIT licensed.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
sillybos/__init__.py,sha256=5ls6TUJg91yFOvJ6V5V4uxUtU4hjpFeGcu47TBQTRls,1814
|
|
2
|
+
sillybos/_version.py,sha256=n_5vdJsPNu7wZ57LGuRL585uvll-hiuvZUBWzdG0RQU,520
|
|
3
|
+
sillybos/check.py,sha256=A8Qc_eU5A4ULR9Yj72jyeJ2J3mtacel5GOdlwSB_61I,7973
|
|
4
|
+
sillybos/cli.py,sha256=hYfTPEtRXLMUT4xqIvUWCcWMjUqnbdDcpezLI0z79hQ,4017
|
|
5
|
+
sillybos/headers.py,sha256=1rP-yePk0LCUIhq33RbHPDlwxdG8du9zwvH6TWAHE10,9733
|
|
6
|
+
sillybos/pdf.py,sha256=UWXXGHHOy2i5ywTdVGsudhUzjh_ZuSSKPnypC2aL4s4,4835
|
|
7
|
+
sillybos/pillow.py,sha256=mlYGHuAnY265jEYq5KjpcDSdcGJGY7vfsb_MRuQDNeY,1861
|
|
8
|
+
sillybos/policy.py,sha256=D1EIcFZNutwF7efMsdhzG-d6toUGn5bStiZrRwI9uT8,3940
|
|
9
|
+
sillybos/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
sillybos/sniff.py,sha256=BUOAa1L9UvHC-dHKVPzi7j45wB42xNQ9n9t804C2vKg,3924
|
|
11
|
+
sillybos-0.1.0.dist-info/METADATA,sha256=mMblXYIb64-9eY7-6m-Dyh75A-EH0JkETY0z0DkY0DE,11471
|
|
12
|
+
sillybos-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
13
|
+
sillybos-0.1.0.dist-info/entry_points.txt,sha256=0IFtaDvnaSup0uBvFMwA8-f1hI3L7NPOUmEInKHdUwU,47
|
|
14
|
+
sillybos-0.1.0.dist-info/licenses/LICENSE,sha256=C9aF2rOOTEyD4yzG0tzy-QZcuWfUVZySFtIN0lCBm10,1093
|
|
15
|
+
sillybos-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Stephen Masters and Yvette Mankerian
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|