uploadshield 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uploadshield/__init__.py +49 -0
- uploadshield/__main__.py +17 -0
- uploadshield/checks/__init__.py +65 -0
- uploadshield/checks/archive.py +316 -0
- uploadshield/checks/basic.py +476 -0
- uploadshield/checks/image.py +107 -0
- uploadshield/checks/magika.py +129 -0
- uploadshield/checks/markup.py +181 -0
- uploadshield/checks/office.py +174 -0
- uploadshield/checks/pdf.py +277 -0
- uploadshield/checks/scanners.py +205 -0
- uploadshield/checks/xml.py +123 -0
- uploadshield/cli/__init__.py +276 -0
- uploadshield/cli/_commands.py +401 -0
- uploadshield/cli/_json.py +80 -0
- uploadshield/cli/_render.py +319 -0
- uploadshield/cli/_scan.py +106 -0
- uploadshield/cli/_style.py +95 -0
- uploadshield/cli/_verdicts.py +363 -0
- uploadshield/cli/_walk.py +159 -0
- uploadshield/core/__init__.py +4 -0
- uploadshield/core/_deconstruct.py +39 -0
- uploadshield/core/check.py +79 -0
- uploadshield/core/containers.py +131 -0
- uploadshield/core/result.py +63 -0
- uploadshield/core/sniffing.py +432 -0
- uploadshield/core/types.py +319 -0
- uploadshield/core/upload.py +192 -0
- uploadshield/core/violations.py +79 -0
- uploadshield/django/__init__.py +36 -0
- uploadshield/django/apps.py +13 -0
- uploadshield/django/checks.py +148 -0
- uploadshield/django/cli.py +36 -0
- uploadshield/django/conf.py +189 -0
- uploadshield/django/decorators.py +204 -0
- uploadshield/django/exceptions.py +39 -0
- uploadshield/django/forms.py +40 -0
- uploadshield/django/handlers.py +322 -0
- uploadshield/django/responses.py +92 -0
- uploadshield/django/service.py +473 -0
- uploadshield/django/signals.py +16 -0
- uploadshield/django/storage.py +204 -0
- uploadshield/django/templates/uploadshield/rejected.html +15 -0
- uploadshield/django/validators.py +48 -0
- uploadshield/django/views.py +40 -0
- uploadshield/engine.py +230 -0
- uploadshield/fastapi/__init__.py +23 -0
- uploadshield/fastapi/shield.py +299 -0
- uploadshield/flask/__init__.py +42 -0
- uploadshield/flask/core.py +359 -0
- uploadshield/flask/decorators.py +69 -0
- uploadshield/flask/wtforms.py +58 -0
- uploadshield/policy.py +137 -0
- uploadshield/presets.py +65 -0
- uploadshield/py.typed +0 -0
- uploadshield/sanitizers.py +132 -0
- uploadshield/web.py +218 -0
- uploadshield-2.0.0.dist-info/METADATA +239 -0
- uploadshield-2.0.0.dist-info/RECORD +62 -0
- uploadshield-2.0.0.dist-info/WHEEL +4 -0
- uploadshield-2.0.0.dist-info/entry_points.txt +3 -0
- uploadshield-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"""A deterministic registry of the file types this library can reason about.
|
|
2
|
+
|
|
3
|
+
The standard library's :mod:`mimetypes` reads ``/etc/mime.types`` and the Windows
|
|
4
|
+
registry, so its answers differ between machines. Validation results must not depend
|
|
5
|
+
on the host, so every type we detect or allow is declared here explicitly.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import threading
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import TYPE_CHECKING
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from collections.abc import Iterable, Iterator
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"GB",
|
|
19
|
+
"KB",
|
|
20
|
+
"MB",
|
|
21
|
+
"FileType",
|
|
22
|
+
"GiB",
|
|
23
|
+
"KiB",
|
|
24
|
+
"MiB",
|
|
25
|
+
"all_types",
|
|
26
|
+
"get_type",
|
|
27
|
+
"register_type",
|
|
28
|
+
"resolve_mime",
|
|
29
|
+
"types_for_extension",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
KB = 1000
|
|
33
|
+
MB = 1000 * KB
|
|
34
|
+
GB = 1000 * MB
|
|
35
|
+
KiB = 1024
|
|
36
|
+
MiB = 1024 * KiB
|
|
37
|
+
GiB = 1024 * MiB
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True, slots=True)
|
|
41
|
+
class FileType:
|
|
42
|
+
"""A file format identified by its canonical MIME type.
|
|
43
|
+
|
|
44
|
+
``extensions`` are lowercase, start with a dot and the first one is canonical.
|
|
45
|
+
``aliases`` are non-canonical MIME strings that clients commonly send.
|
|
46
|
+
``container`` names the physical container (``"zip"``/``"ole"``) so that
|
|
47
|
+
container-level checks can apply to every format built on it.
|
|
48
|
+
``parent`` marks a *refinement*: content can prove a file is ``text/plain`` but not
|
|
49
|
+
that it is CSV, so a ``.csv`` name may narrow ``text/plain`` to ``text/csv``.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
mime: str
|
|
53
|
+
extensions: tuple[str, ...]
|
|
54
|
+
label: str
|
|
55
|
+
aliases: tuple[str, ...] = ()
|
|
56
|
+
container: str | None = None
|
|
57
|
+
is_text: bool = False
|
|
58
|
+
parent: str | None = None
|
|
59
|
+
executable: bool = field(default=False, compare=False)
|
|
60
|
+
|
|
61
|
+
def __post_init__(self) -> None:
|
|
62
|
+
if self.mime != self.mime.lower() or "/" not in self.mime:
|
|
63
|
+
raise ValueError(f"MIME type must be lowercase 'type/subtype': {self.mime!r}")
|
|
64
|
+
for ext in self.extensions:
|
|
65
|
+
if not ext.startswith(".") or ext != ext.lower():
|
|
66
|
+
raise ValueError(f"Extensions must be lowercase and start with '.': {ext!r}")
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def category(self) -> str:
|
|
70
|
+
"""The top-level media type, e.g. ``"image"`` for ``image/png``."""
|
|
71
|
+
return self.mime.split("/", 1)[0]
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def canonical_extension(self) -> str:
|
|
75
|
+
return self.extensions[0] if self.extensions else ""
|
|
76
|
+
|
|
77
|
+
def matches(self, mime: str) -> bool:
|
|
78
|
+
"""Whether ``mime`` (canonical or alias, parameters ignored) denotes this type."""
|
|
79
|
+
bare = normalize_mime(mime)
|
|
80
|
+
return bare == self.mime or bare in self.aliases
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def normalize_mime(mime: str) -> str:
|
|
84
|
+
"""Strip parameters and whitespace and lowercase a MIME string."""
|
|
85
|
+
return mime.split(";", 1)[0].strip().lower()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def format_size(num_bytes: int) -> str:
|
|
89
|
+
"""Human-readable decimal size: ``format_size(2_500_000) == "2.5 MB"``."""
|
|
90
|
+
value = float(num_bytes)
|
|
91
|
+
for unit in ("bytes", "KB", "MB", "GB"):
|
|
92
|
+
if value < 1000 or unit == "GB":
|
|
93
|
+
return f"{int(value)} {unit}" if unit == "bytes" else f"{value:.3g} {unit}"
|
|
94
|
+
value /= 1000
|
|
95
|
+
raise AssertionError("unreachable") # pragma: no cover
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
_OOXML = "application/vnd.openxmlformats-officedocument"
|
|
99
|
+
_ODF = "application/vnd.oasis.opendocument"
|
|
100
|
+
|
|
101
|
+
_BUILTIN_TYPES: tuple[FileType, ...] = (
|
|
102
|
+
# --- raster & vector images -------------------------------------------------------
|
|
103
|
+
FileType(
|
|
104
|
+
"image/jpeg",
|
|
105
|
+
(".jpg", ".jpeg", ".jpe", ".jfif"),
|
|
106
|
+
"JPEG image",
|
|
107
|
+
aliases=("image/jpg", "image/pjpeg"),
|
|
108
|
+
),
|
|
109
|
+
FileType("image/png", (".png",), "PNG image", aliases=("image/x-png",)),
|
|
110
|
+
FileType("image/gif", (".gif",), "GIF image"),
|
|
111
|
+
FileType("image/webp", (".webp",), "WebP image"),
|
|
112
|
+
FileType("image/bmp", (".bmp", ".dib"), "BMP image", aliases=("image/x-ms-bmp", "image/x-bmp")),
|
|
113
|
+
FileType("image/tiff", (".tif", ".tiff"), "TIFF image"),
|
|
114
|
+
FileType("image/vnd.microsoft.icon", (".ico",), "ICO icon", aliases=("image/x-icon",)),
|
|
115
|
+
FileType("image/avif", (".avif",), "AVIF image"),
|
|
116
|
+
FileType("image/heic", (".heic", ".heif"), "HEIC image", aliases=("image/heif",)),
|
|
117
|
+
FileType("image/svg+xml", (".svg",), "SVG image", is_text=True),
|
|
118
|
+
# --- documents --------------------------------------------------------------------
|
|
119
|
+
FileType("application/pdf", (".pdf",), "PDF document", aliases=("application/x-pdf",)),
|
|
120
|
+
FileType(f"{_OOXML}.wordprocessingml.document", (".docx",), "Word document", container="zip"),
|
|
121
|
+
FileType(f"{_OOXML}.spreadsheetml.sheet", (".xlsx",), "Excel workbook", container="zip"),
|
|
122
|
+
FileType(
|
|
123
|
+
f"{_OOXML}.presentationml.presentation",
|
|
124
|
+
(".pptx",),
|
|
125
|
+
"PowerPoint presentation",
|
|
126
|
+
container="zip",
|
|
127
|
+
),
|
|
128
|
+
FileType(
|
|
129
|
+
"application/vnd.ms-word.document.macroenabled.12",
|
|
130
|
+
(".docm",),
|
|
131
|
+
"Word macro-enabled document",
|
|
132
|
+
container="zip",
|
|
133
|
+
),
|
|
134
|
+
FileType(
|
|
135
|
+
"application/vnd.ms-excel.sheet.macroenabled.12",
|
|
136
|
+
(".xlsm",),
|
|
137
|
+
"Excel macro-enabled workbook",
|
|
138
|
+
container="zip",
|
|
139
|
+
),
|
|
140
|
+
FileType(
|
|
141
|
+
"application/vnd.ms-powerpoint.presentation.macroenabled.12",
|
|
142
|
+
(".pptm",),
|
|
143
|
+
"PowerPoint macro-enabled presentation",
|
|
144
|
+
container="zip",
|
|
145
|
+
),
|
|
146
|
+
FileType(f"{_ODF}.text", (".odt",), "OpenDocument text", container="zip"),
|
|
147
|
+
FileType(f"{_ODF}.spreadsheet", (".ods",), "OpenDocument spreadsheet", container="zip"),
|
|
148
|
+
FileType(f"{_ODF}.presentation", (".odp",), "OpenDocument presentation", container="zip"),
|
|
149
|
+
FileType("application/epub+zip", (".epub",), "EPUB e-book", container="zip"),
|
|
150
|
+
FileType(
|
|
151
|
+
"application/x-ole-storage",
|
|
152
|
+
(".doc", ".xls", ".ppt", ".msg"),
|
|
153
|
+
"OLE2 compound document (legacy Office)",
|
|
154
|
+
container="ole",
|
|
155
|
+
aliases=(
|
|
156
|
+
"application/msword",
|
|
157
|
+
"application/vnd.ms-excel",
|
|
158
|
+
"application/vnd.ms-powerpoint",
|
|
159
|
+
"application/vnd.ms-outlook",
|
|
160
|
+
),
|
|
161
|
+
),
|
|
162
|
+
FileType("application/rtf", (".rtf",), "Rich Text Format", aliases=("text/rtf",)),
|
|
163
|
+
FileType("application/postscript", (".ps", ".eps"), "PostScript"),
|
|
164
|
+
# --- text -------------------------------------------------------------------------
|
|
165
|
+
FileType("text/plain", (".txt", ".text", ".log"), "Plain text", is_text=True),
|
|
166
|
+
FileType(
|
|
167
|
+
"text/csv",
|
|
168
|
+
(".csv",),
|
|
169
|
+
"CSV",
|
|
170
|
+
is_text=True,
|
|
171
|
+
parent="text/plain",
|
|
172
|
+
aliases=("application/csv",),
|
|
173
|
+
),
|
|
174
|
+
FileType("text/tab-separated-values", (".tsv",), "TSV", is_text=True, parent="text/plain"),
|
|
175
|
+
FileType("text/markdown", (".md", ".markdown"), "Markdown", is_text=True, parent="text/plain"),
|
|
176
|
+
FileType("application/json", (".json",), "JSON", is_text=True, parent="text/plain"),
|
|
177
|
+
FileType("text/html", (".html", ".htm", ".xhtml"), "HTML document", is_text=True),
|
|
178
|
+
FileType("application/xml", (".xml",), "XML document", is_text=True, aliases=("text/xml",)),
|
|
179
|
+
FileType(
|
|
180
|
+
"application/x-sh",
|
|
181
|
+
(".sh",),
|
|
182
|
+
"Shell script",
|
|
183
|
+
is_text=True,
|
|
184
|
+
aliases=("text/x-shellscript",),
|
|
185
|
+
executable=True,
|
|
186
|
+
),
|
|
187
|
+
FileType(
|
|
188
|
+
"application/x-httpd-php",
|
|
189
|
+
(".php",),
|
|
190
|
+
"PHP script",
|
|
191
|
+
is_text=True,
|
|
192
|
+
aliases=("application/x-php", "text/x-php"),
|
|
193
|
+
executable=True,
|
|
194
|
+
),
|
|
195
|
+
# --- archives ---------------------------------------------------------------------
|
|
196
|
+
FileType(
|
|
197
|
+
"application/zip",
|
|
198
|
+
(".zip",),
|
|
199
|
+
"ZIP archive",
|
|
200
|
+
container="zip",
|
|
201
|
+
aliases=("application/x-zip-compressed",),
|
|
202
|
+
),
|
|
203
|
+
FileType("application/gzip", (".gz", ".tgz"), "gzip archive", aliases=("application/x-gzip",)),
|
|
204
|
+
FileType("application/x-tar", (".tar",), "tar archive"),
|
|
205
|
+
FileType("application/x-bzip2", (".bz2",), "bzip2 archive"),
|
|
206
|
+
FileType("application/x-xz", (".xz",), "xz archive"),
|
|
207
|
+
FileType("application/x-7z-compressed", (".7z",), "7-Zip archive"),
|
|
208
|
+
FileType(
|
|
209
|
+
"application/vnd.rar", (".rar",), "RAR archive", aliases=("application/x-rar-compressed",)
|
|
210
|
+
),
|
|
211
|
+
FileType(
|
|
212
|
+
"application/java-archive", (".jar",), "Java archive", container="zip", executable=True
|
|
213
|
+
),
|
|
214
|
+
FileType(
|
|
215
|
+
"application/vnd.android.package-archive",
|
|
216
|
+
(".apk",),
|
|
217
|
+
"Android package",
|
|
218
|
+
container="zip",
|
|
219
|
+
executable=True,
|
|
220
|
+
),
|
|
221
|
+
# --- audio ------------------------------------------------------------------------
|
|
222
|
+
FileType("audio/mpeg", (".mp3",), "MP3 audio", aliases=("audio/mp3",)),
|
|
223
|
+
FileType(
|
|
224
|
+
"audio/wav", (".wav",), "WAV audio", aliases=("audio/x-wav", "audio/wave", "audio/vnd.wave")
|
|
225
|
+
),
|
|
226
|
+
FileType("audio/ogg", (".ogg", ".oga", ".opus"), "Ogg audio"),
|
|
227
|
+
FileType("audio/flac", (".flac",), "FLAC audio", aliases=("audio/x-flac",)),
|
|
228
|
+
FileType("audio/mp4", (".m4a",), "MPEG-4 audio", aliases=("audio/x-m4a",)),
|
|
229
|
+
FileType("audio/aac", (".aac",), "AAC audio"),
|
|
230
|
+
# --- video ------------------------------------------------------------------------
|
|
231
|
+
FileType("video/mp4", (".mp4", ".m4v"), "MPEG-4 video"),
|
|
232
|
+
FileType("video/quicktime", (".mov", ".qt"), "QuickTime video"),
|
|
233
|
+
FileType("video/webm", (".webm",), "WebM video"),
|
|
234
|
+
FileType("video/x-matroska", (".mkv",), "Matroska video"),
|
|
235
|
+
FileType("video/x-msvideo", (".avi",), "AVI video", aliases=("video/avi", "video/msvideo")),
|
|
236
|
+
FileType("video/mpeg", (".mpeg", ".mpg"), "MPEG video"),
|
|
237
|
+
FileType("video/3gpp", (".3gp",), "3GPP video"),
|
|
238
|
+
# --- executables & other binaries (detected so they can be named and refused) -----
|
|
239
|
+
FileType(
|
|
240
|
+
"application/vnd.microsoft.portable-executable",
|
|
241
|
+
(".exe", ".dll", ".sys", ".scr"),
|
|
242
|
+
"Windows executable",
|
|
243
|
+
aliases=("application/x-msdownload", "application/x-dosexec"),
|
|
244
|
+
executable=True,
|
|
245
|
+
),
|
|
246
|
+
FileType("application/x-elf", (".elf", ".so"), "ELF executable", executable=True),
|
|
247
|
+
FileType("application/x-mach-binary", (".dylib",), "Mach-O executable", executable=True),
|
|
248
|
+
FileType("application/java-vm", (".class",), "Java class file", executable=True),
|
|
249
|
+
FileType("application/wasm", (".wasm",), "WebAssembly module", executable=True),
|
|
250
|
+
FileType("application/vnd.sqlite3", (".sqlite", ".sqlite3", ".db"), "SQLite database"),
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class _Registry:
|
|
255
|
+
"""Thread-safe lookup tables built from the declared types."""
|
|
256
|
+
|
|
257
|
+
def __init__(self, types: Iterable[FileType]) -> None:
|
|
258
|
+
self._lock = threading.Lock()
|
|
259
|
+
self._by_mime: dict[str, FileType] = {}
|
|
260
|
+
self._by_alias: dict[str, FileType] = {}
|
|
261
|
+
self._by_ext: dict[str, list[FileType]] = {}
|
|
262
|
+
for ft in types:
|
|
263
|
+
self.register(ft)
|
|
264
|
+
|
|
265
|
+
def register(self, file_type: FileType) -> None:
|
|
266
|
+
with self._lock:
|
|
267
|
+
existing = self._by_mime.get(file_type.mime)
|
|
268
|
+
if existing is not None and existing != file_type:
|
|
269
|
+
raise ValueError(f"A different FileType is already registered for {file_type.mime}")
|
|
270
|
+
self._by_mime[file_type.mime] = file_type
|
|
271
|
+
for alias in file_type.aliases:
|
|
272
|
+
self._by_alias.setdefault(alias, file_type)
|
|
273
|
+
for ext in file_type.extensions:
|
|
274
|
+
bucket = self._by_ext.setdefault(ext, [])
|
|
275
|
+
if file_type not in bucket:
|
|
276
|
+
bucket.append(file_type)
|
|
277
|
+
|
|
278
|
+
def get(self, mime: str) -> FileType | None:
|
|
279
|
+
bare = normalize_mime(mime)
|
|
280
|
+
return self._by_mime.get(bare) or self._by_alias.get(bare)
|
|
281
|
+
|
|
282
|
+
def for_extension(self, ext: str) -> tuple[FileType, ...]:
|
|
283
|
+
return tuple(self._by_ext.get(ext.lower(), ()))
|
|
284
|
+
|
|
285
|
+
def __iter__(self) -> Iterator[FileType]:
|
|
286
|
+
return iter(tuple(self._by_mime.values()))
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
_registry = _Registry(_BUILTIN_TYPES)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def get_type(mime: str) -> FileType | None:
|
|
293
|
+
"""Look up a type by canonical MIME or alias (case and parameters ignored)."""
|
|
294
|
+
return _registry.get(mime)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def resolve_mime(mime: str) -> FileType:
|
|
298
|
+
"""Like :func:`get_type` but raise a helpful ``ValueError`` for unknown types."""
|
|
299
|
+
ft = _registry.get(mime)
|
|
300
|
+
if ft is None:
|
|
301
|
+
raise ValueError(
|
|
302
|
+
f"Unknown file type {mime!r}. Use a canonical MIME type such as 'image/jpeg', "
|
|
303
|
+
"a wildcard such as 'image/*', or register custom types with register_type()."
|
|
304
|
+
)
|
|
305
|
+
return ft
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def types_for_extension(ext: str) -> tuple[FileType, ...]:
|
|
309
|
+
"""All registered types that use ``ext`` (e.g. ``".jpg"``)."""
|
|
310
|
+
return _registry.for_extension(ext)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def all_types() -> tuple[FileType, ...]:
|
|
314
|
+
return tuple(_registry)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def register_type(file_type: FileType) -> None:
|
|
318
|
+
"""Register a custom type, e.g. one produced by a custom :class:`Sniffer`."""
|
|
319
|
+
_registry.register(file_type)
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Framework-agnostic representation of an uploaded file."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import io
|
|
7
|
+
import logging
|
|
8
|
+
import shutil
|
|
9
|
+
import tempfile
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from functools import cached_property
|
|
13
|
+
from typing import IO, TYPE_CHECKING, Any, cast
|
|
14
|
+
|
|
15
|
+
from .types import FileType, MiB, types_for_extension
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from collections.abc import Generator, Iterator
|
|
19
|
+
|
|
20
|
+
from .sniffing import Sniffer
|
|
21
|
+
|
|
22
|
+
__all__ = ["HEAD_SIZE", "ContentTooLargeError", "InspectedFile", "Upload"]
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger("uploadshield")
|
|
25
|
+
|
|
26
|
+
#: Bytes read from the start of a file for signature and text detection.
|
|
27
|
+
HEAD_SIZE = 8 * 1024
|
|
28
|
+
_CHUNK_SIZE = 1 * MiB
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ContentTooLargeError(Exception):
|
|
32
|
+
"""Raised when a check asks for more content than it is allowed to buffer."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True, slots=True)
|
|
36
|
+
class Upload:
|
|
37
|
+
"""The raw facts about an upload, as provided by the web framework.
|
|
38
|
+
|
|
39
|
+
``name`` and ``declared_type`` are client-controlled and therefore untrusted.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
stream: IO[bytes]
|
|
43
|
+
name: str
|
|
44
|
+
size: int | None = None
|
|
45
|
+
declared_type: str | None = None
|
|
46
|
+
field: str | None = None
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def from_bytes(
|
|
50
|
+
cls,
|
|
51
|
+
data: bytes,
|
|
52
|
+
name: str,
|
|
53
|
+
*,
|
|
54
|
+
declared_type: str | None = None,
|
|
55
|
+
field: str | None = None,
|
|
56
|
+
) -> Upload:
|
|
57
|
+
return cls(io.BytesIO(data), name, len(data), declared_type, field)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class InspectedFile:
|
|
61
|
+
"""A read-only, lazily analysed view over an :class:`Upload`.
|
|
62
|
+
|
|
63
|
+
Facts that several checks need (head bytes, detected type, hashes, container
|
|
64
|
+
listings) are computed at most once. The underlying stream position is restored
|
|
65
|
+
after every read so the application can still consume the file afterwards.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def __init__(self, upload: Upload, sniffer: Sniffer) -> None:
|
|
69
|
+
self.upload = upload
|
|
70
|
+
self.name = upload.name or ""
|
|
71
|
+
self.field = upload.field
|
|
72
|
+
self.declared_type = upload.declared_type
|
|
73
|
+
self._sniffer = sniffer
|
|
74
|
+
self._stream = _ensure_seekable(upload.stream)
|
|
75
|
+
self._owns_stream = self._stream is not upload.stream
|
|
76
|
+
self._origin = self._stream.tell()
|
|
77
|
+
self.size = upload.size if upload.size is not None else self._measure()
|
|
78
|
+
self._head = b""
|
|
79
|
+
#: Scratch space so checks can share expensive parse results.
|
|
80
|
+
self.cache: dict[str, Any] = {}
|
|
81
|
+
|
|
82
|
+
def __repr__(self) -> str:
|
|
83
|
+
return f"<InspectedFile {self.name!r} size={self.size}>"
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def stream(self) -> IO[bytes]:
|
|
87
|
+
"""The seekable stream (a private copy if the upload's stream was not seekable)."""
|
|
88
|
+
return self._stream
|
|
89
|
+
|
|
90
|
+
def close(self) -> None:
|
|
91
|
+
"""Release the private copy made for non-seekable streams, if any."""
|
|
92
|
+
if self._owns_stream:
|
|
93
|
+
self._stream.close()
|
|
94
|
+
|
|
95
|
+
# -- raw access -------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
@contextmanager
|
|
98
|
+
def open(self) -> Generator[IO[bytes], None, None]:
|
|
99
|
+
"""Yield the stream positioned at the start; restore the position afterwards."""
|
|
100
|
+
try:
|
|
101
|
+
self._stream.seek(0)
|
|
102
|
+
yield self._stream
|
|
103
|
+
finally:
|
|
104
|
+
self._stream.seek(self._origin)
|
|
105
|
+
|
|
106
|
+
def head(self, n: int = HEAD_SIZE) -> bytes:
|
|
107
|
+
if len(self._head) < n and len(self._head) < self.size:
|
|
108
|
+
with self.open() as fh:
|
|
109
|
+
self._head = fh.read(max(n, HEAD_SIZE))
|
|
110
|
+
return self._head[:n]
|
|
111
|
+
|
|
112
|
+
def tail(self, n: int) -> bytes:
|
|
113
|
+
with self.open() as fh:
|
|
114
|
+
fh.seek(max(0, self.size - n))
|
|
115
|
+
return fh.read(n)
|
|
116
|
+
|
|
117
|
+
def read(self, limit: int) -> bytes:
|
|
118
|
+
"""Return the whole content, refusing to buffer more than ``limit`` bytes."""
|
|
119
|
+
if self.size > limit:
|
|
120
|
+
raise ContentTooLargeError(f"{self.size} bytes exceeds the buffer limit of {limit}")
|
|
121
|
+
with self.open() as fh:
|
|
122
|
+
return fh.read(limit + 1)[:limit]
|
|
123
|
+
|
|
124
|
+
def iter_chunks(self, chunk_size: int = _CHUNK_SIZE) -> Iterator[bytes]:
|
|
125
|
+
with self.open() as fh:
|
|
126
|
+
while chunk := fh.read(chunk_size):
|
|
127
|
+
yield chunk
|
|
128
|
+
|
|
129
|
+
# -- derived facts ----------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
@cached_property
|
|
132
|
+
def extensions(self) -> tuple[str, ...]:
|
|
133
|
+
"""Every dot-separated suffix, lowercased: ``"a.tar.GZ"`` -> ``(".tar", ".gz")``."""
|
|
134
|
+
stem = self.name.lstrip(".")
|
|
135
|
+
parts = stem.split(".")[1:]
|
|
136
|
+
return tuple(f".{p.lower()}" for p in parts if p)
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def extension(self) -> str:
|
|
140
|
+
"""The final extension (``""`` if there is none)."""
|
|
141
|
+
return self.extensions[-1] if self.extensions else ""
|
|
142
|
+
|
|
143
|
+
@cached_property
|
|
144
|
+
def detected_type(self) -> FileType | None:
|
|
145
|
+
"""The type proven by the file's content, or ``None`` if unidentifiable.
|
|
146
|
+
|
|
147
|
+
A crashing sniffer is logged and treated as "unidentifiable", which every
|
|
148
|
+
allow-list rejects (fail closed).
|
|
149
|
+
"""
|
|
150
|
+
try:
|
|
151
|
+
return self._sniffer.sniff(self)
|
|
152
|
+
except Exception:
|
|
153
|
+
logger.exception("Sniffer %r crashed on %r", self._sniffer, self.name)
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
@cached_property
|
|
157
|
+
def file_type(self) -> FileType | None:
|
|
158
|
+
"""The detected type, narrowed by the extension where content cannot decide.
|
|
159
|
+
|
|
160
|
+
Content proves ``text/plain``; a ``.csv`` name may narrow it to ``text/csv``.
|
|
161
|
+
The extension can only ever *refine* the detected type, never contradict it.
|
|
162
|
+
"""
|
|
163
|
+
detected = self.detected_type
|
|
164
|
+
if detected is None or not self.extension:
|
|
165
|
+
return detected
|
|
166
|
+
for candidate in types_for_extension(self.extension):
|
|
167
|
+
if candidate.parent == detected.mime:
|
|
168
|
+
return candidate
|
|
169
|
+
return detected
|
|
170
|
+
|
|
171
|
+
@cached_property
|
|
172
|
+
def sha256(self) -> str:
|
|
173
|
+
digest = hashlib.sha256()
|
|
174
|
+
for chunk in self.iter_chunks():
|
|
175
|
+
digest.update(chunk)
|
|
176
|
+
return digest.hexdigest()
|
|
177
|
+
|
|
178
|
+
# -- internals --------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
def _measure(self) -> int:
|
|
181
|
+
end = self._stream.seek(0, io.SEEK_END)
|
|
182
|
+
self._stream.seek(self._origin)
|
|
183
|
+
return end
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _ensure_seekable(stream: IO[bytes]) -> IO[bytes]:
|
|
187
|
+
if stream.seekable():
|
|
188
|
+
return stream
|
|
189
|
+
spooled = tempfile.SpooledTemporaryFile(max_size=8 * MiB) # noqa: SIM115 - returned to caller
|
|
190
|
+
shutil.copyfileobj(stream, spooled)
|
|
191
|
+
spooled.seek(0)
|
|
192
|
+
return cast("IO[bytes]", spooled)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Structured reasons why a file was rejected."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
from types import MappingProxyType
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
|
|
13
|
+
__all__ = ["Code", "Violation"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Code(StrEnum):
|
|
17
|
+
"""Stable, machine-readable violation codes.
|
|
18
|
+
|
|
19
|
+
Codes are part of the public API: clients, log pipelines and translations key on
|
|
20
|
+
them. Custom checks may use their own string codes.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
FILE_EMPTY = "file_empty"
|
|
24
|
+
FILE_TOO_LARGE = "file_too_large"
|
|
25
|
+
FILENAME_INVALID = "filename_invalid"
|
|
26
|
+
FILENAME_TOO_LONG = "filename_too_long"
|
|
27
|
+
FILENAME_HIDDEN = "filename_hidden"
|
|
28
|
+
FILENAME_RESERVED = "filename_reserved"
|
|
29
|
+
FILENAME_DANGEROUS_EXTENSION = "filename_dangerous_extension"
|
|
30
|
+
FILENAME_MISSING_EXTENSION = "filename_missing_extension"
|
|
31
|
+
TYPE_UNKNOWN = "type_unknown"
|
|
32
|
+
TYPE_NOT_ALLOWED = "type_not_allowed"
|
|
33
|
+
EXTENSION_MISMATCH = "extension_mismatch"
|
|
34
|
+
CONTENT_MISMATCH = "content_mismatch"
|
|
35
|
+
DECLARED_TYPE_MISMATCH = "declared_type_mismatch"
|
|
36
|
+
IMAGE_CORRUPT = "image_corrupt"
|
|
37
|
+
IMAGE_TOO_LARGE = "image_too_large"
|
|
38
|
+
PDF_ACTIVE_CONTENT = "pdf_active_content"
|
|
39
|
+
ARCHIVE_CORRUPT = "archive_corrupt"
|
|
40
|
+
ARCHIVE_UNSAFE_PATH = "archive_unsafe_path"
|
|
41
|
+
ARCHIVE_BOMB = "archive_bomb"
|
|
42
|
+
ARCHIVE_ENCRYPTED = "archive_encrypted"
|
|
43
|
+
ARCHIVE_DANGEROUS_MEMBER = "archive_dangerous_member"
|
|
44
|
+
OFFICE_MACROS = "office_macros"
|
|
45
|
+
OFFICE_EXTERNAL_REFERENCE = "office_external_reference"
|
|
46
|
+
SVG_ACTIVE_CONTENT = "svg_active_content"
|
|
47
|
+
XML_DTD = "xml_dtd"
|
|
48
|
+
EMBEDDED_CODE = "embedded_code"
|
|
49
|
+
POLYGLOT = "polyglot"
|
|
50
|
+
MALWARE_DETECTED = "malware_detected"
|
|
51
|
+
SIGNATURE_MATCH = "signature_match"
|
|
52
|
+
SCANNER_UNAVAILABLE = "scanner_unavailable"
|
|
53
|
+
SANITIZATION_FAILED = "sanitization_failed"
|
|
54
|
+
CHECK_FAILED = "check_failed"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True, slots=True)
|
|
58
|
+
class Violation:
|
|
59
|
+
"""One reason a file failed validation.
|
|
60
|
+
|
|
61
|
+
``message`` is a short English sentence that is safe to show to the uploader.
|
|
62
|
+
``details`` carries diagnostics (detected type, matched signatures, ...) that are
|
|
63
|
+
meant for logs and must not be exposed to clients by default.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
code: str
|
|
67
|
+
message: str
|
|
68
|
+
check: str = ""
|
|
69
|
+
details: Mapping[str, Any] = field(default_factory=dict, compare=False)
|
|
70
|
+
|
|
71
|
+
def __post_init__(self) -> None:
|
|
72
|
+
object.__setattr__(self, "details", MappingProxyType(dict(self.details)))
|
|
73
|
+
|
|
74
|
+
def as_dict(self, *, include_details: bool = False) -> dict[str, Any]:
|
|
75
|
+
data: dict[str, Any] = {"code": str(self.code), "message": self.message}
|
|
76
|
+
if include_details:
|
|
77
|
+
data["check"] = self.check
|
|
78
|
+
data["details"] = dict(self.details)
|
|
79
|
+
return data
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Django integration, built on Django's own extension points.
|
|
2
|
+
|
|
3
|
+
* ``FILE_UPLOAD_HANDLERS``: :class:`~uploadshield.django.handlers.ValidatingUploadHandler`
|
|
4
|
+
validates every uploaded file while the body is parsed (views, forms, admin, DRF).
|
|
5
|
+
* :func:`validate_uploads` / :func:`skip_upload_validation`: per-view policies.
|
|
6
|
+
* :class:`FileValidator` / :class:`ValidatedFileField`: per-field validation with
|
|
7
|
+
inline errors in forms, model ``full_clean`` and DRF serializers.
|
|
8
|
+
* ``STORAGES``: :class:`~uploadshield.django.storage.ValidatingStorage`
|
|
9
|
+
validates everything that is persisted, inside or outside requests.
|
|
10
|
+
* ``handler400``: :func:`uploadshield.django.views.bad_request` renders
|
|
11
|
+
rejections as 413/415/400 JSON or HTML.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from .decorators import skip_upload_validation, validate_uploads
|
|
15
|
+
from .exceptions import UploadRejected
|
|
16
|
+
from .forms import ValidatedFileField
|
|
17
|
+
from .responses import redirect_with_messages
|
|
18
|
+
from .service import FileOutcome, RejectedUpload, UploadValidation, get_upload_validation
|
|
19
|
+
from .signals import file_rejected, upload_rejected, upload_validated
|
|
20
|
+
from .validators import FileValidator
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"FileOutcome",
|
|
24
|
+
"FileValidator",
|
|
25
|
+
"RejectedUpload",
|
|
26
|
+
"UploadRejected",
|
|
27
|
+
"UploadValidation",
|
|
28
|
+
"ValidatedFileField",
|
|
29
|
+
"file_rejected",
|
|
30
|
+
"get_upload_validation",
|
|
31
|
+
"redirect_with_messages",
|
|
32
|
+
"skip_upload_validation",
|
|
33
|
+
"upload_rejected",
|
|
34
|
+
"upload_validated",
|
|
35
|
+
"validate_uploads",
|
|
36
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from django.apps import AppConfig
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class UploadShieldConfig(AppConfig):
|
|
5
|
+
"""Registers system checks and ships the overridable rejection template."""
|
|
6
|
+
|
|
7
|
+
name = "uploadshield.django"
|
|
8
|
+
label = "uploadshield"
|
|
9
|
+
verbose_name = "UploadShield"
|
|
10
|
+
|
|
11
|
+
def ready(self) -> None:
|
|
12
|
+
# Importing the module registers the system checks.
|
|
13
|
+
from uploadshield.django import checks # noqa: F401
|