uploadshield 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uploadshield/__init__.py +49 -0
- uploadshield/__main__.py +17 -0
- uploadshield/checks/__init__.py +65 -0
- uploadshield/checks/archive.py +316 -0
- uploadshield/checks/basic.py +476 -0
- uploadshield/checks/image.py +107 -0
- uploadshield/checks/magika.py +129 -0
- uploadshield/checks/markup.py +181 -0
- uploadshield/checks/office.py +174 -0
- uploadshield/checks/pdf.py +277 -0
- uploadshield/checks/scanners.py +205 -0
- uploadshield/checks/xml.py +123 -0
- uploadshield/cli/__init__.py +276 -0
- uploadshield/cli/_commands.py +401 -0
- uploadshield/cli/_json.py +80 -0
- uploadshield/cli/_render.py +319 -0
- uploadshield/cli/_scan.py +106 -0
- uploadshield/cli/_style.py +95 -0
- uploadshield/cli/_verdicts.py +363 -0
- uploadshield/cli/_walk.py +159 -0
- uploadshield/core/__init__.py +4 -0
- uploadshield/core/_deconstruct.py +39 -0
- uploadshield/core/check.py +79 -0
- uploadshield/core/containers.py +131 -0
- uploadshield/core/result.py +63 -0
- uploadshield/core/sniffing.py +432 -0
- uploadshield/core/types.py +319 -0
- uploadshield/core/upload.py +192 -0
- uploadshield/core/violations.py +79 -0
- uploadshield/django/__init__.py +36 -0
- uploadshield/django/apps.py +13 -0
- uploadshield/django/checks.py +148 -0
- uploadshield/django/cli.py +36 -0
- uploadshield/django/conf.py +189 -0
- uploadshield/django/decorators.py +204 -0
- uploadshield/django/exceptions.py +39 -0
- uploadshield/django/forms.py +40 -0
- uploadshield/django/handlers.py +322 -0
- uploadshield/django/responses.py +92 -0
- uploadshield/django/service.py +473 -0
- uploadshield/django/signals.py +16 -0
- uploadshield/django/storage.py +204 -0
- uploadshield/django/templates/uploadshield/rejected.html +15 -0
- uploadshield/django/validators.py +48 -0
- uploadshield/django/views.py +40 -0
- uploadshield/engine.py +230 -0
- uploadshield/fastapi/__init__.py +23 -0
- uploadshield/fastapi/shield.py +299 -0
- uploadshield/flask/__init__.py +42 -0
- uploadshield/flask/core.py +359 -0
- uploadshield/flask/decorators.py +69 -0
- uploadshield/flask/wtforms.py +58 -0
- uploadshield/policy.py +137 -0
- uploadshield/presets.py +65 -0
- uploadshield/py.typed +0 -0
- uploadshield/sanitizers.py +132 -0
- uploadshield/web.py +218 -0
- uploadshield-2.0.0.dist-info/METADATA +239 -0
- uploadshield-2.0.0.dist-info/RECORD +62 -0
- uploadshield-2.0.0.dist-info/WHEEL +4 -0
- uploadshield-2.0.0.dist-info/entry_points.txt +3 -0
- uploadshield-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,473 @@
|
|
|
1
|
+
"""Validation logic shared by the upload handler, decorators, fields and storage."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import dataclasses
|
|
6
|
+
import hashlib
|
|
7
|
+
import io
|
|
8
|
+
import logging
|
|
9
|
+
import re
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from types import MappingProxyType
|
|
12
|
+
from typing import IO, TYPE_CHECKING, Any, Final, cast
|
|
13
|
+
|
|
14
|
+
from django.core.exceptions import ValidationError
|
|
15
|
+
from django.core.files.uploadedfile import InMemoryUploadedFile, UploadedFile
|
|
16
|
+
from django.db.models.fields.files import FieldFile
|
|
17
|
+
|
|
18
|
+
from uploadshield import web
|
|
19
|
+
from uploadshield.core.upload import Upload
|
|
20
|
+
from uploadshield.engine import validate
|
|
21
|
+
from uploadshield.presets import DEFAULT
|
|
22
|
+
|
|
23
|
+
from .conf import MODES, Config, PolicyRef, get_config
|
|
24
|
+
from .signals import file_rejected, upload_rejected, upload_validated
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from collections.abc import Callable, Mapping
|
|
28
|
+
|
|
29
|
+
from django.core.files import File
|
|
30
|
+
from django.http import HttpRequest, HttpResponse
|
|
31
|
+
|
|
32
|
+
from uploadshield.core.result import ValidationResult
|
|
33
|
+
from uploadshield.core.violations import Violation
|
|
34
|
+
from uploadshield.policy import Policy
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"SKIP",
|
|
38
|
+
"FileOutcome",
|
|
39
|
+
"RejectedUpload",
|
|
40
|
+
"UploadValidation",
|
|
41
|
+
"ViewSpec",
|
|
42
|
+
"accept",
|
|
43
|
+
"already_reported",
|
|
44
|
+
"build_replacement",
|
|
45
|
+
"close_replaced",
|
|
46
|
+
"field_rejection",
|
|
47
|
+
"finish",
|
|
48
|
+
"get_upload_validation",
|
|
49
|
+
"get_view_spec",
|
|
50
|
+
"message_for",
|
|
51
|
+
"rejection_response",
|
|
52
|
+
"spec_for_request",
|
|
53
|
+
"validate_file",
|
|
54
|
+
"validate_request",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
logger = logging.getLogger("uploadshield")
|
|
58
|
+
|
|
59
|
+
VIEW_ATTR: Final = "_uploadshield_spec"
|
|
60
|
+
DONE_ATTR: Final = "_uploadshield_done"
|
|
61
|
+
#: The spec the request was validated against (the handler may run before routing).
|
|
62
|
+
SPEC_ATTR: Final = "_uploadshield_validated_spec"
|
|
63
|
+
#: Public: views can read ``request.upload_validation`` after validation.
|
|
64
|
+
RESULT_ATTR: Final = "upload_validation"
|
|
65
|
+
_CACHE_ATTR: Final = "_uploadshield_results"
|
|
66
|
+
_REPORTED_ATTR: Final = "_uploadshield_reported"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def already_reported(file: Any) -> bool:
|
|
70
|
+
"""Whether a rejection of ``file`` was already logged and signalled upstream."""
|
|
71
|
+
return bool(getattr(file, "__dict__", {}).get(_REPORTED_ATTR))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class _UseDefault:
|
|
75
|
+
"""Sentinel: use ``DEFAULT_POLICY`` (or the built-in default)."""
|
|
76
|
+
|
|
77
|
+
def __repr__(self) -> str:
|
|
78
|
+
return "USE_DEFAULT"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
USE_DEFAULT: Final = _UseDefault()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class ViewSpec:
|
|
86
|
+
"""How uploads to one view are validated (set by decorators or settings).
|
|
87
|
+
|
|
88
|
+
``on_reject`` is ``None`` (use ``ON_REJECT``), ``"reject"``, ``"field"``, or a
|
|
89
|
+
callable ``(request, validation) -> HttpResponse`` that renders the rejection.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
policy: PolicyRef | _UseDefault | None = USE_DEFAULT
|
|
93
|
+
fields: Mapping[str, PolicyRef | None] = field(default_factory=dict)
|
|
94
|
+
mode: str | None = None
|
|
95
|
+
on_reject: Callable[[HttpRequest, UploadValidation], HttpResponse] | str | None = None
|
|
96
|
+
|
|
97
|
+
def __post_init__(self) -> None:
|
|
98
|
+
if self.mode is not None and self.mode not in MODES:
|
|
99
|
+
raise ValueError(f"mode must be one of {MODES}, got {self.mode!r}")
|
|
100
|
+
if not (self.on_reject in (None, "reject", "field") or callable(self.on_reject)):
|
|
101
|
+
raise ValueError("on_reject must be 'reject', 'field' or a callable")
|
|
102
|
+
object.__setattr__(self, "fields", MappingProxyType(dict(self.fields)))
|
|
103
|
+
|
|
104
|
+
def policy_for(self, field_name: str, config: Config) -> Policy | None:
|
|
105
|
+
if field_name in self.fields:
|
|
106
|
+
ref = self.fields[field_name]
|
|
107
|
+
else:
|
|
108
|
+
# DRF reads "photos[0]" and "photos.0" into "photos": such aliases get that
|
|
109
|
+
# field's policy - but they can never skip validation that way.
|
|
110
|
+
base = self.fields.get(_base_field(field_name))
|
|
111
|
+
ref = base if base is not None else self.policy
|
|
112
|
+
if ref is None:
|
|
113
|
+
return None
|
|
114
|
+
if isinstance(ref, _UseDefault):
|
|
115
|
+
return config.default_policy or DEFAULT
|
|
116
|
+
return config.resolve(ref)
|
|
117
|
+
|
|
118
|
+
def effective_mode(self, config: Config) -> str:
|
|
119
|
+
return self.mode or config.mode
|
|
120
|
+
|
|
121
|
+
def inline_errors(self, config: Config) -> bool:
|
|
122
|
+
"""Whether rejected files become placeholders for forms to report."""
|
|
123
|
+
if self.on_reject is None:
|
|
124
|
+
return config.on_reject == "field"
|
|
125
|
+
return self.on_reject == "field"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
_FIELD_INDEX = re.compile(r"[\[.]")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _base_field(name: str) -> str:
|
|
132
|
+
return _FIELD_INDEX.split(name, maxsplit=1)[0]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
#: Marks a view as exempt from upload validation.
|
|
136
|
+
SKIP: Final = ViewSpec(policy=None, mode="off")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class FileOutcome:
|
|
141
|
+
field: str
|
|
142
|
+
index: int
|
|
143
|
+
result: ValidationResult
|
|
144
|
+
policy_name: str
|
|
145
|
+
original: UploadedFile | None = None
|
|
146
|
+
#: Set when the original was hashed and closed early (it was replaced).
|
|
147
|
+
digest: str | None = None
|
|
148
|
+
|
|
149
|
+
@property
|
|
150
|
+
def ok(self) -> bool:
|
|
151
|
+
return self.result.ok
|
|
152
|
+
|
|
153
|
+
def sha256(self) -> str | None:
|
|
154
|
+
if self.digest is not None:
|
|
155
|
+
return self.digest
|
|
156
|
+
if self.original is None or isinstance(self.original, RejectedUpload):
|
|
157
|
+
return None
|
|
158
|
+
digest = hashlib.sha256()
|
|
159
|
+
for chunk in self.original.chunks():
|
|
160
|
+
digest.update(chunk)
|
|
161
|
+
self.original.seek(0)
|
|
162
|
+
return digest.hexdigest()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class UploadValidation:
|
|
167
|
+
"""Everything that happened to the files of one request."""
|
|
168
|
+
|
|
169
|
+
outcomes: tuple[FileOutcome, ...]
|
|
170
|
+
mode: str
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def ok(self) -> bool:
|
|
174
|
+
return all(o.ok for o in self.outcomes)
|
|
175
|
+
|
|
176
|
+
@property
|
|
177
|
+
def rejected(self) -> tuple[FileOutcome, ...]:
|
|
178
|
+
return tuple(o for o in self.outcomes if not o.ok)
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def enforced(self) -> bool:
|
|
182
|
+
return self.mode == "enforce"
|
|
183
|
+
|
|
184
|
+
def for_field(self, name: str) -> tuple[FileOutcome, ...]:
|
|
185
|
+
return tuple(o for o in self.outcomes if o.field == name)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
# --------------------------------------------------------------------------- #
|
|
189
|
+
# Single files
|
|
190
|
+
# --------------------------------------------------------------------------- #
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class RejectedUpload(InMemoryUploadedFile):
|
|
194
|
+
"""Stands in for a rejected file when a view reports errors inline.
|
|
195
|
+
|
|
196
|
+
It has no content, so the rejected bytes never reach application code. Reading
|
|
197
|
+
``size`` raises the :class:`~django.core.exceptions.ValidationError`: Django's and
|
|
198
|
+
DRF's file fields read it while cleaning, so *every* file field (plain, model,
|
|
199
|
+
admin, serializer) reports the precise reason next to the field.
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
def __init__(self, result: ValidationResult, *, field_name: str | None, config: Config) -> None:
|
|
203
|
+
self.result = result
|
|
204
|
+
self._error = ValidationError(
|
|
205
|
+
[ValidationError(message_for(v, config), code=str(v.code)) for v in result.violations]
|
|
206
|
+
)
|
|
207
|
+
super().__init__(
|
|
208
|
+
io.BytesIO(b""), field_name, result.name, "application/octet-stream", 0, None
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def size(self) -> int | None:
|
|
213
|
+
raise self._error
|
|
214
|
+
|
|
215
|
+
@size.setter
|
|
216
|
+
def size(self, value: int | None) -> None:
|
|
217
|
+
"""Ignored: ``UploadedFile.__init__`` assigns it."""
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _uploaded_file(value: Any, *, stored: bool = False) -> File | None:
|
|
221
|
+
"""The new upload behind ``value``; ``None`` for files already in storage.
|
|
222
|
+
|
|
223
|
+
With ``stored`` a file already in storage counts too: storage validates whatever it
|
|
224
|
+
is given to save, also a stored file copied under a new name.
|
|
225
|
+
"""
|
|
226
|
+
if isinstance(value, FieldFile):
|
|
227
|
+
if getattr(value, "_committed", True):
|
|
228
|
+
return value.file if stored and value else None
|
|
229
|
+
return value.file
|
|
230
|
+
return value if hasattr(value, "read") else None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def validate_file(
|
|
234
|
+
value: Any,
|
|
235
|
+
policy: Policy,
|
|
236
|
+
*,
|
|
237
|
+
field: str | None = None,
|
|
238
|
+
sanitize: bool = True,
|
|
239
|
+
name: str | None = None,
|
|
240
|
+
stored: bool = False,
|
|
241
|
+
) -> ValidationResult | None:
|
|
242
|
+
"""Validate a Django file object, memoising the result on the object.
|
|
243
|
+
|
|
244
|
+
``name`` overrides the file's own name, e.g. with the name it will be stored under.
|
|
245
|
+
``stored`` also validates files already in storage (see :func:`_uploaded_file`).
|
|
246
|
+
"""
|
|
247
|
+
uploaded = _uploaded_file(value, stored=stored)
|
|
248
|
+
if uploaded is None:
|
|
249
|
+
return None
|
|
250
|
+
if isinstance(uploaded, RejectedUpload):
|
|
251
|
+
return uploaded.result # the bytes are gone; the original verdict stands
|
|
252
|
+
name = (uploaded.name or "") if name is None else name
|
|
253
|
+
# Results are keyed by the name they were judged under: a file renamed after
|
|
254
|
+
# validation (f.name = "doc.html") is judged again.
|
|
255
|
+
cache: dict[tuple[Any, ...], ValidationResult] = uploaded.__dict__.setdefault(_CACHE_ATTR, {})
|
|
256
|
+
for key in ((policy, True, name), (policy, sanitize, name)):
|
|
257
|
+
if key in cache:
|
|
258
|
+
return cache[key]
|
|
259
|
+
inner = getattr(uploaded, "file", None)
|
|
260
|
+
# Django's File proxies read/seek/tell/seekable to the underlying file object.
|
|
261
|
+
stream = cast("IO[bytes]", inner if inner is not None else uploaded)
|
|
262
|
+
result = validate(
|
|
263
|
+
Upload(
|
|
264
|
+
stream=stream,
|
|
265
|
+
name=name,
|
|
266
|
+
size=uploaded.size,
|
|
267
|
+
declared_type=getattr(uploaded, "content_type", None),
|
|
268
|
+
field=field,
|
|
269
|
+
),
|
|
270
|
+
policy,
|
|
271
|
+
sanitize=sanitize,
|
|
272
|
+
)
|
|
273
|
+
cache[(policy, sanitize, name)] = result
|
|
274
|
+
return result
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def build_replacement(
|
|
278
|
+
original: UploadedFile, result: ValidationResult, policy: Policy
|
|
279
|
+
) -> UploadedFile:
|
|
280
|
+
"""The file to hand to the application after a passing validation.
|
|
281
|
+
|
|
282
|
+
Applies the sanitized name/content and the detected content type, and records the
|
|
283
|
+
outcome on the returned object so later layers (fields, storage) neither validate
|
|
284
|
+
nor sanitize it again.
|
|
285
|
+
"""
|
|
286
|
+
sanitized = result.sanitized
|
|
287
|
+
if sanitized is None or sanitized.data is None:
|
|
288
|
+
if sanitized is not None:
|
|
289
|
+
original.name = sanitized.name
|
|
290
|
+
if result.file_type is not None and get_config().normalize_content_type:
|
|
291
|
+
original.content_type = result.file_type.mime
|
|
292
|
+
replacement = original
|
|
293
|
+
else:
|
|
294
|
+
replacement = InMemoryUploadedFile(
|
|
295
|
+
file=io.BytesIO(sanitized.data),
|
|
296
|
+
field_name=getattr(original, "field_name", None),
|
|
297
|
+
name=sanitized.name,
|
|
298
|
+
content_type=sanitized.content_type,
|
|
299
|
+
size=len(sanitized.data),
|
|
300
|
+
charset=None,
|
|
301
|
+
content_type_extra=getattr(original, "content_type_extra", None),
|
|
302
|
+
)
|
|
303
|
+
done = dataclasses.replace(result, sanitized=None, name=replacement.name or result.name)
|
|
304
|
+
judged = replacement.name or ""
|
|
305
|
+
replacement.__dict__[_CACHE_ATTR] = {
|
|
306
|
+
(policy, True, judged): done,
|
|
307
|
+
(policy, False, judged): done,
|
|
308
|
+
}
|
|
309
|
+
return replacement
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def field_rejection(result: ValidationResult, policy: Policy, config: Config) -> ValidationError:
|
|
313
|
+
"""Log and signal a field-level rejection; return the error for the form/model."""
|
|
314
|
+
logger.warning(
|
|
315
|
+
"Upload rejected by field validator: name=%r type=%s codes=%s",
|
|
316
|
+
result.name,
|
|
317
|
+
result.file_type.mime if result.file_type else None,
|
|
318
|
+
",".join(result.codes),
|
|
319
|
+
extra={"upload": result.as_dict(include_details=True) | {"policy": policy.name}},
|
|
320
|
+
)
|
|
321
|
+
file_rejected.send(sender=None, result=result, policy=policy)
|
|
322
|
+
return ValidationError(
|
|
323
|
+
[ValidationError(message_for(v, config), code=str(v.code)) for v in result.violations]
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def message_for(violation: Violation, config: Config | None = None) -> str:
|
|
328
|
+
"""The user-facing message, honouring ``MESSAGES`` overrides (may be lazy)."""
|
|
329
|
+
return web.message_for(violation, (config or get_config()).messages)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
# --------------------------------------------------------------------------- #
|
|
333
|
+
# Requests
|
|
334
|
+
# --------------------------------------------------------------------------- #
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def get_upload_validation(request: HttpRequest) -> UploadValidation | None:
|
|
338
|
+
"""The :class:`UploadValidation` of ``request``, if its files were validated."""
|
|
339
|
+
return getattr(request, RESULT_ATTR, None)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def get_view_spec(view: Any, method: str | None = None) -> ViewSpec | None:
|
|
343
|
+
"""The spec attached by ``@validate_uploads``/``@skip_upload_validation``, if any.
|
|
344
|
+
|
|
345
|
+
Looks at the view, its class, and - for ``method`` - the class's handler method,
|
|
346
|
+
where ``method_decorator(validate_uploads(...), name="post")`` leaves it.
|
|
347
|
+
"""
|
|
348
|
+
spec = getattr(view, VIEW_ATTR, None)
|
|
349
|
+
view_class = getattr(view, "view_class", None) or getattr(view, "cls", None)
|
|
350
|
+
if spec is None:
|
|
351
|
+
spec = getattr(view_class, VIEW_ATTR, None)
|
|
352
|
+
if spec is None and view_class is not None and method:
|
|
353
|
+
spec = getattr(getattr(view_class, method.lower(), None), VIEW_ATTR, None)
|
|
354
|
+
return spec if isinstance(spec, ViewSpec) else None
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def spec_for_request(
|
|
358
|
+
request: HttpRequest, view: Any, config: Config, *, match: Any = None
|
|
359
|
+
) -> ViewSpec:
|
|
360
|
+
"""Decorator spec, else the first matching ``VIEW_POLICIES`` entry, else default."""
|
|
361
|
+
if (spec := get_view_spec(view, getattr(request, "method", None))) is not None:
|
|
362
|
+
return spec
|
|
363
|
+
match = match if match is not None else getattr(request, "resolver_match", None)
|
|
364
|
+
matched, policy = config.policy_for_view_name(getattr(match, "view_name", None))
|
|
365
|
+
if matched:
|
|
366
|
+
return SKIP if policy is None else ViewSpec(policy=policy)
|
|
367
|
+
return ViewSpec(policy=config.default_policy)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def finish(request: HttpRequest, validation: UploadValidation, spec: ViewSpec) -> None:
|
|
371
|
+
"""Record the outcome on the request and report it (logs and signals)."""
|
|
372
|
+
setattr(request, RESULT_ATTR, validation)
|
|
373
|
+
setattr(request, SPEC_ATTR, spec)
|
|
374
|
+
setattr(request, DONE_ATTR, True)
|
|
375
|
+
_report(request, validation)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def accept(outcome: FileOutcome, policy: Policy, config: Config, *, inline: bool) -> UploadedFile:
|
|
379
|
+
"""The file object the application receives for one validated upload.
|
|
380
|
+
|
|
381
|
+
Passing files are sanitized; rejected ones become a :class:`RejectedUpload` when
|
|
382
|
+
``inline`` is set, and are returned untouched otherwise (report mode). Replaced
|
|
383
|
+
originals stay open until :func:`close_replaced` (reporting still hashes them).
|
|
384
|
+
"""
|
|
385
|
+
original, result = outcome.original, outcome.result
|
|
386
|
+
if original is None:
|
|
387
|
+
raise ValueError("outcome has no file")
|
|
388
|
+
if result.ok:
|
|
389
|
+
return build_replacement(original, result, policy)
|
|
390
|
+
if inline:
|
|
391
|
+
return RejectedUpload(result, field_name=outcome.field, config=config)
|
|
392
|
+
return original
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def close_replaced(pairs: list[tuple[UploadedFile, UploadedFile]]) -> None:
|
|
396
|
+
"""Close originals that were swapped for a sanitized file or a placeholder."""
|
|
397
|
+
for original, delivered in pairs:
|
|
398
|
+
if delivered is not original:
|
|
399
|
+
original.close()
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def validate_request(request: HttpRequest, spec: ViewSpec) -> UploadValidation:
|
|
403
|
+
"""Validate every file in ``request.FILES`` after the body has been parsed.
|
|
404
|
+
|
|
405
|
+
This is the fallback for projects that use ``@validate_uploads`` without
|
|
406
|
+
``ValidatingUploadHandler``. With the handler installed the files were already
|
|
407
|
+
validated while they streamed in, and this returns that result - unless the body
|
|
408
|
+
was read before URL routing, when the handler could only apply the default policy:
|
|
409
|
+
then the view's own spec is applied to the files now.
|
|
410
|
+
"""
|
|
411
|
+
if getattr(request, DONE_ATTR, False) and getattr(request, SPEC_ATTR, None) == spec:
|
|
412
|
+
return getattr(request, RESULT_ATTR)
|
|
413
|
+
config = get_config()
|
|
414
|
+
mode = spec.effective_mode(config)
|
|
415
|
+
inline = mode == "enforce" and spec.inline_errors(config)
|
|
416
|
+
outcomes: list[FileOutcome] = []
|
|
417
|
+
replaced: list[tuple[UploadedFile, UploadedFile]] = []
|
|
418
|
+
if mode != "off":
|
|
419
|
+
files = request.FILES
|
|
420
|
+
for field_name in list(files):
|
|
421
|
+
policy = spec.policy_for(field_name, config)
|
|
422
|
+
if policy is None:
|
|
423
|
+
continue
|
|
424
|
+
delivered = []
|
|
425
|
+
for index, uploaded in enumerate(files.getlist(field_name)):
|
|
426
|
+
result = validate_file(uploaded, policy, field=field_name)
|
|
427
|
+
if result is None:
|
|
428
|
+
delivered.append(uploaded)
|
|
429
|
+
continue
|
|
430
|
+
outcome = FileOutcome(field_name, index, result, policy.name, uploaded)
|
|
431
|
+
outcomes.append(outcome)
|
|
432
|
+
delivered.append(accept(outcome, policy, config, inline=inline))
|
|
433
|
+
replaced.append((uploaded, delivered[-1]))
|
|
434
|
+
files.setlist(field_name, delivered)
|
|
435
|
+
validation = UploadValidation(tuple(outcomes), mode)
|
|
436
|
+
finish(request, validation, spec)
|
|
437
|
+
close_replaced(replaced)
|
|
438
|
+
return validation
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def rejection_response(
|
|
442
|
+
request: HttpRequest, validation: UploadValidation, spec: ViewSpec
|
|
443
|
+
) -> HttpResponse | None:
|
|
444
|
+
"""The response to send instead of calling the view, or ``None`` to proceed."""
|
|
445
|
+
config = get_config()
|
|
446
|
+
if validation.ok or not validation.enforced or spec.inline_errors(config):
|
|
447
|
+
return None
|
|
448
|
+
on_reject = spec.on_reject
|
|
449
|
+
if on_reject is not None and not isinstance(on_reject, str):
|
|
450
|
+
return on_reject(request, validation)
|
|
451
|
+
return config.rejection_handler(request, validation)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _report(request: HttpRequest, validation: UploadValidation) -> None:
|
|
455
|
+
for outcome in validation.rejected:
|
|
456
|
+
if outcome.original is not None:
|
|
457
|
+
outcome.original.__dict__[_REPORTED_ATTR] = True # later layers stay quiet
|
|
458
|
+
web.log_rejection(
|
|
459
|
+
outcome.result,
|
|
460
|
+
field=outcome.field,
|
|
461
|
+
policy_name=outcome.policy_name,
|
|
462
|
+
mode=validation.mode,
|
|
463
|
+
path=request.path,
|
|
464
|
+
sha256=outcome.sha256(),
|
|
465
|
+
)
|
|
466
|
+
for outcome in validation.outcomes:
|
|
467
|
+
if outcome.ok:
|
|
468
|
+
logger.debug("Upload accepted: field=%r name=%r", outcome.field, outcome.result.name)
|
|
469
|
+
upload_validated.send(sender=None, request=request, validation=validation)
|
|
470
|
+
if not validation.ok:
|
|
471
|
+
upload_rejected.send(
|
|
472
|
+
sender=None, request=request, validation=validation, enforced=validation.enforced
|
|
473
|
+
)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Signals for auditing, metrics and alerting.
|
|
2
|
+
|
|
3
|
+
``upload_validated``: every validated request; kwargs ``request``, ``validation``.
|
|
4
|
+
``upload_rejected``: requests with at least one bad file; kwargs ``request``,
|
|
5
|
+
``validation`` and ``enforced`` (``False`` in report mode).
|
|
6
|
+
``file_rejected``: a form/model/serializer field rejected a file (``FileValidator``,
|
|
7
|
+
``ValidatedFileField``); kwargs ``result`` and ``policy``. No request is available.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from django.dispatch import Signal
|
|
11
|
+
|
|
12
|
+
__all__ = ["file_rejected", "upload_rejected", "upload_validated"]
|
|
13
|
+
|
|
14
|
+
upload_validated = Signal()
|
|
15
|
+
upload_rejected = Signal()
|
|
16
|
+
file_rejected = Signal()
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Validate files where they are persisted: the last line of defense.
|
|
2
|
+
|
|
3
|
+
Every file that reaches storage passes ``Storage.save()``, whether it came from a
|
|
4
|
+
form, the admin, a legacy view writing ``request.FILES`` directly, a Celery task or a
|
|
5
|
+
management command. Configure it in ``STORAGES``, wrapping any backend::
|
|
6
|
+
|
|
7
|
+
STORAGES = {
|
|
8
|
+
"default": {
|
|
9
|
+
"BACKEND": "uploadshield.django.storage.ValidatingStorage",
|
|
10
|
+
"OPTIONS": {
|
|
11
|
+
"backend": "django.core.files.storage.FileSystemStorage", # or S3, ...
|
|
12
|
+
"options": {},
|
|
13
|
+
"policy": "default",
|
|
14
|
+
"path_policies": {"avatars/": "sanitized_images", "thumbnails/": None},
|
|
15
|
+
},
|
|
16
|
+
},
|
|
17
|
+
"staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Files already validated by the upload handler or a field validator against the same
|
|
21
|
+
policy are not checked again: the result travels with the file object.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import posixpath
|
|
27
|
+
from types import MappingProxyType
|
|
28
|
+
from typing import IO, TYPE_CHECKING, Any
|
|
29
|
+
|
|
30
|
+
from django.core.files import File
|
|
31
|
+
from django.core.files.base import ContentFile
|
|
32
|
+
from django.core.files.storage import Storage
|
|
33
|
+
from django.utils.deconstruct import deconstructible
|
|
34
|
+
from django.utils.module_loading import import_string
|
|
35
|
+
|
|
36
|
+
from .conf import PolicyRef, get_config
|
|
37
|
+
from .exceptions import UploadRejected
|
|
38
|
+
from .service import (
|
|
39
|
+
FileOutcome,
|
|
40
|
+
RejectedUpload,
|
|
41
|
+
UploadValidation,
|
|
42
|
+
already_reported,
|
|
43
|
+
validate_file,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
if TYPE_CHECKING:
|
|
47
|
+
from collections.abc import Mapping
|
|
48
|
+
|
|
49
|
+
from uploadshield.policy import Policy
|
|
50
|
+
|
|
51
|
+
__all__ = ["ValidatingStorage", "ValidatingStorageMixin"]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ValidatingStorageMixin(Storage):
|
|
55
|
+
"""Add upload validation to a storage class (``class S3(ValidatingStorageMixin, S3Storage)``).
|
|
56
|
+
|
|
57
|
+
``upload_policy`` applies to every save; ``path_policies`` maps name prefixes to
|
|
58
|
+
other policies (``None`` disables validation, e.g. for generated thumbnails).
|
|
59
|
+
The longest matching prefix wins.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
upload_policy: PolicyRef | None = "default"
|
|
63
|
+
path_policies: Mapping[str, PolicyRef | None] = MappingProxyType({})
|
|
64
|
+
|
|
65
|
+
def policy_for_name(self, name: str) -> Policy | None:
|
|
66
|
+
normalized = name.replace("\\", "/").lstrip("/")
|
|
67
|
+
ref: PolicyRef | None = self.upload_policy
|
|
68
|
+
best = -1
|
|
69
|
+
for prefix, candidate in self.path_policies.items():
|
|
70
|
+
if normalized.startswith(prefix) and len(prefix) > best:
|
|
71
|
+
ref, best = candidate, len(prefix)
|
|
72
|
+
return None if ref is None else get_config().resolve(ref)
|
|
73
|
+
|
|
74
|
+
def save(self, name: str | None, content: IO[Any], max_length: int | None = None) -> str:
|
|
75
|
+
if name is None:
|
|
76
|
+
name = content.name
|
|
77
|
+
# Django's Storage.save() does the same wrapping for plain file objects.
|
|
78
|
+
file = content if isinstance(content, File) else File(content, name)
|
|
79
|
+
name, file = self.validate_upload(str(name), file)
|
|
80
|
+
return self._store(name, file, max_length)
|
|
81
|
+
|
|
82
|
+
def validate_upload(self, name: str, content: File) -> tuple[str, File]:
|
|
83
|
+
"""Validate ``content``; return the (possibly sanitized) name and content.
|
|
84
|
+
|
|
85
|
+
Follows the global ``MODE`` like the upload handler: ``report`` logs and signals
|
|
86
|
+
but stores anyway, ``off`` stores without checking.
|
|
87
|
+
"""
|
|
88
|
+
mode = get_config().mode
|
|
89
|
+
policy = self.policy_for_name(name)
|
|
90
|
+
if policy is None or mode == "off":
|
|
91
|
+
return name, content
|
|
92
|
+
if isinstance(content, RejectedUpload):
|
|
93
|
+
self._reject(content.result, policy, enforce=True) # placeholders hold no data
|
|
94
|
+
# Judge the name the file is stored and served under (upload_to may rename it).
|
|
95
|
+
stored_name = posixpath.basename(name.replace("\\", "/"))
|
|
96
|
+
result = validate_file(content, policy, field=None, name=stored_name, stored=True)
|
|
97
|
+
if result is None: # nothing to read (an empty FieldFile)
|
|
98
|
+
return name, content
|
|
99
|
+
if not result.ok:
|
|
100
|
+
self._reject(result, policy, enforce=mode == "enforce", quiet=already_reported(content))
|
|
101
|
+
return name, content
|
|
102
|
+
sanitized = result.sanitized
|
|
103
|
+
if sanitized is None:
|
|
104
|
+
return name, content
|
|
105
|
+
name = posixpath.join(posixpath.dirname(name), sanitized.name)
|
|
106
|
+
if sanitized.data is not None:
|
|
107
|
+
content = ContentFile(sanitized.data, name=sanitized.name)
|
|
108
|
+
return name, content
|
|
109
|
+
|
|
110
|
+
def open(self, name: str, mode: str = "rb") -> File:
|
|
111
|
+
"""Open for reading; writing would bypass validation and is refused.
|
|
112
|
+
|
|
113
|
+
Backends that support write modes (S3, ...) would store whatever is written
|
|
114
|
+
without passing :meth:`save`. Paths without a policy (``path_policies`` maps
|
|
115
|
+
them to ``None``) can still be opened for writing.
|
|
116
|
+
"""
|
|
117
|
+
if any(flag in mode for flag in "wax+") and self.policy_for_name(name) is not None:
|
|
118
|
+
raise PermissionError(
|
|
119
|
+
f"{type(self).__name__} validates files in save(); opening {name!r} with "
|
|
120
|
+
f"mode {mode!r} would bypass it. Use save(), or a path_policies entry of "
|
|
121
|
+
"None for files your code generates."
|
|
122
|
+
)
|
|
123
|
+
return super().open(name, mode)
|
|
124
|
+
|
|
125
|
+
def _reject(self, result: Any, policy: Policy, *, enforce: bool, quiet: bool = False) -> None:
|
|
126
|
+
from .service import field_rejection
|
|
127
|
+
|
|
128
|
+
if not quiet:
|
|
129
|
+
field_rejection(result, policy, get_config()) # logs and sends file_rejected
|
|
130
|
+
if enforce:
|
|
131
|
+
outcome = FileOutcome("", 0, result, policy.name)
|
|
132
|
+
raise UploadRejected(UploadValidation((outcome,), "enforce"))
|
|
133
|
+
|
|
134
|
+
def _store(self, name: str, content: File, max_length: int | None) -> str:
|
|
135
|
+
return super().save(name, content, max_length=max_length)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@deconstructible(path="uploadshield.django.storage.ValidatingStorage")
|
|
139
|
+
class ValidatingStorage(ValidatingStorageMixin):
|
|
140
|
+
"""Wrap any storage backend (configured by dotted path) with upload validation."""
|
|
141
|
+
|
|
142
|
+
def __init__(
|
|
143
|
+
self,
|
|
144
|
+
backend: str = "django.core.files.storage.FileSystemStorage",
|
|
145
|
+
options: dict[str, Any] | None = None,
|
|
146
|
+
policy: PolicyRef | None = "default",
|
|
147
|
+
path_policies: Mapping[str, PolicyRef | None] | None = None,
|
|
148
|
+
) -> None:
|
|
149
|
+
self.backend = backend
|
|
150
|
+
self.wrapped: Storage = import_string(backend)(**(options or {}))
|
|
151
|
+
self.upload_policy = policy
|
|
152
|
+
self.path_policies = MappingProxyType(dict(path_policies or {}))
|
|
153
|
+
|
|
154
|
+
def _store(self, name: str, content: File, max_length: int | None) -> str:
|
|
155
|
+
return self.wrapped.save(name, content, max_length=max_length)
|
|
156
|
+
|
|
157
|
+
# Storage API: delegate everything to the wrapped backend.
|
|
158
|
+
def _open(self, name: str, mode: str = "rb") -> File:
|
|
159
|
+
return self.wrapped.open(name, mode)
|
|
160
|
+
|
|
161
|
+
def _save(self, name: str, content: File) -> str: # pragma: no cover - save() delegates
|
|
162
|
+
return self.wrapped.save(name, content)
|
|
163
|
+
|
|
164
|
+
def delete(self, name: str) -> None:
|
|
165
|
+
self.wrapped.delete(name)
|
|
166
|
+
|
|
167
|
+
def exists(self, name: str) -> bool:
|
|
168
|
+
return self.wrapped.exists(name)
|
|
169
|
+
|
|
170
|
+
def listdir(self, path: str) -> tuple[list[str], list[str]]:
|
|
171
|
+
return self.wrapped.listdir(path)
|
|
172
|
+
|
|
173
|
+
def size(self, name: str) -> int:
|
|
174
|
+
return self.wrapped.size(name)
|
|
175
|
+
|
|
176
|
+
def url(self, name: str | None) -> str:
|
|
177
|
+
return self.wrapped.url(name)
|
|
178
|
+
|
|
179
|
+
def path(self, name: str) -> str:
|
|
180
|
+
return self.wrapped.path(name)
|
|
181
|
+
|
|
182
|
+
def get_valid_name(self, name: str) -> str:
|
|
183
|
+
return self.wrapped.get_valid_name(name)
|
|
184
|
+
|
|
185
|
+
def get_available_name(self, name: str, max_length: int | None = None) -> str:
|
|
186
|
+
return self.wrapped.get_available_name(name, max_length=max_length)
|
|
187
|
+
|
|
188
|
+
def generate_filename(self, filename: Any) -> str:
|
|
189
|
+
return self.wrapped.generate_filename(filename)
|
|
190
|
+
|
|
191
|
+
def get_accessed_time(self, name: str) -> Any:
|
|
192
|
+
return self.wrapped.get_accessed_time(name)
|
|
193
|
+
|
|
194
|
+
def get_created_time(self, name: str) -> Any:
|
|
195
|
+
return self.wrapped.get_created_time(name)
|
|
196
|
+
|
|
197
|
+
def get_modified_time(self, name: str) -> Any:
|
|
198
|
+
return self.wrapped.get_modified_time(name)
|
|
199
|
+
|
|
200
|
+
def __getattr__(self, attr: str) -> Any:
|
|
201
|
+
# Backend-specific attributes (bucket_name, location, ...).
|
|
202
|
+
if attr == "wrapped": # not yet initialised (e.g. during unpickling)
|
|
203
|
+
raise AttributeError(attr)
|
|
204
|
+
return getattr(self.wrapped, attr)
|