uploadshield 2.0.0__py3-none-any.whl

uploadshield/django/templates/uploadshield/rejected.html

@@ -0,0 +1,15 @@
+{% load i18n %}<!doctype html>
+<html lang="en">
+<head><meta charset="utf-8"><title>{% translate "Upload rejected" %}</title></head>
+<body>
+  <h1>{{ rejection.message }}</h1>
+  <ul>
+    {% for file in rejection.files %}
+      <li><strong>{{ file.name }}</strong>
+        <ul>{% for error in file.errors %}<li>{{ error.message }}</li>{% endfor %}</ul>
+      </li>
+    {% endfor %}
+  </ul>
+  <p><a href="javascript:history.back()">{% translate "Go back" %}</a></p>
+</body>
+</html>

uploadshield/django/validators.py

@@ -0,0 +1,48 @@
+"""Field validators for models, forms and DRF serializers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.utils.deconstruct import deconstructible
+
+from .conf import PolicyRef, get_config
+from .service import field_rejection, validate_file
+
+__all__ = ["FileValidator"]
+
+
+@deconstructible(path="uploadshield.django.FileValidator")
+class FileValidator:
+    """Validate a file field against a policy::
+
+        avatar = models.ImageField(validators=[FileValidator("images")])
+        cv = forms.FileField(validators=[FileValidator(PDF.replace(max_size=5 * MB))])
+
+    Validators always enforce, independent of the global ``MODE``: adding one is an
+    explicit decision for that field.  They cannot replace the file; use
+    :class:`~uploadshield.django.forms.ValidatedFileField` or a decorator for
+    sanitization.  Files already in storage (re-saving a model) are not re-checked.
+
+    Prefer policy *names* in model fields: they keep migrations short and let you
+    change the policy in settings without a new migration.
+    """
+
+    def __init__(self, policy: PolicyRef = "default") -> None:
+        self.policy = policy
+
+    def __call__(self, value: Any) -> None:
+        config = get_config()
+        policy = config.resolve(self.policy)
+        result = validate_file(value, policy, sanitize=False)
+        if result is not None and not result.ok:
+            raise field_rejection(result, policy, config)
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, FileValidator) and self.policy == other.policy
+
+    def __hash__(self) -> int:
+        return hash(self.policy)
+
+    def __repr__(self) -> str:
+        return f"FileValidator({self.policy!r})"

uploadshield/django/views.py

@@ -0,0 +1,40 @@
+"""Error views.
+
+Point Django's ``handler400`` at :func:`bad_request` in your root URLconf::
+
+    handler400 = "uploadshield.django.views.bad_request"
+
+Rejected uploads then get 413/415/400 responses with a JSON or HTML body (or a view's
+own ``on_reject`` handler); every other bad request falls through to Django's default.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from django.views import defaults
+
+from .conf import get_config
+from .exceptions import UploadRejected
+from .service import get_view_spec
+
+if TYPE_CHECKING:
+    from django.http import HttpRequest, HttpResponse
+
+__all__ = ["bad_request"]
+
+
+def bad_request(request: HttpRequest, exception: Exception | None = None) -> HttpResponse:
+    if not isinstance(exception, UploadRejected):
+        return defaults.bad_request(request, exception)  # ty: ignore[invalid-argument-type]
+    match = getattr(request, "resolver_match", None)
+    spec = get_view_spec(match.func, request.method) if match is not None else None
+    on_reject = spec.on_reject if spec is not None else None
+    if on_reject is not None and not isinstance(on_reject, str):
+        response = on_reject(request, exception.validation)
+    else:
+        response = get_config().rejection_handler(request, exception.validation)
+    # The rejection was already logged with full details on the "uploadshield"
+    # logger; this stops Django from logging it again with a traceback.
+    setattr(response, "_has_been_logged", True)  # noqa: B010 - private Django flag
+    return response

uploadshield/engine.py

@@ -0,0 +1,230 @@
+"""The validation engine: run a policy against one upload."""
+
+from __future__ import annotations
+
+import io
+import logging
+from typing import TYPE_CHECKING
+
+from .core.result import SanitizedFile, ValidationResult
+from .core.upload import ContentTooLargeError, InspectedFile, Upload
+from .core.violations import Code, Violation
+
+if TYPE_CHECKING:
+    from collections.abc import Collection
+
+    from .core.check import Check, Stage
+    from .core.types import FileType
+    from .policy import Policy
+
+__all__ = ["SanitizationError", "preflight", "validate"]
+
+logger = logging.getLogger("uploadshield")
+
+
+class SanitizationError(Exception):
+    """A sanitizer produced output that does not match the validated file."""
+
+
+def validate(upload: Upload, policy: Policy, *, sanitize: bool = True) -> ValidationResult:
+    """Validate ``upload`` against ``policy`` and, if it passes, sanitize it.
+
+    Never raises for bad input: crashing checks and sanitizers are reported as
+    violations (fail closed), so a parser bug cannot become a bypass.
+    """
+    file = InspectedFile(upload, policy.sniffer)
+    try:
+        return _validate(file, policy, sanitize=sanitize)
+    finally:
+        file.close()
+
+
+def _validate(file: InspectedFile, policy: Policy, *, sanitize: bool) -> ValidationResult:
+    violations: list[Violation] = []
+    checks_run: list[str] = []
+
+    for check in policy.pipeline:
+        found = _run_check(check, file)
+        if found is None:
+            continue
+        checks_run.append(check.name)
+        violations.extend(found)
+        if found and policy.fail_fast:
+            break
+
+    sanitized: SanitizedFile | None = None
+    if not violations and sanitize and policy.sanitizers:
+        try:
+            sanitized = _sanitize(file, policy)
+        except Exception as exc:
+            logger.exception("Sanitizing %r failed", file.name)
+            violations.append(
+                Violation(
+                    code=Code.SANITIZATION_FAILED,
+                    message="The file could not be processed.",
+                    check="sanitizer",
+                    details={"error": f"{type(exc).__name__}: {exc}"},
+                )
+            )
+
+    if (
+        sanitized is not None
+        and sanitized.name != file.name
+        and (found := _recheck_name(sanitized.name, file, policy))
+    ):
+        violations.extend(found)
+        sanitized = None
+    if (
+        sanitized is not None
+        and sanitized.data is not None
+        and (found := _recheck(sanitized.data, sanitized.name, file, policy))
+    ):
+        violations.extend(found)
+        sanitized = None
+
+    return ValidationResult(
+        name=file.name,
+        field=file.field,
+        size=file.size,
+        file_type=_safe_file_type(file),
+        declared_type=file.declared_type,
+        violations=tuple(violations),
+        checks_run=tuple(checks_run),
+        sanitized=sanitized,
+    )
+
+
+def preflight(
+    policy: Policy,
+    *,
+    stages: Collection[Stage],
+    name: str,
+    head: bytes = b"",
+    size: int = 0,
+    declared_type: str | None = None,
+    field: str | None = None,
+) -> ValidationResult:
+    """Run the checks of ``stages`` against an upload that is still arriving.
+
+    ``head`` holds the first bytes and ``size`` the bytes received so far.  Type checks
+    are skipped for container formats (ZIP, OLE2): telling a DOCX from a macro-enabled
+    DOCM needs the complete file, so they are decided by :func:`validate` later.
+    """
+    file = InspectedFile(Upload(io.BytesIO(head), name, size, declared_type, field), policy.sniffer)
+    wanted = set(stages)
+    if "type" in wanted:
+        detected = _safe_file_type(file)
+        if detected is not None and detected.container is not None:
+            wanted.discard("type")
+    violations: list[Violation] = []
+    checks_run: list[str] = []
+    for check in policy.pipeline:
+        if check.stage not in wanted:
+            continue
+        found = _run_check(check, file)
+        if found is None:
+            continue
+        checks_run.append(check.name)
+        violations.extend(found)
+        if found and policy.fail_fast:
+            break
+    return ValidationResult(
+        name=file.name,
+        field=field,
+        size=size,
+        file_type=_safe_file_type(file) if "type" in wanted else None,
+        declared_type=declared_type,
+        violations=tuple(violations),
+        checks_run=tuple(checks_run),
+    )
+
+
+def _run_check(check: Check, file: InspectedFile) -> list[Violation] | None:
+    """Run one check; ``None`` means "not applicable"."""
+    try:
+        if not check.applies(file):
+            return None
+        return list(check.run(file))
+    except ContentTooLargeError as exc:
+        logger.warning("Check %r could not inspect %r: %s", check.name, file.name, exc)
+        error = str(exc)
+        reason = "too_large"
+        message = "The file is too large to be verified."
+    except Exception as exc:
+        logger.exception("Check %r crashed on %r", check.name, file.name)
+        error = f"{type(exc).__name__}: {exc}"
+        reason = "crashed"
+        message = "The file could not be verified."
+    details = {"error": error, "reason": reason}
+    return [Violation(code=Code.CHECK_FAILED, message=message, check=check.name, details=details)]
+
+
+def _recheck_name(name: str, file: InspectedFile, policy: Policy) -> list[Violation]:
+    """Names produced by sanitizers must pass the same filename rules as uploaded ones."""
+    probe = InspectedFile(Upload(io.BytesIO(b""), name, 0, None, file.field), policy.sniffer)
+    return [
+        Violation(v.code, v.message, v.check, {**v.details, "in_sanitized_output": True})
+        for v in _run_check(policy.filename, probe) or []
+    ]
+
+
+def _recheck(data: bytes, name: str, file: InspectedFile, policy: Policy) -> list[Violation]:
+    """Run the content checks on what the sanitizers produced.
+
+    Re-encoding must not *create* what the checks refuse: payloads can be crafted to
+    appear only after an image library compresses the pixels again.
+    """
+    output = InspectedFile(
+        Upload(io.BytesIO(data), name, len(data), file.declared_type, file.field), policy.sniffer
+    )
+    found: list[Violation] = []
+    for check in policy.content_checks:
+        for violation in _run_check(check, output) or []:
+            details = {**violation.details, "in_sanitized_output": True}
+            found.append(Violation(violation.code, violation.message, violation.check, details))
+    return found
+
+
+def _sanitize(file: InspectedFile, policy: Policy) -> SanitizedFile | None:
+    current = file
+    data: bytes | None = None
+    applied: list[str] = []
+    for sanitizer in policy.sanitizers:
+        if not sanitizer.applies(current):
+            continue
+        replacement = sanitizer.sanitize(current)
+        applied.append(sanitizer.name)
+        if replacement.data is None and replacement.name is None:
+            continue
+        if replacement.data is not None:
+            data = replacement.data
+        stream = io.BytesIO(data) if data is not None else current.stream
+        current = InspectedFile(
+            Upload(
+                stream=stream,
+                name=replacement.name or current.name,
+                size=len(data) if data is not None else current.size,
+                declared_type=current.declared_type,
+                field=current.field,
+            ),
+            policy.sniffer,
+        )
+    if not applied:
+        return None
+
+    original, result = file.file_type, current.file_type
+    if result is None or (original is not None and result.mime != original.mime):
+        raise SanitizationError(
+            f"sanitized output is {result.mime if result else 'unidentifiable'}, "
+            f"expected {original.mime if original else 'unknown'}"
+        )
+    return SanitizedFile(
+        name=current.name, content_type=result.mime, data=data, applied=tuple(applied)
+    )
+
+
+def _safe_file_type(file: InspectedFile) -> FileType | None:
+    try:
+        return file.file_type
+    except Exception:
+        return None

uploadshield/fastapi/__init__.py

@@ -0,0 +1,23 @@
+"""FastAPI integration: validated uploads as dependencies.
+
+::
+
+    from fastapi import Depends, FastAPI, UploadFile
+    from uploadshield.fastapi import UploadShield
+
+    shield = UploadShield()
+    app = FastAPI(dependencies=[Depends(shield.all_files())])  # optional safety net
+
+    @app.post("/avatar")
+    async def avatar(file: Annotated[UploadFile, Depends(shield.file("sanitized_images"))]):
+        ...  # `file` is validated and sanitized; its content type comes from its bytes
+
+FastAPI parses the request body before dependencies run, so files are checked once
+they have been received (there is no per-file streaming hook as in Django).  Limit
+request sizes at the server or proxy.  Validation runs in Starlette's thread pool, so
+the event loop is never blocked.
+"""
+
+from .shield import UploadShield
+
+__all__ = ["UploadShield"]

uploadshield/fastapi/shield.py

@@ -0,0 +1,299 @@
+"""The :class:`UploadShield` dependency factory for FastAPI."""
+
+from __future__ import annotations
+
+import inspect
+import io
+from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
+from typing import TYPE_CHECKING, Any, cast
+
+from fastapi import File, HTTPException, Request, UploadFile, params
+from starlette.concurrency import run_in_threadpool
+from starlette.datastructures import Headers
+from starlette.datastructures import UploadFile as StarletteUploadFile
+from starlette.requests import HTTPConnection  # noqa: TC002 - FastAPI injects by annotation
+
+from uploadshield import web
+from uploadshield.core.upload import Upload
+from uploadshield.engine import validate
+from uploadshield.presets import BUILTIN_POLICIES
+
+if TYPE_CHECKING:
+    from typing import IO
+
+    from uploadshield.core.result import ValidationResult
+    from uploadshield.policy import Policy
+
+__all__ = ["UploadShield"]
+
+MODES = ("enforce", "report", "off")
+#: Set on the dependencies :meth:`UploadShield.file`/``files`` build:
+#: ``(form field, whether it takes every file of the field)``.
+FIELD_ATTR = "__uploadshield_field__"
+#: ``(request, [(field, result), ...])``, called for refused files (sync or async).
+RejectedCallback = Callable[[Request, list[tuple[str, "ValidationResult"]]], Awaitable[None] | None]
+
+
+class UploadShield:
+    """Builds FastAPI dependencies that validate uploads against policies.
+
+    ``policies`` are added to the built-in presets (``"images"``, ``"pdf"``, ...);
+    any dependency accepts a :class:`~uploadshield.Policy`, a registered name, or a
+    dotted import path.  ``mode`` is ``"enforce"`` (refuse with 400/413/415),
+    ``"report"`` (log and call ``on_rejected`` only) or ``"off"``.
+    """
+
+    def __init__(
+        self,
+        *,
+        policies: Mapping[str, Policy] | None = None,
+        default_policy: Any = "default",
+        mode: str = "enforce",
+        expose_details: bool = False,
+        messages: Mapping[str, str] | None = None,
+        on_rejected: RejectedCallback | None = None,
+    ) -> None:
+        if mode not in MODES:
+            raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
+        self.policies: dict[str, Policy] = {**BUILTIN_POLICIES, **(policies or {})}
+        self.default_policy = self.resolve(default_policy)
+        self.mode = mode
+        self.expose_details = expose_details
+        self.messages = dict(messages or {})
+        self.on_rejected = on_rejected
+
+    def resolve(self, ref: Any) -> Policy:
+        try:
+            return web.resolve_policy(ref, self.policies)
+        except web.PolicyLookupError as exc:
+            raise ValueError(str(exc)) from exc
+
+    # -- dependency factories ---------------------------------------------------------
+
+    def file(
+        self, policy: Any = None, *, alias: str = "file"
+    ) -> Callable[..., Awaitable[UploadFile]]:
+        """A dependency for one file field; returns the validated file, sanitized in place.
+
+        The ``UploadFile`` itself is updated (bytes, name, content type), so a route
+        that also receives the field directly, or declares the dependency in
+        ``dependencies=[...]``, sees the sanitized file as well.
+        """
+        resolved = self.default_policy if policy is None else self.resolve(policy)
+
+        async def dependency(request: Request, upload: UploadFile) -> UploadFile:
+            results = await self._check(request, [(alias, upload)], resolved, sanitize=True)
+            return upload if results is None else _deliver(upload, results[0])
+
+        return _with_signature(dependency, File(alias=alias), UploadFile, field=(alias, False))
+
+    def files(
+        self, policy: Any = None, *, alias: str = "files"
+    ) -> Callable[..., Awaitable[list[UploadFile]]]:
+        """A dependency for a multi-file field; returns the files, sanitized in place."""
+        resolved = self.default_policy if policy is None else self.resolve(policy)
+
+        async def dependency(request: Request, upload: list[UploadFile]) -> list[UploadFile]:
+            items = [(alias, u) for u in upload]
+            results = await self._check(request, items, resolved, sanitize=True)
+            if results is None:
+                return upload
+            return [_deliver(u, r) for u, r in zip(upload, results, strict=True)]
+
+        return _with_signature(dependency, File(alias=alias), list[UploadFile], field=(alias, True))
+
+    def all_files(self, policy: Any = None) -> Callable[..., AsyncIterator[None]]:
+        """A safety-net dependency for ``FastAPI(dependencies=...)`` or a router.
+
+        It validates every file of a multipart request to the routes it is attached to
+        against ``policy`` (default: ``default_policy``) and refuses the request if one
+        fails.  Fields that the endpoint validates itself with :meth:`file`/:meth:`files`
+        are left to that policy, as a view's own policy wins over the default in Django
+        and Flask.  Passing files get the detected ``content_type`` (in place: Starlette's
+        form data is immutable); use :meth:`file` for sanitization.  A part without a
+        filename sent to a field the route declares as ``File()`` is checked too (and
+        refused for its missing name): FastAPI would hand its content to a ``bytes``
+        parameter.  WebSocket routes are left alone, and so are apps mounted inside.
+
+        A form the safety net parses itself (the endpoint declares no form parameters)
+        is closed once the response is done, as FastAPI does with forms it parses.
+        """
+        resolved = self.default_policy if policy is None else self.resolve(policy)
+
+        async def dependency(connection: HTTPConnection) -> AsyncIterator[None]:
+            if self.mode == "off" or not (
+                isinstance(connection, Request) and _is_multipart(connection)
+            ):
+                yield
+                return
+            request = connection
+            # Starlette caches the parsed form here; set when FastAPI parsed it already.
+            owned = getattr(request, "_form", None) is None
+            form = await request.form()  # cached: the endpoint sees the same files
+            try:
+                items = _unchecked_files(request, form)
+                results = await self._check(request, items, resolved, sanitize=False)
+                for (_, upload), result in zip(items, results or [], strict=False):
+                    if result.ok and result.file_type is not None:
+                        _set_content_type(upload, result.file_type.mime)
+                yield
+            finally:
+                if owned:
+                    await request.close()  # its temporary files
+
+        return dependency
+
+    # -- internals --------------------------------------------------------------------
+
+    async def _check(
+        self,
+        request: Request,
+        items: list[tuple[str, Any]],
+        policy: Policy,
+        *,
+        sanitize: bool,
+    ) -> list[ValidationResult] | None:
+        """Validate ``items``; refuse (enforce) or report.  ``None`` when mode is off."""
+        if self.mode == "off":
+            return None
+        results = [
+            await run_in_threadpool(_validate, upload, policy, field, sanitize=sanitize)
+            for field, upload in items
+        ]
+        rejected = [
+            (field, result)
+            for (field, _), result in zip(items, results, strict=True)
+            if not result.ok
+        ]
+        for (field, upload), result in zip(items, results, strict=True):
+            if not result.ok:
+                digest = await run_in_threadpool(web.sha256_of, upload.file)
+                web.log_rejection(
+                    result,
+                    field=field,
+                    policy_name=policy.name,
+                    mode=self.mode,
+                    path=request.url.path,
+                    sha256=digest,
+                )
+        if rejected:
+            if self.on_rejected is not None:
+                outcome = self.on_rejected(request, rejected)
+                if inspect.isawaitable(outcome):
+                    await outcome
+            if self.mode == "enforce":
+                raise HTTPException(
+                    status_code=web.status_code_for(r for _, r in rejected),
+                    detail=web.rejection_payload(
+                        (r for _, r in rejected),
+                        include_details=self.expose_details,
+                        messages=self.messages,
+                        fields=[f for f, _ in rejected],
+                    ),
+                )
+        return results
+
+
+def _validate(upload: Any, policy: Policy, field: str, *, sanitize: bool) -> ValidationResult:
+    stream = cast("IO[bytes]", upload.file)
+    return validate(
+        Upload(
+            stream=stream,
+            name=upload.filename or "",
+            size=upload.size,
+            declared_type=upload.content_type,
+            field=field,
+        ),
+        policy,
+        sanitize=sanitize,
+    )
+
+
+def _deliver(upload: Any, result: ValidationResult) -> UploadFile:
+    """Sanitize ``upload`` in place: bytes, name and the detected content type.
+
+    Updating the object (rather than returning a new one) means every reference to it -
+    the dependency's return value, the route's own parameter, ``request.form()`` - sees
+    the sanitized file.
+    """
+    if not result.ok or result.file_type is None:
+        return upload  # report mode: untouched
+    sanitized = result.sanitized
+    content_type = result.file_type.mime
+    if sanitized is not None:
+        upload.filename = sanitized.name
+        content_type = sanitized.content_type
+        if sanitized.data is not None:
+            original = upload.file
+            upload.file = io.BytesIO(sanitized.data)
+            upload.size = len(sanitized.data)
+            original.close()  # the form now closes the replacement instead
+    _set_content_type(upload, content_type)
+    return upload
+
+
+def _is_multipart(request: Request) -> bool:
+    """Decide "multipart" the way Starlette's parser does: case- and space-insensitive."""
+    media_type = request.headers.get("content-type", "").split(";", 1)[0]
+    return media_type.strip().lower() == "multipart/form-data"
+
+
+def _unchecked_files(request: Request, form: Any) -> list[tuple[str, Any]]:
+    """The parts the safety net must check: every file no endpoint dependency takes.
+
+    A field taken by :meth:`UploadShield.file` only exempts the part FastAPI hands it
+    (the last one); further parts under the same name are checked here.  Parts without
+    a filename that a ``File()`` parameter would receive as bytes are checked as files.
+    """
+    own, file_fields = _route_fields(request)
+    items: list[tuple[str, Any]] = []
+    for key, value in form.multi_items():
+        if isinstance(value, StarletteUploadFile):
+            many = own.get(key)
+            if many is True or (many is False and value is form.get(key)):
+                continue
+            if not web.is_unused_file_input(value.filename, value.file):
+                items.append((key, value))
+        elif isinstance(value, str) and value and key in file_fields:
+            data = value.encode("utf-8", "surrogateescape")
+            items.append((key, StarletteUploadFile(io.BytesIO(data), size=len(data))))
+    return items
+
+
+def _set_content_type(upload: StarletteUploadFile, mime: str) -> None:
+    """Replace the client-declared type in place (the safety net cannot swap files)."""
+    raw = [(k, v) for k, v in upload.headers.raw if k.lower() != b"content-type"]
+    upload.headers = Headers(raw=[*raw, (b"content-type", mime.encode("latin-1"))])
+
+
+def _route_fields(request: Request) -> tuple[dict[str, bool], set[str]]:
+    """``(fields file()/files() dependencies take -> takes all parts, File() fields)``."""
+    root = getattr(request.scope.get("route"), "dependant", None)
+    own: dict[str, bool] = {}
+    file_fields: set[str] = set()
+    pending = [root] if root is not None else []
+    while pending:
+        dependant = pending.pop()
+        if (field := getattr(dependant.call, FIELD_ATTR, None)) is not None:
+            alias, many = field
+            own[alias] = own.get(alias, False) or many
+        for body_field in getattr(dependant, "body_params", ()):
+            if isinstance(getattr(body_field, "field_info", None), params.File):
+                file_fields.add(body_field.alias)
+        pending.extend(dependant.dependencies)
+    return own, file_fields
+
+
+def _with_signature(
+    dependency: Any, default: Any, annotation: Any, *, field: tuple[str, bool]
+) -> Any:
+    """Give the dependency an ``upload`` parameter bound to the form field ``alias``."""
+    kw = inspect.Parameter.KEYWORD_ONLY
+    dependency.__signature__ = inspect.Signature(
+        [
+            inspect.Parameter("request", kw, annotation=Request),
+            inspect.Parameter("upload", kw, default=default, annotation=annotation),
+        ]
+    )
+    setattr(dependency, FIELD_ATTR, field)
+    return dependency