uploadshield 2.0.0__py3-none-any.whl

uploadshield/checks/magika.py

@@ -0,0 +1,129 @@
+"""A machine-learning second opinion from Google Magika (``uv add "uploadshield[magika]"``).
+
+Magika classifies 200+ formats, including the text and source-code formats that have
+no magic number (it tells CSV from Python, PowerShell or batch).  It is used here as a
+**veto only**: it can refuse a file the deterministic checks accepted, never accept one
+they refused.  The reasons:
+
+* it is a probabilistic classifier with no claims about adversarial robustness;
+* it only looks at the first and last KiB of a file, so whoever writes those bytes
+  steers its verdict;
+* a model upgrade changes its answers.
+
+A bad verdict can therefore cost availability (a false positive) but never safety.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from uploadshield.core.check import Check
+from uploadshield.core.violations import Code, Violation
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from uploadshield.core.upload import InspectedFile
+
+__all__ = ["DEFAULT_MAGIKA_FORBIDDEN", "MAGIKA_EQUIVALENTS", "MagikaCheck"]
+
+#: Magika labels for content that executes or renders when opened or served.
+DEFAULT_MAGIKA_FORBIDDEN = frozenset({
+    # executables, installers, bytecode, shortcuts
+    "pebin", "elf", "macho", "coff", "msi", "lnk", "dex", "apk", "jar", "javabytecode",
+    "pythonbytecode", "wasm", "chm", "swf", "crx", "xpi", "internetshortcut", "winregistry",
+    # scripts
+    "shell", "batch", "powershell", "vba", "autohotkey", "autoit", "javascript",
+    "typescript", "coffeescript", "python", "perl", "ruby", "php", "asp", "lua", "tcl",
+    # active markup and server configuration
+    "html", "svg", "mht", "htaccess",
+})  # fmt: skip
+
+#: Labels that correspond to a registered type: if a policy explicitly allows that
+#: type (e.g. ``text/html``), Magika agreeing with it is not a reason to refuse.
+MAGIKA_EQUIVALENTS: dict[str, str] = {
+    "html": "text/html",
+    "svg": "image/svg+xml",
+    "php": "application/x-httpd-php",
+    "shell": "application/x-sh",
+    "pebin": "application/vnd.microsoft.portable-executable",
+    "elf": "application/x-elf",
+    "macho": "application/x-mach-binary",
+    "jar": "application/java-archive",
+    "apk": "application/vnd.android.package-archive",
+    "wasm": "application/wasm",
+    "javabytecode": "application/java-vm",
+}
+
+_models: dict[str, Any] = {}
+_models_lock = threading.Lock()
+
+
+def _model(prediction_mode: str) -> Any:
+    """One ``Magika`` instance per process and mode, created on first use."""
+    if (model := _models.get(prediction_mode)) is None:
+        with _models_lock:
+            if (model := _models.get(prediction_mode)) is None:
+                from magika import Magika, PredictionMode
+
+                model = Magika(prediction_mode=PredictionMode(prediction_mode))
+                _models[prediction_mode] = model
+    return model
+
+
+@dataclass(frozen=True, slots=True)
+class MagikaCheck(Check):
+    """Refuse files that Magika confidently recognises as code, executables or markup.
+
+    Typical catch: a ``.csv`` or ``.txt`` that is really a PowerShell, batch or Python
+    script; deterministic sniffing can only prove it is text.  Only labels in
+    ``forbidden`` with a score of at least ``min_score`` refuse a file, and never when
+    the label matches the type the policy already accepted (see
+    :data:`MAGIKA_EQUIVALENTS`).  ``prediction_mode`` is ``"high_confidence"``,
+    ``"medium_confidence"`` or ``"best_guess"``.
+    """
+
+    _deconstruct_module = "uploadshield.checks"
+    name = "magika"
+    requires = ("magika",)
+
+    forbidden: frozenset[str] = DEFAULT_MAGIKA_FORBIDDEN
+    min_score: float = 0.9
+    prediction_mode: str = "high_confidence"
+
+    def __post_init__(self) -> None:
+        if self.prediction_mode not in ("high_confidence", "medium_confidence", "best_guess"):
+            raise ValueError(f"Unknown Magika prediction mode {self.prediction_mode!r}")
+        if not 0.0 <= self.min_score <= 1.0:
+            raise ValueError("min_score must be between 0 and 1")
+        object.__setattr__(self, "forbidden", frozenset(self.forbidden))
+
+    def applies(self, file: InspectedFile) -> bool:
+        return file.file_type is not None
+
+    def run(self, file: InspectedFile) -> Iterator[Violation]:
+        model = _model(self.prediction_mode)
+        with file.open() as fh:
+            result = model.identify_stream(fh)
+        if not result.ok:
+            raise RuntimeError(f"Magika could not analyse the file: {result.status}")
+        output = result.output
+        label = str(output.label)
+        accepted = file.file_type.mime if file.file_type else None
+        if (
+            label in self.forbidden
+            and result.score >= self.min_score
+            and MAGIKA_EQUIVALENTS.get(label) != accepted
+        ):
+            yield self.violation(
+                Code.CONTENT_MISMATCH,
+                f"The file's content looks like {output.description}, which is not allowed here.",
+                magika_label=label,
+                score=round(float(result.score), 3),
+                group=str(output.group),
+                mime_type=str(output.mime_type),
+                model=str(model.get_model_name()),
+                detected_type=accepted,
+            )

uploadshield/checks/markup.py

@@ -0,0 +1,181 @@
+"""Script-capable markup (SVG) and code or polyglot payloads hidden in media files."""
+
+from __future__ import annotations
+
+import html
+import re
+import struct
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from uploadshield.core.check import Check
+from uploadshield.core.types import GiB, MiB
+from uploadshield.core.upload import ContentTooLargeError
+from uploadshield.core.violations import Code, Violation
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from uploadshield.core.upload import InspectedFile
+
+__all__ = ["EmbeddedContent", "SvgActiveContent"]
+
+# "<" cannot occur inside an XML attribute value, so "<[^<]*" spans a whole start tag -
+# also when an attribute value contains ">" - and the scan stays linear.
+_SVG_RULES: dict[str, re.Pattern[str]] = {
+    "script element": re.compile(r"<\s*(?:[a-z0-9_-]+:)?script\b"),
+    "script URI": re.compile(r"(?:java|vb)script:|data:text/html"),
+    "foreignObject": re.compile(r"<\s*(?:[a-z0-9_-]+:)?foreignobject\b"),
+    "embedded document": re.compile(r"<\s*(?:iframe|embed|object|frame)\b"),
+    "entity declaration": re.compile(r"<!entity|<!doctype[^<]*\["),
+    "XSLT stylesheet": re.compile(r"<\?xml-stylesheet"),
+    "external resource": re.compile(r"href\s*=\s*[\"']?\s*(?:https?:)?//"),
+    "href animation": re.compile(
+        r"<\s*(?:set|animate)\b[^<]*attributename\s*=\s*[\"']?(?:xlink:)?href"
+    ),
+}
+#: An ``on...=`` attribute inside a start tag.  Attribute values are skipped as quoted
+#: strings, so a ">" inside a value does not end the tag, while text after the tag is
+#: never mistaken for an attribute.  Matched on the text *before* character references
+#: are decoded: "&quot;>" inside a value must not look like the end of the tag.  Every
+#: part stops at "<", ">" or a quote, so the scan stays linear.
+_EVENT_HANDLER = re.compile(
+    r"<[a-z][^\s/<>]*"
+    r"(?:[\s/]+[^\s/<>=\"']+(?:\s*=\s*(?:\"[^\"]*\"|'[^']*'|[^\s<>\"']*))?)*?"
+    r"(?:[\s/]+|(?<=[\"']))on[a-z]+\s*="
+)
+_URL_INVISIBLES = re.compile(r"[\t\n\r\x00]")
+
+
+def _decode_markup(data: bytes) -> str:
+    if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+        return data.decode("utf-16", "replace").lower()
+    return data.decode("utf-8", "replace").lower()
+
+
+def _normalize_markup(text: str) -> str:
+    # Browsers decode character references and ignore tabs/newlines inside URLs, so
+    # "&#106;ava&#x09;script:" must be seen as "javascript:".
+    return _URL_INVISIBLES.sub("", html.unescape(text)).lower()
+
+
+@dataclass(frozen=True, slots=True)
+class SvgActiveContent(Check):
+    """Refuse SVG images that can run script or load external resources.
+
+    SVG is XML that browsers execute like HTML; when served from your origin it is a
+    stored-XSS vector.  Prefer rasterising user SVGs; if you accept them, this check
+    rejects scripts, event handlers, ``javascript:`` URIs, ``foreignObject``,
+    entities (XXE), XSLT and external references.
+    """
+
+    _deconstruct_module = "uploadshield.checks"
+    name = "svg"
+
+    max_bytes: int = 10 * MiB
+
+    def applies(self, file: InspectedFile) -> bool:
+        return file.file_type is not None and file.file_type.mime == "image/svg+xml"
+
+    def run(self, file: InspectedFile) -> Iterator[Violation]:
+        raw = _decode_markup(file.read(self.max_bytes))
+        text = _normalize_markup(raw)
+        found = [label for label, rule in _SVG_RULES.items() if rule.search(text)]
+        if _EVENT_HANDLER.search(raw):
+            found.insert(0, "event handler attribute")
+        if found:
+            yield self.violation(
+                Code.SVG_ACTIVE_CONTENT, "The SVG image contains active content.", found=found
+            )
+
+
+# Markers specific enough that they do not occur by chance in compressed media data
+# (a 3-byte "<?=" alone would, about once per 20 MB): PHP open and short-echo tags,
+# HTML scripts, JSP/ASP directives, server-side includes and PHAR stubs (polyglot
+# images that PHP deserializes through phar://).
+_CODE_MARKERS = re.compile(
+    rb"<\?php\s|<\?=\s{0,3}(?:[$`]|[a-z_]\w{0,30}\s*\()|<script[\s>]"
+    rb"|<%@\s*(?:page|language|import)\b|<jsp:"
+    rb"|<!--\s*#\s*(?:exec|include|echo|config|set|printenv)\b"
+    rb"|__halt_compiler\s*\(\s*\)",
+    re.IGNORECASE,
+)
+#: What a PHP ``include`` runs from *any* file, text included: open and short-echo tags.
+_INCLUDABLE_CODE = re.compile(rb"<\?php\s|<\?=\s{0,3}(?:[$`]|[a-z_]\w{0,30}\s*\()", re.IGNORECASE)
+_OVERLAP = 64
+_EOCD = b"PK\x05\x06"
+_CENTRAL_DIRECTORY = b"PK\x01\x02"
+_EOCD_SEARCH_WINDOW = 65_557  # 22-byte record + maximum comment length
+
+
+def _embedded_zip_offset(tail: bytes) -> int | None:
+    """Locate a *structurally valid* ZIP end record (not just its 4 magic bytes)."""
+    pos = tail.rfind(_EOCD)
+    while pos != -1:
+        if pos + 22 <= len(tail):
+            disk, cd_disk, on_disk, total, cd_size = struct.unpack_from("<HHHHI", tail, pos + 4)
+            cd_start = pos - cd_size
+            if (
+                disk == 0
+                and cd_disk == 0
+                and on_disk == total
+                and total > 0
+                and cd_start >= 0
+                and tail[cd_start : cd_start + 4] == _CENTRAL_DIRECTORY
+            ):
+                return pos
+        pos = tail.rfind(_EOCD, 0, pos)
+    return None
+
+
+@dataclass(frozen=True, slots=True)
+class EmbeddedContent(Check):
+    """Refuse files that smuggle server-side code or a second file format.
+
+    In media it catches the classic "PHP in EXIF comment" image, HTML/script payloads
+    and polyglots: an image with a ZIP appended (GIFAR), or a PDF header hidden near the
+    start where PDF readers still find it.  In text files (``scan_text``) it refuses PHP
+    tags anywhere, because PHP's ``include`` runs them from a ``.txt`` or ``.csv`` too.
+    Files larger than ``max_scan_bytes`` are refused rather than scanned in part.
+    """
+
+    _deconstruct_module = "uploadshield.checks"
+    name = "embedded"
+
+    categories: frozenset[str] = frozenset({"image", "audio", "video"})
+    scan_text: bool = True
+    max_scan_bytes: int = 1 * GiB
+
+    def applies(self, file: InspectedFile) -> bool:
+        file_type = file.file_type
+        if file_type is None:
+            return False
+        if file_type.is_text:
+            return self.scan_text
+        return file_type.category in self.categories
+
+    def run(self, file: InspectedFile) -> Iterator[Violation]:
+        if file.size > self.max_scan_bytes:
+            raise ContentTooLargeError(f"{file.size} bytes exceeds {self.max_scan_bytes}")
+        text = file.file_type is not None and file.file_type.is_text
+        if marker := self._find_code(file, _INCLUDABLE_CODE if text else _CODE_MARKERS):
+            yield self.violation(
+                Code.EMBEDDED_CODE, "The file contains embedded code.", marker=marker
+            )
+            return
+        if text:
+            return
+        if b"%PDF-" in file.head(1024):
+            yield self.violation(Code.POLYGLOT, "The file also contains a PDF document.")
+        elif _embedded_zip_offset(file.tail(_EOCD_SEARCH_WINDOW)) is not None:
+            yield self.violation(Code.POLYGLOT, "The file also contains a ZIP archive.")
+
+    @staticmethod
+    def _find_code(file: InspectedFile, markers: re.Pattern[bytes]) -> str | None:
+        carry = b""
+        for chunk in file.iter_chunks():
+            window = carry + chunk
+            if match := markers.search(window):
+                return match.group(0).decode("latin-1").strip()
+            carry = window[-_OVERLAP:]
+        return None

uploadshield/checks/office.py

@@ -0,0 +1,174 @@
+"""Macros and remote references in Office documents (OOXML, ODF, legacy OLE2)."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from uploadshield.core.check import Check
+from uploadshield.core.containers import zip_entries, zip_member, zip_read, zip_search
+from uploadshield.core.types import MiB
+from uploadshield.core.upload import ContentTooLargeError
+from uploadshield.core.violations import Code, Violation
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from uploadshield.core.upload import InspectedFile
+
+__all__ = ["OFFICE_ZIP_TYPES", "OfficeMacros"]
+
+OFFICE_ZIP_TYPES = frozenset(
+    {
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "application/vnd.ms-word.document.macroenabled.12",
+        "application/vnd.ms-excel.sheet.macroenabled.12",
+        "application/vnd.ms-powerpoint.presentation.macroenabled.12",
+        "application/vnd.oasis.opendocument.text",
+        "application/vnd.oasis.opendocument.spreadsheet",
+        "application/vnd.oasis.opendocument.presentation",
+    }
+)
+
+#: Relationship types that make Office fetch and run or embed remote content
+#: ("remote template injection").  Plain hyperlinks are external too but harmless.
+_REMOTE_RELATIONSHIPS = ("attachedtemplate", "oleobject", "frame", "subdocument")
+# Every pattern stops at the next "<" (or cell delimiter), so scanning stays linear
+# whatever the input: no quantifier can run across a tag boundary and backtrack.
+_RELATIONSHIP = re.compile(rb"<relationship\b[^<>]*>", re.IGNORECASE)
+_DDE_FIELD = re.compile(
+    rb"(?:instrtext[^<>]*>|instr=\")[ \t\r\n]*\"?[ \t\r\n]*dde(?:auto)?\b", re.IGNORECASE
+)
+#: Excel dynamic data exchange: ``=cmd|' /C calc'!A0`` runs a program when the sheet
+#: opens.  "|" is no Excel operator, so ``application|topic!item`` is a DDE reference.
+_DDE_LINK = re.compile(rb"<ddeLink\b", re.IGNORECASE)
+_DDE_FORMULA = re.compile(rb"<f\b[^<>]*>[^<|]*\|[^<]*!", re.IGNORECASE)
+_DDE_CELL = re.compile(rb"(?:^|[,;\t\r\n])[ \t]*\"?[=+\-@][^,;\t\r\n|]*\|[^,;\t\r\n]*!")
+_CSV_OVERLAP = 4096
+_VBA_PROJECT_UTF16 = "_VBA_PROJECT".encode("utf-16-le")
+_OLE_SCAN_CHUNK = 4 * MiB
+
+
+def _is_macro_member(name: str) -> bool:
+    lowered = name.lower()
+    return (
+        lowered.endswith(("vbaproject.bin", "vbadata.xml"))
+        or "/activex/" in f"/{lowered}"
+        or lowered.startswith(("basic/", "scripts/"))  # OpenDocument macros
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class OfficeMacros(Check):
+    """Refuse Office documents with VBA macros, ActiveX controls, DDE (Word fields, Excel
+    links and formulas, and formulas in CSV files, which open in Excel) or remote
+    templates.
+
+    For legacy OLE2 files (``.doc``/``.xls``) the presence of a ``_VBA_PROJECT``
+    stream is detected from the directory entries; XLM (Excel 4.0) macros in OLE2
+    files are not detected - use :class:`~uploadshield.checks.YaraRules` or
+    :class:`~uploadshield.checks.ClamAV` if you must accept those formats.
+    """
+
+    _deconstruct_module = "uploadshield.checks"
+    name = "office"
+
+    block_external_references: bool = True
+    max_part_bytes: int = 16 * MiB
+
+    def applies(self, file: InspectedFile) -> bool:
+        file_type = file.file_type
+        return file_type is not None and (
+            file_type.mime in OFFICE_ZIP_TYPES
+            or file_type.container == "ole"
+            or file_type.mime == "text/csv"
+        )
+
+    def run(self, file: InspectedFile) -> Iterator[Violation]:
+        file_type = file.file_type
+        if file_type is not None and file_type.container == "ole":
+            yield from self._run_ole(file)
+        elif file_type is not None and file_type.mime == "text/csv":
+            yield from self._run_csv(file)
+        else:
+            yield from self._run_zip(file)
+
+    def _run_csv(self, file: InspectedFile) -> Iterator[Violation]:
+        """CSV opens in Excel, which evaluates DDE formulas in cells."""
+        carry = b""
+        for chunk in file.iter_chunks():
+            window = carry + chunk
+            if _DDE_CELL.search(window):
+                yield self._dde()
+                return
+            carry = window[-_CSV_OVERLAP:]
+
+    def _run_ole(self, file: InspectedFile) -> Iterator[Violation]:
+        carry = b""
+        for chunk in file.iter_chunks(_OLE_SCAN_CHUNK):
+            if _VBA_PROJECT_UTF16 in carry + chunk:
+                yield self.violation(Code.OFFICE_MACROS, "Documents with macros are not allowed.")
+                return
+            carry = chunk[-len(_VBA_PROJECT_UTF16) :]
+
+    def _run_zip(self, file: InspectedFile) -> Iterator[Violation]:
+        entries = zip_entries(file) or []
+        names = [e.filename for e in entries]
+        if macros := [n for n in names if _is_macro_member(n)]:
+            yield self.violation(
+                Code.OFFICE_MACROS, "Documents with macros are not allowed.", parts=macros[:10]
+            )
+            return
+        # Fields run from every part Word renders: body, headers, footers, notes, ...
+        word_parts = [n for n in names if n.startswith("word/") and n.endswith(".xml")]
+        for part in word_parts:
+            if zip_search(file, part, _DDE_FIELD):
+                yield self.violation(
+                    Code.OFFICE_MACROS, "Documents with DDE fields are not allowed.", parts=[part]
+                )
+                return
+        spreadsheet = [
+            n
+            for n in names
+            if n.startswith(("xl/externalLinks/", "xl/worksheets/")) and n.endswith(".xml")
+        ]
+        for part in spreadsheet:
+            pattern = _DDE_LINK if part.startswith("xl/externalLinks/") else _DDE_FORMULA
+            if zip_search(file, part, pattern):
+                yield self._dde(parts=[part])
+                return
+        if self.block_external_references and (targets := self._remote_targets(file, names)):
+            yield self.violation(
+                Code.OFFICE_EXTERNAL_REFERENCE,
+                "The document loads content from external locations.",
+                targets=targets[:10],
+            )
+
+    def _dde(self, **details: object) -> Violation:
+        return self.violation(
+            Code.OFFICE_MACROS,
+            "Spreadsheets with DDE links or formulas are not allowed.",
+            **details,
+        )
+
+    def _remote_targets(self, file: InspectedFile, names: list[str]) -> list[str]:
+        targets: list[str] = []
+        for rels in (n for n in names if n.endswith(".rels")):
+            info = zip_member(file, rels)
+            if info is not None and info.file_size > self.max_part_bytes:
+                # Unreadable means unchecked: refuse rather than skip (fails closed).
+                raise ContentTooLargeError(f"{rels} exceeds {self.max_part_bytes} bytes")
+            xml = zip_read(file, rels, self.max_part_bytes) or b""
+            for element in _RELATIONSHIP.findall(xml):
+                lowered = element.lower()
+                if b'targetmode="external"' not in lowered:
+                    continue
+                type_match = re.search(rb'type="([^"]*)"', lowered)
+                rel_type = type_match.group(1).decode("latin-1") if type_match else ""
+                if rel_type.rsplit("/", 1)[-1] in _REMOTE_RELATIONSHIPS:
+                    target = re.search(rb'target="([^"]*)"', element, re.IGNORECASE)
+                    targets.append(target.group(1).decode("latin-1") if target else rels)
+        return targets