uploadshield 2.0.0__py3-none-any.whl

uploadshield/cli/_json.py

@@ -0,0 +1,80 @@
+"""The ``--json`` document: one object with every file, a summary and the exit status."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from uploadshield.web import json_safe
+
+from ._verdicts import Verdict
+
+if TYPE_CHECKING:
+    from ._verdicts import Outcome, Summary
+
+__all__ = ["SCHEMA", "document", "dumps", "json_safe"]
+
+#: Bumped when a field changes meaning or disappears; new fields do not bump it.
+SCHEMA = 1
+
+
+def file_entry(outcome: Outcome) -> dict[str, Any]:
+    entry: dict[str, Any] = {"path": outcome.display, "verdict": outcome.verdict.value}
+    if outcome.skipped is not None:
+        return {"path": outcome.display, "verdict": None, "skipped": outcome.skipped}
+    if outcome.error is not None:
+        entry["error"] = outcome.error
+    if outcome.result is not None:
+        entry["sha256"] = outcome.sha256
+        result = outcome.result
+        entry |= result.as_dict(include_details=True)
+        entry.pop("field", None)  # always None outside a web request
+        for error, verdict in zip(entry["errors"], outcome.verdicts, strict=True):
+            error["verdict"] = verdict.value
+        if (sanitized := result.sanitized) is not None:
+            entry["sanitized_as"] = {
+                "name": sanitized.name,
+                "content_type": sanitized.content_type,
+                "size": len(sanitized.data) if sanitized.data is not None else result.size,
+                "content_changed": sanitized.data is not None,
+            }
+    if outcome.saved is not None:
+        entry["saved"] = outcome.saved
+    if outcome.save_error is not None:
+        entry["save_error"] = outcome.save_error
+    return entry
+
+
+def document(
+    *,
+    command: str,
+    policy: str | None,
+    outcomes: list[Outcome],
+    summary: Summary,
+    warnings: list[str],
+    interrupted: bool,
+    exit_code: int,
+) -> dict[str, Any]:
+    from uploadshield import __version__
+
+    return {
+        "schema": SCHEMA,
+        "version": __version__,
+        "command": command,
+        "policy": policy,
+        "files": [file_entry(o) for o in outcomes],
+        "summary": {
+            "files": summary.files,
+            **{verdict.value: summary.counts[verdict] for verdict in Verdict},
+            "skipped": sum(summary.skipped.values()),
+            "seconds": round(summary.seconds, 3),
+        },
+        "warnings": warnings,
+        "interrupted": interrupted,
+        "exit_code": exit_code,
+    }
+
+
+def dumps(data: dict[str, Any]) -> str:
+    # ASCII escapes keep undecodable names (lone surrogates) valid and printable.
+    return json.dumps(json_safe(data), indent=2, ensure_ascii=True)

uploadshield/cli/_render.py

@@ -0,0 +1,319 @@
+"""Human-readable output for ``scan`` and ``validate``, and the ``--json`` reporter."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Protocol, TextIO
+
+from uploadshield.core.types import format_size
+from uploadshield.core.violations import Code
+
+from ._json import document, dumps
+from ._style import Style, safe
+from ._verdicts import Verdict
+
+if TYPE_CHECKING:
+    from uploadshield.core.check import Check, Sanitizer
+    from uploadshield.core.result import ValidationResult
+    from uploadshield.policy import Policy
+
+    from ._verdicts import Outcome, Summary
+
+__all__ = ["HumanReporter", "JsonReporter", "Reporter", "missing_dependency"]
+
+_COLORS = {
+    Verdict.CLEAN: "green",
+    Verdict.REFUSED: "yellow",
+    Verdict.NOT_VERIFIED: "magenta",
+    Verdict.DANGEROUS: "red",
+}
+#: Details worth a glance in a scan (``validate`` shows all of them).
+_HIGHLIGHTS = (
+    "keywords",
+    "members",
+    "found",
+    "marker",
+    "signature",
+    "rules",
+    "magika_label",
+    "read_as",
+)
+#: Optional dependencies without which a component silently does nothing.
+_NEEDS: dict[str, tuple[str, str]] = {
+    "ImageIntegrity": ("PIL", "images"),
+    "ReencodeImage": ("PIL", "images"),
+    "MagikaCheck": ("magika", "magika"),
+    "YaraRules": ("yara", "yara"),
+}
+
+
+def missing_dependency(component: Check | Sanitizer) -> str | None:
+    """The extra a check or sanitizer needs but cannot find (it is then inactive)."""
+    needed = _NEEDS.get(type(component).__name__)
+    if needed is None or importlib.util.find_spec(needed[0]) is not None:
+        return None
+    return needed[1]
+
+
+class Reporter(Protocol):
+    def file(self, outcome: Outcome) -> None: ...
+    def warn(self, text: str) -> None: ...
+    def finish(self, summary: Summary, *, interrupted: bool, exit_code: int) -> None: ...
+
+
+class JsonReporter:
+    """Collects everything and prints one document at the end."""
+
+    def __init__(self, out: TextIO, *, command: str, policy: str | None) -> None:
+        self.out = out
+        self.command = command
+        self.policy = policy
+        self.outcomes: list[Outcome] = []
+        self.warnings: list[str] = []
+
+    def file(self, outcome: Outcome) -> None:
+        self.outcomes.append(outcome)
+
+    def warn(self, text: str) -> None:
+        self.warnings.append(text)
+
+    def finish(self, summary: Summary, *, interrupted: bool, exit_code: int) -> None:
+        data = document(
+            command=self.command,
+            policy=self.policy,
+            outcomes=self.outcomes,
+            summary=summary,
+            warnings=self.warnings,
+            interrupted=interrupted,
+            exit_code=exit_code,
+        )
+        print(dumps(data), file=self.out)
+
+
+class HumanReporter:
+    """Prints each file as it is inspected: a line per file (scan) or a trace (validate)."""
+
+    def __init__(
+        self,
+        out: TextIO,
+        err: TextIO,
+        style: Style,
+        *,
+        policy: Policy,
+        detailed: bool,
+        quiet: bool,
+    ) -> None:
+        self.out, self.err, self.s = out, err, style
+        self.policy = policy
+        self.detailed = detailed
+        self.quiet = quiet
+        self._root: str | None = None
+
+    # -- per file ---------------------------------------------------------------------
+
+    def file(self, outcome: Outcome) -> None:
+        if outcome.skipped is not None:
+            if not self.quiet:
+                self._heading(outcome)
+                line = f"  - {safe(outcome.label)}  skipped: {outcome.skipped}"
+                self._print(self.s.dim(line))
+            return
+        if self.quiet and outcome.verdict is Verdict.CLEAN and outcome.save_error is None:
+            return
+        self._heading(outcome)
+        if self.detailed:
+            self._trace(outcome)
+        else:
+            self._line(outcome)
+
+    def _heading(self, outcome: Outcome) -> None:
+        """Files found in a directory are listed under it, relative to it."""
+        if outcome.root != self._root:
+            self._root = outcome.root
+            if outcome.root is not None:
+                self._print(self.s.bold(safe(outcome.root.rstrip("/\\") + "/")))
+
+    def _glyph(self, verdict: Verdict) -> str:
+        glyph = {
+            Verdict.CLEAN: self.s.ok,
+            Verdict.REFUSED: "!",
+            Verdict.NOT_VERIFIED: "?",
+            Verdict.DANGEROUS: self.s.bad,
+        }[verdict]
+        return self.s.paint(glyph, _COLORS[verdict], "bold")
+
+    def _describe(self, result: ValidationResult) -> str:
+        kind = result.file_type.label if result.file_type is not None else "unknown format"
+        return f"{kind} {self.s.dot} {format_size(result.size)}"
+
+    def _line(self, outcome: Outcome) -> None:
+        name = self.s.bold(safe(outcome.label))
+        if outcome.result is None:
+            reason = outcome.error or "could not be read"
+            self._print(f"  {self._glyph(outcome.verdict)} {name}  not verified: {reason}")
+            return
+        result = outcome.result
+        self._print(
+            f"  {self._glyph(outcome.verdict)} {name}  {self.s.dim(self._describe(result))}"
+        )
+        for violation, verdict in zip(result.violations, outcome.verdicts, strict=True):
+            # A line per file shows what makes it this verdict; --json lists everything.
+            if verdict is not outcome.verdict:
+                continue
+            message = violation.message
+            if violation.code == Code.TYPE_NOT_ALLOWED and verdict is Verdict.NOT_VERIFIED:
+                message = (
+                    "Its contents cannot be inspected: unpack it and scan the files, "
+                    "or add --clamav."
+                )
+            extra = _highlights(violation.details, self.s.ellipsis)
+            code = self.s.paint(str(violation.code), _COLORS[verdict])
+            self._print(f"      {code}  {safe(message)}{self.s.dim(extra)}")
+
+    def _trace(self, outcome: Outcome) -> None:
+        s = self.s
+        name = s.bold(safe(outcome.label))
+        policy = f'"{self.policy.name or "custom"}"'
+        if outcome.result is None:
+            self._print(f"{self._glyph(outcome.verdict)} {name}  not verified: {outcome.error}")
+            self._print("")
+            return
+        result = outcome.result
+        self._print(f"{self._glyph(outcome.verdict)} {name}  {self._head(outcome, policy)}")
+        if result.name != Path(outcome.label).name:
+            self._row("as", safe(result.name))
+        detected = (
+            f"{result.file_type.label} ({result.file_type.mime}), {format_size(result.size)}"
+            if result.file_type is not None
+            else f"unknown format, {format_size(result.size)}"
+        )
+        self._row("detected", detected)
+        if result.declared_type:
+            declared = safe(result.declared_type)
+            if not self.policy.match_declared_type:
+                declared += s.dim("  (not compared: the policy has match_declared_type=False)")
+            self._row("declared", declared)
+        self._checks(result)
+        self._violations(outcome)
+        if result.sanitized is not None:
+            sanitized = result.sanitized
+            size = len(sanitized.data) if sanitized.data is not None else result.size
+            changed = "renamed only" if sanitized.data is None else "rewritten"
+            self._row(
+                "sanitized",
+                f"{', '.join(sanitized.applied)} {s.arrow} {safe(sanitized.name)} "
+                f"({sanitized.content_type}, {format_size(size)}, {changed})",
+            )
+        if outcome.saved is not None:
+            self._row("saved", safe(outcome.saved))
+        if outcome.save_error is not None:
+            self._row("not saved", s.paint(outcome.save_error, "red"))
+        if not result.ok:
+            codes = " ".join(dict.fromkeys(str(v.code) for v in result.violations))
+            self._row("explain", s.dim(f"uploadshield explain {codes}"))
+        self._print("")
+
+    def _head(self, outcome: Outcome, policy: str) -> str:
+        s = self.s
+        if outcome.verdict is Verdict.CLEAN:
+            return s.paint(f"accepted by {policy}", "green", "bold")
+        head = s.paint(f"refused by {policy}", _COLORS[outcome.verdict], "bold")
+        if outcome.verdict in (Verdict.DANGEROUS, Verdict.NOT_VERIFIED):
+            head += s.paint(f" ({outcome.verdict.label})", _COLORS[outcome.verdict])
+        return head
+
+    def _violations(self, outcome: Outcome) -> None:
+        s = self.s
+        violations = outcome.result.violations if outcome.result is not None else ()
+        for violation, verdict in zip(violations, outcome.verdicts, strict=True):
+            code = s.paint(f"{self._glyph(verdict)} {violation.code}", _COLORS[verdict])
+            self._print(f"  {code}  {safe(violation.message)}")
+            for key, value in violation.details.items():
+                if value not in (None, "", [], ()):
+                    self._print(s.dim(f"      {key}: {_format(value, s.ellipsis, limit=300)}"))
+
+    def _checks(self, result: ValidationResult) -> None:
+        s = self.s
+        failed = {v.check for v in result.violations}
+        ran: list[str] = []
+        idle: list[str] = []
+        not_run: list[str] = []
+        inactive: list[str] = []
+        stopped = False
+        for check in self.policy.pipeline:
+            if check.name in result.checks_run:
+                if check.name in failed:
+                    ran.append(s.paint(f"{s.bad} {check.name}", "red"))
+                    stopped = self.policy.fail_fast
+                else:
+                    ran.append(s.paint(f"{s.ok} {check.name}", "green"))
+            elif stopped:
+                not_run.append(check.name)
+            elif (extra := missing_dependency(check)) is not None:
+                inactive.append(f"{check.name} (needs uploadshield[{extra}])")
+            else:
+                idle.append(check.name)
+        self._row("checks", "  ".join(ran) or "none")
+        if idle:
+            self._row("", s.dim(f"not applicable: {', '.join(idle)}"))
+        if not_run:
+            self._row("", s.dim(f"not run after the first failure: {', '.join(not_run)}"))
+            self._row("", s.dim("  (--all-checks runs every check)"))
+        if inactive:
+            self._row("", s.paint(f"inactive: {', '.join(inactive)}", "yellow"))
+
+    # -- summary ----------------------------------------------------------------------
+
+    def warn(self, text: str) -> None:
+        print(self.s.paint(f"warning: {text}", "yellow"), file=self.err)
+
+    def finish(self, summary: Summary, *, interrupted: bool, exit_code: int) -> None:
+        s = self.s
+        parts = [
+            s.paint(f"{summary.counts[v]} {v.label}", _COLORS[v])
+            for v in (Verdict.CLEAN, Verdict.DANGEROUS, Verdict.REFUSED, Verdict.NOT_VERIFIED)
+            if summary.counts[v] or v is Verdict.CLEAN
+        ]
+        files = f"{summary.files} file{'' if summary.files == 1 else 's'}"
+        line = f"{files} in {summary.seconds:.1f} s: {', '.join(parts)}"
+        if skipped := sum(summary.skipped.values()):
+            reasons = ", ".join(f"{n} {why}" for why, n in sorted(summary.skipped.items()))
+            line += s.dim(f"; {skipped} skipped ({reasons})")
+        if not self.detailed or summary.files > 1 or interrupted:
+            self._print(("" if self.detailed else "\n") + s.bold(line + "."))
+        if interrupted:
+            self._print(
+                s.paint("Interrupted: the files above are all that was inspected.", "yellow")
+            )
+        elif exit_code == 1 and not self.detailed:
+            self._print(s.dim("What a code means: uploadshield explain <code>"))
+
+    # -- helpers ----------------------------------------------------------------------
+
+    def _row(self, label: str, value: str) -> None:
+        self._print(f"  {self.s.dim(f'{label:<10}')} {value}")
+
+    def _print(self, text: str) -> None:
+        print(text, file=self.out)
+
+
+def _highlights(details: Any, ellipsis: str) -> str:
+    shown = [
+        _format(details[key], ellipsis, limit=80)
+        for key in _HIGHLIGHTS
+        if details.get(key) not in (None, "", [], ())
+    ]
+    if details.get("in_sanitized_output"):
+        shown.append("in the sanitized output")
+    return f"  {'; '.join(shown)}" if shown else ""
+
+
+def _format(value: Any, ellipsis: str, *, limit: int) -> str:
+    if isinstance(value, (list, tuple, set, frozenset)):
+        items = sorted(value, key=str) if isinstance(value, (set, frozenset)) else value
+        text = ", ".join(str(v) for v in items)
+    else:
+        text = str(value)
+    text = safe(text)
+    return text if len(text) <= limit else text[: limit - 1] + ellipsis

uploadshield/cli/_scan.py

@@ -0,0 +1,106 @@
+"""The policy behind ``uploadshield scan``, and the optional scanners."""
+
+from __future__ import annotations
+
+import dataclasses
+import importlib.util
+from typing import TYPE_CHECKING
+
+from uploadshield.checks import (
+    DEFAULT_CONTENT_CHECKS,
+    ClamAV,
+    EmbeddedContent,
+    FilenameCheck,
+    MagikaCheck,
+    YaraRules,
+)
+from uploadshield.checks.scanners import clamd_ping
+from uploadshield.core.types import all_types
+from uploadshield.policy import Policy
+
+if TYPE_CHECKING:
+    from uploadshield.core.check import Check
+
+__all__ = ["SetupError", "scan_policy", "scanner_checks"]
+
+#: Formats a scan refuses outright: dangerous to accept whatever they contain.
+ACTIVE_TYPES = frozenset({"text/html"})
+#: Archives whose members the checks cannot see (only ZIP containers are opened).
+OPAQUE_ARCHIVES = frozenset(
+    {
+        "application/gzip",
+        "application/x-tar",
+        "application/x-bzip2",
+        "application/x-xz",
+        "application/x-7z-compressed",
+        "application/vnd.rar",
+    }
+)
+
+
+class SetupError(Exception):
+    """The command cannot run as asked (exit status 2)."""
+
+
+def scan_policy(extra_checks: tuple[Check, ...] = ()) -> Policy:
+    """Accept every format uploadshield can inspect, and judge each one by its content.
+
+    Built when the command runs, so formats an application registered count too.
+    Executables and HTML are refused (they are dangerous as uploads whatever they
+    contain); archives it cannot open are reported as not verified.  Names are judged
+    for what makes them dangerous - not for being hidden or lacking an extension - and
+    there is no size limit: files too large for a check are reported, not skipped.
+    """
+    allowed = {
+        t.mime
+        for t in all_types()
+        if not t.executable and t.mime not in ACTIVE_TYPES | OPAQUE_ARCHIVES
+    }
+    content_checks = tuple(
+        dataclasses.replace(check, max_scan_bytes=1 << 62)
+        if isinstance(check, EmbeddedContent)
+        else check
+        for check in DEFAULT_CONTENT_CHECKS
+    )
+    return Policy(
+        name="scan",
+        allowed_types=allowed,
+        max_size=None,
+        allow_empty=True,
+        filename=FilenameCheck(allow_hidden=True, require_extension=False),
+        content_checks=content_checks,
+        checks=extra_checks,
+        fail_fast=False,
+    )
+
+
+def scanner_checks(
+    *, clamav: str | None, yara: list[str] | None, magika: bool
+) -> tuple[Check, ...]:
+    """The optional scanners, verified before the first file is read."""
+    checks: list[Check] = []
+    if clamav:
+        if not clamd_ping(clamav):
+            raise SetupError(f"clamd does not answer at {clamav} (is it running?)")
+        checks.append(ClamAV(address=clamav))
+    if yara:
+        _require("yara", "yara")
+        rules = YaraRules(rules=tuple(yara))
+        try:
+            rules.compile()
+        except Exception as exc:
+            raise SetupError(f"YARA rules do not compile: {exc}") from exc
+        checks.append(rules)
+    if magika:
+        _require("magika", "magika")
+        checks.append(MagikaCheck())
+    return tuple(checks)
+
+
+def _require(module: str, extra: str) -> None:
+    if importlib.util.find_spec(module) is None:
+        raise SetupError(
+            f"--{extra} needs the {extra} extra: "
+            f'uvx --from "uploadshield[{extra}]" uploadshield ..., '
+            f"or uv add 'uploadshield[{extra}]' in your project"
+        )

uploadshield/cli/_style.py

@@ -0,0 +1,95 @@
+"""Terminal styling: colour decisions, glyphs, and escaping of untrusted names."""
+
+from __future__ import annotations
+
+import os
+import sys
+import unicodedata
+from typing import TextIO
+
+__all__ = ["Style", "color_enabled", "safe"]
+
+_BIDI_CONTROLS = frozenset(
+    "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069"
+)
+_CODES = {
+    "bold": "1",
+    "dim": "2",
+    "red": "31",
+    "green": "32",
+    "yellow": "33",
+    "blue": "34",
+    "magenta": "35",
+    "cyan": "36",
+}
+
+
+def safe(text: str) -> str:
+    """``text`` with everything that could act on a terminal made visible.
+
+    File names are attacker-controlled: an ESC byte would let a name rewrite the
+    screen, CR/LF would forge lines, bidi overrides would reverse what is shown, and
+    undecodable bytes (lone surrogates) would crash the output stream.
+    """
+    if text.isprintable() and not _BIDI_CONTROLS.intersection(text):
+        return text
+    out: list[str] = []
+    for char in text:
+        if char in _BIDI_CONTROLS or unicodedata.category(char) in ("Cc", "Cs"):
+            point = ord(char)
+            out.append(f"\\x{point:02x}" if point < 0x100 else f"\\u{point:04x}")
+        else:
+            out.append(char)
+    return "".join(out)
+
+
+def color_enabled(choice: str, stream: TextIO) -> bool:
+    """``--color`` first, then ``NO_COLOR``, ``FORCE_COLOR``, and finally a TTY check."""
+    if choice != "auto":
+        return choice == "always"
+    if os.environ.get("NO_COLOR"):
+        return False
+    if os.environ.get("FORCE_COLOR", "0") not in ("", "0"):
+        return True
+    if os.environ.get("TERM") == "dumb" or not _isatty(stream):
+        return False
+    return sys.platform != "win32" or "WT_SESSION" in os.environ
+
+
+def _isatty(stream: TextIO) -> bool:
+    try:
+        return stream.isatty()
+    except (AttributeError, ValueError):
+        return False
+
+
+def _can_encode(stream: TextIO, text: str) -> bool:
+    try:
+        text.encode(getattr(stream, "encoding", None) or "ascii")
+    except (UnicodeEncodeError, LookupError):
+        return False
+    return True
+
+
+class Style:
+    """Colours and glyphs for one output stream (plain text when colour is off)."""
+
+    def __init__(self, stream: TextIO, *, color: bool) -> None:
+        self.color = color
+        fancy = _can_encode(stream, "\u2713\u2717\u2192\u00b7\u2026\u2500")
+        self.ok, self.bad = ("\u2713", "\u2717") if fancy else ("OK", "X")
+        self.arrow = "\u2192" if fancy else "->"
+        self.dot = "\u00b7" if fancy else "-"
+        self.ellipsis = "\u2026" if fancy else "..."
+
+    def paint(self, text: str, *styles: str) -> str:
+        if not self.color or not styles or not text:
+            return text
+        codes = ";".join(_CODES[s] for s in styles)
+        return f"\x1b[{codes}m{text}\x1b[0m"
+
+    def bold(self, text: str) -> str:
+        return self.paint(text, "bold")
+
+    def dim(self, text: str) -> str:
+        return self.paint(text, "dim")