whiff-python 0.1.0__cp311-abi3-win_amd64.whl

whiff/options.py

@@ -0,0 +1,238 @@
+"""Validate public options before any native call.
+
+Each entry point has its own allow-list, so an option that a function would
+ignore is rejected instead of being dropped.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from whiff.errors import InvalidOptionError
+
+# Mirrors LIMIT_CAPS in native/src/lib.rs. A test compares both.
+LIMIT_CAPS: dict[str, int] = {
+    "max_read_bytes": 4 * 1024 * 1024 * 1024,
+    "max_decompressed_bytes": 4 * 1024 * 1024 * 1024,
+    "max_depth": 8,
+    "max_children": 500,
+    "file_timeout_ms": 600_000,
+    "extractor_timeout_ms": 600_000,
+    "max_text_bytes": 64 * 1024 * 1024,
+    "hash_max_bytes": 4 * 1024 * 1024 * 1024,
+    "max_parse_bytes": 256 * 1024 * 1024,
+}
+
+_NONZERO = {
+    "max_read_bytes",
+    "max_decompressed_bytes",
+    "file_timeout_ms",
+    "extractor_timeout_ms",
+}
+
+_NATIVE = {
+    "profile",
+    "hashes",
+    "privacy",
+    "only",
+    "skip",
+    "format",
+    "provenance",
+    "limits",
+}
+_PIPELINE = {
+    "extractors",
+    "analyzers",
+    "mode",
+    "strict",
+    "plugin_timeout_ms",
+    "override_fields",
+}
+
+_ALLOWED = {
+    "path": _NATIVE | _PIPELINE | {"stage"},
+    "bytes": _NATIVE | _PIPELINE | {"name", "filename", "mime"},
+    "llm_path": _NATIVE | {"tokens", "stage"},
+    "llm_bytes": _NATIVE | {"tokens", "name", "filename"},
+    "extract_many": _NATIVE | _PIPELINE | {"stage", "max_concurrency", "on_error", "timeout"},
+}
+
+_PROFILES = ("minimal", "standard", "full", "llm")
+_PRIVACY = ("strict", "off")
+_MODES = ("fallback", "augment", "override")
+_STAGES = ("never", "local")
+_HASHES = ("sha256", "blake3")
+
+_DEFAULTS = {
+    "profile": "standard",
+    "privacy": "strict",
+    "mode": "fallback",
+    "strict": False,
+    "plugin_timeout_ms": 5_000,
+    "provenance": False,
+    "tokens": 800,
+    "stage": "never",
+}
+
+
+def validate(opts: dict[str, Any], entry: str) -> dict[str, Any]:
+    """Return normalized options for ``entry`` or raise :class:`InvalidOptionError`."""
+    allowed = _ALLOWED[entry]
+    unknown = sorted(key for key in opts if key not in allowed)
+    if unknown:
+        raise InvalidOptionError(
+            f"unknown option(s): {unknown}",
+            details={"unknown": unknown},
+        )
+    if "name" in opts and "filename" in opts:
+        raise InvalidOptionError("pass either name or filename, not both")
+
+    normalized = dict(_DEFAULTS)
+    for key, value in opts.items():
+        normalized[key] = _one(key, value)
+    return normalized
+
+
+def native_payload(opts: dict[str, Any], *, include_tokens: bool) -> dict[str, Any]:
+    """JSON-ready native options. Defaults that match upstream are omitted."""
+    payload: dict[str, Any] = {}
+    if opts.get("profile") not in (None, "standard"):
+        payload["profile"] = opts["profile"]
+    if "hashes" in opts and opts["hashes"] is not None:
+        payload["hashes"] = opts["hashes"]
+    if opts.get("privacy") not in (None, "strict"):
+        payload["privacy"] = opts["privacy"]
+    for key in ("only", "skip", "format"):
+        if key in opts and opts[key] is not None:
+            payload[key] = opts[key]
+    if opts.get("provenance"):
+        payload["provenance"] = True
+    if opts.get("limits"):
+        payload["limits"] = opts["limits"]
+    if include_tokens and "tokens" in opts:
+        payload["tokens"] = opts["tokens"]
+    return payload
+
+
+def _one(key: str, value: Any) -> Any:
+    if key == "profile":
+        return _choice(key, value, _PROFILES)
+    if key == "privacy":
+        return _choice(key, value, _PRIVACY)
+    if key == "mode":
+        return _choice(key, value, _MODES)
+    if key == "stage":
+        return _choice(key, value, _STAGES)
+    if key == "on_error":
+        return _choice(key, value, ("collect", "halt"))
+    if key == "hashes":
+        return _hashes(value)
+    if key in {"only", "skip"}:
+        return _strings(key, value, "be a list of extractor id strings")
+    if key == "override_fields":
+        return _paths(value)
+    if key in {"format", "name", "filename", "mime"}:
+        if isinstance(value, str) and value:
+            return value
+        raise InvalidOptionError(f"{key} must be a non-empty string, got: {value!r}")
+    if key in {"provenance", "strict"}:
+        if isinstance(value, bool):
+            return value
+        raise InvalidOptionError(f"{key} must be a boolean, got: {value!r}")
+    if key == "tokens":
+        return _int_range(key, value, 1, 100_000)
+    if key == "plugin_timeout_ms":
+        return _int_range(key, value, 1, 600_000)
+    if key == "max_concurrency":
+        return _int_range(key, value, 1, 128)
+    if key == "timeout":
+        if value is None:
+            return None
+        return _int_range(key, value, 1, 3_600_000, extra=" or None")
+    if key == "limits":
+        return _limits(value)
+    if key in {"extractors", "analyzers"}:
+        return _plugins(key, value)
+    raise InvalidOptionError(f"unknown option(s): [{key!r}]")
+
+
+def _choice(key: str, value: Any, allowed: tuple[str, ...]) -> str:
+    if isinstance(value, str) and value in allowed:
+        return value
+    raise InvalidOptionError(f"{key} must be one of {list(allowed)}, got: {value!r}")
+
+
+def _hashes(value: Any) -> str | list[str]:
+    if isinstance(value, str) and value in {"all", "none"}:
+        return value
+    if isinstance(value, (list, tuple)) and value and all(item in _HASHES for item in value):
+        seen: list[str] = []
+        for item in value:
+            if item not in seen:
+                seen.append(item)
+        return seen
+    raise InvalidOptionError(
+        f"hashes must be 'all', 'none', or a non-empty list of 'sha256' / 'blake3', got: {value!r}"
+    )
+
+
+def _strings(key: str, value: Any, expectation: str) -> list[str]:
+    if isinstance(value, (list, tuple)) and all(isinstance(item, str) and item for item in value):
+        return list(value)
+    raise InvalidOptionError(f"{key} must {expectation}, got: {value!r}")
+
+
+def _paths(value: Any) -> list[str]:
+    if isinstance(value, (list, tuple)) and all(_dotted(item) for item in value):
+        return list(value)
+    raise InvalidOptionError(
+        f'override_fields must be a list of dotted paths such as "content.title", got: {value!r}'
+    )
+
+
+def _dotted(value: object) -> bool:
+    if not isinstance(value, str) or "." not in value:
+        return False
+    parts = value.split(".")
+    return len(parts) >= 2 and all(parts)
+
+
+def _int_range(key: str, value: Any, low: int, high: int, extra: str = "") -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high:
+        raise InvalidOptionError(
+            f"{key} must be an integer in {low}..{high}{extra}, got: {value!r}"
+        )
+    return value
+
+
+def _limits(value: Any) -> dict[str, int]:
+    if not isinstance(value, dict):
+        raise InvalidOptionError(f"limits must be a dict, got: {value!r}")
+    unknown = sorted(key for key in value if key not in LIMIT_CAPS)
+    if unknown:
+        raise InvalidOptionError(f"unknown limit key(s): {unknown}")
+    out: dict[str, int] = {}
+    for key, item in value.items():
+        if isinstance(item, bool) or not isinstance(item, int) or item < 0:
+            raise InvalidOptionError(f"{key} must be a non-negative integer, got: {item!r}")
+        if item == 0 and key in _NONZERO:
+            raise InvalidOptionError(f"{key} must be greater than 0")
+        cap = LIMIT_CAPS[key]
+        if item > cap:
+            raise InvalidOptionError(f"{key} exceeds cap {cap}")
+        out[key] = item
+    return out
+
+
+def _plugins(key: str, value: Any) -> list[Any]:
+    if not isinstance(value, (list, tuple)):
+        raise InvalidOptionError(f"{key} must be a list")
+    needed = ("id", "supports", "extract") if key == "extractors" else ("id", "analyze")
+    bad = [item for item in value if any(not hasattr(item, attr) for attr in needed)]
+    if bad:
+        names = ", ".join(needed)
+        raise InvalidOptionError(
+            f"{key} must provide {names}: {bad!r}",
+            details={"invalid": [repr(item) for item in bad]},
+        )
+    return list(value)

whiff/plugins.py

@@ -0,0 +1,315 @@
+"""Run Python extractors, then analyzers, after native extraction.
+
+Plugins run on a worker thread with ``plugin_timeout_ms``. A failure is
+recorded in ``diagnostics.extractors`` unless ``strict`` is set, in which
+case it is raised. Plugins are not a security boundary: a native crash inside
+a plugin still takes down the process. Error text stored in the report is
+privacy-scrubbed and truncated, and does not include arbitrary exception
+messages.
+"""
+
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FuturesTimeout
+from typing import Any
+
+from whiff.errors import (
+    AnalyzerError,
+    ExtractorError,
+    WhiffError,
+    WhiffTimeoutError,
+    error_for,
+)
+from whiff.input import Input
+from whiff.merge import merge_tracked, validate_patch
+from whiff.privacy import scrub_patch, scrub_string
+from whiff.report import append_extractor, extractor_ids, native_covered, unreadable
+
+
+class Extractor:
+    """Base class for a plugin that adds fields after native extraction.
+
+    ``supports`` is called with the :class:`~whiff.input.Input` and the report
+    so far. ``extract`` returns a patch dict. With ``mode="fallback"`` (the
+    default) ``extract`` runs only when native extraction did not cover a
+    readable file.
+    """
+
+    id: str = ""
+    version: int = 1
+
+    def supports(self, source: Input, report: dict[str, Any]) -> bool:
+        return False
+
+    def extract(self, source: Input, report: dict[str, Any]) -> dict[str, Any]:
+        raise NotImplementedError
+
+
+class Analyzer:
+    """Base class for a plugin that runs after extractors.
+
+    ``analyze`` returns a patch merged in ``augment`` mode: it can add fields
+    and append keywords, risks, and warnings, and it does not replace values
+    already set.
+    """
+
+    id: str = ""
+    version: int = 1
+
+    def analyze(self, report: dict[str, Any], source: Input) -> dict[str, Any]:
+        raise NotImplementedError
+
+
+_MAX_ID_BYTES = 128
+_MAX_VERSION = 4_294_967_295
+_MAX_ERROR_BYTES = 256
+_META_TIMEOUT_MS = 1_000
+
+
+def run(source: Input, report: dict[str, Any], opts: dict[str, Any]) -> dict[str, Any]:
+    ctx = _Ctx(
+        mode=opts.get("mode", "fallback"),
+        strict=bool(opts.get("strict", False)),
+        timeout_ms=int(opts.get("plugin_timeout_ms", 5_000)),
+        privacy=opts.get("privacy", "strict"),
+        provenance=bool(opts.get("provenance", False)),
+        override_fields=list(opts.get("override_fields") or []),
+        native_ids=set(extractor_ids(report)),
+        covered=native_covered(report),
+        unreadable=unreadable(report),
+    )
+    report, owned = _reduce(
+        opts.get("extractors") or [], report, set(), source, ctx, kind="extractor"
+    )
+    report, _owned = _reduce(
+        opts.get("analyzers") or [], report, owned, source, ctx, kind="analyzer"
+    )
+    return report
+
+
+class _Ctx:
+    def __init__(
+        self,
+        *,
+        mode: str,
+        strict: bool,
+        timeout_ms: int,
+        privacy: str,
+        provenance: bool,
+        override_fields: list[str],
+        native_ids: set[str],
+        covered: bool,
+        unreadable: bool,
+    ) -> None:
+        self.mode = mode
+        self.strict = strict
+        self.timeout_ms = timeout_ms
+        self.privacy = privacy
+        self.provenance = provenance
+        self.override_fields = override_fields
+        self.native_ids = native_ids
+        self.covered = covered
+        self.unreadable = unreadable
+
+
+def _reduce(plugins, report, owned, source, ctx, *, kind: str):
+    for plugin in plugins:
+        report, owned = _one(plugin, source, report, owned, ctx, kind)
+    return report, owned
+
+
+def _one(plugin, source, report, owned, ctx: _Ctx, kind: str):
+    obj = plugin() if isinstance(plugin, type) else plugin
+    meta, err = _metadata(obj, kind, ctx)
+    if err is not None:
+        return _failure(report, owned, err, meta, ctx)
+    if kind == "extractor":
+        supported, err = _call(
+            lambda: obj.supports(source, report) is True,
+            timeout_ms=ctx.timeout_ms,
+            kind=kind,
+            plugin_id=meta["id"],
+        )
+        if err is not None:
+            return _failure(report, owned, err, meta, ctx)
+        if not supported:
+            return report, owned
+        if ctx.mode == "fallback" and (ctx.covered or ctx.unreadable):
+            return report, owned
+        started = time.perf_counter()
+        result, err = _call(
+            lambda: obj.extract(source, report),
+            timeout_ms=ctx.timeout_ms,
+            kind=kind,
+            plugin_id=meta["id"],
+        )
+        meta = {**meta, "ms": (time.perf_counter() - started) * 1000}
+        mode = ctx.mode
+    else:
+        started = time.perf_counter()
+        result, err = _call(
+            lambda: obj.analyze(report, source),
+            timeout_ms=ctx.timeout_ms,
+            kind=kind,
+            plugin_id=meta["id"],
+        )
+        meta = {**meta, "ms": (time.perf_counter() - started) * 1000}
+        mode = "augment"
+    if err is not None:
+        return _failure(report, owned, err, meta, ctx)
+    if not isinstance(result, dict):
+        reason = "analyzer_error" if kind == "analyzer" else "extractor_error"
+        err = error_for(reason, f"{kind} returned {_shape(result)}; expected a dict")
+        return _failure(report, owned, err, meta, ctx)
+    return _apply(report, owned, result, mode, meta, ctx)
+
+
+def _metadata(obj, kind: str, ctx: _Ctx) -> tuple[dict[str, Any], WhiffError | None]:
+    fallback = {"id": type(obj).__name__, "version": 0, "ms": 0.0, "kind": kind}
+
+    def read():
+        plugin_id = obj.id() if callable(obj.id) else obj.id
+        version = (
+            obj.version() if callable(getattr(obj, "version", None)) else getattr(obj, "version", 0)
+        )
+        return plugin_id, version
+
+    try:
+        plugin_id, version = _invoke(read, min(ctx.timeout_ms, _META_TIMEOUT_MS))
+    except WhiffTimeoutError as err:
+        return fallback, err
+    except WhiffError as err:
+        return fallback, err
+    except Exception as err:
+        reason = "analyzer_error" if kind == "analyzer" else "extractor_error"
+        return fallback, error_for(
+            reason, f"plugin raised {type(err).__name__}", details={"plugin": fallback["id"]}
+        )
+
+    reason = "analyzer_error" if kind == "analyzer" else "extractor_error"
+    if not _valid_id(plugin_id):
+        return fallback, error_for(
+            reason,
+            f"id must be a non-empty string (max {_MAX_ID_BYTES} bytes)",
+        )
+    if (
+        isinstance(version, bool)
+        or not isinstance(version, int)
+        or not 0 <= version <= _MAX_VERSION
+    ):
+        return fallback, error_for(reason, f"version must be an integer in 0..{_MAX_VERSION}")
+    if plugin_id in ctx.native_ids:
+        return fallback, error_for(
+            "conflict", f"plugin id `{plugin_id}` collides with a native extractor id"
+        )
+    return {**fallback, "id": plugin_id, "version": version}, None
+
+
+def _valid_id(plugin_id: object) -> bool:
+    return (
+        isinstance(plugin_id, str)
+        and plugin_id != ""
+        and len(plugin_id.encode("utf-8")) <= _MAX_ID_BYTES
+    )
+
+
+def _call(fn, *, timeout_ms: int, kind: str, plugin_id: str):
+    try:
+        return _invoke(fn, timeout_ms), None
+    except WhiffError as err:
+        err.details = {**err.details, "plugin": plugin_id}
+        return None, err
+    except FuturesTimeout:
+        return None, WhiffTimeoutError(
+            f"plugin timed out after {timeout_ms}ms",
+            details={"plugin": plugin_id},
+        )
+    except Exception as err:
+        reason_cls = AnalyzerError if kind == "analyzer" else ExtractorError
+        return None, reason_cls(
+            f"plugin raised {type(err).__name__}",
+            details={"plugin": plugin_id, "class": type(err).__name__},
+        )
+
+
+def _invoke(fn, timeout_ms: int):
+    pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="whiff-plugin")
+    try:
+        future = pool.submit(fn)
+        try:
+            return future.result(timeout=timeout_ms / 1000)
+        except FuturesTimeout:
+            raise WhiffTimeoutError(f"plugin timed out after {timeout_ms}ms") from None
+    finally:
+        pool.shutdown(wait=False, cancel_futures=True)
+
+
+def _apply(report, owned: set[str], patch: dict, mode: str, meta: dict, ctx: _Ctx):
+    try:
+        validate_patch(patch)
+        patch = scrub_patch(patch, ctx.privacy)
+        merged, written = merge_tracked(
+            report,
+            patch,
+            mode=mode,
+            override_fields=ctx.override_fields,
+            plugin_id=meta["id"],
+            provenance=ctx.provenance,
+            owned=owned,
+        )
+    except WhiffError as err:
+        return _failure(report, owned, err, meta, ctx)
+    entry = {"id": meta["id"], "version": meta["version"], "ms": meta["ms"], "status": "ok"}
+    return append_extractor(merged, entry), owned | set(written)
+
+
+def _failure(report, owned, err: WhiffError, meta: dict, ctx: _Ctx):
+    err = _sanitize(err, ctx)
+    err.details = {**err.details, "plugin": meta["id"]}
+    if ctx.strict:
+        raise err
+    entry = {
+        "id": meta["id"],
+        "version": meta["version"],
+        "ms": meta["ms"],
+        "status": "error",
+        "error_kind": err.reason,
+        "error": err.message,
+    }
+    return append_extractor(report, entry), owned
+
+
+def _sanitize(err: WhiffError, ctx: _Ctx) -> WhiffError:
+    message = scrub_string(err.message, ctx.privacy) if isinstance(err.message, str) else ""
+    err.message = _truncate(message)
+    err.args = (err._format(),)
+    return err
+
+
+def _truncate(message: str) -> str:
+    raw = message.encode("utf-8")
+    if len(raw) <= _MAX_ERROR_BYTES:
+        return message
+    cut = raw[:_MAX_ERROR_BYTES]
+    while cut:
+        try:
+            return cut.decode("utf-8") + "..."
+        except UnicodeDecodeError:
+            cut = cut[:-1]
+    return "..."
+
+
+def _shape(value: object) -> str:
+    if isinstance(value, tuple):
+        return f"a tuple of size {len(value)}"
+    if isinstance(value, dict):
+        return "a dict"
+    if isinstance(value, list):
+        return "a list"
+    if isinstance(value, str):
+        return "a str"
+    if value is None:
+        return "None"
+    return type(value).__name__