extrospect 0.1.0__py3-none-any.whl

extrospect/__init__.py

@@ -0,0 +1,3 @@
+"""Extrospect: local browser-extension risk analysis with code evidence."""
+
+__version__ = "0.1.0"

extrospect/__main__.py

@@ -0,0 +1,3 @@
+from extrospect.cli import main
+
+main()

extrospect/acquire/__init__.py

@@ -0,0 +1 @@
+"""Acquiring extensions: local browser profiles, Chrome Web Store downloads, CRX unpacking."""

extrospect/acquire/crx_download.py

@@ -0,0 +1,88 @@
+"""Download the current CRX of an extension from Chrome's public update endpoint.
+
+Only the *latest* published version is available; removed extensions cannot be
+downloaded this way (the endpoint answers 204/404) - the evaluation set needs other
+sources for those. Requests are rate-limited (1/s by default) and identify the client.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import quote
+
+from extrospect.acquire.unpack import CRX_MAGIC, UnpackError, parse_crx_header
+from extrospect.config import Settings
+from extrospect.http import HttpClient
+from extrospect.models import EXTENSION_ID_RE
+from extrospect.util import sha256_bytes
+
+UPDATE_ENDPOINT = "https://clients2.google.com/service/update2/crx"
+
+
+class DownloadError(RuntimeError):
+    pass
+
+
+class ExtensionUnavailable(DownloadError):
+    """The store does not serve this id (removed, private, or wrong id)."""
+
+
+@dataclass
+class DownloadResult:
+    ext_id: str
+    path: Path
+    sha256: str
+    size: int
+    crx_format: str
+
+
+def download_url(ext_id: str, prodversion: str) -> str:
+    x = quote(f"id={ext_id}&uc", safe="")
+    return (
+        f"{UPDATE_ENDPOINT}?response=redirect&prodversion={prodversion}"
+        f"&acceptformat=crx2,crx3&x={x}"
+    )
+
+
+def download_crx(
+    ext_id: str, settings: Settings, http: HttpClient, dest_dir: Path | None = None
+) -> DownloadResult:
+    ext_id = ext_id.strip().lower()
+    if not EXTENSION_ID_RE.match(ext_id):
+        raise DownloadError(f"invalid extension id: {ext_id!r} (expected 32 letters a-p)")
+    url = download_url(ext_id, settings.network.crx_prodversion)
+    response = http.get(
+        url,
+        rate_key="chrome-webstore",
+        per_second=settings.network.webstore_requests_per_s,
+        timeout=max(60.0, settings.network.timeout_s),
+    )
+    if response.status_code in {204, 404}:
+        raise ExtensionUnavailable(f"{ext_id} is not available from the Chrome Web Store")
+    if response.status_code >= 400:
+        raise DownloadError(f"download failed with HTTP {response.status_code}")
+    data = response.content
+    if not data.startswith(CRX_MAGIC):
+        raise ExtensionUnavailable(f"{ext_id}: the update server did not return a CRX file")
+    try:
+        info = parse_crx_header(data)
+    except UnpackError as exc:
+        raise DownloadError(f"{ext_id}: corrupt CRX ({exc})") from exc
+    target_dir = dest_dir or settings.paths.downloads_dir
+    target_dir.mkdir(parents=True, exist_ok=True)
+    digest = sha256_bytes(data)
+    path = target_dir / f"{ext_id}_{digest[:12]}.crx"
+    path.write_bytes(data)
+    return DownloadResult(
+        ext_id=ext_id, path=path, sha256=digest, size=len(data), crx_format=info.format
+    )
+
+
+__all__ = [
+    "DownloadError",
+    "DownloadResult",
+    "ExtensionUnavailable",
+    "download_crx",
+    "download_url",
+]

extrospect/acquire/local_profile.py

@@ -0,0 +1,285 @@
+"""Discover extensions installed in local Chromium-family browser profiles.
+
+Reads ``Local State`` (profile list), ``Secure Preferences`` / ``Preferences``
+(``extensions.settings``: enabled state, install location, unpacked paths) and scans
+``<profile>/Extensions/<id>/<version>_0/``. Nothing is sent anywhere.
+
+Preference field names differ across browser versions (``state`` vs
+``disable_reasons``); both are handled. Set ``EXTROSPECT_BROWSER_ROOTS`` to
+``name=path`` entries (``os.pathsep``-separated) to scan custom locations.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from extrospect.models import EXTENSION_ID_RE, ExtensionRef, InstalledExtension
+from extrospect.static.manifest import load_json_file, summarize_manifest
+
+_LINUX = {
+    "chrome": [
+        "~/.config/google-chrome",
+        "~/.config/google-chrome-beta",
+        "~/.config/google-chrome-unstable",
+        "~/.var/app/com.google.Chrome/config/google-chrome",
+    ],
+    "chromium": [
+        "~/.config/chromium",
+        "~/snap/chromium/common/chromium",
+        "~/.var/app/org.chromium.Chromium/config/chromium",
+    ],
+    "edge": [
+        "~/.config/microsoft-edge",
+        "~/.config/microsoft-edge-beta",
+        "~/.config/microsoft-edge-dev",
+        "~/.var/app/com.microsoft.Edge/config/microsoft-edge",
+    ],
+    "brave": [
+        "~/.config/BraveSoftware/Brave-Browser",
+        "~/.var/app/com.brave.Browser/config/BraveSoftware/Brave-Browser",
+    ],
+    "vivaldi": ["~/.config/vivaldi"],
+}
+_MACOS = {
+    "chrome": [
+        "~/Library/Application Support/Google/Chrome",
+        "~/Library/Application Support/Google/Chrome Beta",
+        "~/Library/Application Support/Google/Chrome Canary",
+    ],
+    "chromium": ["~/Library/Application Support/Chromium"],
+    "edge": ["~/Library/Application Support/Microsoft Edge"],
+    "brave": ["~/Library/Application Support/BraveSoftware/Brave-Browser"],
+    "vivaldi": ["~/Library/Application Support/Vivaldi"],
+}
+_WINDOWS = {
+    "chrome": [
+        "%LOCALAPPDATA%/Google/Chrome/User Data",
+        "%LOCALAPPDATA%/Google/Chrome Beta/User Data",
+        "%LOCALAPPDATA%/Google/Chrome SxS/User Data",
+    ],
+    "chromium": ["%LOCALAPPDATA%/Chromium/User Data"],
+    "edge": ["%LOCALAPPDATA%/Microsoft/Edge/User Data"],
+    "brave": ["%LOCALAPPDATA%/BraveSoftware/Brave-Browser/User Data"],
+    "vivaldi": ["%LOCALAPPDATA%/Vivaldi/User Data"],
+}
+
+# Manifest::Location enum (chrome/common/extensions/manifest.h)
+LOCATIONS = {
+    1: "webstore",  # INTERNAL: installed from a CRX (normally the Web Store)
+    2: "external",
+    3: "external",
+    4: "unpacked",
+    5: "component",
+    6: "external",
+    7: "policy",
+    8: "command_line",
+    9: "policy",
+    10: "component",
+}
+# Built-in component extensions that are not user-installed.
+COMPONENT_IDS = {
+    "nmmhkkegccagdldgiimedpiccmgmieda",  # Chrome Web Store Payments
+    "pkedcjkdefgpdelpbcmbmeomcjbeemfm",  # Chrome Media Router
+    "mhjfbmdgcfjbbpaeojofohoefgiehjai",  # Chrome PDF Viewer
+    "ahfgeienlihckogmohjhadlkjgocpleb",  # Web Store app
+    "kmendfapggjehodndflmmgagdbamhnfd",  # CryptoTokenExtension
+    "neajdppkdcdipfabeoofebfddakdcjhd",  # Google Network Speech
+    "nkeimhogjdpnpccoofpliimaahmaaome",  # Google Hangouts
+}
+_VERSION_DIR_RE = re.compile(r"^(\d+(?:\.\d+)*)(?:_\d+)?$")
+
+
+@dataclass
+class BrowserRoot:
+    browser: str
+    path: Path
+
+
+def default_roots() -> list[BrowserRoot]:
+    env = os.environ.get("EXTROSPECT_BROWSER_ROOTS")
+    if env:
+        roots = []
+        for item in env.split(os.pathsep):
+            if not item.strip():
+                continue
+            name, sep, path = item.partition("=")
+            if not sep:
+                name, path = "chromium", item
+            roots.append(BrowserRoot(name.strip() or "chromium", Path(path).expanduser()))
+        return roots
+    table = _WINDOWS if sys.platform == "win32" else _MACOS if sys.platform == "darwin" else _LINUX
+    roots = []
+    for browser, paths in table.items():
+        for raw in paths:
+            expanded = Path(os.path.expandvars(os.path.expanduser(raw)))
+            if "%" not in str(expanded) and expanded.is_dir():
+                roots.append(BrowserRoot(browser, expanded))
+    return roots
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        data = load_json_file(path)
+    except (OSError, ValueError, UnicodeError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _profiles(root: Path) -> list[tuple[str, str | None]]:
+    """(profile directory, display name) pairs."""
+    local_state = _read_json(root / "Local State")
+    cache = (local_state.get("profile") or {}).get("info_cache") or {}
+    profiles: dict[str, str | None] = {}
+    if isinstance(cache, dict):
+        for directory, info in cache.items():
+            if (root / directory).is_dir():
+                profiles[directory] = (info or {}).get("name") if isinstance(info, dict) else None
+    for child in sorted(root.iterdir()) if root.is_dir() else []:
+        if child.is_dir() and (child.name == "Default" or child.name.startswith("Profile ")):
+            profiles.setdefault(child.name, None)
+    return sorted(profiles.items(), key=lambda kv: (kv[0] != "Default", kv[0]))
+
+
+def _extension_settings(profile_dir: Path) -> dict[str, dict[str, Any]]:
+    merged: dict[str, dict[str, Any]] = {}
+    for name in ("Preferences", "Secure Preferences"):
+        prefs = _read_json(profile_dir / name)
+        settings = (prefs.get("extensions") or {}).get("settings") or {}
+        if isinstance(settings, dict):
+            for ext_id, entry in settings.items():
+                if isinstance(entry, dict):
+                    merged.setdefault(ext_id, {}).update(entry)
+    return merged
+
+
+def _version_key(name: str) -> tuple[int, ...]:
+    match = _VERSION_DIR_RE.match(name)
+    if not match:
+        return (-1,)
+    return tuple(int(p) for p in match.group(1).split("."))
+
+
+def _installed_dir(profile_dir: Path, ext_id: str, settings: dict[str, Any]) -> Path | None:
+    location = settings.get("location")
+    path_value = settings.get("path")
+    if isinstance(path_value, str) and path_value:
+        candidate = Path(path_value)
+        if location == 4 or candidate.is_absolute():
+            return candidate if (candidate / "manifest.json").is_file() else None
+        candidate = profile_dir / "Extensions" / path_value
+        if (candidate / "manifest.json").is_file():
+            return candidate
+    ext_root = profile_dir / "Extensions" / ext_id
+    if not ext_root.is_dir():
+        return None
+    versions = [d for d in ext_root.iterdir() if d.is_dir() and (d / "manifest.json").is_file()]
+    if not versions:
+        return None
+    return max(versions, key=lambda d: _version_key(d.name))
+
+
+def _enabled(settings: dict[str, Any]) -> bool | None:
+    if "disable_reasons" in settings:
+        reasons = settings.get("disable_reasons")
+        if isinstance(reasons, list):
+            return len(reasons) == 0
+        if isinstance(reasons, int):
+            return reasons == 0
+    if "state" in settings:
+        return settings.get("state") == 1
+    return None if not settings else True
+
+
+def discover_extensions(
+    roots: Iterable[BrowserRoot] | None = None,
+    include_components: bool = False,
+    ui_language: str | None = None,
+) -> list[InstalledExtension]:
+    found: list[InstalledExtension] = []
+    for root in roots if roots is not None else default_roots():
+        if not root.path.is_dir():
+            continue
+        for profile, profile_name in _profiles(root.path):
+            profile_dir = root.path / profile
+            settings_map = _extension_settings(profile_dir)
+            ids = set(settings_map)
+            ext_dir_root = profile_dir / "Extensions"
+            if ext_dir_root.is_dir():
+                ids |= {
+                    d.name
+                    for d in ext_dir_root.iterdir()
+                    if d.is_dir() and EXTENSION_ID_RE.match(d.name)
+                }
+            for ext_id in sorted(ids):
+                settings = settings_map.get(ext_id, {})
+                location_code = settings.get("location")
+                location = LOCATIONS.get(location_code) if isinstance(location_code, int) else None
+                is_component = location == "component" or ext_id in COMPONENT_IDS
+                if is_component and not include_components:
+                    continue
+                directory = _installed_dir(profile_dir, ext_id, settings)
+                if directory is None:
+                    continue
+                raw = _read_json(directory / "manifest.json")
+                if not raw:
+                    continue
+                summary = summarize_manifest(directory, raw, ui_language)
+                if (
+                    raw.get("theme") is not None
+                    and not raw.get("background")
+                    and not raw.get("content_scripts")
+                ):
+                    location = location or "theme"
+                found.append(
+                    InstalledExtension(
+                        ref=ExtensionRef(
+                            id=ext_id,
+                            version=summary.version or None,
+                            source="local",
+                            path=directory,
+                            browser=root.browser,
+                            profile=profile,
+                        ),
+                        name=summary.name,
+                        enabled=_enabled(settings),
+                        install_location=location
+                        or ("unpacked" if directory.parent.name != ext_id else None),
+                        from_webstore=settings.get("from_webstore")
+                        if isinstance(settings.get("from_webstore"), bool)
+                        else None,
+                        profile_name=profile_name,
+                        manifest_version=summary.manifest_version,
+                    )
+                )
+    found.sort(key=lambda e: (e.ref.browser or "", e.ref.profile or "", e.name.lower()))
+    return found
+
+
+def find_installed(
+    ext_id: str, profile: str | None = None, browser: str | None = None
+) -> InstalledExtension | None:
+    for ext in discover_extensions(include_components=True):
+        if ext.ref.id != ext_id:
+            continue
+        if profile and ext.ref.profile != profile:
+            continue
+        if browser and ext.ref.browser != browser:
+            continue
+        return ext
+    return None
+
+
+__all__ = [
+    "COMPONENT_IDS",
+    "LOCATIONS",
+    "BrowserRoot",
+    "default_roots",
+    "discover_extensions",
+    "find_installed",
+]

extrospect/acquire/unpack.py

@@ -0,0 +1,199 @@
+"""CRX2/CRX3 parsing and safe extraction (zip-slip and zip-bomb protection).
+
+CRX3 layout: ``"Cr24"`` | uint32 LE version (3) | uint32 LE header length | protobuf
+``CrxFileHeader`` | ZIP. The extension id is the ``crx_id`` inside the signed header data
+(first 16 bytes of the SHA-256 of the publisher key), mapped to ``a``-``p``.
+"""
+
+from __future__ import annotations
+
+import io
+import shutil
+import stat
+import struct
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from extrospect.static.manifest import id_from_bytes, id_from_public_key
+from extrospect.util import PathEscapeError, safe_join
+
+CRX_MAGIC = b"Cr24"
+ZIP_MAGIC = b"PK\x03\x04"
+# Real extensions reach several hundred MB unpacked (ad-block filter lists, WASM models).
+DEFAULT_MAX_BYTES = 1024 * 1024 * 1024
+DEFAULT_MAX_FILES = 20_000
+# Zip-bomb guard: a large member that expands far beyond its compressed size.
+BOMB_MIN_BYTES = 64 * 1024 * 1024
+BOMB_RATIO = 200
+
+
+class UnpackError(ValueError):
+    """The package is malformed, unsafe or too large."""
+
+
+@dataclass
+class CrxInfo:
+    format: str  # "crx3" / "crx2" / "zip"
+    zip_offset: int
+    extension_id: str | None = None
+    public_key: bytes | None = None
+
+
+def _read_varint(data: bytes, pos: int) -> tuple[int, int]:
+    result = shift = 0
+    while True:
+        if pos >= len(data):
+            raise UnpackError("truncated varint in CRX header")
+        byte = data[pos]
+        pos += 1
+        result |= (byte & 0x7F) << shift
+        if not byte & 0x80:
+            return result, pos
+        shift += 7
+        if shift > 63:
+            raise UnpackError("varint too long in CRX header")
+
+
+def parse_protobuf(data: bytes) -> dict[int, list[bytes | int]]:
+    """Minimal protobuf wire-format reader (enough for CrxFileHeader / SignedData)."""
+    fields: dict[int, list[bytes | int]] = {}
+    pos = 0
+    while pos < len(data):
+        key, pos = _read_varint(data, pos)
+        number, wire_type = key >> 3, key & 7
+        if wire_type == 0:
+            value, pos = _read_varint(data, pos)
+            fields.setdefault(number, []).append(value)
+        elif wire_type == 2:
+            length, pos = _read_varint(data, pos)
+            if pos + length > len(data):
+                raise UnpackError("truncated field in CRX header")
+            fields.setdefault(number, []).append(data[pos : pos + length])
+            pos += length
+        elif wire_type == 1:
+            pos += 8
+        elif wire_type == 5:
+            pos += 4
+        else:
+            raise UnpackError(f"unsupported protobuf wire type {wire_type}")
+    return fields
+
+
+def parse_crx_header(data: bytes) -> CrxInfo:
+    if data.startswith((ZIP_MAGIC, b"PK")):
+        return CrxInfo(format="zip", zip_offset=0)
+    if not data.startswith(CRX_MAGIC) or len(data) < 12:
+        raise UnpackError("not a CRX or ZIP file")
+    version = struct.unpack_from("<I", data, 4)[0]
+    if version == 3:
+        header_len = struct.unpack_from("<I", data, 8)[0]
+        header_end = 12 + header_len
+        if header_end > len(data):
+            raise UnpackError("CRX3 header exceeds file size")
+        header = parse_protobuf(data[12:header_end])
+        info = CrxInfo(format="crx3", zip_offset=header_end)
+        signed = header.get(10000)
+        if signed and isinstance(signed[0], bytes):
+            crx_id = parse_protobuf(signed[0]).get(1)
+            if crx_id and isinstance(crx_id[0], bytes) and len(crx_id[0]) >= 16:
+                info.extension_id = id_from_bytes(crx_id[0])
+        for proof_field in (2, 3):  # sha256_with_rsa, sha256_with_ecdsa
+            for proof in header.get(proof_field, []):
+                if isinstance(proof, bytes):
+                    key = parse_protobuf(proof).get(1)
+                    if key and isinstance(key[0], bytes):
+                        info.public_key = info.public_key or key[0]
+        if info.extension_id is None and info.public_key:
+            info.extension_id = id_from_public_key(info.public_key)
+        return info
+    if version == 2:
+        key_len, sig_len = struct.unpack_from("<II", data, 8)
+        start = 16
+        public_key = data[start : start + key_len]
+        return CrxInfo(
+            format="crx2",
+            zip_offset=start + key_len + sig_len,
+            public_key=public_key,
+            extension_id=id_from_public_key(public_key) if public_key else None,
+        )
+    raise UnpackError(f"unsupported CRX version {version}")
+
+
+def extract_zip(
+    zip_bytes: bytes,
+    dest: Path,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    max_files: int = DEFAULT_MAX_FILES,
+) -> int:
+    """Extract safely; returns the number of files written."""
+    dest.mkdir(parents=True, exist_ok=True)
+    total = 0
+    count = 0
+    try:
+        archive = zipfile.ZipFile(io.BytesIO(zip_bytes))
+    except zipfile.BadZipFile as exc:
+        raise UnpackError(f"invalid zip archive: {exc}") from exc
+    with archive:
+        for info in archive.infolist():
+            name = info.filename.replace("\\", "/")
+            if info.is_dir() or name.endswith("/"):
+                continue
+            mode = info.external_attr >> 16
+            if stat.S_ISLNK(mode):
+                continue  # never materialise symlinks from an archive
+            try:
+                target = safe_join(dest, name)
+            except PathEscapeError as exc:
+                raise UnpackError(f"unsafe path in archive: {name}") from exc
+            count += 1
+            if count > max_files:
+                raise UnpackError(f"archive has more than {max_files} files")
+            target.parent.mkdir(parents=True, exist_ok=True)
+            written = 0
+            with archive.open(info) as src, target.open("wb") as dst:
+                while chunk := src.read(1 << 16):
+                    total += len(chunk)
+                    written += len(chunk)
+                    if total > max_bytes:
+                        raise UnpackError(f"archive expands beyond {max_bytes} bytes")
+                    if written > BOMB_MIN_BYTES and written > BOMB_RATIO * max(
+                        info.compress_size, 1
+                    ):
+                        raise UnpackError(f"suspicious compression ratio in {name}")
+                    dst.write(chunk)
+    return count
+
+
+def unpack_package(
+    data: bytes,
+    dest: Path,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    max_files: int = DEFAULT_MAX_FILES,
+) -> CrxInfo:
+    """Unpack a CRX/ZIP into ``dest`` (replacing it). Returns header information."""
+    info = parse_crx_header(data)
+    if dest.exists():
+        shutil.rmtree(dest)
+    try:
+        extract_zip(data[info.zip_offset :], dest, max_bytes=max_bytes, max_files=max_files)
+    except UnpackError:
+        shutil.rmtree(dest, ignore_errors=True)
+        raise
+    return info
+
+
+def unpack_file(path: Path, dest: Path, **kwargs: int) -> CrxInfo:
+    return unpack_package(path.read_bytes(), dest, **kwargs)
+
+
+__all__ = [
+    "CRX_MAGIC",
+    "CrxInfo",
+    "UnpackError",
+    "extract_zip",
+    "parse_crx_header",
+    "parse_protobuf",
+    "unpack_file",
+    "unpack_package",
+]

extrospect/baseline/__init__.py

@@ -0,0 +1 @@
+"""Non-LLM baselines: the deterministic rule score and a LightGBM model."""