deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/web_source/robots.py

@@ -0,0 +1,182 @@
+"""Per-crawl robots rules and host pacing for documentation sync (#1429)."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from fnmatch import translate
+import math
+import re
+import time
+from urllib.parse import quote, urljoin, urlsplit
+
+import httpx
+
+from deeptutor.tools.web_fetch import (
+    DEFAULT_USER_AGENT,
+    MAX_RESPONSE_BYTES,
+    _bounded_read,
+    _is_disallowed_host,
+)
+
+DEFAULT_REQUEST_INTERVAL_S = 0.125
+#: A hostile robots.txt can otherwise park a crawl (and the shared per-host
+#: pace lock) on an arbitrarily long sleep; delays beyond a minute are treated
+#: as "no directive" and the default interval applies instead.
+MAX_HONORED_CRAWL_DELAY_S = 60.0
+_UNRESERVED = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~")
+
+
+def _normalise_path(value: str) -> str:
+    # Compare UTF-8 octets while preserving escaped reserved characters.
+    value = quote(value, safe="/%:@!$&'()*+,;=-._~?")
+    return re.sub(
+        r"%([0-9a-fA-F]{2})",
+        lambda match: (
+            chr(int(match[1], 16)) if chr(int(match[1], 16)) in _UNRESERVED else match[0].upper()
+        ),
+        value,
+    )
+
+
+@dataclass(frozen=True)
+class RobotsPolicy:
+    rules: tuple[tuple[str, bool], ...] = ()
+    crawl_delay_s: float = 0.0
+    available: bool = True
+
+    def permits(self, url: str) -> bool:
+        if not self.available:
+            return False
+        parsed = urlsplit(url)
+        path = _normalise_path((parsed.path or "/") + ("?" + parsed.query if parsed.query else ""))
+        if parsed.path == "/robots.txt":
+            return True
+        matches: list[tuple[int, bool]] = []
+        for rule, allowed in self.rules:
+            anchored = rule.endswith("$")
+            pattern = rule[:-1] if anchored else rule
+            # fnmatch's atomic wildcard matching avoids exponential regex
+            # backtracking on an untrusted rule with many stars. Robots only
+            # gives '*' wildcard meaning; brackets and '?' remain literal.
+            glob = pattern.replace("[", "[[]").replace("?", "[?]")
+            expression = translate(glob + ("" if anchored else "*"))
+            if re.match(expression, path):
+                matches.append((len(rule.encode("utf-8")), allowed))
+        # Longest match wins; Allow wins ties (RFC 9309).
+        return max(matches, default=(0, True))[1]
+
+
+def parse_robots_txt(raw: str) -> RobotsPolicy:
+    groups: list[tuple[list[str], list[tuple[str, bool]], list[float]]] = []
+    agents: list[str] = []
+    rules: list[tuple[str, bool]] = []
+    delays: list[float] = []
+    has_directive = False
+    for line in raw.splitlines():
+        key, sep, value = line.split("#", 1)[0].partition(":")
+        if not sep:
+            continue
+        key, value = key.strip().lower(), value.strip()
+        if key == "user-agent":
+            if has_directive:
+                groups.append((agents, rules, delays))
+                agents, rules, delays = [], [], []
+                has_directive = False
+            if value:
+                agents.append(value.lower())
+        elif agents:
+            has_directive = True
+            if key in {"allow", "disallow"} and value.startswith("/"):
+                rules.append((_normalise_path(value), key == "allow"))
+            elif key == "crawl-delay":
+                try:
+                    delay = float(value)
+                except ValueError:
+                    continue
+                # Ignore absurd delays instead of honoring them: a hostile
+                # robots.txt could otherwise park every request to this host
+                # on one unbounded sleep (RFC 9309 lets crawlers cap
+                # unreasonable values).
+                if math.isfinite(delay) and 0 <= delay <= MAX_HONORED_CRAWL_DELAY_S:
+                    delays.append(delay)
+    groups.append((agents, rules, delays))
+    product = DEFAULT_USER_AGENT.split("/", 1)[0].lower()
+    selected = [group for group in groups if product in group[0]]
+    selected = selected or [group for group in groups if "*" in group[0]]
+    return RobotsPolicy(
+        rules=tuple(rule for _, rules, _ in selected for rule in rules),
+        crawl_delay_s=max((delay for _, _, delays in selected for delay in delays), default=0.0),
+    )
+
+
+class CrawlAccess:
+    """Cache by origin; serialize request starts per host, including redirects."""
+
+    def __init__(self, client: httpx.AsyncClient) -> None:
+        self.client = client
+        self._policies: dict[str, RobotsPolicy] = {}
+        self._policy_locks: dict[str, asyncio.Lock] = {}
+        self._host_locks: dict[str, asyncio.Lock] = {}
+        self._last_request: dict[str, float] = {}
+        self._intervals: dict[str, float] = {}
+
+    async def _pace(self, url: str, delay: float = 0.0) -> None:
+        host = urlsplit(url).hostname or ""
+        self._intervals[host] = max(self._intervals.get(host, DEFAULT_REQUEST_INTERVAL_S), delay)
+        async with self._host_locks.setdefault(host, asyncio.Lock()):
+            last = self._last_request.get(host)
+            if last is not None:
+                wait = self._intervals[host] - (time.monotonic() - last)
+                if wait > 0:
+                    await asyncio.sleep(wait)
+            self._last_request[host] = time.monotonic()
+
+    async def _fetch_policy(self, url: str) -> RobotsPolicy:
+        # robots redirects do not recursively request another robots file, but
+        # every hop retains the same SSRF checks as page requests.
+        for _ in range(6):
+            parsed = urlsplit(url)
+            if (
+                parsed.scheme not in {"http", "https"}
+                or not parsed.hostname
+                or _is_disallowed_host(parsed.hostname)
+            ):
+                return RobotsPolicy(available=False)
+            await self._pace(url)
+            try:
+                async with self.client.stream(
+                    "GET",
+                    url,
+                    headers={"User-Agent": DEFAULT_USER_AGENT, "Accept": "text/plain,*/*"},
+                    follow_redirects=False,
+                ) as response:
+                    if response.status_code in {301, 302, 303, 307, 308}:
+                        location = response.headers.get("location", "").strip()
+                        if not location:
+                            return RobotsPolicy(available=False)
+                        url = urljoin(url, location)
+                        continue
+                    if response.status_code >= 500 or response.status_code == 429:
+                        return RobotsPolicy(available=False)
+                    if response.status_code >= 400:
+                        return RobotsPolicy()
+                    return parse_robots_txt(await _bounded_read(response, MAX_RESPONSE_BYTES))
+            except httpx.HTTPError:
+                return RobotsPolicy(available=False)
+        return RobotsPolicy(available=False)
+
+    async def policy_for(self, url: str) -> RobotsPolicy:
+        parsed = urlsplit(url)
+        origin = f"{parsed.scheme}://{parsed.netloc}"
+        async with self._policy_locks.setdefault(origin, asyncio.Lock()):
+            if origin not in self._policies:
+                self._policies[origin] = await self._fetch_policy(origin + "/robots.txt")
+        return self._policies[origin]
+
+    async def permits_request(self, url: str) -> bool:
+        policy = await self.policy_for(url)
+        if not policy.permits(url):
+            return False
+        await self._pace(url, policy.crawl_delay_s)
+        return True

deeptutor/services/web_source/scheduler.py

@@ -0,0 +1,336 @@
+"""Durable background scheduler for account-scoped web sources."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+import logging
+import time
+from typing import Any
+import uuid
+
+from deeptutor.services.web_source.repository import (
+    SQLiteWebSourceSyncRepository,
+    WebSourceSyncJob,
+)
+
+logger = logging.getLogger(__name__)
+
+WEB_SYNC_MIN_INTERVAL_HOURS = 1
+WEB_SYNC_MAX_INTERVAL_HOURS = 168
+WEB_SYNC_INTERVAL_HOURS = 24
+WEB_SYNC_CHECK_SECONDS = 15
+WEB_SYNC_LEASE_SECONDS = 60
+WEB_SYNC_LEASE_RENEW_SECONDS = 20
+WEB_SYNC_MAX_CONCURRENCY = 2
+
+
+def _now_ms() -> int:
+    return int(time.time() * 1000)
+
+
+def _hours_ms(hours: int) -> int:
+    return hours * 60 * 60 * 1000
+
+
+def default_repository() -> SQLiteWebSourceSyncRepository:
+    from deeptutor.multi_user.paths import SYSTEM_ROOT
+
+    return SQLiteWebSourceSyncRepository(SYSTEM_ROOT / "web-source-sync.sqlite")
+
+
+def normalize_sync_interval(value: Any) -> int:
+    try:
+        interval = int(value)
+    except (TypeError, ValueError):
+        return WEB_SYNC_INTERVAL_HOURS
+    return max(WEB_SYNC_MIN_INTERVAL_HOURS, min(WEB_SYNC_MAX_INTERVAL_HOURS, interval))
+
+
+class WebSourceSyncScheduler:
+    """Run due web-source jobs while preserving state across restarts."""
+
+    def __init__(
+        self,
+        *,
+        repository: SQLiteWebSourceSyncRepository | None = None,
+        manager_factory: Callable[[], Any] | None = None,
+        check_interval_s: float = WEB_SYNC_CHECK_SECONDS,
+        max_concurrency: int = WEB_SYNC_MAX_CONCURRENCY,
+    ) -> None:
+        self._repository = repository
+        self._manager_factory = manager_factory
+        self._check_interval_s = check_interval_s
+        self._max_concurrency = max_concurrency
+        self._runner_id = f"web-sync-{uuid.uuid4().hex}"
+        self._task: asyncio.Task[None] | None = None
+        self._running = False
+        self._run_tasks: dict[tuple[str, str, str], asyncio.Task[None]] = {}
+        self._sources: dict[tuple[str, str, str], dict[str, Any]] = {}
+
+    @property
+    def repo(self) -> SQLiteWebSourceSyncRepository:
+        if self._repository is None:
+            self._repository = default_repository()
+        return self._repository
+
+    def _make_manager(self) -> Any:
+        if self._manager_factory is not None:
+            return self._manager_factory()
+        from deeptutor.knowledge.manager import KnowledgeBaseManager
+        from deeptutor.multi_user.knowledge_access import current_kb_base_dir
+
+        return KnowledgeBaseManager(base_dir=str(current_kb_base_dir()))
+
+    async def start(self) -> None:
+        if self._running:
+            return
+        self._running = True
+        self.repo.recover_interrupted(self._runner_id)
+        self._task = asyncio.create_task(self._loop(), name="web-source-sync:scheduler")
+        logger.info("Web source sync scheduler started")
+
+    async def stop(self) -> None:
+        self._running = False
+        if self._task and not self._task.done():
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+        self._task = None
+        if self._run_tasks:
+            for task in self._run_tasks.values():
+                task.cancel()
+            await asyncio.gather(*self._run_tasks.values(), return_exceptions=True)
+            self._run_tasks.clear()
+
+    async def _loop(self) -> None:
+        while self._running:
+            try:
+                await self._synchronize_sources()
+                await asyncio.to_thread(self.repo.recover_interrupted, self._runner_id)
+                self._start_due_jobs()
+            except Exception:
+                logger.exception("Web source scheduler cycle failed")
+            await asyncio.sleep(self._check_interval_s)
+
+    def _manager_for_owner(self, owner_id: str) -> Any:
+        from deeptutor.multi_user.context import reset_current_user, set_current_user
+        from deeptutor.multi_user.models import CurrentUser
+        from deeptutor.multi_user.paths import (
+            LOCAL_ADMIN_ID,
+            ensure_scope_workspace,
+            local_admin_user,
+            scope_for_user,
+        )
+
+        user = (
+            local_admin_user()
+            if owner_id == LOCAL_ADMIN_ID
+            else CurrentUser(
+                id=owner_id,
+                username=owner_id,
+                role="user",
+                scope=scope_for_user(owner_id, is_admin=False),
+            )
+        )
+        if user.scope.kind == "user":
+            ensure_scope_workspace(user.scope)
+        token = set_current_user(user)
+        try:
+            return self._make_manager()
+        finally:
+            reset_current_user(token)
+
+    async def _synchronize_sources(self) -> None:
+        from deeptutor.multi_user.paths import LOCAL_ADMIN_ID, USERS_ROOT
+
+        manager = self._manager_for_owner(LOCAL_ADMIN_ID)
+        owner_ids = [LOCAL_ADMIN_ID, *(path.name for path in USERS_ROOT.glob("*") if path.is_dir())]
+        # The first manager call above also validates the configured admin path.
+        del manager
+        source_keys: set[tuple[str, str, str]] = set()
+        sources: dict[tuple[str, str, str], dict[str, Any]] = {}
+        scanned_owner_ids: set[str] = set()
+        for owner_id in owner_ids:
+            owner_sources: dict[tuple[str, str, str], dict[str, Any]] = {}
+            try:
+                owner_manager = await asyncio.to_thread(self._manager_for_owner, owner_id)
+                for kb_name, source in owner_manager.get_all_web_sources():
+                    if not source.get("enabled", True) or not source.get("auto_sync_enabled", True):
+                        continue
+                    source_id = str(source.get("id") or "")
+                    kb_key = str(kb_name)
+                    if source_id and kb_key:
+                        owner_sources[owner_id, kb_key, source_id] = dict(source)
+            except Exception:
+                logger.exception("Failed to enumerate web sources for owner %s", owner_id)
+                continue
+            scanned_owner_ids.add(owner_id)
+            sources.update(owner_sources)
+            source_keys.update(owner_sources)
+        self._sources = sources
+        await asyncio.to_thread(
+            self.repo.reconcile_sources, source_keys, scanned_owner_ids=scanned_owner_ids
+        )
+
+    def _start_due_jobs(self) -> None:
+        for job in self.repo.due_jobs():
+            key = self.repo.key(job)
+            if key in self._run_tasks or key not in self._sources:
+                continue
+            if len(self._run_tasks) >= self._max_concurrency:
+                return
+            task = asyncio.create_task(self._run_job(job), name=f"web-source-sync:{key[2]}")
+            self._run_tasks[key] = task
+            task.add_done_callback(lambda _task, item=key: self._run_tasks.pop(item, None))  # type: ignore[misc]
+
+    async def _run_job(self, scheduled: WebSourceSyncJob) -> None:
+        from deeptutor.multi_user.context import reset_current_user, set_current_user
+        from deeptutor.multi_user.models import CurrentUser
+        from deeptutor.multi_user.paths import (
+            LOCAL_ADMIN_ID,
+            ensure_scope_workspace,
+            local_admin_user,
+            scope_for_user,
+        )
+        from deeptutor.services.web_source.sync import sync_source
+
+        key = self.repo.key(scheduled)
+        lease_until = _now_ms() + WEB_SYNC_LEASE_SECONDS * 1000
+        claimed = await asyncio.to_thread(
+            self.repo.claim,
+            scheduled,
+            runner_id=f"{self._runner_id}:{uuid.uuid4().hex}",
+            lease_until_ms=lease_until,
+        )
+        if claimed is None:
+            return
+        source = self._sources.get(key)
+        if source is None:
+            # A scan may have failed after this job was queued. Keep the job
+            # retryable until its owner's inventory is read successfully.
+            self.repo.mark_interrupted(claimed)
+            return
+
+        user = (
+            local_admin_user()
+            if claimed.owner_id == LOCAL_ADMIN_ID
+            else CurrentUser(
+                id=claimed.owner_id,
+                username=claimed.owner_id,
+                role="user",
+                scope=scope_for_user(claimed.owner_id, is_admin=False),
+            )
+        )
+        if user.scope.kind == "user":
+            ensure_scope_workspace(user.scope)
+        token = set_current_user(user)
+        owner_task = asyncio.current_task()
+        assert owner_task is not None
+        lease_task = asyncio.create_task(
+            self._renew_lease(claimed, owner_task), name=f"web-source-sync:lease:{key[2]}"
+        )
+        try:
+            current = await asyncio.to_thread(self.repo.get, key)
+            if current is not None and current.cancel_requested:
+                self.repo.mark_cancelled(current)
+                return
+            result = await sync_source(
+                kb_name=claimed.kb_name,
+                source=source,
+                base_dir=str(self._make_manager().base_dir),
+                max_depth=source.get("max_depth"),
+                max_pages=source.get("max_pages"),
+            )
+            current = await asyncio.to_thread(self.repo.get, key)
+            if current is not None and current.cancel_requested:
+                self.repo.mark_cancelled(current)
+                return
+            if result.ok:
+                interval = normalize_sync_interval(source.get("sync_interval_hours"))
+                self.repo.mark_success(claimed, _now_ms() + _hours_ms(interval))
+            else:
+                self._mark_failure(claimed, source, result.error or "Web synchronization failed")
+        except asyncio.CancelledError:
+            current = self.repo.get(key)
+            if current is not None and current.cancel_requested:
+                self.repo.mark_cancelled(claimed, _now_ms())
+            else:
+                self.repo.mark_interrupted(claimed)
+            raise
+        except Exception as exc:
+            logger.exception("Scheduled web source sync failed for %s", source.get("url"))
+            self._mark_failure(claimed, source, str(exc))
+        finally:
+            lease_task.cancel()
+            await asyncio.gather(lease_task, return_exceptions=True)
+            reset_current_user(token)
+
+    async def _renew_lease(self, job: WebSourceSyncJob, owner_task: asyncio.Task[None]) -> None:
+        while True:
+            await asyncio.sleep(WEB_SYNC_LEASE_RENEW_SECONDS)
+            renewed = await asyncio.to_thread(
+                self.repo.renew_lease, job, _now_ms() + WEB_SYNC_LEASE_SECONDS * 1000
+            )
+            if not renewed:
+                owner_task.cancel()
+                return
+
+    def _mark_failure(
+        self,
+        job: WebSourceSyncJob,
+        source: dict[str, Any],
+        error: str,
+    ) -> None:
+        interval = normalize_sync_interval(source.get("sync_interval_hours"))
+        backoff = min(interval, WEB_SYNC_MIN_INTERVAL_HOURS * max(1, job.attempt + 1))
+        self.repo.mark_failure(
+            job,
+            error=error,
+            next_run_at_ms=_now_ms() + _hours_ms(backoff),
+        )
+
+    async def request_cancel(self, owner_id: str, kb_name: str, source_id: str) -> bool:
+        key = owner_id, kb_name, source_id
+        if not await asyncio.to_thread(self.repo.request_cancel, key):
+            return False
+        task = self._run_tasks.get(key)
+        if task is not None and not task.done():
+            task.cancel()
+        return True
+
+    async def retry(self, owner_id: str, kb_name: str, source_id: str) -> bool:
+        job = await asyncio.to_thread(self.repo.retry, (owner_id, kb_name, source_id))
+        return job is not None
+
+
+_scheduler: WebSourceSyncScheduler | None = None
+
+
+def get_web_source_sync_scheduler() -> WebSourceSyncScheduler:
+    global _scheduler
+    if _scheduler is None:
+        _scheduler = WebSourceSyncScheduler()
+    return _scheduler
+
+
+async def start_web_source_sync_scheduler() -> None:
+    await get_web_source_sync_scheduler().start()
+
+
+async def stop_web_source_sync_scheduler() -> None:
+    await get_web_source_sync_scheduler().stop()
+
+
+__all__ = [
+    "WEB_SYNC_INTERVAL_HOURS",
+    "WEB_SYNC_MAX_INTERVAL_HOURS",
+    "WEB_SYNC_MIN_INTERVAL_HOURS",
+    "WebSourceSyncScheduler",
+    "get_web_source_sync_scheduler",
+    "normalize_sync_interval",
+    "start_web_source_sync_scheduler",
+    "stop_web_source_sync_scheduler",
+]

deeptutor/services/web_source/snapshot_assets.py

@@ -0,0 +1,171 @@
+"""Safe, local image assets for captured web Markdown."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+import hashlib
+import logging
+import re
+from typing import Any
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+
+from deeptutor.tools.web_fetch import (
+    ALLOWED_SCHEMES,
+    DEFAULT_TIMEOUT_S,
+    DEFAULT_USER_AGENT,
+    MAX_REDIRECTS,
+    _is_disallowed_host,
+)
+
+logger = logging.getLogger(__name__)
+
+MAX_SNAPSHOT_IMAGES = 24
+MAX_SNAPSHOT_IMAGE_BYTES = 8 * 1024 * 1024
+_IMAGE = re.compile(
+    r"!\[([^\]]*)\]\(\s*(https?://[^\s)]+)(?:\s+[\"'][^)]*[\"'])?\s*\)",
+    re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class SnapshotAsset:
+    data: bytes
+    mime: str
+    extension: str
+
+
+ImageFetcher = Callable[[str], Awaitable[SnapshotAsset | None]]
+
+
+async def localize_snapshot_images(
+    markdown: str,
+    material_id: str,
+    *,
+    fetcher: ImageFetcher | None = None,
+) -> tuple[str, dict[str, bytes]]:
+    """Cache bounded raster images and replace hotlinks with local API URLs."""
+    matches = list(_IMAGE.finditer(markdown))
+    urls = list(dict.fromkeys(match.group(2) for match in matches))[:MAX_SNAPSHOT_IMAGES]
+    if not urls:
+        return markdown, {}
+
+    resolved_fetcher = fetcher or fetch_snapshot_image
+    semaphore = asyncio.Semaphore(4)
+
+    async def load(url: str) -> tuple[str, SnapshotAsset | None]:
+        try:
+            async with semaphore:
+                return url, await resolved_fetcher(url)
+        except Exception:
+            logger.info("Snapshot image could not be localized: %s", url, exc_info=True)
+            return url, None
+
+    fetched = dict(await asyncio.gather(*(load(url) for url in urls)))
+    assets: dict[str, bytes] = {}
+    replacements: dict[str, str | None] = {}
+    for url, asset in fetched.items():
+        if asset is None:
+            replacements[url] = None
+            continue
+        digest = hashlib.sha256(url.encode("utf-8") + asset.data).hexdigest()[:20]
+        name = f"{digest}.{asset.extension}"
+        assets[name] = asset.data
+        replacements[url] = f"/api/reading/materials/{material_id}/assets/{name}"
+
+    def replace(match: re.Match[str]) -> str:
+        alt, url = match.group(1), match.group(2)
+        local = replacements.get(url)
+        if local:
+            return f"![{alt}]({local})"
+        label = alt.strip() or "image"
+        return f"*Image unavailable: {label}*"
+
+    return _IMAGE.sub(replace, markdown), assets
+
+
+async def fetch_snapshot_image(
+    url: str,
+    *,
+    timeout_s: float = DEFAULT_TIMEOUT_S,
+    client_factory: Any = None,
+    host_validator: Any = None,
+) -> SnapshotAsset | None:
+    """Fetch one public raster image with redirect, MIME, and size guards."""
+    current_url = str(url or "").strip()
+    validator = host_validator or _is_disallowed_host
+    factory = client_factory or _default_client_factory
+    try:
+        async with factory(timeout=timeout_s) as client:
+            for hop in range(MAX_REDIRECTS + 1):
+                parsed = urlsplit(current_url)
+                host = (parsed.hostname or "").strip()
+                if parsed.scheme.lower() not in ALLOWED_SCHEMES or not host:
+                    return None
+                if validator(host):
+                    return None
+                async with client.stream(
+                    "GET",
+                    current_url,
+                    headers={
+                        "User-Agent": DEFAULT_USER_AGENT,
+                        "Accept": "image/png,image/jpeg,image/gif,image/webp;q=0.9",
+                    },
+                    follow_redirects=False,
+                ) as response:
+                    location = response.headers.get("location", "")
+                    if response.status_code in {301, 302, 303, 307, 308} and location:
+                        if hop >= MAX_REDIRECTS:
+                            return None
+                        current_url = urljoin(current_url, location)
+                        continue
+                    if response.status_code >= 400:
+                        return None
+                    data = await _bounded_image_read(response)
+                    return _sniff_raster(data)
+    except httpx.HTTPError:
+        return None
+    return None
+
+
+def _default_client_factory(*, timeout: float) -> httpx.AsyncClient:
+    return httpx.AsyncClient(timeout=timeout, max_redirects=MAX_REDIRECTS)
+
+
+async def _bounded_image_read(response: httpx.Response) -> bytes:
+    data = bytearray()
+    async for chunk in response.aiter_bytes():
+        data.extend(chunk)
+        if len(data) > MAX_SNAPSHOT_IMAGE_BYTES:
+            return b""
+    return bytes(data)
+
+
+def _sniff_raster(data: bytes) -> SnapshotAsset | None:
+    if data.startswith(b"\x89PNG\r\n\x1a\n"):
+        return SnapshotAsset(data, "image/png", "png")
+    if data.startswith(b"\xff\xd8\xff"):
+        return SnapshotAsset(data, "image/jpeg", "jpg")
+    if data.startswith((b"GIF87a", b"GIF89a")):
+        return SnapshotAsset(data, "image/gif", "gif")
+    if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+        return SnapshotAsset(data, "image/webp", "webp")
+    return None
+
+
+def snapshot_asset_mime(data: bytes) -> str | None:
+    asset = _sniff_raster(data)
+    return asset.mime if asset else None
+
+
+__all__ = [
+    "MAX_SNAPSHOT_IMAGE_BYTES",
+    "MAX_SNAPSHOT_IMAGES",
+    "SnapshotAsset",
+    "fetch_snapshot_image",
+    "localize_snapshot_images",
+    "snapshot_asset_mime",
+]