dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/routing.py

@@ -0,0 +1,67 @@
+"""Lightweight label dispatch for ordinary developer process functions."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from .context import CrawlContext
+from .errors import ConfigurationError
+from .models import CaptureResult
+
+Handler = Callable[[CaptureResult, CrawlContext], Any]
+
+
+class RouteAlreadyRegistered(ConfigurationError):
+    """Raised when a label is registered more than once."""
+
+
+class RouteNotFound(ConfigurationError):
+    """Raised when a capture has neither a matching route nor a default."""
+
+
+class Router:
+    """Dispatch a capture to a sync or async handler using its request label.
+
+    The router is itself an ordinary ``process(capture, context)`` callable. It
+    deliberately returns the selected handler's value unchanged; the crawler's
+    existing process semantics await async results and pass through sync results.
+    """
+
+    def __init__(
+        self,
+        routes: Mapping[str, Handler] | None = None,
+        *,
+        default: Handler | None = None,
+    ) -> None:
+        if default is not None and not callable(default):
+            raise ConfigurationError("Default route must be callable")
+        self._routes: dict[str, Handler] = {}
+        self._default = default
+        for label, handler in (routes or {}).items():
+            self.register(label, handler)
+
+    def register(self, label: str, handler: Handler) -> Handler:
+        """Register one exact label and return the handler for decorator use."""
+        if not isinstance(label, str):
+            raise ConfigurationError("Route label must be a string")
+        if not callable(handler):
+            raise ConfigurationError("Route handler must be callable")
+        if label in self._routes:
+            raise RouteAlreadyRegistered(f"Route label {label!r} is already registered")
+        self._routes[label] = handler
+        return handler
+
+    def route(self, label: str) -> Callable[[Handler], Handler]:
+        """Return a small decorator that registers an exact label."""
+
+        def decorate(handler: Handler) -> Handler:
+            return self.register(label, handler)
+
+        return decorate
+
+    def __call__(self, capture: CaptureResult, context: CrawlContext) -> Any:
+        handler = self._routes.get(capture.request.label, self._default)
+        if handler is None:
+            raise RouteNotFound(f"No route registered for request label {capture.request.label!r}")
+        return handler(capture, context)

dataapp_crawler/scheduling.py

@@ -0,0 +1,425 @@
+"""Optional resource-aware concurrency and conservative acquisition selection."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import math
+import os
+import sys
+import time
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, replace
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any, Literal, Protocol
+
+try:  # The stdlib module is unavailable on Windows.
+    import resource as _resource_module
+except ImportError:  # pragma: no cover - exercised through the platform-independent helper test
+    _resource: Any = None
+else:
+    _resource = _resource_module
+
+from .errors import ConfigurationError
+from .models import CaptureResult, Request
+
+
+@dataclass(frozen=True)
+class ResourceSample:
+    """Normalized pressure plus scope; pressure may exceed one under contention."""
+
+    cpu: float
+    memory: float
+    cpu_scope: Literal["cgroup", "local"] = "local"
+    memory_scope: Literal["cgroup", "local", "process", "unavailable"] = "local"
+
+    def __post_init__(self) -> None:
+        for name in ("cpu", "memory"):
+            value = getattr(self, name)
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                raise ConfigurationError(f"ResourceSample.{name} must be numeric")
+            if not math.isfinite(value) or value < 0:
+                raise ConfigurationError(f"ResourceSample.{name} must be finite and nonnegative")
+        if self.cpu_scope not in {"cgroup", "local"}:
+            raise ConfigurationError("ResourceSample.cpu_scope is invalid")
+        if self.memory_scope not in {"cgroup", "local", "process", "unavailable"}:
+            raise ConfigurationError("ResourceSample.memory_scope is invalid")
+
+
+class ResourceSampler(Protocol):
+    def __call__(self) -> ResourceSample | Awaitable[ResourceSample]: ...
+
+
+def _read_number(path: str, *, allow_zero: bool = False) -> float | None:
+    try:
+        value = Path(path).read_text(encoding="ascii").strip()
+        if value == "max":
+            return None
+        number = float(value)
+        minimum_ok = number >= 0 if allow_zero else number > 0
+        return number if math.isfinite(number) and minimum_ok else None
+    except (OSError, ValueError):
+        return None
+
+
+def _cgroup_cpu_count(root: Path) -> float | None:
+    try:
+        quota_text, period_text = (root / "cpu.max").read_text(encoding="ascii").split()[:2]
+        if quota_text != "max" and float(quota_text) > 0 and float(period_text) > 0:
+            return float(quota_text) / float(period_text)
+    except (OSError, ValueError, IndexError):
+        pass
+    quota_value = _read_number(str(root / "cpu/cpu.cfs_quota_us"))
+    period_value = _read_number(str(root / "cpu/cpu.cfs_period_us"))
+    return (
+        quota_value / period_value
+        if quota_value is not None and period_value is not None
+        else None
+    )
+
+
+def _effective_cpu_count(root: Path) -> float:
+    local = float(os.cpu_count() or 1)
+    cgroup = _cgroup_cpu_count(root)
+    return max(0.01, min(local, cgroup)) if cgroup is not None else local
+
+
+def _cgroup_cpu_usage(root: Path) -> float | None:
+    """Return aggregate cgroup CPU seconds, including browser child processes."""
+    try:
+        values = dict(
+            line.split(maxsplit=1)
+            for line in (root / "cpu.stat").read_text(encoding="ascii").splitlines()
+            if len(line.split(maxsplit=1)) == 2
+        )
+        usage = float(values["usage_usec"]) / 1_000_000
+        return usage if math.isfinite(usage) and usage >= 0 else None
+    except (KeyError, OSError, ValueError):
+        pass
+    usage_ns = _read_number(str(root / "cpuacct/cpuacct.usage"), allow_zero=True)
+    return usage_ns / 1_000_000_000 if usage_ns is not None else None
+
+
+def _current_rss(proc_root: Path) -> float:
+    if sys.platform.startswith("linux"):
+        try:
+            fields = (proc_root / "self/statm").read_text(encoding="ascii").split()
+            return float(fields[1]) * os.sysconf("SC_PAGE_SIZE")
+        except (AttributeError, OSError, ValueError, IndexError):
+            pass
+    if _resource is None:
+        return 0.0
+    peak = float(_resource.getrusage(_resource.RUSAGE_SELF).ru_maxrss)
+    return peak if sys.platform == "darwin" else peak * 1024
+
+
+def _local_memory_pressure(proc_root: Path) -> float | None:
+    try:
+        values = {
+            fields[0].rstrip(":"): float(fields[1]) * 1024
+            for line in (proc_root / "meminfo").read_text(encoding="ascii").splitlines()
+            if len(fields := line.split()) >= 2
+        }
+        total, available = values["MemTotal"], values["MemAvailable"]
+        if total > 0:
+            return max(0.0, min(1.0, (total - available) / total))
+    except (KeyError, OSError, ValueError):
+        pass
+    return None
+
+
+def _memory_values(
+    root: Path, proc_root: Path
+) -> tuple[float, Literal["cgroup", "local", "process", "unavailable"]]:
+    current = _read_number(str(root / "memory.current"), allow_zero=True)
+    limit = _read_number(str(root / "memory.max"))
+    if current is None:
+        current = _read_number(
+            str(root / "memory/memory.usage_in_bytes"), allow_zero=True
+        )
+    if limit is None:
+        limit = _read_number(str(root / "memory/memory.limit_in_bytes"))
+    if current is not None and limit is not None:
+        return current / max(limit, 1.0), "cgroup"
+    local = _local_memory_pressure(proc_root)
+    if local is not None:
+        return local, "local"
+    rss = _current_rss(proc_root)
+    try:
+        physical = float(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES"))
+    except (AttributeError, OSError, ValueError):
+        physical = 0.0
+    if rss > 0 and physical > 0:
+        # Darwin's ru_maxrss fallback is peak process RSS; it excludes browser children.
+        return rss / physical, "process"
+    return 0.0, "unavailable"
+
+
+class LocalResourceSampler:
+    """Sample cgroup-wide pressure, falling back to explicitly scoped local metrics.
+
+    Cgroup CPU and memory include browser subprocesses. Outside a cgroup, CPU uses
+    the larger of process CPU and host load average; Linux memory uses MemAvailable.
+    The final portable memory fallback is process RSS (peak RSS on Darwin).
+    """
+
+    def __init__(
+        self,
+        *,
+        cgroup_root: str | Path = "/sys/fs/cgroup",
+        proc_root: str | Path = "/proc",
+    ) -> None:
+        self._cgroup_root = Path(cgroup_root)
+        self._proc_root = Path(proc_root)
+        self._wall = time.monotonic()
+        self._cpu = time.process_time()
+        self._cgroup_cpu = _cgroup_cpu_usage(self._cgroup_root)
+
+    def __call__(self) -> ResourceSample:
+        wall, cpu = time.monotonic(), time.process_time()
+        elapsed = max(wall - self._wall, 1e-9)
+        cgroup_cpu = _cgroup_cpu_usage(self._cgroup_root)
+        effective_cpus = _effective_cpu_count(self._cgroup_root)
+        if cgroup_cpu is not None and self._cgroup_cpu is not None:
+            cpu_pressure = max(cgroup_cpu - self._cgroup_cpu, 0.0) / (
+                elapsed * effective_cpus
+            )
+            cpu_scope: Literal["cgroup", "local"] = "cgroup"
+        else:
+            process_pressure = max(cpu - self._cpu, 0.0) / (elapsed * effective_cpus)
+            try:
+                load_pressure = max(float(os.getloadavg()[0]), 0.0) / effective_cpus
+            except (AttributeError, OSError):
+                load_pressure = 0.0
+            cpu_pressure = max(process_pressure, load_pressure)
+            cpu_scope = "local"
+        self._wall, self._cpu = wall, cpu
+        self._cgroup_cpu = cgroup_cpu
+        memory_pressure, memory_scope = _memory_values(self._cgroup_root, self._proc_root)
+        return ResourceSample(cpu_pressure, memory_pressure, cpu_scope, memory_scope)
+
+
+@dataclass(frozen=True)
+class AutoConcurrency:
+    """Configuration for a bounded, acquisition-driven concurrency controller."""
+
+    min_concurrency: int = 1
+    max_concurrency: int | None = None
+    sample_interval: float = 0.5
+    target_cpu: float = 0.75
+    target_memory: float = 0.80
+    decrease_ratio: float = 0.5
+    sampler: ResourceSampler | None = None
+
+    def __post_init__(self) -> None:
+        for name in ("min_concurrency", "max_concurrency"):
+            value = getattr(self, name)
+            if value is not None and (
+                isinstance(value, bool) or not isinstance(value, int) or value < 1
+            ):
+                raise ConfigurationError(f"{name} must be a positive integer or None")
+        if self.max_concurrency is not None and self.min_concurrency > self.max_concurrency:
+            raise ConfigurationError("min_concurrency cannot exceed max_concurrency")
+        for name in ("sample_interval", "target_cpu", "target_memory", "decrease_ratio"):
+            value = getattr(self, name)
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                raise ConfigurationError(f"{name} must be numeric")
+            if not math.isfinite(value) or value <= 0:
+                raise ConfigurationError(f"{name} must be finite and positive")
+        if self.target_cpu > 1 or self.target_memory > 1 or self.decrease_ratio >= 1:
+            raise ConfigurationError("resource targets must be <= 1 and decrease_ratio must be < 1")
+
+    def controller(self, ceiling: int) -> ConcurrencyController:
+        return ConcurrencyController(self, ceiling=ceiling)
+
+
+class ConcurrencyController:
+    """Resizable async permit gate that never revokes active acquisitions."""
+
+    def __init__(self, config: AutoConcurrency, *, ceiling: int) -> None:
+        if isinstance(ceiling, bool) or not isinstance(ceiling, int) or ceiling < 1:
+            raise ConfigurationError("concurrency ceiling must be a positive integer")
+        configured = config.max_concurrency if config.max_concurrency is not None else ceiling
+        self._ceiling = min(ceiling, configured)
+        if config.min_concurrency > self._ceiling:
+            raise ConfigurationError("min_concurrency cannot exceed the concurrency ceiling")
+        self._config = config
+        self._sampler = config.sampler or LocalResourceSampler()
+        self._condition = asyncio.Condition()
+        self._sample_lock = asyncio.Lock()
+        self._limit = config.min_concurrency
+        self._active = 0
+        self._last_sample_at = -math.inf
+        self._last_sample: ResourceSample | None = None
+
+    @property
+    def current_limit(self) -> int:
+        return self._limit
+
+    @property
+    def active(self) -> int:
+        return self._active
+
+    @property
+    def last_sample(self) -> ResourceSample | None:
+        return self._last_sample
+
+    async def _sample_if_due(self) -> None:
+        now = time.monotonic()
+        if now - self._last_sample_at < self._config.sample_interval:
+            return
+        async with self._sample_lock:
+            now = time.monotonic()
+            if now - self._last_sample_at < self._config.sample_interval:
+                return
+            value = self._sampler()
+            sample = await value if inspect.isawaitable(value) else value
+            if not isinstance(sample, ResourceSample):
+                raise ConfigurationError("resource sampler must return ResourceSample")
+            pressured = (
+                sample.cpu >= self._config.target_cpu
+                or sample.memory >= self._config.target_memory
+            )
+            async with self._condition:
+                if pressured:
+                    reduced = max(
+                        self._config.min_concurrency,
+                        math.floor(self._limit * self._config.decrease_ratio),
+                    )
+                    self._limit = min(self._limit, reduced)
+                elif self._limit < self._ceiling:
+                    self._limit += 1
+                self._last_sample = sample
+                self._last_sample_at = now
+                self._condition.notify_all()
+
+    async def acquire(self) -> None:
+        await self._sample_if_due()
+        async with self._condition:
+            await self._condition.wait_for(lambda: self._active < self._limit)
+            self._active += 1
+
+    async def release(self) -> None:
+        async with self._condition:
+            if self._active < 1:
+                raise RuntimeError("Concurrency permit released without an active acquisition")
+            self._active -= 1
+            self._condition.notify_all()
+
+    async def __aenter__(self) -> ConcurrencyController:
+        await self.acquire()
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.release()
+
+
+@dataclass(frozen=True)
+class AdaptiveDecision:
+    use_browser: bool
+    reason: str
+
+
+Validation = Callable[[CaptureResult], bool | Awaitable[bool]]
+
+
+class _VisibleText(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self._hidden = 0
+        self.parts: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag.lower() in {"script", "style", "noscript", "template"}:
+            self._hidden += 1
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag.lower() in {"script", "style", "noscript", "template"} and self._hidden:
+            self._hidden -= 1
+
+    def handle_data(self, data: str) -> None:
+        if not self._hidden:
+            self.parts.append(data)
+
+
+def _looks_like_javascript_shell(capture: CaptureResult) -> bool:
+    content_type = capture.headers.get("content-type", "").lower()
+    if "html" not in content_type and not capture.body.lstrip().lower().startswith(b"<!doctype"):
+        return False
+    text = capture.text
+    parser = _VisibleText()
+    try:
+        parser.feed(text)
+    except Exception:
+        return False
+    visible = " ".join(" ".join(parser.parts).split())
+    lowered = text.lower()
+    root_marker = any(marker in lowered for marker in ('id="root"', "id='root'", "__next"))
+    return len(visible) < 24 and root_marker and "<script" in lowered
+
+
+class AdaptivePolicy:
+    """Choose a browser retry only when a safe HTTP GET is demonstrably incomplete."""
+
+    def __init__(self, validate: Validation | None = None) -> None:
+        if validate is not None and not callable(validate):
+            raise ConfigurationError("adaptive validate must be callable or None")
+        self._validate = validate
+
+    def initial_request(self, request: Request) -> Request:
+        self._check_request(request)
+        return replace(request, mode="http")
+
+    def browser_request(self, request: Request) -> Request:
+        self._check_request(request)
+        if request.method != "GET":
+            raise ConfigurationError("Only adaptive GET requests can use browser fallback")
+        return replace(request, mode="browser")
+
+    async def decide(self, capture: CaptureResult) -> AdaptiveDecision:
+        method = capture.request.method
+        if method == "HEAD":
+            return AdaptiveDecision(False, "head_http_only")
+        if method != "GET":
+            return AdaptiveDecision(False, "method_not_replayable")
+        status = capture.status
+        if status is None or not 200 <= status < 300:
+            return AdaptiveDecision(False, "http_status_not_eligible")
+        lowered = capture.body.lower()
+        if any(
+            marker in lowered
+            for marker in (
+                b"captcha",
+                b"verify you are human",
+                b"/cdn-cgi/challenge",
+                b"cf-chl-",
+            )
+        ):
+            return AdaptiveDecision(False, "access_challenge_detected")
+        if self._validate is not None:
+            result = self._validate(capture)
+            valid = await result if inspect.isawaitable(result) else result
+            if not isinstance(valid, bool):
+                raise ConfigurationError("adaptive validate must return a boolean")
+            return AdaptiveDecision(not valid, "validation_failed" if not valid else "validated")
+        if status == 200 and not capture.body.strip():
+            return AdaptiveDecision(True, "empty_http_body")
+        if _looks_like_javascript_shell(capture):
+            return AdaptiveDecision(True, "javascript_shell")
+        return AdaptiveDecision(False, "http_content_present")
+
+    @staticmethod
+    def _check_request(request: Request) -> None:
+        if request.mode != "auto":
+            raise ConfigurationError("AdaptivePolicy requires Request.mode='auto'")
+        if request.method not in {"GET", "HEAD"}:
+            raise ConfigurationError("Adaptive acquisition supports only GET or HEAD")
+        if request.headers:
+            raise ConfigurationError(
+                "Adaptive browser fallback requires origin-scoped Identity.headers"
+            )
+        if request.content is not None or request.json is not None:
+            raise ConfigurationError("Adaptive acquisition does not replay request bodies")
+        if request.method == "GET" and not request.can_retry:
+            raise ConfigurationError("Adaptive GET requests must be replayable")