webskrap 0.1.0__py3-none-any.whl

webskrap/__init__.py

@@ -0,0 +1,26 @@
+from webskrap.client import WebSkrapClient, WebSkrapError, WebSkrapSession
+from webskrap.models import (
+    BrowserProfile,
+    FetchResult,
+    ProxyConfig,
+    ResourcePolicy,
+    SessionConfig,
+    StealthConfig,
+    Viewport,
+)
+from webskrap.profiles import get_profile, list_profiles
+
+__all__ = [
+    "BrowserProfile",
+    "FetchResult",
+    "ProxyConfig",
+    "ResourcePolicy",
+    "SessionConfig",
+    "StealthConfig",
+    "Viewport",
+    "WebSkrapClient",
+    "WebSkrapError",
+    "WebSkrapSession",
+    "get_profile",
+    "list_profiles",
+]

webskrap/cli.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Annotated, cast
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from webskrap.client import WaitUntil, WebSkrapClient
+from webskrap.models import ResourcePolicy, SessionConfig
+from webskrap.profiles import get_profile, list_profiles
+
+app = typer.Typer(help="WebSkrap browser scraping toolkit.")
+console = Console()
+
+
+@app.command("profiles")
+def profiles_command() -> None:
+    table = Table(title="WebSkrap Profiles")
+    table.add_column("Name")
+    table.add_column("Viewport")
+    table.add_column("Locale")
+    table.add_column("Timezone")
+    table.add_column("Mobile")
+
+    for profile in list_profiles():
+        table.add_row(
+            profile.name,
+            f"{profile.viewport.width}x{profile.viewport.height}",
+            profile.locale,
+            profile.timezone_id,
+            "yes" if profile.is_mobile else "no",
+        )
+
+    console.print(table)
+
+
+@app.command("doctor")
+def doctor_command() -> None:
+    asyncio.run(_doctor())
+
+
+async def _doctor() -> None:
+    try:
+        from playwright.async_api import async_playwright
+    except Exception as exc:
+        console.print(f"[red]Playwright import failed:[/red] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    try:
+        manager = async_playwright()
+        playwright = await manager.start()
+        browser = await playwright.chromium.launch(headless=True)
+        await browser.close()
+        await playwright.stop()
+    except Exception as exc:
+        console.print("[yellow]Playwright is installed, but Chromium did not launch.[/yellow]")
+        console.print(str(exc))
+        console.print("Run: python -m playwright install chromium")
+        raise typer.Exit(code=1) from exc
+
+    console.print("[green]Playwright and Chromium are ready.[/green]")
+
+
+@app.command("fetch")
+def fetch_command(
+    url: Annotated[str, typer.Argument(help="URL to fetch.")],
+    profile: Annotated[
+        str,
+        typer.Option("--profile", "-p", help="Bundled profile name."),
+    ] = "desktop-chrome",
+    headed: Annotated[bool, typer.Option("--headed", help="Run with a visible browser.")] = False,
+    channel: Annotated[
+        str | None,
+        typer.Option("--channel", help="Browser channel, e.g. chrome."),
+    ] = None,
+    screenshot: Annotated[
+        Path | None,
+        typer.Option("--screenshot", help="Write a full-page screenshot to this path."),
+    ] = None,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="Write HTML to this file."),
+    ] = None,
+    wait_until: Annotated[
+        str,
+        typer.Option("--wait-until", help="commit, domcontentloaded, load, or networkidle."),
+    ] = "domcontentloaded",
+    timeout_ms: Annotated[
+        float,
+        typer.Option("--timeout-ms", min=1, help="Navigation timeout."),
+    ] = 30_000,
+    resource_policy: Annotated[
+        ResourcePolicy,
+        typer.Option("--resource-policy", help="Resource routing preset."),
+    ] = ResourcePolicy.ALL,
+    no_stealth: Annotated[
+        bool,
+        typer.Option("--no-stealth", help="Disable browser hardening."),
+    ] = False,
+) -> None:
+    asyncio.run(
+        _fetch(
+            url=url,
+            profile=profile,
+            headed=headed,
+            channel=channel,
+            screenshot=screenshot,
+            output=output,
+            wait_until=wait_until,
+            timeout_ms=timeout_ms,
+            resource_policy=resource_policy,
+            no_stealth=no_stealth,
+        )
+    )
+
+
+async def _fetch(
+    *,
+    url: str,
+    profile: str,
+    headed: bool,
+    channel: str | None,
+    screenshot: Path | None,
+    output: Path | None,
+    wait_until: str,
+    timeout_ms: float,
+    resource_policy: ResourcePolicy,
+    no_stealth: bool,
+) -> None:
+    selected_profile = get_profile(profile)
+    config = SessionConfig(
+        headless=not headed,
+        channel=channel,
+        navigation_timeout_ms=timeout_ms,
+        resource_policy=resource_policy,
+    )
+    config.stealth.enabled = not no_stealth
+
+    async with WebSkrapClient() as client:
+        result = await client.fetch(
+            url,
+            profile=selected_profile,
+            config=config,
+            wait_until=_parse_wait_until(wait_until),
+            screenshot=screenshot or False,
+            timeout_ms=timeout_ms,
+        )
+
+    if output:
+        output.parent.mkdir(parents=True, exist_ok=True)
+        output.write_text(result.text, encoding="utf-8")
+
+    console.print(f"[bold]Status:[/bold] {result.status}")
+    console.print(f"[bold]Final URL:[/bold] {result.final_url}")
+    console.print(f"[bold]Title:[/bold] {result.title}")
+    if result.screenshot_path:
+        console.print(f"[bold]Screenshot:[/bold] {result.screenshot_path}")
+    if output:
+        console.print(f"[bold]HTML:[/bold] {output}")
+
+
+def _parse_wait_until(value: str) -> WaitUntil:
+    valid = ("commit", "domcontentloaded", "load", "networkidle")
+    if value not in valid:
+        allowed = ", ".join(valid)
+        raise typer.BadParameter(f"must be one of: {allowed}")
+    return cast(WaitUntil, value)

webskrap/client.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Literal
+from uuid import uuid4
+
+from playwright.async_api import Browser, BrowserContext, Page, Playwright, async_playwright
+
+from webskrap.models import BrowserProfile, FetchResult, ResourcePolicy, SessionConfig
+from webskrap.profiles import get_profile
+from webskrap.stealth import apply_stealth
+
+WaitUntil = Literal["commit", "domcontentloaded", "load", "networkidle"]
+
+
+class WebSkrapError(RuntimeError):
+    pass
+
+
+class WebSkrapSession:
+    def __init__(
+        self,
+        *,
+        name: str,
+        context: BrowserContext,
+        config: SessionConfig,
+        profile: BrowserProfile,
+        browser: Browser | None = None,
+    ) -> None:
+        self.name = name
+        self.context = context
+        self.config = config
+        self.profile = profile
+        self.browser = browser
+        self._closed = False
+
+    async def __aenter__(self) -> WebSkrapSession:
+        return self
+
+    async def __aexit__(self, exc_type: object, exc: object, traceback: object) -> None:
+        await self.close()
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        wait_until: WaitUntil = "domcontentloaded",
+        screenshot: bool | str | Path = False,
+        timeout_ms: float | None = None,
+    ) -> FetchResult:
+        self._ensure_open()
+        started = time.perf_counter()
+        page = await self.context.new_page()
+        try:
+            _configure_page_timeouts(page, self.config)
+            response = await page.goto(
+                url,
+                wait_until=wait_until,
+                timeout=timeout_ms or self.config.navigation_timeout_ms,
+            )
+            title = await page.title()
+            text = await page.content()
+            screenshot_path = await _maybe_screenshot(page, screenshot)
+            cookies = await self.context.cookies()
+            elapsed_ms = (time.perf_counter() - started) * 1000
+            status = response.status if response else None
+            headers = dict(response.headers) if response else {}
+            return FetchResult(
+                url=url,
+                final_url=page.url,
+                status=status,
+                ok=status is not None and 200 <= status < 400,
+                headers=headers,
+                text=text,
+                title=title,
+                cookies=cookies,
+                timings={"elapsed_ms": elapsed_ms},
+                screenshot_path=screenshot_path,
+            )
+        finally:
+            await page.close()
+
+    async def close(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        await self.context.close()
+        if self.browser is not None:
+            await self.browser.close()
+
+    def _ensure_open(self) -> None:
+        if self._closed:
+            msg = f"session '{self.name}' is closed"
+            raise WebSkrapError(msg)
+
+
+class WebSkrapClient:
+    def __init__(
+        self,
+        *,
+        default_config: SessionConfig | None = None,
+        profiles: Mapping[str, BrowserProfile] | None = None,
+    ) -> None:
+        self.default_config = default_config or SessionConfig()
+        self.profiles = dict(profiles or {})
+        self._playwright_manager = None
+        self._playwright: Playwright | None = None
+        self._sessions: dict[str, WebSkrapSession] = {}
+
+    async def __aenter__(self) -> WebSkrapClient:
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type: object, exc: object, traceback: object) -> None:
+        await self.close()
+
+    async def start(self) -> None:
+        if self._playwright is not None:
+            return
+        self._playwright_manager = async_playwright()
+        self._playwright = await self._playwright_manager.start()
+
+    async def close(self) -> None:
+        for session in list(self._sessions.values()):
+            await session.close()
+        self._sessions.clear()
+        if self._playwright is not None:
+            await self._playwright.stop()
+        self._playwright_manager = None
+        self._playwright = None
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        profile: str | BrowserProfile | None = None,
+        config: SessionConfig | None = None,
+        wait_until: WaitUntil = "domcontentloaded",
+        screenshot: bool | str | Path = False,
+        timeout_ms: float | None = None,
+    ) -> FetchResult:
+        name = f"_single_{uuid4().hex}"
+        session = await self.session(name, config=config, profile=profile)
+        try:
+            return await session.fetch(
+                url,
+                wait_until=wait_until,
+                screenshot=screenshot,
+                timeout_ms=timeout_ms,
+            )
+        finally:
+            await session.close()
+            self._sessions.pop(name, None)
+
+    async def session(
+        self,
+        name: str,
+        *,
+        config: SessionConfig | None = None,
+        profile: str | BrowserProfile | None = None,
+    ) -> WebSkrapSession:
+        if name in self._sessions:
+            return self._sessions[name]
+        await self.start()
+        resolved_config = config or self.default_config
+        resolved_profile = self._resolve_profile(profile)
+        session = await self._create_session(name, resolved_config, resolved_profile)
+        self._sessions[name] = session
+        return session
+
+    def _resolve_profile(self, profile: str | BrowserProfile | None) -> BrowserProfile:
+        if isinstance(profile, BrowserProfile):
+            return profile
+        if profile in self.profiles:
+            return self.profiles[profile].model_copy(deep=True)
+        return get_profile(profile)
+
+    async def _create_session(
+        self,
+        name: str,
+        config: SessionConfig,
+        profile: BrowserProfile,
+    ) -> WebSkrapSession:
+        if self._playwright is None:
+            msg = "client is not started"
+            raise WebSkrapError(msg)
+
+        browser_type = getattr(self._playwright, config.browser)
+        context_options = config.context_options(profile)
+
+        if config.user_data_dir is not None:
+            config.user_data_dir.mkdir(parents=True, exist_ok=True)
+            context = await browser_type.launch_persistent_context(
+                str(config.user_data_dir),
+                **config.launch_options(),
+                **context_options,
+            )
+            browser = None
+        else:
+            browser = await browser_type.launch(**config.launch_options())
+            context = await browser.new_context(**context_options)
+
+        context.set_default_timeout(config.default_timeout_ms)
+        context.set_default_navigation_timeout(config.navigation_timeout_ms)
+
+        if config.resource_policy != ResourcePolicy.ALL:
+            await context.route("**/*", _resource_route_handler(config.resource_policy))
+        if config.stealth.enabled:
+            await apply_stealth(context, profile, config.stealth)
+
+        return WebSkrapSession(
+            name=name,
+            context=context,
+            config=config,
+            profile=profile,
+            browser=browser,
+        )
+
+
+def _resource_route_handler(policy: ResourcePolicy):
+    blocked = {
+        ResourcePolicy.LITE: {"image", "font", "media"},
+        ResourcePolicy.DOCUMENTS: {"image", "font", "media", "stylesheet"},
+    }[policy]
+
+    async def handle(route) -> None:
+        if route.request.resource_type in blocked:
+            await route.abort()
+        else:
+            await route.continue_()
+
+    return handle
+
+
+def _configure_page_timeouts(page: Page, config: SessionConfig) -> None:
+    page.set_default_timeout(config.default_timeout_ms)
+    page.set_default_navigation_timeout(config.navigation_timeout_ms)
+
+
+async def _maybe_screenshot(page: Page, screenshot: bool | str | Path) -> Path | None:
+    if not screenshot:
+        return None
+    path = (
+        Path(f"webskrap-{int(time.time() * 1000)}.png")
+        if screenshot is True
+        else Path(screenshot)
+    )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    await page.screenshot(path=str(path), full_page=True)
+    return path

webskrap/models.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+class ResourcePolicy(StrEnum):
+    ALL = "all"
+    LITE = "lite"
+    DOCUMENTS = "documents"
+
+
+class Viewport(BaseModel):
+    width: int = Field(gt=0)
+    height: int = Field(gt=0)
+
+    def to_playwright(self) -> dict[str, int]:
+        return {"width": self.width, "height": self.height}
+
+
+class ProxyConfig(BaseModel):
+    server: str
+    bypass: str | None = None
+    username: str | None = None
+    password: str | None = None
+
+    @field_validator("server")
+    @classmethod
+    def validate_server(cls, value: str) -> str:
+        allowed_prefixes = ("http://", "https://", "socks4://", "socks5://")
+        if not value.startswith(allowed_prefixes):
+            msg = "proxy server must start with http://, https://, socks4://, or socks5://"
+            raise ValueError(msg)
+        return value
+
+    def to_playwright(self) -> dict[str, str]:
+        payload = {"server": self.server}
+        if self.bypass:
+            payload["bypass"] = self.bypass
+        if self.username:
+            payload["username"] = self.username
+        if self.password:
+            payload["password"] = self.password
+        return payload
+
+
+class StealthConfig(BaseModel):
+    enabled: bool = True
+    patch_webdriver: bool = True
+    patch_chrome_runtime: bool = True
+    patch_permissions: bool = True
+    patch_plugins: bool = True
+    patch_webgl: bool = True
+    patch_canvas: bool = True
+    patch_hardware: bool = True
+
+
+class BrowserProfile(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    user_agent: str | None = None
+    viewport: Viewport = Field(default_factory=lambda: Viewport(width=1365, height=768))
+    screen: Viewport = Field(default_factory=lambda: Viewport(width=1440, height=900))
+    locale: str = "en-US"
+    timezone_id: str = "Europe/Paris"
+    device_scale_factor: float = Field(default=1.0, gt=0)
+    is_mobile: bool = False
+    has_touch: bool = False
+    color_scheme: Literal["dark", "light", "no-preference", "null"] = "light"
+    reduced_motion: Literal["reduce", "no-preference", "null"] = "no-preference"
+    extra_http_headers: dict[str, str] = Field(default_factory=dict)
+    navigator_languages: list[str] = Field(default_factory=lambda: ["en-US", "en"])
+    hardware_concurrency: int = Field(default=8, ge=1, le=64)
+    device_memory: int = Field(default=8, ge=1, le=128)
+    webgl_vendor: str = "Google Inc. (Intel)"
+    webgl_renderer: str = "ANGLE (Intel, Intel(R) Iris(TM) Plus Graphics, OpenGL 4.1)"
+
+    @field_validator("name")
+    @classmethod
+    def validate_name(cls, value: str) -> str:
+        if not value.strip():
+            msg = "profile name cannot be empty"
+            raise ValueError(msg)
+        return value
+
+    @field_validator("locale", "timezone_id")
+    @classmethod
+    def validate_non_empty(cls, value: str) -> str:
+        if not value.strip():
+            msg = "value cannot be empty"
+            raise ValueError(msg)
+        return value
+
+    @model_validator(mode="after")
+    def ensure_language_consistency(self) -> BrowserProfile:
+        if not self.navigator_languages:
+            self.navigator_languages = [self.locale]
+        if self.locale not in self.navigator_languages:
+            self.navigator_languages.insert(0, self.locale)
+        return self
+
+    def accept_language(self) -> str:
+        languages = self.navigator_languages or [self.locale]
+        weighted = [languages[0]]
+        weighted.extend(
+            f"{language};q={max(0.1, 1 - index * 0.1):.1f}"
+            for index, language in enumerate(languages[1:], start=1)
+        )
+        return ",".join(weighted)
+
+    def headers(self) -> dict[str, str]:
+        headers = {
+            "Accept": (
+                "text/html,application/xhtml+xml,application/xml;q=0.9,"
+                "image/avif,image/webp,*/*;q=0.8"
+            ),
+            "Accept-Language": self.accept_language(),
+            "Upgrade-Insecure-Requests": "1",
+        }
+        headers.update(self.extra_http_headers)
+        return headers
+
+    def to_context_options(self) -> dict[str, Any]:
+        options: dict[str, Any] = {
+            "viewport": self.viewport.to_playwright(),
+            "screen": self.screen.to_playwright(),
+            "locale": self.locale,
+            "timezone_id": self.timezone_id,
+            "device_scale_factor": self.device_scale_factor,
+            "is_mobile": self.is_mobile,
+            "has_touch": self.has_touch,
+            "color_scheme": self.color_scheme,
+            "reduced_motion": self.reduced_motion,
+            "extra_http_headers": self.headers(),
+        }
+        if self.user_agent:
+            options["user_agent"] = self.user_agent
+        return options
+
+
+class SessionConfig(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
+
+    browser: Literal["chromium", "firefox", "webkit"] = "chromium"
+    channel: str | None = None
+    headless: bool = True
+    user_data_dir: Path | None = None
+    storage_state: Path | dict[str, Any] | None = None
+    proxy: ProxyConfig | None = None
+    resource_policy: ResourcePolicy = ResourcePolicy.ALL
+    stealth: StealthConfig = Field(default_factory=StealthConfig)
+    ignore_https_errors: bool = False
+    java_script_enabled: bool = True
+    service_workers: Literal["allow", "block"] = "allow"
+    timeout_ms: float = Field(default=30_000, gt=0)
+    navigation_timeout_ms: float = Field(default=30_000, gt=0)
+    default_timeout_ms: float = Field(default=30_000, gt=0)
+    slow_mo_ms: float | None = Field(default=None, ge=0)
+    launch_args: list[str] = Field(default_factory=list)
+
+    def launch_options(self) -> dict[str, Any]:
+        options: dict[str, Any] = {
+            "headless": self.headless,
+            "timeout": self.timeout_ms,
+        }
+        if self.channel:
+            options["channel"] = self.channel
+        if self.slow_mo_ms is not None:
+            options["slow_mo"] = self.slow_mo_ms
+        if self.launch_args:
+            options["args"] = self.launch_args
+        return options
+
+    def context_options(self, profile: BrowserProfile) -> dict[str, Any]:
+        options = profile.to_context_options()
+        options.update(
+            {
+                "ignore_https_errors": self.ignore_https_errors,
+                "java_script_enabled": self.java_script_enabled,
+                "service_workers": self.service_workers,
+            }
+        )
+        if self.proxy:
+            options["proxy"] = self.proxy.to_playwright()
+        if self.storage_state is not None and self.user_data_dir is None:
+            options["storage_state"] = self.storage_state
+        return options
+
+
+class FetchResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    url: str
+    final_url: str
+    status: int | None
+    ok: bool
+    headers: dict[str, str]
+    text: str
+    title: str
+    cookies: list[dict[str, Any]]
+    timings: dict[str, float]
+    screenshot_path: Path | None = None

webskrap/profiles.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from webskrap.models import BrowserProfile, Viewport
+
+_PROFILES: dict[str, BrowserProfile] = {
+    "desktop-chrome": BrowserProfile(
+        name="desktop-chrome",
+        viewport=Viewport(width=1365, height=768),
+        screen=Viewport(width=1440, height=900),
+        locale="en-US",
+        timezone_id="Europe/Paris",
+        device_scale_factor=1,
+        navigator_languages=["en-US", "en"],
+        hardware_concurrency=8,
+        device_memory=8,
+        webgl_vendor="Google Inc. (Intel)",
+        webgl_renderer="ANGLE (Intel, Intel(R) Iris(TM) Plus Graphics, OpenGL 4.1)",
+    ),
+    "desktop-edge": BrowserProfile(
+        name="desktop-edge",
+        viewport=Viewport(width=1440, height=810),
+        screen=Viewport(width=1536, height=864),
+        locale="en-US",
+        timezone_id="Europe/Paris",
+        device_scale_factor=1,
+        navigator_languages=["en-US", "en"],
+        hardware_concurrency=8,
+        device_memory=8,
+        webgl_vendor="Google Inc. (NVIDIA)",
+        webgl_renderer="ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0)",
+    ),
+    "mobile-chrome": BrowserProfile(
+        name="mobile-chrome",
+        user_agent=(
+            "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 "
+            "(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
+        ),
+        viewport=Viewport(width=412, height=915),
+        screen=Viewport(width=412, height=915),
+        locale="en-US",
+        timezone_id="Europe/Paris",
+        device_scale_factor=2.625,
+        is_mobile=True,
+        has_touch=True,
+        navigator_languages=["en-US", "en"],
+        hardware_concurrency=8,
+        device_memory=8,
+        webgl_vendor="Google Inc. (Qualcomm)",
+        webgl_renderer="ANGLE (Qualcomm, Adreno (TM) 730, OpenGL ES 3.2)",
+    ),
+}
+
+
+def list_profiles() -> tuple[BrowserProfile, ...]:
+    return tuple(_PROFILES.values())
+
+
+def get_profile(name: str | BrowserProfile | None = None) -> BrowserProfile:
+    if isinstance(name, BrowserProfile):
+        return name
+    key = name or "desktop-chrome"
+    try:
+        return _PROFILES[key].model_copy(deep=True)
+    except KeyError as exc:
+        available = ", ".join(sorted(_PROFILES))
+        msg = f"unknown profile '{key}'. Available profiles: {available}"
+        raise ValueError(msg) from exc

webskrap/py.typed

@@ -0,0 +1 @@
+

webskrap/stealth.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import json
+
+from playwright.async_api import BrowserContext
+
+from webskrap.models import BrowserProfile, StealthConfig
+
+
+async def apply_stealth(
+    context: BrowserContext,
+    profile: BrowserProfile,
+    config: StealthConfig,
+) -> None:
+    if not config.enabled:
+        return
+    await context.add_init_script(script=build_stealth_script(profile, config))
+
+
+def build_stealth_script(profile: BrowserProfile, config: StealthConfig) -> str:
+    payload = {
+        "languages": profile.navigator_languages,
+        "hardwareConcurrency": profile.hardware_concurrency,
+        "deviceMemory": profile.device_memory,
+        "webglVendor": profile.webgl_vendor,
+        "webglRenderer": profile.webgl_renderer,
+        "patchWebdriver": config.patch_webdriver,
+        "patchChromeRuntime": config.patch_chrome_runtime,
+        "patchPermissions": config.patch_permissions,
+        "patchPlugins": config.patch_plugins,
+        "patchWebgl": config.patch_webgl,
+        "patchCanvas": config.patch_canvas,
+        "patchHardware": config.patch_hardware,
+    }
+    encoded = json.dumps(payload, separators=(",", ":"))
+    return f"""
+(() => {{
+  const profile = {encoded};
+  const defineGetter = (target, prop, getter) => {{
+    try {{
+      Object.defineProperty(target, prop, {{ get: getter, configurable: true }});
+    }} catch (_) {{}}
+  }};
+
+  if (profile.patchWebdriver) {{
+    defineGetter(Navigator.prototype, "webdriver", () => undefined);
+  }}
+
+  defineGetter(Navigator.prototype, "languages", () => profile.languages.slice());
+  defineGetter(Navigator.prototype, "language", () => profile.languages[0]);
+
+  if (profile.patchHardware) {{
+    defineGetter(Navigator.prototype, "hardwareConcurrency", () => profile.hardwareConcurrency);
+    defineGetter(Navigator.prototype, "deviceMemory", () => profile.deviceMemory);
+  }}
+
+  if (profile.patchPlugins) {{
+    defineGetter(Navigator.prototype, "plugins", () => [1, 2, 3, 4, 5]);
+    defineGetter(Navigator.prototype, "mimeTypes", () => [1, 2, 3]);
+  }}
+
+  if (profile.patchChromeRuntime && !window.chrome) {{
+    try {{
+      Object.defineProperty(window, "chrome", {{
+        configurable: true,
+        value: {{
+          app: {{ isInstalled: false }},
+          csi: () => ({{}}),
+          loadTimes: () => ({{}}),
+          runtime: {{}}
+        }}
+      }});
+    }} catch (_) {{}}
+  }}
+
+  if (profile.patchPermissions && navigator.permissions && navigator.permissions.query) {{
+    const originalQuery = navigator.permissions.query.bind(navigator.permissions);
+    navigator.permissions.query = (parameters) => {{
+      if (parameters && parameters.name === "notifications") {{
+        return Promise.resolve({{ state: Notification.permission }});
+      }}
+      return originalQuery(parameters);
+    }};
+  }}
+
+  if (profile.patchWebgl) {{
+    const patchWebGL = (proto) => {{
+      if (!proto || !proto.getParameter) return;
+      const original = proto.getParameter;
+      proto.getParameter = function(parameter) {{
+        if (parameter === 37445) return profile.webglVendor;
+        if (parameter === 37446) return profile.webglRenderer;
+        return original.apply(this, arguments);
+      }};
+    }};
+    patchWebGL(window.WebGLRenderingContext && window.WebGLRenderingContext.prototype);
+    patchWebGL(window.WebGL2RenderingContext && window.WebGL2RenderingContext.prototype);
+  }}
+
+  if (profile.patchCanvas && window.HTMLCanvasElement) {{
+    const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
+    HTMLCanvasElement.prototype.toDataURL = function() {{
+      try {{
+        const context = this.getContext("2d");
+        if (context && this.width && this.height) {{
+          const x = Math.min(1, this.width - 1);
+          const y = Math.min(1, this.height - 1);
+          const imageData = context.getImageData(x, y, 1, 1);
+          imageData.data[0] = (imageData.data[0] + 1) % 255;
+          context.putImageData(imageData, x, y);
+        }}
+      }} catch (_) {{}}
+      return originalToDataURL.apply(this, arguments);
+    }};
+  }}
+}})();
+"""

webskrap-0.1.0.dist-info/METADATA

@@ -0,0 +1,278 @@
+Metadata-Version: 2.4
+Name: webskrap
+Version: 0.1.0
+Summary: A Playwright-based Python scraping framework with coherent browser profiles and session controls.
+Project-URL: Homepage, https://github.com/kacigaya/webskrap
+Project-URL: Repository, https://github.com/kacigaya/webskrap
+Project-URL: Issues, https://github.com/kacigaya/webskrap/issues
+Author: WebSkrap contributors
+License-Expression: MIT
+Keywords: browser-automation,crawler,playwright,scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: playwright>=1.49
+Requires-Dist: pydantic>=2.8
+Requires-Dist: rich>=13.9
+Requires-Dist: typer>=0.15
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="assets/webskrap-logo.png" alt="WebSkrap logo" width="200">
+</p>
+
+<h1 align="center">WebSkrap</h1>
+
+<p align="center">
+   <strong>Async-first Python scraping framework built on Playwright.</strong><br>
+   <em>It provides coherent browser profiles, persistent sessions, resource routing, and configurable browser hardening for data collection workflows that need realistic browser behavior.</em>
+</p>
+
+WebSkrap does not include CAPTCHA solving, login-wall bypassing, credential bypassing, or access-control circumvention. Use it only on targets you are allowed to access.
+
+## Install
+
+```bash
+pip install webskrap
+python -m playwright install chromium
+```
+
+## Quick Start
+
+```python
+import asyncio
+
+from webskrap import WebSkrapClient
+
+
+async def main() -> None:
+    async with WebSkrapClient() as client:
+        result = await client.fetch("https://example.com")
+        print(result.status)
+        print(result.title)
+        print(result.text[:200])
+
+
+asyncio.run(main())
+```
+
+## Persistent Session
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import SessionConfig, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        user_data_dir=Path(".webskrap/sessions/shop"),
+        headless=True,
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("shop", config=config, profile="desktop-chrome")
+        first = await session.fetch("https://example.com")
+        second = await session.fetch("https://example.com/account")
+        print(first.final_url, second.final_url)
+
+
+asyncio.run(main())
+```
+
+## Headed Browser
+
+Use a persistent session when you want the browser to stay open.
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import SessionConfig, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        headless=False,
+        user_data_dir=Path(".webskrap/dev-session"),
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("dev", config=config)
+        page = await session.context.new_page()
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+
+        input("Press Enter to close browser...")
+
+
+asyncio.run(main())
+```
+
+Example for a headed Chrome session with a French desktop profile:
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import BrowserProfile, SessionConfig, Viewport, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        headless=False,
+        channel="chrome",
+        user_data_dir=Path(".webskrap/gmf"),
+        navigation_timeout_ms=90_000,
+        default_timeout_ms=90_000,
+        slow_mo_ms=50,
+        launch_args=[
+            "--start-maximized",
+            "--disable-blink-features=AutomationControlled",
+            "--no-first-run",
+            "--no-default-browser-check",
+        ],
+    )
+    profile = BrowserProfile(
+        name="fr-desktop",
+        viewport=Viewport(width=1440, height=900),
+        screen=Viewport(width=1440, height=900),
+        locale="fr-FR",
+        timezone_id="Europe/Paris",
+        navigator_languages=["fr-FR", "fr", "en-US", "en"],
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("gmf", config=config, profile=profile)
+        page = await session.context.new_page()
+        await page.goto("https://www.gmf.fr/habitation/devis", wait_until="domcontentloaded")
+
+        input("Press Enter to close browser...")
+
+
+asyncio.run(main())
+```
+
+## Custom Profile
+
+```python
+from webskrap import BrowserProfile, Viewport
+
+profile = BrowserProfile(
+    name="workstation",
+    viewport=Viewport(width=1440, height=900),
+    screen=Viewport(width=1440, height=900),
+    locale="en-US",
+    timezone_id="Europe/Paris",
+)
+```
+
+## Session Options
+
+```python
+from pathlib import Path
+
+from webskrap import ProxyConfig, ResourcePolicy, SessionConfig
+
+config = SessionConfig(
+    browser="chromium",
+    channel="chrome",
+    headless=False,
+    user_data_dir=Path(".webskrap/session"),
+    storage_state=None,
+    proxy=ProxyConfig(server="http://127.0.0.1:8080"),
+    resource_policy=ResourcePolicy.LITE,
+    ignore_https_errors=True,
+    java_script_enabled=True,
+    service_workers="allow",
+    timeout_ms=30_000,
+    navigation_timeout_ms=90_000,
+    default_timeout_ms=90_000,
+    slow_mo_ms=50,
+    launch_args=[
+        "--start-maximized",
+        "--disable-blink-features=AutomationControlled",
+        "--disable-dev-shm-usage",
+        "--no-first-run",
+        "--no-default-browser-check",
+    ],
+)
+```
+
+`resource_policy` values:
+
+- `ResourcePolicy.ALL`: allow all resources.
+- `ResourcePolicy.LITE`: block images, fonts, and media.
+- `ResourcePolicy.DOCUMENTS`: block images, fonts, media, and stylesheets.
+
+## Profile Options
+
+```python
+from webskrap import BrowserProfile, Viewport
+
+profile = BrowserProfile(
+    name="fr-desktop",
+    user_agent=None,
+    viewport=Viewport(width=1440, height=900),
+    screen=Viewport(width=1440, height=900),
+    locale="fr-FR",
+    timezone_id="Europe/Paris",
+    device_scale_factor=1,
+    is_mobile=False,
+    has_touch=False,
+    color_scheme="light",
+    reduced_motion="no-preference",
+    extra_http_headers={},
+    navigator_languages=["fr-FR", "fr", "en-US", "en"],
+    hardware_concurrency=8,
+    device_memory=8,
+    webgl_vendor="Google Inc. (Intel)",
+    webgl_renderer="ANGLE (Intel, Intel(R) Iris(TM) Plus Graphics, OpenGL 4.1)",
+)
+```
+
+## Stealth Options
+
+```python
+from webskrap import SessionConfig, StealthConfig
+
+config = SessionConfig(
+    stealth=StealthConfig(
+        enabled=True,
+        patch_webdriver=True,
+        patch_chrome_runtime=True,
+        patch_permissions=True,
+        patch_plugins=True,
+        patch_webgl=True,
+        patch_canvas=True,
+        patch_hardware=True,
+    )
+)
+```
+
+## CLI
+
+```bash
+webskrap profiles
+webskrap doctor
+webskrap fetch https://example.com --profile desktop-chrome
+webskrap fetch https://example.com --headed --screenshot example.png
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+ruff check .
+```

webskrap-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+webskrap/__init__.py,sha256=JSQts2TI_3G41T5avnltIK3GRttbSZ8YtfcCxC-zG6k,551
+webskrap/cli.py,sha256=TnKdDYQCE8zJgsPTXQGKwwXXwlzAl437o-dk9CLXplc,5187
+webskrap/client.py,sha256=wVcipoyX-EsMPD7tv08lDojS_nA1zTIczU_4RmyArII,8232
+webskrap/models.py,sha256=1Hhbzt_GJsv2SCWIK6msy04gmgrL-H-XnwC4wieng4M,7159
+webskrap/profiles.py,sha256=jRPqsSRf-bpTLsBVQ9w_Fb6UU7v3Gmgc0b3h2dRhUrg,2366
+webskrap/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+webskrap/stealth.py,sha256=btCeOFJTGIKTGZpUjp4zyD-bxmf3CQOSoX4o3qAMaVE,4082
+webskrap-0.1.0.dist-info/METADATA,sha256=W9iPOKgXCig4TAVNBVAQStw3Fc6dyjjy0EP_wAqI2HU,7260
+webskrap-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+webskrap-0.1.0.dist-info/entry_points.txt,sha256=nKdAJqYIMy8Ql7TH1MUER8IgiaUcBi7hqU8p4bhIU7A,46
+webskrap-0.1.0.dist-info/RECORD,,

webskrap-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

webskrap-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+webskrap = webskrap.cli:app