crawlix 0.1.0__py3-none-any.whl

crawlix/__init__.py

@@ -0,0 +1,28 @@
+from crawlix.browser import Browser, browse, fetch, get
+from crawlix.element import Element
+from crawlix.exceptions import (
+    BackendError,
+    CrawlixError,
+    JavaScriptError,
+    NavigationError,
+    NetworkError,
+    SelectorError,
+    TimeoutError,
+)
+from crawlix.page import Page
+
+__all__ = [
+    "Browser",
+    "Page",
+    "Element",
+    "get",
+    "fetch",
+    "browse",
+    "CrawlixError",
+    "BackendError",
+    "TimeoutError",
+    "NavigationError",
+    "SelectorError",
+    "NetworkError",
+    "JavaScriptError",
+]

crawlix/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

crawlix/async_api.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Any
+
+from crawlix.backends import _create_backend
+from crawlix.backends.protocol import Backend, PageData
+
+
+class AsyncBrowser:
+    def __init__(
+        self,
+        backend: str = "httpx",
+        headless: bool = True,
+        stealth: bool = True,
+        timeout: int = 30,
+        proxy: str | None = None,
+        locale: str = "en-US",
+        user_agent: str | None = None,
+    ):
+        self._backend: Backend = _create_backend(
+            name=backend,
+            headless=headless,
+            stealth=stealth,
+            timeout=timeout,
+            proxy=proxy,
+            user_agent=user_agent,
+            locale=locale,
+        )
+
+    @property
+    def backend_name(self) -> str:
+        return self._backend.name
+
+    async def open(self, url: str) -> AsyncPage:
+        data = await self._backend.open(url)
+        return AsyncPage(self._backend, data)
+
+    async def close(self) -> None:
+        await self._backend.close()
+
+    async def __aenter__(self) -> AsyncBrowser:
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
+        await self.close()
+
+
+class AsyncPage:
+    def __init__(self, backend, data: PageData | None = None):
+        self._backend = backend
+        self._data = data or PageData()
+
+    @property
+    def url(self) -> str:
+        return self._data.url
+
+    @property
+    def html(self) -> str:
+        return self._data.html
+
+    @property
+    def text(self) -> str:
+        return self._data.text
+
+    @property
+    def status(self) -> int:
+        return self._data.status
+
+    async def json(self) -> dict:
+        import json as _json
+        return _json.loads(self.text)
+
+
+async def aget(url: str, **kwargs: Any) -> AsyncPage:
+    async with AsyncBrowser(**kwargs) as b:
+        return await b.open(url)
+
+
+async def afetch(url: str, **kwargs: Any) -> str:
+    async with AsyncBrowser(**kwargs) as b:
+        page = await b.open(url)
+        return page.html

crawlix/backends/__init__.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crawlix.backends.protocol import Backend
+
+
+def detect_backend(
+    explicit: str | None = None,
+    headless: bool = True,
+    stealth: bool = True,
+    timeout: int = 30,
+    proxy: str | None = None,
+    user_agent: str | None = None,
+    locale: str = "en-US",
+) -> Backend:
+    if explicit is not None:
+        return _create_backend(
+            explicit, headless, stealth, timeout, proxy, user_agent, locale
+        )
+
+    for name in ("playwright", "selenium"):
+        try:
+            return _create_backend(
+                name, headless, stealth, timeout, proxy, user_agent, locale
+            )
+        except ImportError:
+            continue
+
+    from crawlix.backends.requests import RequestsBackend
+    return RequestsBackend(
+        headless=headless,
+        stealth=stealth,
+        timeout=timeout,
+        proxy=proxy,
+        user_agent=user_agent,
+        locale=locale,
+    )
+
+
+def _create_backend(
+    name: str,
+    headless: bool = True,
+    stealth: bool = True,
+    timeout: int = 30,
+    proxy: str | None = None,
+    user_agent: str | None = None,
+    locale: str = "en-US",
+) -> Backend:
+    if name == "requests":
+        from crawlix.backends.requests import RequestsBackend
+        return RequestsBackend(
+            headless=headless, stealth=stealth, timeout=timeout,
+            proxy=proxy, user_agent=user_agent, locale=locale,
+        )
+    elif name == "playwright":
+        from crawlix.backends.playwright import PlaywrightBackend
+        return PlaywrightBackend(
+            headless=headless, stealth=stealth, timeout=timeout,
+            proxy=proxy, user_agent=user_agent, locale=locale,
+        )
+    elif name == "selenium":
+        from crawlix.backends.selenium import SeleniumBackend
+        return SeleniumBackend(
+            headless=headless, stealth=stealth, timeout=timeout,
+            proxy=proxy, user_agent=user_agent, locale=locale,
+        )
+    elif name == "httpx":
+        from crawlix.backends.httpx import HttpxBackend
+        return HttpxBackend(
+            headless=headless, stealth=stealth, timeout=timeout,
+            proxy=proxy, user_agent=user_agent, locale=locale,
+        )
+    else:
+        from crawlix.exceptions import BackendError
+        raise BackendError(
+            f"Unknown backend: {name!r}. Available: requests, playwright, selenium, httpx"
+        )

crawlix/backends/httpx.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from typing import Any
+
+from crawlix.backends.protocol import Backend, ElementData, PageData
+
+
+class HttpxBackend(Backend):
+    def __init__(
+        self,
+        headless: bool = True,
+        stealth: bool = True,
+        timeout: int = 30,
+        proxy: str | None = None,
+        user_agent: str | None = None,
+        locale: str = "en-US",
+        **kwargs: Any,
+    ):
+        self._timeout = timeout
+        self._proxy = proxy
+        self._user_agent = user_agent
+        self._locale = locale
+        self._stealth = stealth
+        self._client = None
+
+    async def _get_client(self):
+        if self._client is None:
+            try:
+                import httpx
+            except ImportError as e:
+                raise ImportError(
+                    "httpx is required for async support.\n"
+                    "Install: pip install crawlix[async]"
+                ) from e
+
+            headers = {}
+            if self._stealth:
+                from crawlix.utils import stealth_headers
+                headers = stealth_headers(self._user_agent, self._locale)
+
+            client_kwargs: dict[str, Any] = {
+                "headers": headers,
+                "timeout": self._timeout,
+            }
+            if self._proxy:
+                client_kwargs["proxies"] = self._proxy
+            self._client = httpx.AsyncClient(**client_kwargs)
+        return self._client
+
+    @property
+    def name(self) -> str:
+        return "httpx"
+
+    @property
+    def supports_js(self) -> bool:
+        return False
+
+    async def open(self, url: str) -> PageData:
+        import httpx
+        from bs4 import BeautifulSoup
+
+        client = await self._get_client()
+        try:
+            resp = await client.get(url)
+        except httpx.RequestError as e:
+            from crawlix.exceptions import NavigationError
+            raise NavigationError(f"Failed to load {url}: {e}") from e
+
+        soup = BeautifulSoup(resp.text, "html.parser")
+        title_tag = soup.find("title")
+        title = title_tag.get_text(strip=True) if title_tag else ""
+        text = soup.get_text(separator=" ", strip=True)
+
+        return PageData(
+            url=str(resp.url),
+            html=resp.text,
+            text=text,
+            title=title,
+            status=resp.status_code,
+            headers=dict(resp.headers),
+        )
+
+    async def close(self) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    async def find(self, selector: str) -> ElementData | None:
+        return None
+
+    async def find_all(self, selector: str) -> list[ElementData]:
+        return []

crawlix/backends/playwright.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+from typing import Any
+
+from crawlix.backends.protocol import Backend, ElementData, PageData
+from crawlix.utils import random_user_agent
+
+
+class PlaywrightBackend(Backend):
+    def __init__(
+        self,
+        headless: bool = True,
+        stealth: bool = True,
+        timeout: int = 30,
+        proxy: str | None = None,
+        user_agent: str | None = None,
+        locale: str = "en-US",
+        **kwargs: Any,
+    ):
+        try:
+            from playwright.sync_api import sync_playwright
+        except ImportError as e:
+            raise ImportError(
+                "playwright is not installed.\n"
+                "Install: pip install crawlix[playwright]"
+            ) from e
+
+        self._headless = headless
+        self._stealth = stealth
+        self._timeout = timeout * 1000
+        self._proxy = proxy
+        self._user_agent = user_agent or random_user_agent()
+        self._locale = locale
+        self._pw = None
+        self._browser = None
+        self._context = None
+        self._page = None
+
+        self._pw = sync_playwright().start()
+        launch_kwargs: dict[str, Any] = {"headless": headless, "timeout": self._timeout}
+        if proxy:
+            launch_kwargs["proxy"] = {"server": proxy}
+        self._browser = self._pw.chromium.launch(**launch_kwargs)
+
+        context_kwargs: dict[str, Any] = {
+            "user_agent": self._user_agent,
+            "locale": locale,
+        }
+        if stealth:
+            context_kwargs.update(self._stealth_context())
+        self._context = self._browser.new_context(**context_kwargs)
+        self._page = self._context.new_page()
+
+    @staticmethod
+    def _stealth_context() -> dict:
+        return {
+            "viewport": {"width": 1920, "height": 1080},
+            "screen": {"width": 1920, "height": 1080},
+        }
+
+    @property
+    def name(self) -> str:
+        return "playwright"
+
+    @property
+    def supports_js(self) -> bool:
+        return True
+
+    def open(self, url: str) -> PageData:
+        self._page.goto(url, timeout=self._timeout)
+        return self._to_page_data()
+
+    def new_page(self) -> PageData:
+        self._page = self._context.new_page()
+        return PageData(url="about:blank")
+
+    def close(self) -> None:
+        try:
+            if self._pw:
+                self._pw.stop()
+        except Exception:
+            pass
+
+    def _to_page_data(self) -> PageData:
+        return PageData(
+            url=self._page.url,
+            html=self._page.content(),
+            text=self._page.inner_text("body") if self._page.query_selector("body") else "",
+            title=self._page.title(),
+            status=200,
+        )
+
+    def goto(self, url: str) -> None:
+        self._page.goto(url, timeout=self._timeout)
+
+    def reload(self) -> None:
+        self._page.reload()
+
+    def back(self) -> None:
+        self._page.go_back()
+
+    def forward(self) -> None:
+        self._page.go_forward()
+
+    def find(self, selector: str) -> ElementData | None:
+        el = self._page.query_selector(selector)
+        if el is None:
+            return None
+        return self._element_to_data(el)
+
+    def find_all(self, selector: str) -> list[ElementData]:
+        els = self._page.query_selector_all(selector)
+        return [self._element_to_data(el) for el in els]
+
+    def _element_to_data(self, el) -> ElementData:
+        try:
+            tag = el.evaluate("el => el.tagName.toLowerCase()")
+        except Exception:
+            tag = ""
+        return ElementData(
+            tag=tag,
+            text=el.inner_text() if el else "",
+            html=el.inner_html() if el else "",
+            outer_html=el.evaluate("el => el.outerHTML") if el else "",
+            attributes=el.evaluate(
+                "el => { const a = {}; for (const attr of el.attributes) a[attr.name] = attr.value; return a; }"
+            ) if el else {},
+            classes=el.evaluate("el => [...el.classList]") if el else [],
+            element_id=el.evaluate("el => el.id") if el else "",
+        )
+
+    def click(self, selector: str) -> None:
+        self._page.click(selector)
+
+    def double_click(self, selector: str) -> None:
+        self._page.dblclick(selector)
+
+    def right_click(self, selector: str) -> None:
+        self._page.click(selector, button="right")
+
+    def type(self, selector: str, text: str) -> None:
+        self._page.fill(selector, text)
+
+    def clear(self, selector: str) -> None:
+        self._page.fill(selector, "")
+
+    def submit(self, selector: str = "form") -> None:
+        el = self._page.query_selector(selector)
+        if el:
+            el.evaluate("el => el.submit()")
+
+    def select(self, selector: str, value: str) -> None:
+        self._page.select_option(selector, value)
+
+    def hover(self, selector: str) -> None:
+        self._page.hover(selector)
+
+    def focus(self, selector: str) -> None:
+        self._page.focus(selector)
+
+    def blur(self, selector: str) -> None:
+        self._page.evaluate(f"document.querySelector('{selector}')?.blur()")
+
+    def scroll_to(self, selector: str) -> None:
+        self._page.evaluate(f"document.querySelector('{selector}')?.scrollIntoView()")
+
+    def key(self, key: str) -> None:
+        self._page.keyboard.press(key)
+
+    def upload(self, selector: str, path: str) -> None:
+        self._page.set_input_files(selector, path)
+
+    def wait_for(self, selector: str, timeout: int = 10) -> None:
+        self._page.wait_for_selector(selector, timeout=timeout * 1000)
+
+    def wait_for_text(self, text: str, timeout: int = 10) -> None:
+        self._page.wait_for_function(
+            f'document.body.innerText.includes("{text}")',
+            timeout=timeout * 1000,
+        )
+
+    def wait_for_url(self, pattern: str, timeout: int = 10) -> None:
+        self._page.wait_for_url(pattern, timeout=timeout * 1000)
+
+    def wait_for_load(self, timeout: int = 30) -> None:
+        self._page.wait_for_load_state("load", timeout=timeout * 1000)
+
+    def wait_for_network_idle(self, timeout: int = 30) -> None:
+        self._page.wait_for_load_state("networkidle", timeout=timeout * 1000)
+
+    def sleep(self, seconds: float) -> None:
+        import time
+        time.sleep(seconds)
+
+    def evaluate(self, js_code: str) -> Any:
+        return self._page.evaluate(js_code)
+
+    def evaluate_on(self, selector: str, js: str) -> Any:
+        return self._page.evaluate(f"document.querySelector('{selector}')?.{js}")
+
+    def screenshot(self, path: str | None = None) -> bytes:
+        return self._page.screenshot(path=path)
+
+    def pdf(self, path: str | None = None) -> bytes:
+        return self._page.pdf(path=path)
+
+    def set_headers(self, headers: dict[str, str]) -> None:
+        self._context.set_extra_http_headers(headers)
+
+    def get_cookies(self) -> list[dict]:
+        return [
+            {"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "")}
+            for c in self._context.cookies()
+        ]
+
+    def set_cookies(self, cookies: list[dict]) -> None:
+        self._context.add_cookies(cookies)
+
+    def clear_cookies(self) -> None:
+        self._context.clear_cookies()
+
+    def intercept(self, pattern: str, handler) -> None:
+        self._page.route(pattern, handler)

crawlix/backends/protocol.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from crawlix.exceptions import BackendError
+
+
+@dataclass
+class PageData:
+    url: str = ""
+    html: str = ""
+    text: str = ""
+    title: str = ""
+    status: int = 200
+    headers: dict[str, str] = field(default_factory=dict)
+    cookies: list[dict[str, str]] = field(default_factory=list)
+
+
+@dataclass
+class ElementData:
+    tag: str = ""
+    text: str = ""
+    html: str = ""
+    outer_html: str = ""
+    attributes: dict[str, str] = field(default_factory=dict)
+    classes: list[str] = field(default_factory=list)
+    element_id: str = ""
+    parent_data: ElementData | None = None
+    children_data: list[ElementData] = field(default_factory=list)
+
+
+class Backend(ABC):
+
+    @property
+    @abstractmethod
+    def name(self) -> str: ...
+
+    @property
+    @abstractmethod
+    def supports_js(self) -> bool: ...
+
+    @abstractmethod
+    def open(self, url: str) -> PageData: ...
+
+    @abstractmethod
+    def close(self) -> None: ...
+
+    # Navigation
+    def goto(self, url: str) -> None:
+        raise BackendError(self._browser_only("goto"))
+
+    def reload(self) -> None:
+        raise BackendError(self._browser_only("reload"))
+
+    def back(self) -> None:
+        raise BackendError(self._browser_only("back"))
+
+    def forward(self) -> None:
+        raise BackendError(self._browser_only("forward"))
+
+    # Querying
+    @abstractmethod
+    def find(self, selector: str) -> ElementData | None: ...
+
+    @abstractmethod
+    def find_all(self, selector: str) -> list[ElementData]: ...
+
+    def find_text(self, text: str) -> ElementData | None:
+        matches = [el for el in self.find_all("*") if text in (el.text or "")]
+        if not matches:
+            return None
+        matches.sort(key=lambda el: len(el.text or ""))
+        return matches[0]
+
+    def xpath(self, expr: str) -> list[ElementData]:
+        raise BackendError(self._browser_only("xpath"))
+
+    # Interaction
+    def click(self, selector: str) -> None:
+        raise BackendError(self._browser_only("click"))
+
+    def double_click(self, selector: str) -> None:
+        raise BackendError(self._browser_only("double_click"))
+
+    def type(self, selector: str, text: str) -> None:
+        raise BackendError(self._browser_only("type"))
+
+    def clear(self, selector: str) -> None:
+        raise BackendError(self._browser_only("clear"))
+
+    def submit(self, selector: str = "form") -> None:
+        raise BackendError(self._browser_only("submit"))
+
+    def select(self, selector: str, value: str) -> None:
+        raise BackendError(self._browser_only("select"))
+
+    def hover(self, selector: str) -> None:
+        raise BackendError(self._browser_only("hover"))
+
+    def focus(self, selector: str) -> None:
+        raise BackendError(self._browser_only("focus"))
+
+    def blur(self, selector: str) -> None:
+        raise BackendError(self._browser_only("blur"))
+
+    def right_click(self, selector: str) -> None:
+        raise BackendError(self._browser_only("right_click"))
+
+    def scroll_to(self, selector: str) -> None:
+        raise BackendError(self._browser_only("scroll_to"))
+
+    def key(self, key: str) -> None:
+        raise BackendError(self._browser_only("key"))
+
+    def upload(self, selector: str, path: str) -> None:
+        raise BackendError(self._browser_only("upload"))
+
+    # Waiting
+    def wait_for(self, selector: str, timeout: int = 10) -> None:
+        raise BackendError(self._browser_only("wait_for"))
+
+    def wait_for_text(self, text: str, timeout: int = 10) -> None:
+        raise BackendError(self._browser_only("wait_for_text"))
+
+    def wait_for_url(self, pattern: str, timeout: int = 10) -> None:
+        raise BackendError(self._browser_only("wait_for_url"))
+
+    def wait_for_load(self, timeout: int = 30) -> None:
+        raise BackendError(self._browser_only("wait_for_load"))
+
+    def wait_for_network_idle(self, timeout: int = 30) -> None:
+        raise BackendError(self._browser_only("wait_for_network_idle"))
+
+    def sleep(self, seconds: float) -> None:
+        raise BackendError(self._browser_only("sleep"))
+
+    # JavaScript
+    def evaluate(self, js_code: str) -> Any:
+        raise BackendError(self._browser_only("evaluate"))
+
+    def evaluate_on(self, selector: str, js: str) -> Any:
+        raise BackendError(self._browser_only("evaluate_on"))
+
+    # Network
+    def set_headers(self, headers: dict[str, str]) -> None:
+        raise BackendError(self._browser_only("set_headers"))
+
+    def set_cookies(self, cookies: list[dict]) -> None:
+        raise BackendError(self._browser_only("set_cookies"))
+
+    def get_cookies(self) -> list[dict]:
+        raise BackendError(self._browser_only("get_cookies"))
+
+    def clear_cookies(self) -> None:
+        raise BackendError(self._browser_only("clear_cookies"))
+
+    def intercept(self, pattern: str, handler) -> None:
+        raise BackendError(self._browser_only("intercept"))
+
+    # Output
+    def screenshot(self, path: str | None = None) -> bytes:
+        raise BackendError(self._browser_only("screenshot"))
+
+    def pdf(self, path: str | None = None) -> bytes:
+        raise BackendError(self._browser_only("pdf"))
+
+    @staticmethod
+    def _browser_only(method: str) -> str:
+        return (
+            f"{method}() requires a browser backend.\n"
+            "Install one:\n"
+            "  pip install crawlix[playwright]   \u2190 recommended\n"
+            "  pip install crawlix[selenium]"
+        )