crawlerkit-core 0.1.0__py3-none-any.whl

crawlerkit/core/__init__.py

@@ -0,0 +1,6 @@
+from .base_crawler import BaseCrawler, RawResponse
+from .base_parser import BaseParser
+from .identity import Profile
+from .transport import Transport
+
+__all__ = ["BaseCrawler", "BaseParser", "RawResponse", "Transport", "Profile"]

crawlerkit/core/base_crawler.py

@@ -0,0 +1,126 @@
+"""BaseCrawler — the crawl stage. A new target fills one hook: flow()."""
+
+import random
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+import structlog
+from bs4 import BeautifulSoup
+
+from .captcha.base import CaptchaRegistry, Challenge, default_registry
+from .errors import BlockedError, PermanentError, TransientError
+from .identity import Profile, pick
+from .proxy import NullProxyProvider, ProxyProvider
+from .transport import Transport
+
+log = structlog.get_logger(__name__)
+
+
+@dataclass
+class RawResponse:
+    url: str
+    status: int
+    text: str
+    headers: dict = field(default_factory=dict)
+
+
+class BaseCrawler(ABC):
+    """Owns transport+identity+proxy+captcha; subclass implements only flow().
+
+    No business logic, no parsing here — crawl and return the raw response.
+    """
+
+    captcha_hint: Challenge | None = None  # known sitekey when the widget isn't inline
+
+    def __init__(
+        self,
+        *,
+        proxy_provider: ProxyProvider | None = None,
+        registry: CaptchaRegistry | None = None,
+        verify: bool = True,
+        profile: Profile | None = None,
+        client_cert: str | None = None,
+        max_attempts: int = 3,
+    ):
+        self._proxy_provider = proxy_provider or NullProxyProvider()
+        self._verify = verify
+        self._client_cert = client_cert
+        self._fixed_profile = profile
+        self.max_attempts = max_attempts
+        self.registry = registry or default_registry()
+        self._build_transport()
+
+    def _build_transport(self) -> None:
+        """(Re)create identity + proxy lease + transport — on init and on each rotation."""
+        self.profile = self._fixed_profile or pick()
+        self.proxy = self._proxy_provider.lease()
+        self.transport = Transport(
+            self.profile, self.proxy, verify=self._verify, client_cert=self._client_cert
+        )
+
+    def _rotate(self) -> None:
+        log.info("rotate_identity_proxy")
+        self._build_transport()
+
+    # --- helpers exposed to flow() ---
+    def get(self, url: str, **kw):
+        return self.transport.get(url, **kw)
+
+    def post(self, url: str, **kw):
+        return self.transport.post(url, **kw)
+
+    def solve_captcha(self, source) -> str | None:
+        """detect+solve; returns a token, None (no challenge), or raises UnsupportedCaptcha."""
+        solved = self.registry.solve(source, self.transport, hint=self.captcha_hint)
+        return solved.token if solved else None
+
+    def hidden_fields(self, html: str) -> dict:
+        """All hidden inputs of the form (JSF ViewState / WebForms __VIEWSTATE postback state)."""
+        try:
+            soup = BeautifulSoup(html, "lxml")
+        except Exception:  # noqa: BLE001
+            soup = BeautifulSoup(html, "html.parser")
+        form = soup.find("form") if soup else None
+        scope = form or soup
+        hidden: dict[str, str] = {}
+        if scope:
+            for inp in scope.find_all("input"):
+                name = inp.get("name")
+                if name and (inp.get("type") == "hidden" or "ViewState" in name or "VIEWSTATE" in name.upper()):
+                    hidden[name] = inp.get("value", "")
+        return hidden
+
+    # --- the only required hook ---
+    @abstractmethod
+    def flow(self, params: dict) -> RawResponse:
+        ...
+
+    def run(self, params: dict) -> RawResponse:
+        """Run flow() with retry + rotation. TransientError -> back off, retry (same identity);
+        BlockedError -> rotate identity+proxy, then retry; PermanentError -> fail fast."""
+        last: Exception | None = None
+        for attempt in range(1, self.max_attempts + 1):
+            try:
+                log.info("crawl_start", crawler=type(self).__name__, attempt=attempt)
+                raw = self.flow(params)
+                log.info("crawl_done", status=raw.status, bytes=len(raw.text))
+                return raw
+            except PermanentError:
+                raise
+            except BlockedError as e:
+                last = e
+                log.warning("blocked", attempt=attempt, error=str(e))
+                if attempt < self.max_attempts:
+                    self._rotate()
+                    self._backoff(attempt)
+            except TransientError as e:
+                last = e
+                log.warning("transient", attempt=attempt, error=str(e))
+                if attempt < self.max_attempts:
+                    self._backoff(attempt)
+        raise last or RuntimeError("crawl failed with no captured error")
+
+    @staticmethod
+    def _backoff(attempt: int, cap: float = 30.0) -> None:
+        time.sleep(min(2.0**attempt + random.uniform(0, 1), cap))

crawlerkit/core/base_parser.py

@@ -0,0 +1,77 @@
+"""BaseParser — the parse stage. A new target fills one hook: parse().
+
+Pure + item-local: no network beyond fetching static assets for the optional PDF, no
+cross-item state. Operates on the RawResponse the crawler returned (or a replayed one).
+"""
+
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+from urllib.parse import urlparse
+
+import structlog
+
+from .base_crawler import RawResponse
+
+log = structlog.get_logger(__name__)
+
+#: What ``parse()`` yields — your own model, a dataclass, a ``dict``, anything.
+#: crawlerkit-core stays dependency-free: it never dictates the output type.
+T = TypeVar("T")
+
+# Print fixups: hide leftover form inputs, landscape, fit wide tables.
+_PDF_FIXUP_CSS = """
+input { display: none !important; }
+@page { size: A4 landscape; margin: 1.2cm; }
+table { font-size: 9px; table-layout: fixed; width: 100%; }
+td, th { overflow-wrap: anywhere; }
+"""
+
+
+def render_pdf(html: str, base_url: str) -> bytes:
+    """HTML -> PDF (WeasyPrint, no browser). Fetches remote CSS over a verified, AIA-repaired
+    TLS connection (curl_cffi + crawlerkit.core.tls). No `requests`."""
+    from curl_cffi import requests as cffi
+    from weasyprint import CSS, HTML, default_url_fetcher
+
+    from . import tls
+
+    def fetcher(url: str, **kw):
+        if url.startswith(("http://", "https://")):
+            host = urlparse(url).hostname or ""
+            try:
+                r = cffi.get(url, verify=tls.build_ca_bundle(host), timeout=30, impersonate="chrome131")
+                ct = r.headers.get("content-type", "")
+                out = {"string": r.content, "redirected_url": str(r.url)}
+                mime = ct.split(";")[0].strip()
+                if mime:
+                    out["mime_type"] = mime
+                return out
+            except Exception as e:  # noqa: BLE001 — a missing asset must not kill the PDF
+                log.warning("pdf_asset_skipped", url=url, error=str(e))
+                return {"string": b"", "mime_type": "text/plain"}
+        return default_url_fetcher(url, **kw)
+
+    return HTML(string=html, base_url=base_url, url_fetcher=fetcher).write_pdf(
+        stylesheets=[CSS(string=_PDF_FIXUP_CSS)]
+    )
+
+
+class BaseParser(ABC, Generic[T]):
+    """Parse stage. Subclass with your own item type: ``class MyParser(BaseParser[MyModel])``
+    (or ``BaseParser[dict]``). ``parse()`` returns ``list[T]``; the type is yours, not the lib's."""
+
+    render_pdf_enabled: bool = True
+
+    @abstractmethod
+    def parse(self, raw: RawResponse) -> list[T]:
+        ...
+
+    def pdf(self, raw: RawResponse) -> bytes | None:
+        if not self.render_pdf_enabled:
+            return None
+        return render_pdf(raw.text, base_url=raw.url)
+
+    def run(self, raw: RawResponse) -> tuple[list[T], bytes | None]:
+        items = self.parse(raw)
+        log.info("parse_done", count=len(items))
+        return items, self.pdf(raw)

crawlerkit/core/captcha/__init__.py

@@ -0,0 +1,34 @@
+from .base import (
+    CaptchaRegistry,
+    CaptchaServiceError,
+    Challenge,
+    Solved,
+    UnsupportedCaptcha,
+    default_registry,
+)
+from .govbr import GovBrSolver
+from .llm_image import LlmImageSolver
+from .mcaptcha import McaptchaPowSolver, mcaptcha_hint
+from .token_adapters import HcaptchaSolver, RecaptchaV2Solver, RecaptchaV3Solver, TokenProvider
+from .turnstile import TurnstileSolver
+
+__all__ = [
+    "Challenge",
+    "Solved",
+    "UnsupportedCaptcha",
+    "CaptchaServiceError",
+    "CaptchaRegistry",
+    "default_registry",
+    # own solvers
+    "McaptchaPowSolver",
+    "mcaptcha_hint",
+    "LlmImageSolver",
+    # browserless stubs (TODO crack)
+    "TurnstileSolver",
+    "GovBrSolver",
+    # optional token-adapters (opt-in)
+    "TokenProvider",
+    "RecaptchaV2Solver",
+    "RecaptchaV3Solver",
+    "HcaptchaSolver",
+]

crawlerkit/core/captcha/base.py

@@ -0,0 +1,90 @@
+"""Captcha detection + a registry of our own solvers.
+
+Three outcomes when a source (HTML or response) is checked:
+  - no challenge        -> registry.solve returns None
+  - challenge + solver  -> Solved{token, expires_at}
+  - challenge, no solver -> raise UnsupportedCaptcha
+
+A solver produces a token; the backend (compute / LLM-image / JS-runtime) is its own business.
+Tokens are single-use and solved on submit (never pre-solved).
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, Protocol, runtime_checkable
+
+
+@dataclass
+class Challenge:
+    kind: str
+    params: dict = field(default_factory=dict)
+
+
+@dataclass
+class Solved:
+    token: str
+    expires_at: float | None = None  # absolute epoch seconds, from the challenge's own ttl
+
+
+class UnsupportedCaptcha(Exception):
+    def __init__(self, kind: str):
+        super().__init__(f"no solver registered for captcha kind: {kind}")
+        self.kind = kind
+
+
+class CaptchaServiceError(Exception):
+    """The captcha backend returned an unexpected/error response (often transient)."""
+
+
+@runtime_checkable
+class Solver(Protocol):
+    kind: str
+
+    @classmethod
+    def detect(cls, text: str) -> Optional[Challenge]:
+        ...
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        ...
+
+
+class CaptchaRegistry:
+    def __init__(self) -> None:
+        self._solvers: dict[str, Solver] = {}
+
+    def register(self, solver: Solver) -> "CaptchaRegistry":
+        self._solvers[solver.kind] = solver
+        return self
+
+    def detect(self, source) -> Optional[Challenge]:
+        text = source if isinstance(source, str) else getattr(source, "text", "") or ""
+        for solver in self._solvers.values():
+            ch = solver.detect(text)
+            if ch is not None:
+                return ch
+        return None
+
+    def solve(self, source, transport, *, hint: Optional[Challenge] = None) -> Optional[Solved]:
+        challenge = self.detect(source) or hint
+        if challenge is None:
+            return None
+        solver = self._solvers.get(challenge.kind)
+        if solver is None:
+            raise UnsupportedCaptcha(challenge.kind)
+        return solver.solve(challenge, transport)
+
+
+def default_registry() -> CaptchaRegistry:
+    """Registry with the built-in own-solvers: mCaptcha PoW (working) + gov.br/Turnstile
+    browserless stubs (detect works, solve raises NotImplementedError until cracked).
+    Optional token-adapters (reCAPTCHA/hCaptcha) and the LLM image solver are opt-in —
+    register them yourself when configured."""
+    from .govbr import GovBrSolver
+    from .mcaptcha import McaptchaPowSolver
+    from .turnstile import TurnstileSolver
+
+    return (
+        CaptchaRegistry()
+        .register(McaptchaPowSolver())
+        .register(TurnstileSolver())
+        .register(GovBrSolver())
+    )

crawlerkit/core/captcha/govbr.py

@@ -0,0 +1,40 @@
+"""gov.br (sso.acesso.gov.br) — BROWSERLESS solver scaffold.
+
+gov.br SSO is the fleet's most common gate (~79 repos) and is browser-only in atlas today.
+`detect()` works now; `solve()` is a TODO for a manual, browserless crack — fails loudly.
+"""
+
+import re
+
+from .base import Challenge, Solved
+
+_SIGNATURE = re.compile(r"sso\.acesso\.gov\.br|acesso\.gov\.br|\bgovbr\b", re.I)
+_SITEKEY_RE = re.compile(r'data-sitekey=["\']([0-9A-Za-z_-]{8,})["\']')
+
+
+class GovBrSolver:
+    kind = "govbr"
+
+    @classmethod
+    def detect(cls, text: str):
+        text = text or ""
+        if not _SIGNATURE.search(text):
+            return None
+        m = _SITEKEY_RE.search(text)  # gov.br embeds hCaptcha/reCAPTCHA
+        return Challenge(kind=cls.kind, params={"sitekey": m.group(1) if m else None})
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        # TODO(crawlerkit): implement the BROWSERLESS gov.br SSO authentication.
+        # gov.br (sso.acesso.gov.br) is JS-heavy and gated by a captcha (hCaptcha/reCAPTCHA) plus
+        # fingerprint checks. Browserless approach to fill in here:
+        #   1. Drive the SSO step sequence with the verified curl_cffi transport, carrying cookies
+        #      across redirects (login -> authorize -> callback).
+        #   2. Solve the embedded captcha via the registry (hCaptcha/reCAPTCHA token solver) OR a
+        #      JS-runtime crack of the gov.br challenge script (QuickJS/Node + DOM shim seeded from
+        #      the active Profile + proxy IP).
+        #   3. Complete the OAuth/SSO redirect; return Solved(token=<session cookie / SSO assertion>).
+        # Note: some gov.br services accept ICP-Brasil mutual-TLS client certs — see crawlerkit.core.tls.
+        raise NotImplementedError(
+            f"browserless gov.br solve is a TODO (params={challenge.params!r}) "
+            "— implement the SSO/JS-runtime crack"
+        )

crawlerkit/core/captcha/llm_image.py

@@ -0,0 +1,46 @@
+"""Own image-captcha solver: fetch the challenge image over the verified transport, classify with
+a pluggable vision LLM, return the answer/token. Provider-agnostic — inject a `classify` callable
+`(image_bytes, prompt) -> str`. Prompts ported from atlas's GPT solver.
+
+Image captchas are target-specific, so the crawler builds the Challenge with the image location
+(`params["image_url"]` or `params["image_bytes"]`); `detect()` returns None.
+"""
+
+from .base import CaptchaServiceError, Challenge, Solved
+
+OCR_PROMPT = (
+    "This image is a CAPTCHA. Read the characters exactly. Respond with ONLY the characters "
+    "(letters/digits), no spaces, no explanation."
+)
+# 3x3 / 4x4 grid-selection prompts (hCaptcha / reCAPTCHA) are available for grid challenges;
+# port the full set from atlas chatgpt_captcha_solver.py when wiring a grid flow.
+GRID_3X3_PROMPT = (
+    "A reference image sits above a 3x3 grid (tiles numbered 1-9, left-to-right, top-to-bottom). "
+    "Return the tile numbers that clearly and fully match the reference, separated by '/', e.g. '2/5/9'. "
+    "If none match, return 'none'. No other text."
+)
+
+
+class LlmImageSolver:
+    kind = "image"
+
+    def __init__(self, classify, *, prompt: str = OCR_PROMPT):
+        # classify: Callable[[bytes, str], str]
+        self._classify = classify
+        self._prompt = prompt
+
+    @classmethod
+    def detect(cls, text: str):
+        return None  # the crawler constructs the image Challenge explicitly
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        img = challenge.params.get("image_bytes")
+        if img is None:
+            url = challenge.params.get("image_url")
+            if not url:
+                raise CaptchaServiceError("LlmImageSolver needs params['image_url'] or ['image_bytes']")
+            img = transport.get(url, timeout=30).content
+        answer = (self._classify(img, challenge.params.get("prompt", self._prompt)) or "").strip()
+        if not answer:
+            raise CaptchaServiceError("vision LLM returned an empty answer")
+        return Solved(token=answer)

crawlerkit/core/captcha/mcaptcha.py

@@ -0,0 +1,142 @@
+"""mCaptcha proof-of-work solver (compute backend, no third-party service).
+
+Ported from the Detran POC. Byte layout confirmed empirically against a captured
+(salt, string, nonce) -> result oracle (guarded by self_test). No sitekey secret needed.
+"""
+
+import hashlib
+import re
+import struct
+import time
+
+from .base import CaptchaServiceError, Challenge, Solved
+
+U128_MAX = (1 << 128) - 1
+
+# Captured oracle (a real, self-consistent challenge) — proves the byte layout.
+_ORACLE_SALT = "8c7b6a5d4e3f2a1b0c9d8e7f6a5b4c3d2e1f0a9b8c7d6e5f4a3b2c1d0e9f8a7b"
+_ORACLE_STRING = "7KoiRWIqZk3qFy7C8Jt96E9KUSGfdbVL"
+_ORACLE_DIFFICULTY = 4_000_000
+_ORACLE_NONCE = 3_539_967
+_ORACLE_RESULT = 340282365527686933810834880601832247926
+
+# matches the mCaptcha widget iframe URL embedded in a page
+_WIDGET_RE = re.compile(r"https?://([\w.-]+)/widget\?sitekey=([\w-]+)")
+
+
+def mcaptcha_hint(host: str, sitekey: str) -> Challenge:
+    """Build a Challenge from a known sitekey when the widget isn't inline in the GET HTML."""
+    return Challenge(
+        kind="mcaptcha",
+        params={"host": host, "sitekey": sitekey, "api_base": f"https://{host}/api/v1/pow"},
+    )
+
+
+def _headers(challenge: Challenge) -> dict:
+    host = challenge.params["host"]
+    sitekey = challenge.params["sitekey"]
+    return {
+        "Content-Type": "application/json",
+        "Accept": "*/*",
+        "Origin": f"https://{host}",
+        "Referer": f"https://{host}/widget?sitekey={sitekey}",
+    }
+
+
+class McaptchaPowSolver:
+    kind = "mcaptcha"
+
+    # ---- detection ----
+    @classmethod
+    def detect(cls, text: str):
+        m = _WIDGET_RE.search(text or "")
+        if not m:
+            return None
+        return mcaptcha_hint(host=m.group(1), sitekey=m.group(2))
+
+    # ---- proof-of-work math ----
+    @staticmethod
+    def _prefix(salt: str, string: str):
+        h = hashlib.sha256()
+        h.update(salt.encode())
+        sb = string.encode()
+        h.update(struct.pack("<Q", len(sb)))  # bincode fixint LE u64 length prefix
+        h.update(sb)
+        return h
+
+    @classmethod
+    def compute_result(cls, salt: str, string: str, nonce: int) -> int:
+        h = cls._prefix(salt, string)
+        h.update(str(nonce).encode())  # nonce as decimal ASCII
+        return int.from_bytes(h.digest()[:16], "big")  # first 16 bytes, big-endian
+
+    @staticmethod
+    def threshold(difficulty: int) -> int:
+        return U128_MAX - U128_MAX // difficulty
+
+    @classmethod
+    def solve_pow(cls, salt, string, difficulty, max_iters=2_000_000_000, max_seconds=120):
+        thr = cls.threshold(difficulty)
+        base = cls._prefix(salt, string)
+        start = time.perf_counter()
+        nonce = 0
+        while nonce < max_iters:
+            nonce += 1
+            h = base.copy()
+            h.update(str(nonce).encode())
+            if int.from_bytes(h.digest()[:16], "big") >= thr:
+                return nonce, str(cls.compute_result(salt, string, nonce)), int(
+                    (time.perf_counter() - start) * 1000
+                )
+            if (nonce & 0x3FFFF) == 0 and (time.perf_counter() - start) > max_seconds:
+                raise TimeoutError(f"PoW unsolved in {max_seconds}s (difficulty {difficulty})")
+        raise RuntimeError("PoW unsolved within iteration cap")
+
+    @classmethod
+    def self_test(cls) -> None:
+        """Oracle gate: assert the layout reproduces the captured result. Instant."""
+        got = cls.compute_result(_ORACLE_SALT, _ORACLE_STRING, _ORACLE_NONCE)
+        if got != _ORACLE_RESULT:
+            raise AssertionError(f"mCaptcha layout self-test FAILED: {got} != {_ORACLE_RESULT}")
+
+    # ---- solve (config -> pow -> verify -> token) ----
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        # mCaptcha's actor backend is intermittently flaky ("Actor mailbox error") on both
+        # /config and /verify — retry the whole config -> pow -> verify a few times.
+        last: Exception | None = None
+        for attempt in range(3):
+            try:
+                return self._solve_once(challenge, transport)
+            except CaptchaServiceError as e:
+                last = e
+                time.sleep(1.0 * (attempt + 1))
+        raise last  # type: ignore[misc]
+
+    def _solve_once(self, challenge: Challenge, transport) -> Solved:
+        api = challenge.params["api_base"]
+        key = challenge.params["sitekey"]
+        headers = _headers(challenge)
+
+        cfg = transport.post(f"{api}/config", json={"key": key}, headers=headers).json()
+        if not (isinstance(cfg, dict) and "salt" in cfg):
+            raise CaptchaServiceError(f"mcaptcha /config returned no challenge: {cfg!r}")
+
+        nonce, result, elapsed_ms = self.solve_pow(
+            cfg["salt"], cfg["string"], cfg["difficulty_factor"]
+        )
+        verify = transport.post(
+            f"{api}/verify",
+            json={
+                "key": key,
+                "nonce": nonce,
+                "result": result,
+                "string": cfg["string"],
+                "time": elapsed_ms,
+                "worker_type": "wasm",
+            },
+            headers=headers,
+        ).json()
+        token = verify.get("token") if isinstance(verify, dict) else None
+        if not token:
+            raise CaptchaServiceError(f"mcaptcha /verify returned no token: {verify!r}")
+        return Solved(token=token, expires_at=None)

crawlerkit/core/captcha/token_adapters.py

@@ -0,0 +1,77 @@
+"""OPTIONAL token-captcha adapters: reCAPTCHA v2/v3 + hCaptcha via a third-party token provider.
+
+Browserless (POST site_key + url -> token; no DOM). These reintroduce a paid third party, so they
+are OPT-IN — NOT registered in `default_registry()`. Wire them only when configured:
+
+    reg.register(RecaptchaV2Solver(provider)).register(HcaptchaSolver(provider))
+
+`provider` is any object implementing `TokenProvider` (e.g. an adapter around atlas's
+AntiCaptcha/2Captcha solvers). `detect()` finds the sitekey; the crawler supplies params["url"]
+(the page URL the provider needs) via a hint.
+"""
+
+import re
+from typing import Protocol, runtime_checkable
+
+from .base import CaptchaServiceError, Challenge, Solved
+
+
+@runtime_checkable
+class TokenProvider(Protocol):
+    def solve_recaptcha_v2(self, site_key: str, url: str, **kw) -> str: ...
+    def solve_recaptcha_v3(self, site_key: str, url: str, **kw) -> str: ...
+    def solve_hcaptcha(self, site_key: str, url: str, **kw) -> str: ...
+
+
+def _sitekey(text: str, marker: str) -> str | None:
+    if marker not in (text or "").lower():
+        return None
+    m = re.search(r'(?:data-sitekey|sitekey|render)["\']?\s*[:=]\s*["\']?([0-9A-Za-z_-]{20,})', text or "")
+    return m.group(1) if m else None
+
+
+class _BaseTokenSolver:
+    kind = ""
+    _marker = ""
+
+    def __init__(self, provider: TokenProvider):
+        self._p = provider
+
+    @classmethod
+    def detect(cls, text: str):
+        sk = _sitekey(text, cls._marker)
+        return Challenge(kind=cls.kind, params={"sitekey": sk}) if sk else None
+
+    def _require(self, challenge: Challenge) -> tuple[str, str]:
+        sk = challenge.params.get("sitekey")
+        url = challenge.params.get("url")  # supplied by the crawler (provider needs the page URL)
+        if not sk or not url:
+            raise CaptchaServiceError(f"{self.kind} needs params['sitekey'] and ['url']")
+        return sk, url
+
+
+class RecaptchaV2Solver(_BaseTokenSolver):
+    kind = "recaptcha_v2"
+    _marker = "recaptcha"
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        sk, url = self._require(challenge)
+        return Solved(token=self._p.solve_recaptcha_v2(sk, url))
+
+
+class RecaptchaV3Solver(_BaseTokenSolver):
+    kind = "recaptcha_v3"
+    _marker = "recaptcha"
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        sk, url = self._require(challenge)
+        return Solved(token=self._p.solve_recaptcha_v3(sk, url, action=challenge.params.get("action")))
+
+
+class HcaptchaSolver(_BaseTokenSolver):
+    kind = "hcaptcha"
+    _marker = "hcaptcha"
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        sk, url = self._require(challenge)
+        return Solved(token=self._p.solve_hcaptcha(sk, url))

crawlerkit/core/captcha/turnstile.py

@@ -0,0 +1,40 @@
+"""Cloudflare Turnstile — BROWSERLESS solver scaffold.
+
+`detect()` works today (finds the widget + sitekey). `solve()` is a TODO for a manual,
+browserless crack — it fails loudly (NotImplementedError), never silently.
+"""
+
+import re
+
+from .base import Challenge, Solved
+
+_SIGNATURE = re.compile(r"challenges\.cloudflare\.com/turnstile|cf-turnstile|turnstile\.render", re.I)
+_SITEKEY_RE = re.compile(r'(?:data-sitekey|sitekey)["\']?\s*[:=]\s*["\']([0-9A-Za-z_-]{8,})["\']')
+
+
+class TurnstileSolver:
+    kind = "turnstile"
+
+    @classmethod
+    def detect(cls, text: str):
+        text = text or ""
+        if not _SIGNATURE.search(text):
+            return None
+        m = _SITEKEY_RE.search(text)
+        return Challenge(kind=cls.kind, params={"sitekey": m.group(1) if m else None})
+
+    def solve(self, challenge: Challenge, transport) -> Solved:
+        # TODO(crawlerkit): implement the BROWSERLESS Cloudflare Turnstile solve.
+        # Turnstile mints a `cf-turnstile-response` token by running obfuscated,
+        # fingerprint-bearing widget JS. Browserless approach to fill in here:
+        #   1. First try the PASSIVE path — a clean impersonated identity (the active Profile +
+        #      proxy IP) often receives a token with no interactive challenge. Attempt that first.
+        #   2. Otherwise fetch the widget bundle (turnstile/v0/api.js + the challenge for
+        #      params["sitekey"]) and execute its JS in a JS runtime (QuickJS via py-mini-racer,
+        #      or a Node subprocess) with a minimal DOM/navigator shim seeded from the active
+        #      Profile (UA, sec-ch-ua, screen, languages) and the leased proxy IP.
+        #   3. return Solved(token=<cf-turnstile-response>, expires_at=now+~300).
+        raise NotImplementedError(
+            f"browserless Turnstile solve is a TODO (params={challenge.params!r}) "
+            "— implement the passive/JS-runtime crack"
+        )

crawlerkit/core/cookies.py

@@ -0,0 +1,40 @@
+"""Optional cookie-jar persistence across crawls.
+
+curl_cffi keeps cookies within a Session automatically (a GET that sets JSESSIONID is reused by the
+following POST — no action needed). These helpers add OPTIONAL cross-run warming: dump the jar to
+disk after a crawl and reload it before the next. Best-effort; never raises on a malformed file.
+"""
+
+import json
+
+
+def save_cookies(transport, path: str) -> int:
+    """Dump the transport's current cookies to `path` (JSON). Returns the count saved."""
+    data = []
+    try:
+        for c in transport._session.cookies.jar:
+            data.append({"name": c.name, "value": c.value, "domain": c.domain, "path": c.path})
+    except Exception:  # noqa: BLE001 — cookie internals vary; best-effort
+        return 0
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f)
+    return len(data)
+
+
+def load_cookies(transport, path: str) -> int:
+    """Load cookies from `path` into the transport's session. Returns the count loaded (0 if absent)."""
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+    except (FileNotFoundError, ValueError):
+        return 0
+    n = 0
+    for c in data:
+        try:
+            transport._session.cookies.set(
+                c["name"], c["value"], domain=c.get("domain"), path=c.get("path", "/")
+            )
+            n += 1
+        except Exception:  # noqa: BLE001
+            continue
+    return n

crawlerkit/core/errors.py

@@ -0,0 +1,48 @@
+"""Crawl error taxonomy + a response block-detector.
+
+`BaseCrawler.run()` reacts by class:
+  - `TransientError`  -> back off, retry with the SAME identity (network blip / timeout / 5xx).
+  - `BlockedError`    -> rotate identity + proxy, then retry (anti-bot block: 403/429/challenge page).
+  - `PermanentError`  -> fail fast, no retry (bad input / unrecoverable).
+Anything else propagates unchanged.
+"""
+
+import re
+
+
+class CrawlerError(Exception):
+    """Base class for crawl-stage errors."""
+
+
+class TransientError(CrawlerError):
+    """A transient failure (network blip, timeout, 5xx) — retry unchanged."""
+
+
+class PermanentError(CrawlerError):
+    """An unrecoverable failure (bad input, hard 4xx) — do not retry."""
+
+
+class BlockedError(CrawlerError):
+    """An anti-bot block (403/429 or a challenge page) — rotate identity+proxy and retry."""
+
+
+# Common interstitial/anti-bot markers (Cloudflare, Akamai, Incapsula, generic).
+_BLOCK_MARKERS = re.compile(
+    r"just a moment|attention required|access denied|/cdn-cgi/challenge|"
+    r"cf-error-details|akamai|incapsula|request unsuccessful",
+    re.I,
+)
+
+
+def raise_for_block(response) -> None:
+    """Opt-in guard: raise `BlockedError`/`TransientError` if a response looks blocked.
+
+    Call from `flow()` after a request whose 200 you don't fully trust. Detects 403/429 and
+    common challenge-page markers (-> blocked) and 5xx (-> transient).
+    """
+    status = getattr(response, "status_code", 0) or 0
+    text = getattr(response, "text", "") or ""
+    if status in (403, 429) or _BLOCK_MARKERS.search(text[:4000]):
+        raise BlockedError(f"anti-bot block detected (status={status})")
+    if status >= 500:
+        raise TransientError(f"server error (status={status})")

crawlerkit/core/identity.py

@@ -0,0 +1,108 @@
+"""Coherent browser identity via browserforge + curl_cffi impersonate.
+
+curl_cffi's `impersonate` target owns the TLS/JA3 + HTTP2 fingerprint and MUST stay coherent with
+the User-Agent — a UA that disagrees with the JA3 is worse than no spoofing. browserforge supplies a
+realistic header SET + ORDER + locale; we pick the nearest supported impersonate target and SNAP the
+UA/sec-ch-ua Chrome version to it, so UA <-> JA3 never drift. Each `generate()` randomizes (rotation),
+and profiles are rotated together with the proxy.
+"""
+
+import re
+from dataclasses import dataclass, field
+
+# curl_cffi impersonate targets we support, ascending by Chrome major.
+_IMPERSONATE_BY_MAJOR: list[tuple[int, str]] = [
+    (120, "chrome120"),
+    (124, "chrome124"),
+    (131, "chrome131"),
+    (133, "chrome133a"),
+]
+DEFAULT_IMPERSONATE = "chrome131"
+
+
+def _impersonate_for_major(major: int) -> tuple[str, int]:
+    """Nearest supported (target, target_major) with target_major <= major; else the lowest."""
+    chosen_major, chosen_target = _IMPERSONATE_BY_MAJOR[0]
+    for mj, target in _IMPERSONATE_BY_MAJOR:
+        if mj <= major:
+            chosen_major, chosen_target = mj, target
+    return chosen_target, chosen_major
+
+
+def _snap_version(headers: dict, gen_major: int, target_major: int) -> None:
+    """Rewrite the UA + sec-ch-ua Chrome version from gen_major to target_major (in place)."""
+    if gen_major == target_major:
+        return
+    if ua := headers.get("User-Agent"):
+        headers["User-Agent"] = re.sub(r"Chrome/\d+", f"Chrome/{target_major}", ua)
+    if sch := headers.get("sec-ch-ua"):  # only the Chrome/Chromium brands carry the major
+        headers["sec-ch-ua"] = sch.replace(f'v="{gen_major}"', f'v="{target_major}"')
+
+
+@dataclass(frozen=True)
+class Profile:
+    impersonate: str
+    _headers: dict = field(default_factory=dict)
+
+    @property
+    def user_agent(self) -> str:
+        return self._headers.get("User-Agent", "")
+
+    def headers(self) -> dict:
+        return dict(self._headers)
+
+
+def _fallback_profile() -> Profile:
+    """Static coherent profile when browserforge is unavailable."""
+    return Profile(
+        impersonate=DEFAULT_IMPERSONATE,
+        _headers={
+            "User-Agent": (
+                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/131.0.0.0 Safari/537.36"
+            ),
+            "Accept": (
+                "text/html,application/xhtml+xml,application/xml;q=0.9,"
+                "image/avif,image/webp,*/*;q=0.8"
+            ),
+            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br, zstd",  # curl_cffi decodes br/zstd natively
+        },
+    )
+
+
+class ProfileGenerator:
+    """Generate coherent Profiles via browserforge, snapped to a curl_cffi impersonate target."""
+
+    def __init__(self, *, browser="chrome", os=("windows", "linux"), device="desktop", locale="pt-BR"):
+        self._hg = None
+        try:
+            from browserforge.headers import HeaderGenerator
+
+            self._hg = HeaderGenerator(browser=browser, os=os, device=device, locale=locale)
+        except Exception:  # noqa: BLE001 — browserforge optional; fall back to a static profile
+            self._hg = None
+
+    def generate(self) -> Profile:
+        if self._hg is None:
+            return _fallback_profile()
+        try:
+            h = dict(self._hg.generate())
+        except Exception:  # noqa: BLE001
+            return _fallback_profile()
+        m = re.search(r"Chrome/(\d+)", h.get("User-Agent", ""))
+        gen_major = int(m.group(1)) if m else 131
+        target, target_major = _impersonate_for_major(gen_major)
+        _snap_version(h, gen_major, target_major)
+        return Profile(impersonate=target, _headers=h)
+
+
+_DEFAULT_GEN: ProfileGenerator | None = None
+
+
+def pick(pool=None, index: int = 0) -> Profile:
+    """Return a freshly generated, coherent Profile (browserforge-randomized = rotation)."""
+    global _DEFAULT_GEN
+    if _DEFAULT_GEN is None:
+        _DEFAULT_GEN = ProfileGenerator()
+    return _DEFAULT_GEN.generate()

crawlerkit/core/proxy.py

@@ -0,0 +1,96 @@
+"""Proxy leasing.
+
+The leased egress is what the transport binds AND what any captcha solver uses — so a
+risk-scored token is minted from the same IP that will submit it. v1 ships Null + Static
+providers; BrightData / sticky-session / VPN-as-proxy come later.
+"""
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ProxyLease:
+    url: str | None  # e.g. "http://user:pass@host:port", or None for direct
+
+
+class ProxyProvider:
+    def lease(self, key: str | None = None) -> ProxyLease:
+        raise NotImplementedError
+
+    def release(self, lease: ProxyLease) -> None:  # noqa: B027 — optional hook
+        pass
+
+
+class NullProxyProvider(ProxyProvider):
+    """Direct egress (no proxy)."""
+
+    def lease(self, key: str | None = None) -> ProxyLease:
+        return ProxyLease(url=None)
+
+
+class StaticProxyProvider(ProxyProvider):
+    """Round-robin a fixed list (arg, or CRAWLERKIT_PROXIES env, comma-separated)."""
+
+    def __init__(self, proxies: list[str] | None = None):
+        self._proxies = proxies or [
+            p.strip() for p in os.environ.get("CRAWLERKIT_PROXIES", "").split(",") if p.strip()
+        ]
+        self._i = 0
+
+    def lease(self, key: str | None = None) -> ProxyLease:
+        if not self._proxies:
+            return ProxyLease(url=None)
+        url = self._proxies[self._i % len(self._proxies)]
+        self._i += 1
+        return ProxyLease(url=url)
+
+
+class BrightDataProxyProvider(ProxyProvider):
+    """BrightData (Luminati) sticky-session proxy. The session id (seeded from `key`, e.g. the
+    crawl item) pins a sticky egress IP so retries reuse it. Creds from args or env
+    BRIGHT_DATA_USER/PASS/HOST/PORT. NOTE: the exact username param syntax varies by zone/plan."""
+
+    def __init__(self, user=None, password=None, host=None, port=None, sticky=True):
+        self.user = user or os.environ.get("BRIGHT_DATA_USER", "")
+        self.password = password or os.environ.get("BRIGHT_DATA_PASS", "")
+        self.host = host or os.environ.get("BRIGHT_DATA_HOST", "")
+        self.port = port or os.environ.get("BRIGHT_DATA_PORT", "22225")
+        self.sticky = sticky
+        self._n = 0
+
+    def lease(self, key: str | None = None) -> ProxyLease:
+        if not (self.user and self.host):
+            return ProxyLease(url=None)
+        user = self.user
+        if self.sticky:
+            sid = key or f"s{self._n}"
+            self._n += 1
+            user = f"{self.user}-session-{sid}"
+        return ProxyLease(url=f"http://{user}:{self.password}@{self.host}:{self.port}")
+
+
+class DataImpulseProxyProvider(ProxyProvider):
+    """DataImpulse rotating/sticky proxy. Creds from args or env DATA_IMPULSE_USER/PASS/HOST/PORT;
+    optional country targeting. Sticky session id seeded from `key`."""
+
+    def __init__(self, user=None, password=None, host=None, port=None, country=None, sticky=True):
+        self.user = user or os.environ.get("DATA_IMPULSE_USER", "")
+        self.password = password or os.environ.get("DATA_IMPULSE_PASS", "")
+        self.host = host or os.environ.get("DATA_IMPULSE_HOST", "gw.dataimpulse.com")
+        self.port = port or os.environ.get("DATA_IMPULSE_PORT", "823")
+        self.country = country
+        self.sticky = sticky
+        self._n = 0
+
+    def lease(self, key: str | None = None) -> ProxyLease:
+        if not (self.user and self.host):
+            return ProxyLease(url=None)
+        user = self.user
+        if self.country:
+            user += f"__cr.{self.country}"
+        if self.sticky:
+            sid = key or f"s{self._n}"
+            self._n += 1
+            user += f";sess-{sid}"
+        return ProxyLease(url=f"http://{user}:{self.password}@{self.host}:{self.port}")

crawlerkit/core/tls.py

@@ -0,0 +1,113 @@
+"""Per-host CA bundle builder with AIA repair.
+
+Some hosts (e.g. Detran) serve only their leaf certificate and omit the intermediate, so
+Python TLS verification fails with "unable to get local issuer certificate". A browser papers
+over this by fetching the missing intermediate from the leaf's AIA "CA Issuers" URL; we do the
+same here, generically: read the leaf's AIA, fetch + follow intermediates up to a trusted root,
+and concatenate with certifi's roots. Verification stays ON. Cached per host.
+"""
+
+import os
+import socket
+import ssl
+import tempfile
+import urllib.request
+
+import certifi
+from cryptography import x509
+from cryptography.hazmat.primitives import serialization
+from cryptography.x509.oid import AuthorityInformationAccessOID, ExtensionOID
+
+_CACHE_DIR = os.environ.get(
+    "CRAWLERKIT_CA_DIR", os.path.join(tempfile.gettempdir(), "crawlerkit-ca")
+)
+
+
+def _leaf_cert(host: str, port: int = 443) -> x509.Certificate:
+    ctx = ssl._create_unverified_context()
+    with socket.create_connection((host, port), timeout=30) as sock:
+        with ctx.wrap_socket(sock, server_hostname=host) as ssock:
+            der = ssock.getpeercert(binary_form=True)
+    return x509.load_der_x509_certificate(der)
+
+
+def _ca_issuer_urls(cert: x509.Certificate) -> list[str]:
+    try:
+        aia = cert.extensions.get_extension_for_oid(
+            ExtensionOID.AUTHORITY_INFORMATION_ACCESS
+        ).value
+    except x509.ExtensionNotFound:
+        return []
+    return [
+        d.access_location.value
+        for d in aia
+        if d.access_method == AuthorityInformationAccessOID.CA_ISSUERS
+    ]
+
+
+def _fetch_cert(url: str) -> x509.Certificate:
+    with urllib.request.urlopen(url, timeout=30) as r:  # noqa: S310 (public CA cert, http ok)
+        raw = r.read()
+    if raw.lstrip().startswith(b"-----BEGIN"):
+        return x509.load_pem_x509_certificate(raw)
+    return x509.load_der_x509_certificate(raw)
+
+
+def build_ca_bundle(host: str, port: int = 443, *, force: bool = False, max_depth: int = 4) -> str:
+    """Return a path to a CA bundle = trusted roots + any intermediates `host` omits.
+
+    Best-effort: if AIA repair fails, falls back to certifi roots only.
+    """
+    os.makedirs(_CACHE_DIR, exist_ok=True)
+    path = os.path.join(_CACHE_DIR, f"{host}_{port}.pem")
+    if os.path.exists(path) and not force:
+        return path
+
+    roots = open(certifi.where(), encoding="utf-8").read()
+    extra: list[str] = []
+    try:
+        cert = _leaf_cert(host, port)
+        seen: set[str] = set()
+        for _ in range(max_depth):
+            urls = [u for u in _ca_issuer_urls(cert) if u.startswith(("http://", "https://"))]
+            if not urls or urls[0] in seen:
+                break
+            seen.add(urls[0])
+            cert = _fetch_cert(urls[0])
+            extra.append(cert.public_bytes(serialization.Encoding.PEM).decode())
+            if cert.issuer == cert.subject:  # reached a self-signed root
+                break
+    except Exception:  # noqa: BLE001 — never let CA discovery crash a crawl; roots-only fallback
+        pass
+
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(roots if roots.endswith("\n") else roots + "\n")
+        for pem in extra:
+            f.write(pem if pem.endswith("\n") else pem + "\n")
+    return path
+
+
+def client_cert_from_pfx(pfx_path: str, password: str | bytes | None, out_path: str | None = None) -> str:
+    """Load an ICP-Brasil / PKCS#12 `.pfx` and write a combined PEM (private key + cert + CA chain)
+    for curl_cffi's `cert=` (mutual TLS). Returns the PEM path. Port of alexandria/pfx_to_pem via
+    `cryptography` (no pyOpenSSL). The output is chmod 600 (contains the private key)."""
+    if isinstance(password, str):
+        password = password.encode()
+    with open(pfx_path, "rb") as f:
+        data = f.read()
+    key, cert, extra = serialization.pkcs12.load_key_and_certificates(data, password)
+    os.makedirs(_CACHE_DIR, exist_ok=True)
+    out_path = out_path or os.path.join(_CACHE_DIR, os.path.basename(pfx_path) + ".pem")
+    with open(out_path, "wb") as f:
+        if key is not None:
+            f.write(key.private_bytes(
+                serialization.Encoding.PEM,
+                serialization.PrivateFormat.TraditionalOpenSSL,
+                serialization.NoEncryption(),
+            ))
+        if cert is not None:
+            f.write(cert.public_bytes(serialization.Encoding.PEM))
+        for ca in (extra or []):
+            f.write(ca.public_bytes(serialization.Encoding.PEM))
+    os.chmod(out_path, 0o600)
+    return out_path

crawlerkit/core/transport.py

@@ -0,0 +1,76 @@
+"""Fingerprinted HTTP transport — the only HTTP path.
+
+A `curl_cffi` Session bound to one Profile (TLS/JA3 + UA + header order) + a proxy lease +
+per-host verified CA bundle (with AIA repair). TLS/JA3 fingerprint is a property of THIS client,
+so it is the foundation, not a plugin. `requests` is intentionally not used (giveaway fingerprint).
+"""
+
+import os
+import random
+import time
+from urllib.parse import urlparse
+
+import structlog
+from curl_cffi import requests as cffi
+from curl_cffi.requests import exceptions as _cffi_exc
+
+from . import tls
+from .errors import TransientError
+from .identity import Profile
+from .proxy import ProxyLease
+
+log = structlog.get_logger(__name__)
+
+
+class Transport:
+    def __init__(self, profile: Profile, proxy: ProxyLease, *, verify: bool = True,
+                 client_cert: str | None = None, min_interval: float | None = None):
+        self.profile = profile
+        self.proxy = proxy
+        self.verify = verify
+        self.client_cert = client_cert  # PEM (cert+key) for ICP-Brasil mutual TLS, or None
+        # politeness: minimum seconds between requests (+ up to 25% jitter). 0/None = off.
+        self.min_interval = float(
+            min_interval if min_interval is not None else os.environ.get("CRAWLERKIT_MIN_INTERVAL", 0)
+        )
+        self._last = 0.0
+        self._ca: dict[str, str] = {}
+        self._session = cffi.Session(impersonate=profile.impersonate)
+        self._session.headers.update(profile.headers())
+        if proxy.url:
+            self._session.proxies = {"http": proxy.url, "https": proxy.url}
+
+    def _verify_for(self, url: str):
+        if self.verify is False:
+            return False
+        host = urlparse(url).hostname or ""
+        if host not in self._ca:
+            self._ca[host] = tls.build_ca_bundle(host)
+        return self._ca[host]
+
+    def _throttle(self) -> None:
+        if self.min_interval <= 0:
+            return
+        wait = self.min_interval - (time.monotonic() - self._last)
+        if wait > 0:
+            time.sleep(wait + random.uniform(0, self.min_interval * 0.25))
+        self._last = time.monotonic()
+
+    def request(self, method: str, url: str, **kw):
+        kw.setdefault("verify", self._verify_for(url))
+        kw.setdefault("impersonate", self.profile.impersonate)
+        kw.setdefault("timeout", 30)
+        if self.client_cert:
+            kw.setdefault("cert", self.client_cert)
+        self._throttle()
+        log.debug("http", method=method, url=url, proxy=bool(self.proxy.url))
+        try:
+            return self._session.request(method, url, **kw)
+        except _cffi_exc.RequestsError as e:  # network/curl failure -> transient (retryable)
+            raise TransientError(f"{method} {url}: {e}") from e
+
+    def get(self, url: str, **kw):
+        return self.request("GET", url, **kw)
+
+    def post(self, url: str, **kw):
+        return self.request("POST", url, **kw)

crawlerkit_core-0.1.0.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: crawlerkit-core
+Version: 0.1.0
+Summary: Browserless crawler base: curl_cffi transport, TLS/AIA, identity, proxy, captcha, BaseCrawler/BaseParser.
+Author-email: Lucas Caovilla <lucasgrisac@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/lucascaovilla/crawlerkit
+Project-URL: Repository, https://github.com/lucascaovilla/crawlerkit
+Project-URL: Documentation, https://github.com/lucascaovilla/crawlerkit#readme
+Project-URL: Issues, https://github.com/lucascaovilla/crawlerkit/issues
+Keywords: crawler,scraping,curl_cffi,tls,fingerprint,captcha,browserless
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: curl_cffi>=0.7
+Requires-Dist: browserforge>=1.2
+Requires-Dist: cryptography>=42
+Requires-Dist: certifi>=2024.0
+Requires-Dist: selectolax>=0.3
+Requires-Dist: lxml>=5.0
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: structlog>=24.1
+Requires-Dist: tenacity>=8.2
+Requires-Dist: weasyprint>=60
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=5.0; extra == "dev"
+Requires-Dist: commitizen>=3.27; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.25; extra == "docs"
+Dynamic: license-file
+
+# crawlerkit-core
+
+[![PyPI version](https://img.shields.io/pypi/v/crawlerkit-core.svg)](https://pypi.org/project/crawlerkit-core/)
+[![Python versions](https://img.shields.io/pypi/pyversions/crawlerkit-core.svg)](https://pypi.org/project/crawlerkit-core/)
+[![CI](https://github.com/lucascaovilla/crawlerkit/actions/workflows/ci.yml/badge.svg)](https://github.com/lucascaovilla/crawlerkit/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+A **standalone, browserless** crawler base (`crawlerkit.core`): fingerprinted **curl_cffi** transport,
+per-host TLS with **AIA repair** + `.pfx` client certs, **browserforge** identity (UA snapped to the
+impersonate target), proxy providers, a pluggable **captcha** registry, an error taxonomy with
+retry+rotation, and the `BaseCrawler.flow()` / `BaseParser.parse()` hooks. Zero non-PyPI dependencies —
+`parse()` returns **your own type**, not one the library dictates.
+
+## Install
+
+```bash
+pip install crawlerkit-core
+```
+
+## Use
+
+```python
+from crawlerkit.core import BaseCrawler, BaseParser, RawResponse, Transport, Profile
+from crawlerkit.core.captcha import default_registry, McaptchaPowSolver, mcaptcha_hint
+from crawlerkit.core.proxy import StaticProxyProvider, BrightDataProxyProvider
+from crawlerkit.core.errors import BlockedError, TransientError, raise_for_block
+```
+
+**HTTP is curl_cffi only — `requests` is never used.** Deps: curl_cffi, browserforge, cryptography,
+certifi, selectolax, lxml, beautifulsoup4, weasyprint, structlog, tenacity.
+
+**Build a crawler:** [GETTING_STARTED.md](GETTING_STARTED.md). **Run the demos:**
+[`examples/`](examples/) (`quotes.py` — a full crawl+parse; `fingerprint_demo.py` — identity proof).
+Reference: [`docs/`](docs/) (identity, transport-tls, proxy, captcha, cracking-govbr-turnstile, errors,
+api). License: MIT.

crawlerkit_core-0.1.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+crawlerkit/core/__init__.py,sha256=aU6V1P8kGOOVFP6nWwgUSBacrJoucW1m3BAlIp7Mfps,230
+crawlerkit/core/base_crawler.py,sha256=xGcQL8kToY1ENl7Q9FFmzFRwxiVtWhuqiEycKcI3Yfc,4666
+crawlerkit/core/base_parser.py,sha256=6Heku1NZ6eTaNeMaFN9hRgLikcETZxIEeVT2oNvKIcw,2900
+crawlerkit/core/cookies.py,sha256=PndaPAjN29m5LpIq0jq2WSNuZugteu4vVmX5v8Inu3g,1449
+crawlerkit/core/errors.py,sha256=oogfj8t6I7SMEHOYirE6r4FRIMI9_TtUlLrJBpziL0A,1744
+crawlerkit/core/identity.py,sha256=2Ga0-GIf3mcnkwCZSPQuRXh1NtuvQhoQ3-DG91b08Ek,4108
+crawlerkit/core/proxy.py,sha256=FPfg45I8PiWI8s-I398_d3xiMJED68w7EokAHhUcqOk,3694
+crawlerkit/core/tls.py,sha256=0xXGsDbR25kdP2n3C_5Mw6r7yazorpgHHCkvy1QMgkM,4550
+crawlerkit/core/transport.py,sha256=i8QJsbY7TunCjvNrzHF44c3HvXd5QZq78dR2lxJG0ao,2913
+crawlerkit/core/captcha/__init__.py,sha256=h5WEzZQ7De_ESGQy-T_lABYyQyhtDx1pEjFJr3dpoHI,843
+crawlerkit/core/captcha/base.py,sha256=ZNwqyOGx2ibG6XUxKpDdyhkhz47S_CUB01WooWRqS2M,2830
+crawlerkit/core/captcha/govbr.py,sha256=4OjczX6K9lhzE_ojpkS0ngfZKqIejJ_j0SFfCMfu4j8,1905
+crawlerkit/core/captcha/llm_image.py,sha256=E3LnA_VlKpPXei2T4HsmaQlbo9cdOqigC-fFv1bSU4A,2081
+crawlerkit/core/captcha/mcaptcha.py,sha256=9fvRz1XgZnIArKPoZTgiVTqYazS0GitIFFxV8gyMyhI,5372
+crawlerkit/core/captcha/token_adapters.py,sha256=-GzRC8vMqNWINzT3tcRQln2lIDAC9L429W_eMKTTbyE,2747
+crawlerkit/core/captcha/turnstile.py,sha256=G4MeD01QlFv0faItUtbFnmMkrQcGplAiVyPJCQZmcPQ,1925
+crawlerkit_core-0.1.0.dist-info/licenses/LICENSE,sha256=kAsK6_g7uDe4qOBRw-suoniIN_7YxJsmXCZs-XoqHfE,1071
+crawlerkit_core-0.1.0.dist-info/METADATA,sha256=LvKXmYcJ-GhsNK1nRZ3bxY3l_i_RkcdTJWddYm_UuNs,3738
+crawlerkit_core-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+crawlerkit_core-0.1.0.dist-info/top_level.txt,sha256=vy8AhdTkmxRHsuY8cQY-yWv1bWpk7JkaAMvF3JuqinU,11
+crawlerkit_core-0.1.0.dist-info/RECORD,,

crawlerkit_core-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

crawlerkit_core-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Lucas Caovilla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlerkit_core-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+crawlerkit