scrapefold 0.1.0a2__py3-none-any.whl

scrapefold/__init__.py

@@ -0,0 +1,67 @@
+"""scrapefold — unified interface for web scraping engines.
+
+Public API (v0.1 — scaffold; engines land in later PRs):
+
+    from scrapefold import scrape, crawl_site, ScrapeOptions, ScrapeResult, ScrapeEngine
+
+    res = await scrape("https://example.com")
+    res = await scrape(url, opts=ScrapeOptions(language="ru", stealth=True))
+    md_path = await crawl_site("https://docs.example.com", opts=ScrapeOptions(max_pages=50))
+"""
+
+from __future__ import annotations
+
+from scrapefold.engines.base import EngineCapabilities, EngineError, ScrapeEngine
+from scrapefold.ladders import (
+    AllEnginesFailed,
+    BudgetExceeded,
+    Policy,
+    RaceStep,
+    SequentialStep,
+    SiteClass,
+    WalkBudget,
+    classify_url,
+    get_ladder,
+)
+from scrapefold.options import ScrapeOptions
+from scrapefold.result import ScrapeResult
+from scrapefold.router import walk as _walk
+
+__version__ = "0.1.0a2"
+
+__all__ = [
+    "AllEnginesFailed",
+    "BudgetExceeded",
+    "EngineCapabilities",
+    "EngineError",
+    "Policy",
+    "RaceStep",
+    "ScrapeEngine",
+    "ScrapeOptions",
+    "ScrapeResult",
+    "SequentialStep",
+    "SiteClass",
+    "WalkBudget",
+    "__version__",
+    "classify_url",
+    "crawl_site",
+    "get_ladder",
+    "scrape",
+]
+
+
+async def scrape(url: str, opts: ScrapeOptions | None = None) -> ScrapeResult:
+    """Single-URL scrape with engine auto-selection.
+
+    Walks the per-site-class ladder via ``scrapefold.router.walk``. Raises
+    ``AllEnginesFailed`` if no step in the ladder succeeds.
+    """
+    return await _walk(url, opts)
+
+
+async def crawl_site(url: str, opts: ScrapeOptions | None = None, **kwargs: object) -> str:
+    """Whole-site crawl → single markdown file.
+
+    Scaffold stub — crawler module lands in S8.
+    """
+    raise NotImplementedError("crawl_site() lands in S8 (crawler module).")

scrapefold/cli.py

@@ -0,0 +1,67 @@
+"""scrapefold CLI — Typer entry point.
+
+Scaffold (S1) — subcommands are stubs. Real implementations land in S9.
+
+Usage examples (post-S9):
+
+    scrapefold scrape https://example.com --engine firecrawl --language ru
+    scrapefold crawl https://docs.example.com --max-pages 50 --output site.md
+    scrapefold list-engines
+    scrapefold inspect-opts firecrawl
+"""
+
+from __future__ import annotations
+
+import sys
+
+import typer
+
+app = typer.Typer(
+    name="scrapefold",
+    help="Unified web scraping CLI. Single URL or whole-site → markdown.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+@app.command()
+def scrape(url: str, engine: str = "auto") -> None:
+    """Scrape a single URL (stub — lands in S9)."""
+    typer.echo(f"[stub] would scrape {url} with engine={engine}", err=True)
+    sys.exit(2)
+
+
+@app.command()
+def crawl(url: str, max_pages: int = 50, output: str = "site.md") -> None:
+    """Crawl a whole site → markdown (stub — lands in S9)."""
+    typer.echo(f"[stub] would crawl {url} max_pages={max_pages} output={output}", err=True)
+    sys.exit(2)
+
+
+@app.command("list-engines")
+def list_engines_cmd() -> None:
+    """List registered engines and their availability (stub — lands in S9)."""
+    from scrapefold.engines import list_engine_names
+
+    names = list_engine_names()
+    if not names:
+        typer.echo("(no engines registered yet — scaffold only)")
+    else:
+        for name in names:
+            typer.echo(name)
+
+
+@app.command("inspect-opts")
+def inspect_opts(engine: str) -> None:
+    """Show which ScrapeOptions an engine supports (stub — lands in S9)."""
+    typer.echo(f"[stub] would inspect opts of {engine}", err=True)
+    sys.exit(2)
+
+
+def main() -> None:
+    """Console-script entry point for ``scrapefold``."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

scrapefold/detection.py

@@ -0,0 +1,206 @@
+"""Response-quality detection for the scrapefold router.
+
+Two public functions:
+
+* :func:`is_suspicious` — decides whether a :class:`~scrapefold.result.ScrapeResult`
+  looks blocked / empty and should trigger ladder escalation.
+* :func:`reclassify_from_response` — matches response signals against
+  :data:`~scrapefold.ladders.SIGNATURES` to determine whether the site should
+  be reclassified into a vendor anti-bot :data:`~scrapefold.ladders.SiteClass`.
+
+Both are pure sync functions; no I/O, no network calls.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+from scrapefold.ladders import SIGNATURES, Signature, SiteClass
+from scrapefold.result import ScrapeResult
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Default antibot phrases
+# ---------------------------------------------------------------------------
+
+DEFAULT_ANTIBOT_PHRASES: tuple[str, ...] = (
+    "Just a moment...",
+    "Verify you are human",
+    "Checking your browser",
+    "Access denied",
+    "Please enable JavaScript",
+    "cf-browser-verification",
+)
+
+# Pre-compiled patterns used by the HTML heuristics.
+_RE_NOSCRIPT = re.compile(r"<noscript[^>]*>.*?</noscript>", re.IGNORECASE | re.DOTALL)
+_RE_SCRIPT = re.compile(r"<script[^>]*>.*?</script>", re.IGNORECASE | re.DOTALL)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def is_suspicious(
+    result: ScrapeResult,
+    *,
+    min_text_chars: int = 200,
+    antibot_phrases: tuple[str, ...] = DEFAULT_ANTIBOT_PHRASES,
+) -> bool:
+    """Return True if the scrape result looks like an anti-bot block or empty page.
+
+    Heuristics applied (any one is sufficient):
+
+    1. **Short text + failure signal** — ``len(result.text) < min_text_chars`` AND
+       either: (a) ``result.text`` is empty / whitespace-only, OR (b)
+       ``meta["status_code"]`` indicates an HTTP error (4xx or 5xx).
+       Short but non-empty text with a 2xx status is NOT suspicious under this
+       rule alone — other rules (antibot phrases, noscript/script domination) can
+       still flag it.
+    2. **Anti-bot phrase** — any phrase in *antibot_phrases* appears (case-
+       insensitive) in ``result.text`` or ``result.html``.
+    3. **Noscript domination** — the ``<noscript>`` content exceeds 50 % of the
+       raw HTML length after stripping noscript tags.
+    4. **Script domination** — the ratio of non-script visible text to total HTML
+       is below 0.1 (script tags make up more than 90 % of the document).
+    5. **Error status + empty text** — ``meta["status_code"]`` is 403 or 503 AND
+       ``result.text`` is empty / whitespace-only.
+    """
+    text: str = result.text or ""
+    html: str | None = result.html
+
+    if len(text) < min_text_chars:
+        status_code = result.status_code
+        is_error_status = status_code is not None and status_code >= 400
+        is_empty = not text.strip()
+        if is_empty or is_error_status:
+            logger.debug(
+                "is_suspicious: short text (%d < %d chars) with %s",
+                len(text),
+                min_text_chars,
+                "empty text" if is_empty else f"status_code={status_code}",
+            )
+            return True
+
+    text_lower = text.lower()
+    html_lower = (html or "").lower()
+    phrases_lower = tuple(p.lower() for p in antibot_phrases)
+    for phrase, phrase_lower in zip(antibot_phrases, phrases_lower, strict=True):
+        if phrase_lower in text_lower or phrase_lower in html_lower:
+            logger.debug("is_suspicious: antibot phrase %r found", phrase)
+            return True
+
+    if html:
+        html_len = len(html)
+        # finditer over findall to avoid allocating the matched block strings —
+        # noscript/script blocks can be MB on script-heavy SPAs.
+        noscript_total = sum(m.end() - m.start() for m in _RE_NOSCRIPT.finditer(html))
+        if noscript_total and html_len - noscript_total < html_len * 0.5:
+            logger.debug(
+                "is_suspicious: noscript domination (noscript=%d, total=%d)",
+                noscript_total,
+                html_len,
+            )
+            return True
+
+        if html_len > 0:
+            script_total = sum(m.end() - m.start() for m in _RE_SCRIPT.finditer(html))
+            ratio = (html_len - script_total) / html_len
+            if ratio < 0.1:
+                logger.debug(
+                    "is_suspicious: script domination (ratio=%.3f, script=%d, total=%d)",
+                    ratio,
+                    script_total,
+                    html_len,
+                )
+                return True
+
+    status_code = result.status_code
+    if status_code in (403, 503) and not text.strip():
+        logger.debug("is_suspicious: status_code=%d with empty text", status_code)
+        return True
+
+    return False
+
+
+def reclassify_from_response(
+    *,
+    body: str | None = None,
+    cookies: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+    status_code: int | None = None,
+    signatures: tuple[Signature, ...] = SIGNATURES,
+) -> SiteClass | None:
+    """Match the response against *signatures*; return the target SiteClass if
+    any :class:`~scrapefold.ladders.Signature` meets its ``min_matches``
+    threshold, else ``None``.
+
+    Walk order matters — the first matching signature wins.  SIGNATURES is
+    ordered so that narrow vendor signatures (Datadome, PerimeterX, Akamai)
+    beat the broader Cloudflare one.
+
+    Matching rules per signature field:
+
+    * ``cookie_names`` — each cookie **name** present in *cookies* scores 1.
+    * ``header_names`` — each header name (compared lower-case) present in
+      *headers* scores 1.
+    * ``body_phrases_all`` — if non-empty, **all** phrases must appear in
+      *body* (case-insensitive); if they do, scores 1 collectively.
+    * ``body_phrases_any`` — if non-empty, at least one phrase must appear in
+      *body* (case-insensitive); scores 1.
+    * ``status_codes`` — if non-empty, *status_code* must be in the set; if it
+      is, scores 1.
+
+    If ``status_codes`` is non-empty and *status_code* is NOT in it, the
+    signature is skipped entirely (acts as a filter, not just a scorer).
+    """
+    body_lower = (body or "").lower()
+    cookies_norm: dict[str, str] = cookies or {}
+    # Normalise header names to lower-case once.
+    headers_lower: dict[str, str] = {k.lower(): v for k, v in (headers or {}).items()}
+
+    for sig in signatures:
+        # status_codes acts as a hard filter when non-empty.
+        if sig.status_codes and status_code not in sig.status_codes:
+            continue
+
+        # status_codes match (or absent) counts as 1 contribution toward score.
+        score = 1 if sig.status_codes else 0
+
+        for name in sig.cookie_names:
+            if name in cookies_norm:
+                score += 1
+
+        for name in sig.header_names:
+            if name.lower() in headers_lower:
+                score += 1
+
+        if sig.body_phrases_all and all(
+            phrase.lower() in body_lower for phrase in sig.body_phrases_all
+        ):
+            score += 1
+
+        if sig.body_phrases_any and any(
+            phrase.lower() in body_lower for phrase in sig.body_phrases_any
+        ):
+            score += 1
+
+        if score >= sig.min_matches:
+            logger.debug(
+                "reclassify_from_response: matched signature %r (score=%d)",
+                sig.target,
+                score,
+            )
+            return sig.target
+
+    return None
+
+
+__all__ = [
+    "DEFAULT_ANTIBOT_PHRASES",
+    "is_suspicious",
+    "reclassify_from_response",
+]

scrapefold/engines/__init__.py

@@ -0,0 +1,123 @@
+"""Engine registry.
+
+Engines are imported lazily so that an optional vendor SDK missing on the
+machine (e.g. ``selenium`` not installed) does not break import of
+``scrapefold`` itself. ``get_engine(name)`` returns a class on demand.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from scrapefold.engines.base import ScrapeEngine
+
+# Lazy registry: name -> import-and-return-class function.
+# Each lambda imports the engine module on first call so missing extras
+# only error out when that engine is actually requested.
+_REGISTRY: dict[str, Callable[[], type[ScrapeEngine]]] = {
+    "requests": lambda: (
+        __import__("scrapefold.engines.requests", fromlist=["RequestsEngine"]).RequestsEngine
+    ),
+    "firecrawl": lambda: (
+        __import__("scrapefold.engines.firecrawl", fromlist=["FirecrawlEngine"]).FirecrawlEngine
+    ),
+    "scrapingbee": lambda: (
+        __import__(
+            "scrapefold.engines.scrapingbee", fromlist=["ScrapingbeeEngine"]
+        ).ScrapingbeeEngine
+    ),
+    "scrapingdog": lambda: (
+        __import__(
+            "scrapefold.engines.scrapingdog", fromlist=["ScrapingdogEngine"]
+        ).ScrapingdogEngine
+    ),
+    "jina": lambda: __import__("scrapefold.engines.jina", fromlist=["JinaEngine"]).JinaEngine,
+    "apify_linkedin": lambda: (
+        __import__(
+            "scrapefold.engines.apify_linkedin", fromlist=["ApifyLinkedInEngine"]
+        ).ApifyLinkedInEngine
+    ),
+    "anysite": lambda: (
+        __import__("scrapefold.engines.anysite", fromlist=["AnySiteEngine"]).AnySiteEngine
+    ),
+    "outscraper": lambda: (
+        __import__("scrapefold.engines.outscraper", fromlist=["OutscraperEngine"]).OutscraperEngine
+    ),
+    "scrapling_stealth": lambda: (
+        __import__(
+            "scrapefold.engines.scrapling_stealth", fromlist=["ScraplingStealthEngine"]
+        ).ScraplingStealthEngine
+    ),
+    "scrapling_fast": lambda: (
+        __import__(
+            "scrapefold.engines.scrapling_fast", fromlist=["ScraplingFastEngine"]
+        ).ScraplingFastEngine
+    ),
+    "cloudflare": lambda: (
+        __import__("scrapefold.engines.cloudflare", fromlist=["CloudflareEngine"]).CloudflareEngine
+    ),
+    "crawl4ai": lambda: (
+        __import__("scrapefold.engines.crawl4ai", fromlist=["Crawl4AIEngine"]).Crawl4AIEngine
+    ),
+    "cloakbrowser": lambda: (
+        __import__(
+            "scrapefold.engines.cloakbrowser", fromlist=["CloakBrowserEngine"]
+        ).CloakBrowserEngine
+    ),
+    "selenium": lambda: (
+        __import__("scrapefold.engines.selenium", fromlist=["SeleniumEngine"]).SeleniumEngine
+    ),
+}
+
+
+# User-facing aliases for multi-mode engines so ``opts.engines=["scrapling"]``
+# resolves to the canonical ``scrapling_stealth``, while
+# ``WalkBudget.engines_tried`` stays keyed by unambiguous canonical names.
+ENGINE_ALIASES: dict[str, str] = {}
+
+
+def register(name: str, loader: Callable[[], type[ScrapeEngine]]) -> None:
+    _REGISTRY[name] = loader
+
+
+def register_alias(alias: str, canonical: str) -> None:
+    """Register ``alias`` as a user-facing name for canonical engine ``canonical``."""
+    ENGINE_ALIASES[alias] = canonical
+
+
+def resolve_alias(name: str) -> str:
+    """Return the canonical engine name for ``name``, or ``name`` if no alias."""
+    return ENGINE_ALIASES.get(name, name)
+
+
+def get_engine(name: str) -> type[ScrapeEngine]:
+    """Return the engine class for ``name`` (alias-resolved). Raises KeyError if unknown."""
+    canonical = resolve_alias(name)
+    try:
+        loader = _REGISTRY[canonical]
+    except KeyError as exc:
+        raise KeyError(
+            f"unknown engine: {name!r} (resolved to {canonical!r}). known: {sorted(_REGISTRY)}"
+        ) from exc
+    return loader()
+
+
+def list_engine_names() -> list[str]:
+    return sorted(_REGISTRY)
+
+
+# Register user-facing aliases for multi-mode engines.
+# "scrapling" resolves to "scrapling_stealth" (default / most capable mode).
+register_alias("scrapling", "scrapling_stealth")
+
+
+__all__ = [
+    "ENGINE_ALIASES",
+    "get_engine",
+    "list_engine_names",
+    "register",
+    "register_alias",
+    "resolve_alias",
+]

scrapefold/engines/anysite.py

@@ -0,0 +1,191 @@
+"""AnySiteEngine — paid REST scraping via api.anysite.com.
+
+AnySite specialises in protected targets: LinkedIn, Twitter/X, Instagram, etc.
+There is no official Python SDK. This engine is a pure REST adapter over
+``httpx.AsyncClient``.
+
+Pinned API contract (as of 2026-05):
+  Method   : POST
+  Endpoint : https://api.anysite.com/v1/scrape
+  Auth     : Authorization: Bearer <api_key>  (request header)
+  Body     : JSON — see ``_adapt()`` for full field mapping.
+  Response : {"data": {"html": "...", "markdown": "...",
+                        "screenshot_b64": null | "<b64>"},
+              "meta": {"status_code": <int>}}
+
+Target-site headers (Accept-Language, User-Agent, Cookie, custom_headers)
+are sent inside the request body under the ``headers`` and ``cookies`` keys
+rather than on the AnySite HTTP call itself. This mirrors how similar
+residential-proxy APIs separate "headers for AnySite" from "headers for the
+target site".
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+import httpx
+
+from scrapefold.engines.base import EngineCapabilities, ScrapeEngine
+from scrapefold.html_to_text import html_to_both, markdown_to_text
+from scrapefold.options import (
+    ScrapeOptions,
+    build_target_headers,
+    cookies_to_header,
+    strip_extra_prefix,
+)
+from scrapefold.result import ScrapeResult
+
+logger = logging.getLogger(__name__)
+
+_ENDPOINT = "https://api.anysite.com/v1/scrape"
+
+
+def _adapt(opts: ScrapeOptions, url: str) -> dict:
+    """Map unified ScrapeOptions to the AnySite POST JSON body.
+
+    Target-site headers (Accept-Language, User-Agent, custom_headers) are
+    collected via ``build_target_headers`` and placed in ``body["headers"]``
+    so AnySite can forward them to the destination site rather than
+    interpreting them on its own API call.
+
+    Cookies are serialised to a ``"Cookie: k=v; …"`` string and placed in
+    ``body["cookies"]``.
+
+    Extra keys prefixed ``anysite_`` are stripped of the prefix and merged
+    into the body at the top level (escape hatch for undocumented AnySite
+    params).
+    """
+    body: dict = {
+        "url": url,
+        "render_js": opts.render_js,
+        "wait_ms": opts.wait_ms,
+    }
+
+    if opts.country is not None:
+        body["country"] = opts.country
+
+    if opts.stealth:
+        body["stealth"] = opts.stealth
+
+    if opts.premium_proxy:
+        body["premium_proxy"] = opts.premium_proxy
+
+    if opts.take_screenshot:
+        body["take_screenshot"] = opts.take_screenshot
+
+    # Target-site headers: language, user-agent, custom_headers.
+    # build_target_headers also handles cookies-to-Cookie-header, but we send
+    # cookies separately via the body["cookies"] key, so skip that here.
+    target_headers = build_target_headers(opts, include_cookies=False)
+    if target_headers:
+        body["headers"] = target_headers
+
+    # Cookies become a serialised Cookie header value in the body.
+    cookie_str = cookies_to_header(opts.cookies)
+    if cookie_str:
+        body["cookies"] = cookie_str
+
+    # Forward anysite_* extras (prefix stripped) into the body.
+    extra_params = strip_extra_prefix(opts.extra, "anysite_")
+    body.update(extra_params)
+
+    return body
+
+
+class AnySiteEngine(ScrapeEngine):
+    """Scraping engine backed by the AnySite REST API.
+
+    API key is read from the constructor argument or the ``ANYSITE_API_KEY``
+    environment variable. ``is_available()`` returns ``False`` when neither is set.
+    """
+
+    NAME = "anysite"
+    CAPABILITIES = EngineCapabilities(
+        requires_api_key=True,
+        estimated_cost_usd=0.002,
+        billing_unit="call",
+        proxy_type="residential",
+        js_rendering=True,
+        stealth=True,
+        output_native_markdown=True,
+    )
+    SUPPORTED_OPTIONS = frozenset(
+        {
+            "language",
+            "country",
+            "render_js",
+            "wait_ms",
+            "stealth",
+            "premium_proxy",
+            "user_agent",
+            "custom_headers",
+            "cookies",
+            "output_format",
+            "take_screenshot",
+            "timeout_s",
+            "extra",
+        }
+    )
+
+    def __init__(self, api_key: str | None = None) -> None:
+        super().__init__(api_key or os.getenv("ANYSITE_API_KEY"))
+
+    async def _fetch(self, url: str, opts: ScrapeOptions) -> ScrapeResult:
+        """Fetch *url* via the AnySite API and return a ``ScrapeResult``."""
+        body = _adapt(opts, url)
+        headers = {
+            "Authorization": f"Bearer {self.api_key or ''}",
+            "Content-Type": "application/json",
+        }
+
+        async with httpx.AsyncClient(timeout=float(opts.timeout_s)) as client:
+            response = await client.post(_ENDPOINT, json=body, headers=headers)
+
+        # Surface upstream failures (401 bad key, 429 throttle, 5xx) — without
+        # this, the engine returned an empty ScrapeResult for any error and
+        # the router could not distinguish "blank page" from "API down".
+        response.raise_for_status()
+
+        payload = response.json()
+        data = payload.get("data", {})
+        meta_block = payload.get("meta", {})
+
+        raw_html: str | None = data.get("html") or None
+        raw_markdown: str | None = data.get("markdown") or None
+        screenshot_b64: str | None = data.get("screenshot_b64") or None
+
+        # Populate text and markdown from whichever form the engine returned.
+        # Golden rule: both slots must always be non-empty when scrape succeeds.
+        text_out: str
+        markdown_out: str
+        html_out: str | None = raw_html
+
+        if raw_html:
+            text_out, markdown_out = html_to_both(raw_html, base_url=url)
+        elif raw_markdown:
+            markdown_out = raw_markdown
+            text_out = markdown_to_text(raw_markdown)
+            html_out = None
+        else:
+            text_out = ""
+            markdown_out = ""
+            html_out = None
+
+        upstream_status = meta_block.get("status_code")
+
+        return ScrapeResult(
+            url=url,
+            text=text_out,
+            markdown=markdown_out,
+            html=html_out,
+            engine=self.NAME,
+            elapsed_ms=0,  # base class fills this in
+            cost_usd=self.CAPABILITIES.estimated_cost_usd,
+            screenshot_b64=screenshot_b64,
+            meta={"status_code": upstream_status},
+        )
+
+
+__all__ = ["AnySiteEngine"]