scrapefold 0.1.0__py3-none-any.whl

scrapefold/__init__.py

@@ -0,0 +1,109 @@
+"""scrapefold — unified interface for web scraping engines.
+
+Public API (v0.1 — scaffold; engines land in later PRs):
+
+    from scrapefold import scrape, crawl_site, ScrapeOptions, ScrapeResult, ScrapeEngine
+
+    res = await scrape("https://example.com")
+    res = await scrape(url, opts=ScrapeOptions(language="ru", stealth=True))
+    md_path = await crawl_site("https://docs.example.com", opts=ScrapeOptions(max_pages=50))
+"""
+
+from __future__ import annotations
+
+from scrapefold.crawler.result import CrawlResult
+from scrapefold.engines.base import (
+    EngineCapabilities,
+    EngineError,
+    RedirectScopeViolation,
+    ScrapeEngine,
+)
+from scrapefold.ladders import (
+    AllEnginesFailed,
+    BudgetExceeded,
+    Policy,
+    RaceStep,
+    SequentialStep,
+    SiteClass,
+    WalkBudget,
+    classify_url,
+    get_ladder,
+)
+from scrapefold.options import ScrapeOptions
+from scrapefold.pool import EnginePool
+from scrapefold.result import ScrapeResult
+from scrapefold.router import walk as _walk
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AllEnginesFailed",
+    "BudgetExceeded",
+    "CrawlResult",
+    "EngineCapabilities",
+    "EngineError",
+    "EnginePool",
+    "Policy",
+    "RaceStep",
+    "RedirectScopeViolation",
+    "ScrapeEngine",
+    "ScrapeOptions",
+    "ScrapeResult",
+    "SequentialStep",
+    "SiteClass",
+    "WalkBudget",
+    "__version__",
+    "classify_url",
+    "crawl_site",
+    "get_ladder",
+    "scrape",
+]
+
+
+async def scrape(
+    url: str,
+    opts: ScrapeOptions | None = None,
+    pool: EnginePool | None = None,
+) -> ScrapeResult:
+    """Single-URL scrape with engine auto-selection.
+
+    Walks the per-site-class ladder via ``scrapefold.router.walk``. Raises
+    ``AllEnginesFailed`` if no step in the ladder succeeds.
+
+    Pass a caller-owned ``pool`` to reuse engine instances across scrape calls
+    (e.g. during a ``crawl_site`` run). When ``None`` (default), an ephemeral
+    pool is created and closed for each call.
+    """
+    return await _walk(url, opts, pool=pool)
+
+
+async def crawl_site(
+    url: str,
+    opts: ScrapeOptions | None = None,
+    output: object | None = None,
+    **_unused: object,
+) -> CrawlResult:
+    """Whole-site crawl → single markdown file.
+
+    Discovers URLs from ``url`` (sitemap → robots → BFS), scrapes each
+    via ``scrape()``, and writes a stitched .md file at ``output``
+    (defaults to a unique temp path under ``/tmp``). Returns a
+    :class:`CrawlResult` with the per-URL pages, the stitched markdown
+    path, and per-URL failure strings.
+
+    Pass ``opts.extra["cache_dir"]`` to enable disk caching. The cache
+    TTL is read from ``opts.extra["cache_ttl_days"]`` (default: 7 days).
+    Set ``opts.skip_cache=True`` to bypass the cache for both reads and writes.
+
+    Unknown keyword arguments are accepted and silently ignored for
+    forward compatibility (logged at DEBUG).
+    """
+    import logging as _logging
+
+    if _unused:
+        _logging.getLogger(__name__).debug(
+            "crawl_site: ignoring unimplemented kwargs: %s", sorted(_unused.keys())
+        )
+    from scrapefold.crawler import crawl
+
+    return await crawl(url, opts=opts, output=output)  # type: ignore[arg-type]

scrapefold/_host_utils.py

@@ -0,0 +1,58 @@
+"""Host-comparison utilities shared by the crawler and the requests engine.
+
+Kept in a top-level private module to avoid a circular dependency between
+``scrapefold.crawler.sitemap`` (which needs host checking) and
+``scrapefold.engines.requests`` (which also needs host checking for redirect
+scope enforcement).
+"""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+import httpx
+import tldextract
+
+
+def _is_invalid_location_error(exc: httpx.RemoteProtocolError) -> bool:
+    """Return True if a RemoteProtocolError came from a malformed Location header.
+
+    httpx's message for this case is stable across versions tested:
+    'Invalid URL in location header: ...'. Any other RemoteProtocolError
+    (server disconnects, malformed response bodies) returns False.
+    """
+    return "Invalid URL in location header" in str(exc)
+
+
+def same_host(url: str, root: str, follow_subdomains: bool) -> bool:
+    """Return True if *url* is on the same host as *root*.
+
+    When *follow_subdomains* is ``False`` (the default), only an exact
+    host match (www/apex-normalised) passes.  When ``True``, all subdomains
+    of the same registered domain pass.
+
+    Returns ``False`` for non-HTTP/HTTPS URLs and for unparseable inputs.
+    """
+    try:
+        url_parsed = urlparse(url)
+        root_parsed = urlparse(root)
+    except Exception:
+        return False
+
+    if url_parsed.scheme not in ("http", "https"):
+        return False
+
+    if follow_subdomains:
+
+        def _reg_domain(u: str) -> str:
+            ext = tldextract.extract(u)
+            return f"{ext.domain}.{ext.suffix}" if ext.suffix else ext.domain
+
+        return _reg_domain(url) == _reg_domain(root)
+    else:
+        url_host = url_parsed.netloc.lower().removeprefix("www.")
+        root_host = root_parsed.netloc.lower().removeprefix("www.")
+        return url_host == root_host
+
+
+__all__ = ["_is_invalid_location_error", "same_host"]

scrapefold/cache.py

@@ -0,0 +1,197 @@
+"""Disk-backed TTL cache for ScrapeResult.
+
+Key derivation: sha256(url) + sha256(canonical-json(opts)). One file per
+key under ``<cache_dir>/<first-2-of-key>/<rest-of-key>.json``. TTL via
+file mtime. Atomic writes via ``os.replace``. Corrupt files are treated
+as misses and removed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import hashlib
+import json
+import logging
+import os
+import time
+from dataclasses import asdict, is_dataclass
+from pathlib import Path
+from typing import Any
+
+from scrapefold.options import ScrapeOptions
+from scrapefold.result import ScrapeResult
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_TTL_DAYS = 7
+
+
+_PRIMITIVES: tuple[type, ...] = (str, int, float, bool, type(None))
+
+
+def _canonicalize(obj: Any) -> Any:
+    """Recursively transform *obj* into a JSON-serializable, dict-key-sorted form.
+
+    Strict by design (Codex round-1 HIGH #3): unknown types raise
+    ``ValueError`` so the caller can bypass the cache with a logged
+    warning rather than producing a non-deterministic key via
+    ``default=str``.
+    """
+    if isinstance(obj, _PRIMITIVES):
+        return obj
+    if isinstance(obj, (list, tuple)):
+        return [_canonicalize(x) for x in obj]
+    if isinstance(obj, (set, frozenset)):
+        items = [_canonicalize(x) for x in obj]
+        try:
+            return sorted(items)
+        except TypeError as exc:
+            raise ValueError(f"set with mixed-comparable elements: {exc}") from exc
+    if isinstance(obj, dict):
+        out: dict[str, Any] = {}
+        for k in sorted(obj):
+            if not isinstance(k, str):
+                raise ValueError(f"non-string dict key: {k!r} ({type(k).__name__})")
+            out[k] = _canonicalize(obj[k])
+        return out
+    if is_dataclass(obj) and not isinstance(obj, type):
+        return _canonicalize(asdict(obj))
+    raise ValueError(f"opts contains non-canonicalizable type: {type(obj).__name__}")
+
+
+def _canonical_opts(opts: ScrapeOptions) -> str:
+    """Produce a stable JSON string for opts. Raises ``ValueError`` on failure."""
+    return json.dumps(_canonicalize(asdict(opts)), sort_keys=True)
+
+
+def make_key(url: str, opts: ScrapeOptions) -> str | None:
+    """Return the 64-char hex sha256 key for (url, opts), or ``None`` if opts
+    contains a non-canonicalizable value (cache must bypass).
+    """
+    try:
+        canonical = _canonical_opts(opts)
+    except ValueError as exc:
+        logger.warning(
+            "cache: opts not canonicalizable (%s); cache will bypass for url=%s",
+            exc,
+            url,
+        )
+        return None
+    h = hashlib.sha256()
+    h.update(url.encode("utf-8"))
+    h.update(b"\x00")
+    h.update(canonical.encode("utf-8"))
+    return h.hexdigest()
+
+
+def _read_text_sync(path: Path) -> str:
+    """Synchronous file read — intended for use via asyncio.to_thread."""
+    return path.read_text()
+
+
+def _stat_and_read_sync(path: Path) -> tuple[float, str] | None:
+    """Stat + read in one syscall pair. Returns ``None`` if the file is gone."""
+    try:
+        mtime = path.stat().st_mtime
+        return mtime, path.read_text()
+    except FileNotFoundError:
+        return None
+
+
+def _write_atomic_sync(tmp: Path, path: Path, content: str) -> None:
+    """Synchronous atomic write — intended for use via asyncio.to_thread."""
+    tmp.write_text(content)
+    os.replace(tmp, path)
+
+
+class Cache:
+    """sha256-keyed disk cache for ScrapeResult, TTL'd via file mtime."""
+
+    def __init__(self, dir: Path | str, ttl_days: int = _DEFAULT_TTL_DAYS) -> None:
+        self.dir = Path(dir)
+        self.ttl_days = ttl_days
+        self.dir.mkdir(parents=True, exist_ok=True)
+
+    def _path_for(self, key: str) -> Path:
+        # Sharded by first two hex chars to avoid one giant flat directory.
+        return self.dir / key[:2] / f"{key[2:]}.json"
+
+    async def get(self, key: str, opts: ScrapeOptions | None = None) -> ScrapeResult | None:
+        """Return cached result for *key*, or ``None`` on miss / expiry / skip.
+
+        Pass *opts* to honour ``opts.skip_cache=True`` — in that case this
+        method always returns ``None`` without touching the disk.
+        """
+        if opts is not None and opts.skip_cache:
+            return None
+
+        path = self._path_for(key)
+        try:
+            stat_read = await asyncio.to_thread(_stat_and_read_sync, path)
+        except OSError as exc:
+            logger.debug("cache: read failed %s (%s)", path, exc)
+            return None
+        if stat_read is None:
+            return None
+        mtime, raw = stat_read
+        if time.time() - mtime > self.ttl_days * 86400:
+            return None
+
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as exc:
+            logger.debug("cache: corrupt file %s (%s); removing", path, exc)
+            with contextlib.suppress(OSError):
+                path.unlink(missing_ok=True)
+            return None
+
+        try:
+            return _result_from_dict(data)
+        except (KeyError, TypeError) as exc:
+            logger.debug("cache: shape mismatch in %s (%s); removing", path, exc)
+            path.unlink(missing_ok=True)
+            return None
+
+    async def set(self, key: str, result: ScrapeResult, opts: ScrapeOptions | None = None) -> None:
+        """Store *result* under *key*.
+
+        Pass *opts* to honour ``opts.skip_cache=True`` — in that case this
+        method is a no-op and does **not** write to disk.
+        """
+        if opts is not None and opts.skip_cache:
+            return
+
+        if not isinstance(result, ScrapeResult):
+            raise TypeError(f"Cache.set expects ScrapeResult, got {type(result).__name__}")
+
+        path = self._path_for(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        content = json.dumps(_result_to_dict(result), default=str)
+        await asyncio.to_thread(_write_atomic_sync, tmp, path, content)
+
+
+def _result_to_dict(r: ScrapeResult) -> dict[str, Any]:
+    return asdict(r)
+
+
+def _result_from_dict(d: dict[str, Any]) -> ScrapeResult:
+    # Be strict about the required fields; tolerate missing optional ones
+    return ScrapeResult(
+        url=d["url"],
+        text=d["text"],
+        markdown=d["markdown"],
+        html=d.get("html"),
+        json=d.get("json"),
+        screenshot_b64=d.get("screenshot_b64"),
+        engine=d["engine"],
+        elapsed_ms=d["elapsed_ms"],
+        cost_usd=d.get("cost_usd", 0.0),
+        meta=d.get("meta", {}),
+        failures=d.get("failures", []),
+    )
+
+
+__all__ = ["Cache", "make_key"]

scrapefold/cli.py

@@ -0,0 +1,184 @@
+"""scrapefold — Typer CLI entry point.
+
+Four subcommands: scrape, crawl, list-engines, classify.
+--json flag everywhere; errors fatal with non-zero exit code.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+import typer
+
+import scrapefold
+from scrapefold import AllEnginesFailed, ScrapeOptions, classify_url
+from scrapefold.engines import list_engine_names
+from scrapefold.result import ScrapeResult
+
+app = typer.Typer(
+    help="scrapefold — unified web scraping CLI",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(scrapefold.__version__)
+        raise typer.Exit()
+
+
+@app.callback()
+def _root(
+    version: bool = typer.Option(
+        False,
+        "--version",
+        callback=_version_callback,
+        is_eager=True,
+        help="Print version and exit.",
+    ),
+) -> None:
+    pass
+
+
+def _engines_arg(engines: str | None) -> tuple[str, ...] | None:
+    if engines is None:
+        return None
+    return tuple(e.strip() for e in engines.split(",") if e.strip())
+
+
+# ---------------------------------------------------------------------------
+# scrape
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def scrape(
+    url: str = typer.Argument(..., help="URL to scrape."),
+    engines: str | None = typer.Option(
+        None,
+        "--engines",
+        help="Comma-separated engine override (e.g. 'jina,firecrawl').",
+    ),
+    json_out: bool = typer.Option(False, "--json", help="Emit JSON to stdout."),
+    output: Path | None = typer.Option(  # noqa: B008
+        None, "--output", help="Write markdown to PATH instead of stdout."
+    ),
+) -> None:
+    """Scrape a single URL and print the markdown (or full JSON with --json)."""
+    opts = ScrapeOptions(engines=_engines_arg(engines))
+
+    try:
+        result: ScrapeResult = asyncio.run(scrapefold.scrape(url, opts))
+    except AllEnginesFailed as exc:
+        typer.echo(f"all engines failed: {exc}", err=True)
+        raise typer.Exit(code=1) from exc
+
+    if json_out:
+        typer.echo(json.dumps(asdict(result), default=str))
+        return
+
+    if output is not None:
+        output.write_text(result.markdown)
+        return
+
+    typer.echo(result.markdown)
+
+
+# ---------------------------------------------------------------------------
+# crawl
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def crawl(
+    url: str = typer.Argument(..., help="Root URL to crawl."),
+    max_pages: int = typer.Option(100, "--max-pages", help="Maximum pages to fetch (default 100)."),
+    output: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--output",
+        help="Output .md file path for the stitched crawl result.",
+    ),
+    per_page_dir: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--per-page-dir",
+        help="Write each crawled page as <sha256(url)[:16]>.md into DIR.",
+    ),
+    json_out: bool = typer.Option(False, "--json", help="Emit JSON summary to stdout."),
+) -> None:
+    """Crawl a site → produce one stitched markdown file and/or per-page files."""
+    opts = ScrapeOptions(max_pages=max_pages)
+
+    try:
+        crawl_result: Any = asyncio.run(scrapefold.crawl_site(url, opts=opts, output=output))
+    except Exception as exc:
+        typer.echo(f"crawl failed for {url}: {exc}", err=True)
+        raise typer.Exit(code=1) from exc
+
+    # Write per-page files if requested
+    if per_page_dir is not None:
+        per_page_dir.mkdir(parents=True, exist_ok=True)
+        for page in crawl_result.pages:
+            slug = hashlib.sha256(page.url.encode()).hexdigest()[:16]
+            page_path = per_page_dir / f"{slug}.md"
+            page_path.write_text(page.markdown)
+            typer.echo(f"wrote {page_path} ({page.url})", err=True)
+
+    stitched_path: Path | None = getattr(crawl_result, "stitched_path", None)
+
+    if json_out:
+        typer.echo(json.dumps({"output": str(stitched_path) if stitched_path else None}))
+        return
+
+    if stitched_path is not None:
+        typer.echo(str(stitched_path))
+
+
+# ---------------------------------------------------------------------------
+# list-engines
+# ---------------------------------------------------------------------------
+
+
+@app.command("list-engines")
+def list_engines_cmd(
+    json_out: bool = typer.Option(False, "--json", help="Emit JSON list to stdout."),
+) -> None:
+    """Print every engine registered in the lazy registry."""
+    names = list_engine_names()
+    if json_out:
+        typer.echo(json.dumps(names))
+        return
+    for name in names:
+        typer.echo(name)
+
+
+# ---------------------------------------------------------------------------
+# classify
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def classify(
+    url: str = typer.Argument(..., help="URL to classify."),
+    json_out: bool = typer.Option(False, "--json", help="Emit JSON to stdout."),
+) -> None:
+    """Print the SiteClass scrapefold's router would assign to a URL."""
+    site_class = classify_url(url)
+    if json_out:
+        typer.echo(json.dumps({"url": url, "site_class": site_class}))
+        return
+    typer.echo(site_class)
+
+
+def main() -> None:
+    """Console-script entry point — referenced by pyproject `[project.scripts]`."""
+    app()
+
+
+if __name__ == "__main__":
+    main()