scrapecore-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapecore/__init__.py +11 -0
- scrapecore/_async_client.py +161 -0
- scrapecore/_base.py +89 -0
- scrapecore/_client.py +168 -0
- scrapecore/_errors.py +99 -0
- scrapecore/_version.py +1 -0
- scrapecore/py.typed +0 -0
- scrapecore_sdk-0.1.0.dist-info/METADATA +83 -0
- scrapecore_sdk-0.1.0.dist-info/RECORD +10 -0
- scrapecore_sdk-0.1.0.dist-info/WHEEL +4 -0
scrapecore/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""ScrapeCore Python SDK: web scraping, site maps, crawls and screenshots via the ScrapeCore API v1."""
|
|
2
|
+
from ._async_client import AsyncScrapeCore
|
|
3
|
+
from ._client import ScrapeCore
|
|
4
|
+
from ._errors import (APIConnectionError, APIError, AuthenticationError, InsufficientCreditsError,
|
|
5
|
+
InvalidRequestError, JobFailedError, JobTimeoutError, NotFoundError,
|
|
6
|
+
PermissionDeniedError, RateLimitError, ScrapeCoreError)
|
|
7
|
+
from ._version import __version__
|
|
8
|
+
|
|
9
|
+
__all__ = ["ScrapeCore", "AsyncScrapeCore", "ScrapeCoreError", "AuthenticationError", "InsufficientCreditsError",
|
|
10
|
+
"PermissionDeniedError", "NotFoundError", "InvalidRequestError", "RateLimitError", "APIError",
|
|
11
|
+
"APIConnectionError", "JobFailedError", "JobTimeoutError", "__version__"]
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Asynchronous client (asyncio). Same methods as ScrapeCore, awaitable."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
import time
|
|
6
|
+
from typing import Any, AsyncIterator, Sequence
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
from . import _base, _errors
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AsyncScrapeCore:
|
|
14
|
+
"""Async ScrapeCore client: `async with AsyncScrapeCore() as sc: page = await sc.scrape(url)`."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, api_key: str | None = None, *, base_url: str | None = None, timeout: float = 120.0,
|
|
17
|
+
max_retries: int = 2, user_agent: str | None = None, http_client: httpx.AsyncClient | None = None):
|
|
18
|
+
self._s = _base.Settings(api_key, base_url, timeout, max_retries, user_agent)
|
|
19
|
+
self._http = http_client or httpx.AsyncClient(timeout=timeout)
|
|
20
|
+
self._own = http_client is None
|
|
21
|
+
|
|
22
|
+
# ── plumbing ──────────────────────────────────────────────────────────────────────────
|
|
23
|
+
async def close(self) -> None:
|
|
24
|
+
if self._own:
|
|
25
|
+
await self._http.aclose()
|
|
26
|
+
|
|
27
|
+
async def __aenter__(self) -> "AsyncScrapeCore":
|
|
28
|
+
return self
|
|
29
|
+
|
|
30
|
+
async def __aexit__(self, *exc) -> None:
|
|
31
|
+
await self.close()
|
|
32
|
+
|
|
33
|
+
async def request(self, method: str, path: str, *, json: Any = None, params: dict | None = None,
|
|
34
|
+
idempotent_key: str | None = None) -> Any:
|
|
35
|
+
"""Any /api/v1 endpoint. Retries what is safe to retry."""
|
|
36
|
+
headers = dict(self._s.headers)
|
|
37
|
+
if idempotent_key:
|
|
38
|
+
headers["Idempotency-Key"] = idempotent_key
|
|
39
|
+
attempt = 0
|
|
40
|
+
while True:
|
|
41
|
+
try:
|
|
42
|
+
r = await self._http.request(method, self._s.url(path), json=json, params=params, headers=headers,
|
|
43
|
+
timeout=self._s.timeout)
|
|
44
|
+
return _base.parse(r.status_code, r.headers, r.content)
|
|
45
|
+
except _errors.ScrapeCoreError as e:
|
|
46
|
+
err = e
|
|
47
|
+
except httpx.TimeoutException as e:
|
|
48
|
+
err = _errors.APIConnectionError(f"timed out: {e}", code="TIMEOUT", retryable=True)
|
|
49
|
+
except httpx.HTTPError as e:
|
|
50
|
+
err = _errors.APIConnectionError(f"could not reach the API: {e}", code="CONNECTION_ERROR",
|
|
51
|
+
retryable=True)
|
|
52
|
+
if not _base.should_retry(err, attempt, self._s.max_retries):
|
|
53
|
+
raise err
|
|
54
|
+
await asyncio.sleep(_base.backoff(attempt, err))
|
|
55
|
+
attempt += 1
|
|
56
|
+
|
|
57
|
+
# ── scraping ──────────────────────────────────────────────────────────────────────────
|
|
58
|
+
async def scrape(self, url: str, *, format: str = "html", js_render: bool | None = None, use_proxy: bool | None = None,
|
|
59
|
+
waf: bool | None = None, max_age: int | None = None) -> dict:
|
|
60
|
+
"""Scrape one page. format: "html", "markdown" or "text". max_age (seconds): a stored
|
|
61
|
+
result at most this old is fine (faster)."""
|
|
62
|
+
return await self.request("POST", "scrape", json=_base.scrape_body(url, format, js_render, use_proxy, waf, max_age))
|
|
63
|
+
|
|
64
|
+
async def markdown(self, url: str, **kw) -> str:
|
|
65
|
+
"""The page as Markdown (just the text)."""
|
|
66
|
+
return (await self.scrape(url, format="markdown", **kw)).get("markdown") or ""
|
|
67
|
+
|
|
68
|
+
async def batch(self, urls: Sequence[str], *, js_render: bool | None = None, use_proxy: bool | None = None,
|
|
69
|
+
waf: bool | None = None) -> dict:
|
|
70
|
+
"""Up to 50 URLs in one call (plan feature "batch")."""
|
|
71
|
+
return await self.request("POST", "batch", json=_base._clean({"urls": list(urls), "js_render": js_render,
|
|
72
|
+
"use_proxy": use_proxy, "waf": waf}))
|
|
73
|
+
|
|
74
|
+
async def screenshot(self, url: str, *, full_page: bool = False, wait: bool = True, timeout: float = 120, **options) -> dict:
|
|
75
|
+
"""A screenshot (base64 image in result["image_base64"]). Options: device, width, height,
|
|
76
|
+
format ("png"/"jpeg"), selector, wait_for, dark_mode, block_ads, hide_cookie_banners, ..."""
|
|
77
|
+
body = {"url": url, "full_page": full_page, "sync": False, **options}
|
|
78
|
+
job = await self.request("POST", "screenshot", json=body, idempotent_key=_base.idempotency_key())
|
|
79
|
+
if "job_id" not in job or job.get("status") != "queued":
|
|
80
|
+
return job # ran directly
|
|
81
|
+
return (await self.wait_for_job(job["job_id"], timeout=timeout))["result"] if wait else job
|
|
82
|
+
|
|
83
|
+
async def extract(self, url: str, selectors: dict, *, js_render: bool | None = None) -> dict:
|
|
84
|
+
"""CSS selectors -> values, e.g. {"title": "h1", "prices": ".price"}."""
|
|
85
|
+
return await self.request("POST", "extract", json=_base._clean({"url": url, "selectors": selectors,
|
|
86
|
+
"js_render": js_render, "sync": True}))
|
|
87
|
+
|
|
88
|
+
# ── site discovery and crawling ───────────────────────────────────────────────────────
|
|
89
|
+
async def map(self, url: str, *, limit: int | None = None, search: str | None = None,
|
|
90
|
+
include_subdomains: bool | None = None, respect_robots: bool | None = None) -> dict:
|
|
91
|
+
"""Every URL of a site (sitemaps + links). Returns {"urls": [...], "count": n, ...}."""
|
|
92
|
+
return await self.request("POST", "map", json=_base.map_body(url, limit, search, include_subdomains, respect_robots))
|
|
93
|
+
|
|
94
|
+
async def crawl(self, url: str, *, max_depth: int | None = None, limit: int | None = None,
|
|
95
|
+
include_paths: Sequence[str] | None = None, exclude_paths: Sequence[str] | None = None,
|
|
96
|
+
include_subdomains: bool | None = None, respect_robots: bool | None = None,
|
|
97
|
+
js_render: bool | None = None, use_proxy: bool | None = None,
|
|
98
|
+
wait: bool = False, timeout: float = 1800) -> dict:
|
|
99
|
+
"""Start a crawl job (returns {"job_id", ...}). wait=True: block until it ends and return
|
|
100
|
+
the finished job; read the pages with crawl_pages(job_id)."""
|
|
101
|
+
body = _base.crawl_body(url, max_depth, limit, list(include_paths) if include_paths else None,
|
|
102
|
+
list(exclude_paths) if exclude_paths else None, include_subdomains,
|
|
103
|
+
respect_robots, js_render, use_proxy)
|
|
104
|
+
job = await self.request("POST", "crawl", json=body, idempotent_key=_base.idempotency_key())
|
|
105
|
+
return await self.wait_for_job(job["job_id"], timeout=timeout, raise_on_failure=False) if wait else job
|
|
106
|
+
|
|
107
|
+
async def crawl_pages(self, job_id: str, *, only_ok: bool = False, page_size: int = 100) -> AsyncIterator[dict]:
|
|
108
|
+
"""Every page of a crawl (url, title, markdown, status_code, ...), fetched page by page."""
|
|
109
|
+
offset = 0
|
|
110
|
+
while True:
|
|
111
|
+
r = await self.request("GET", f"crawl/{job_id}/pages",
|
|
112
|
+
params={"offset": offset, "limit": page_size, "only_ok": str(only_ok).lower()})
|
|
113
|
+
pages = r.get("pages") or []
|
|
114
|
+
for p in pages:
|
|
115
|
+
yield p
|
|
116
|
+
offset += len(pages)
|
|
117
|
+
if not pages or offset >= int(r.get("total") or 0):
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
# ── jobs ──────────────────────────────────────────────────────────────────────────────
|
|
121
|
+
async def scrape_async(self, url: str, **kw) -> dict:
|
|
122
|
+
"""Queue a scrape; returns {"job_id", "status": "queued", ...}. See wait_for_job()."""
|
|
123
|
+
body = _base.scrape_body(url, kw.pop("format", None), kw.pop("js_render", None), kw.pop("use_proxy", None),
|
|
124
|
+
kw.pop("waf", None), None)
|
|
125
|
+
return await self.request("POST", "async", json=body, idempotent_key=_base.idempotency_key())
|
|
126
|
+
|
|
127
|
+
async def get_job(self, job_id: str) -> dict:
|
|
128
|
+
return await self.request("GET", f"job/{job_id}")
|
|
129
|
+
|
|
130
|
+
async def wait_for_job(self, job_id: str, *, timeout: float = 300, poll_interval: float = 1.0,
|
|
131
|
+
raise_on_failure: bool = True) -> dict:
|
|
132
|
+
"""Poll until the job is success / failure / cancelled. Raises JobFailedError (failure,
|
|
133
|
+
cancelled) unless raise_on_failure=False, and JobTimeoutError after `timeout` seconds."""
|
|
134
|
+
deadline = time.monotonic() + timeout
|
|
135
|
+
delay = poll_interval
|
|
136
|
+
while True:
|
|
137
|
+
job = await self.get_job(job_id)
|
|
138
|
+
if job.get("status") in _base.TERMINAL:
|
|
139
|
+
if raise_on_failure and job["status"] != "success":
|
|
140
|
+
raise _base.job_failed(job)
|
|
141
|
+
return job
|
|
142
|
+
if time.monotonic() + delay > deadline:
|
|
143
|
+
raise _errors.JobTimeoutError(f"job {job_id} still {job.get('status')} after {timeout}s", job,
|
|
144
|
+
code="JOB_TIMEOUT")
|
|
145
|
+
await asyncio.sleep(delay)
|
|
146
|
+
delay = min(delay * 1.5, 10.0)
|
|
147
|
+
|
|
148
|
+
async def cancel_job(self, job_id: str) -> dict:
|
|
149
|
+
"""Cancel a queued job, or stop a running crawl after its current page."""
|
|
150
|
+
return await self.request("POST", f"job/{job_id}/cancel")
|
|
151
|
+
|
|
152
|
+
async def jobs(self, *, limit: int = 50, offset: int = 0, status: str | None = None, type: str | None = None) -> list:
|
|
153
|
+
return await self.request("GET", "jobs", params=_base._clean({"limit": limit, "offset": offset,
|
|
154
|
+
"status": status, "type": type}))
|
|
155
|
+
|
|
156
|
+
# ── account ───────────────────────────────────────────────────────────────────────────
|
|
157
|
+
async def usage(self) -> dict:
|
|
158
|
+
return await self.request("GET", "usage")
|
|
159
|
+
|
|
160
|
+
async def rate_limit(self) -> dict:
|
|
161
|
+
return await self.request("GET", "rate-limit")
|
scrapecore/_base.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""What the sync and async clients share: settings, request building, retry policy, parsing."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import random
|
|
6
|
+
import uuid
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from . import _errors
|
|
10
|
+
from ._version import __version__
|
|
11
|
+
|
|
12
|
+
DEFAULT_BASE_URL = "https://api.scrapecore.com"
|
|
13
|
+
TERMINAL = ("success", "failure", "cancelled")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _clean(d: dict) -> dict:
|
|
17
|
+
return {k: v for k, v in d.items() if v is not None}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Settings:
|
|
21
|
+
def __init__(self, api_key: str | None, base_url: str | None, timeout: float, max_retries: int,
|
|
22
|
+
user_agent: str | None):
|
|
23
|
+
self.api_key = api_key or os.environ.get("SCRAPECORE_API_KEY")
|
|
24
|
+
if not self.api_key:
|
|
25
|
+
raise _errors.AuthenticationError(
|
|
26
|
+
"No API key: pass api_key=... or set SCRAPECORE_API_KEY (create one at "
|
|
27
|
+
"https://scrapecore.com/dashboard/api-keys)", code="INVALID_API_KEY")
|
|
28
|
+
self.base_url = (base_url or os.environ.get("SCRAPECORE_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
|
|
29
|
+
self.timeout = timeout
|
|
30
|
+
self.max_retries = max(0, int(max_retries))
|
|
31
|
+
ua = f"scrapecore-python/{__version__}"
|
|
32
|
+
self.headers = {"X-API-Key": self.api_key, "User-Agent": f"{user_agent} {ua}" if user_agent else ua,
|
|
33
|
+
"Accept": "application/json"}
|
|
34
|
+
|
|
35
|
+
def url(self, path: str) -> str:
|
|
36
|
+
return f"{self.base_url}/api/v1/{path.lstrip('/')}"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def backoff(attempt: int, err: _errors.ScrapeCoreError | None) -> float:
|
|
40
|
+
if err is not None and err.retry_after is not None:
|
|
41
|
+
return min(float(err.retry_after), 60.0)
|
|
42
|
+
return min(0.5 * (2 ** attempt), 8.0) * (0.75 + random.random() / 2)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def should_retry(err: _errors.ScrapeCoreError, attempt: int, max_retries: int) -> bool:
|
|
46
|
+
if attempt >= max_retries:
|
|
47
|
+
return False
|
|
48
|
+
if isinstance(err, _errors.APIConnectionError):
|
|
49
|
+
return True
|
|
50
|
+
# concurrency / per-minute / queue limits clear quickly; a daily limit does not
|
|
51
|
+
return err.retryable and err.code != "DAILY_LIMIT"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def parse(status: int, headers, content: bytes) -> Any:
|
|
55
|
+
import json
|
|
56
|
+
try:
|
|
57
|
+
body = json.loads(content) if content else None
|
|
58
|
+
except ValueError:
|
|
59
|
+
body = {"detail": content[:500].decode("utf-8", "replace")}
|
|
60
|
+
if status >= 400:
|
|
61
|
+
raise _errors.from_response(status, body, headers)
|
|
62
|
+
return body
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def idempotency_key() -> str:
|
|
66
|
+
return str(uuid.uuid4())
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# ── request bodies (one place, so both clients send the same thing) ─────────────────────────
|
|
70
|
+
def scrape_body(url, format, js_render, use_proxy, waf, max_age):
|
|
71
|
+
return _clean({"url": url, "format": format, "js_render": js_render, "use_proxy": use_proxy,
|
|
72
|
+
"waf": waf, "max_age": max_age})
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def map_body(url, limit, search, include_subdomains, respect_robots):
|
|
76
|
+
return _clean({"url": url, "limit": limit, "search": search, "include_subdomains": include_subdomains,
|
|
77
|
+
"respect_robots": respect_robots})
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def crawl_body(url, max_depth, limit, include_paths, exclude_paths, include_subdomains, respect_robots,
|
|
81
|
+
js_render, use_proxy):
|
|
82
|
+
return _clean({"url": url, "max_depth": max_depth, "limit": limit, "include_paths": include_paths,
|
|
83
|
+
"exclude_paths": exclude_paths, "include_subdomains": include_subdomains,
|
|
84
|
+
"respect_robots": respect_robots, "js_render": js_render, "use_proxy": use_proxy})
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def job_failed(job: dict) -> _errors.JobFailedError:
|
|
88
|
+
return _errors.JobFailedError(f"job {job.get('job_id')} ended as {job.get('status')}: {job.get('error') or ''}".strip(),
|
|
89
|
+
job, code="JOB_" + str(job.get("status", "failed")).upper())
|
scrapecore/_client.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Synchronous client."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
from typing import Any, Iterator, Sequence
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from . import _base, _errors
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ScrapeCore:
|
|
13
|
+
"""ScrapeCore API client.
|
|
14
|
+
|
|
15
|
+
from scrapecore import ScrapeCore
|
|
16
|
+
sc = ScrapeCore() # reads SCRAPECORE_API_KEY
|
|
17
|
+
page = sc.scrape("https://example.com", format="markdown")
|
|
18
|
+
print(page["markdown"])
|
|
19
|
+
|
|
20
|
+
Failed scrapes are not charged. Rate-limit and server errors are retried automatically
|
|
21
|
+
(max_retries, honouring Retry-After); every other error raises a ScrapeCoreError subclass.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(self, api_key: str | None = None, *, base_url: str | None = None, timeout: float = 120.0,
|
|
25
|
+
max_retries: int = 2, user_agent: str | None = None, http_client: httpx.Client | None = None):
|
|
26
|
+
self._s = _base.Settings(api_key, base_url, timeout, max_retries, user_agent)
|
|
27
|
+
self._http = http_client or httpx.Client(timeout=timeout)
|
|
28
|
+
self._own = http_client is None
|
|
29
|
+
|
|
30
|
+
# ── plumbing ──────────────────────────────────────────────────────────────────────────
|
|
31
|
+
def close(self) -> None:
|
|
32
|
+
if self._own:
|
|
33
|
+
self._http.close()
|
|
34
|
+
|
|
35
|
+
def __enter__(self) -> "ScrapeCore":
|
|
36
|
+
return self
|
|
37
|
+
|
|
38
|
+
def __exit__(self, *exc) -> None:
|
|
39
|
+
self.close()
|
|
40
|
+
|
|
41
|
+
def request(self, method: str, path: str, *, json: Any = None, params: dict | None = None,
|
|
42
|
+
idempotent_key: str | None = None) -> Any:
|
|
43
|
+
"""Any /api/v1 endpoint. Retries what is safe to retry."""
|
|
44
|
+
headers = dict(self._s.headers)
|
|
45
|
+
if idempotent_key:
|
|
46
|
+
headers["Idempotency-Key"] = idempotent_key
|
|
47
|
+
attempt = 0
|
|
48
|
+
while True:
|
|
49
|
+
try:
|
|
50
|
+
r = self._http.request(method, self._s.url(path), json=json, params=params, headers=headers,
|
|
51
|
+
timeout=self._s.timeout)
|
|
52
|
+
return _base.parse(r.status_code, r.headers, r.content)
|
|
53
|
+
except _errors.ScrapeCoreError as e:
|
|
54
|
+
err = e
|
|
55
|
+
except httpx.TimeoutException as e:
|
|
56
|
+
err = _errors.APIConnectionError(f"timed out: {e}", code="TIMEOUT", retryable=True)
|
|
57
|
+
except httpx.HTTPError as e:
|
|
58
|
+
err = _errors.APIConnectionError(f"could not reach the API: {e}", code="CONNECTION_ERROR",
|
|
59
|
+
retryable=True)
|
|
60
|
+
if not _base.should_retry(err, attempt, self._s.max_retries):
|
|
61
|
+
raise err
|
|
62
|
+
time.sleep(_base.backoff(attempt, err))
|
|
63
|
+
attempt += 1
|
|
64
|
+
|
|
65
|
+
# ── scraping ──────────────────────────────────────────────────────────────────────────
|
|
66
|
+
def scrape(self, url: str, *, format: str = "html", js_render: bool | None = None, use_proxy: bool | None = None,
|
|
67
|
+
waf: bool | None = None, max_age: int | None = None) -> dict:
|
|
68
|
+
"""Scrape one page. format: "html", "markdown" or "text". max_age (seconds): a stored
|
|
69
|
+
result at most this old is fine (faster)."""
|
|
70
|
+
return self.request("POST", "scrape", json=_base.scrape_body(url, format, js_render, use_proxy, waf, max_age))
|
|
71
|
+
|
|
72
|
+
def markdown(self, url: str, **kw) -> str:
|
|
73
|
+
"""The page as Markdown (just the text)."""
|
|
74
|
+
return self.scrape(url, format="markdown", **kw).get("markdown") or ""
|
|
75
|
+
|
|
76
|
+
def batch(self, urls: Sequence[str], *, js_render: bool | None = None, use_proxy: bool | None = None,
|
|
77
|
+
waf: bool | None = None) -> dict:
|
|
78
|
+
"""Up to 50 URLs in one call (plan feature "batch")."""
|
|
79
|
+
return self.request("POST", "batch", json=_base._clean({"urls": list(urls), "js_render": js_render,
|
|
80
|
+
"use_proxy": use_proxy, "waf": waf}))
|
|
81
|
+
|
|
82
|
+
def screenshot(self, url: str, *, full_page: bool = False, wait: bool = True, timeout: float = 120, **options) -> dict:
|
|
83
|
+
"""A screenshot (base64 image in result["image_base64"]). Options: device, width, height,
|
|
84
|
+
format ("png"/"jpeg"), selector, wait_for, dark_mode, block_ads, hide_cookie_banners, ..."""
|
|
85
|
+
body = {"url": url, "full_page": full_page, "sync": False, **options}
|
|
86
|
+
job = self.request("POST", "screenshot", json=body, idempotent_key=_base.idempotency_key())
|
|
87
|
+
if "job_id" not in job or job.get("status") != "queued":
|
|
88
|
+
return job # ran directly
|
|
89
|
+
return self.wait_for_job(job["job_id"], timeout=timeout)["result"] if wait else job
|
|
90
|
+
|
|
91
|
+
def extract(self, url: str, selectors: dict, *, js_render: bool | None = None) -> dict:
|
|
92
|
+
"""CSS selectors -> values, e.g. {"title": "h1", "prices": ".price"}."""
|
|
93
|
+
return self.request("POST", "extract", json=_base._clean({"url": url, "selectors": selectors,
|
|
94
|
+
"js_render": js_render, "sync": True}))
|
|
95
|
+
|
|
96
|
+
# ── site discovery and crawling ───────────────────────────────────────────────────────
|
|
97
|
+
def map(self, url: str, *, limit: int | None = None, search: str | None = None,
|
|
98
|
+
include_subdomains: bool | None = None, respect_robots: bool | None = None) -> dict:
|
|
99
|
+
"""Every URL of a site (sitemaps + links). Returns {"urls": [...], "count": n, ...}."""
|
|
100
|
+
return self.request("POST", "map", json=_base.map_body(url, limit, search, include_subdomains, respect_robots))
|
|
101
|
+
|
|
102
|
+
def crawl(self, url: str, *, max_depth: int | None = None, limit: int | None = None,
|
|
103
|
+
include_paths: Sequence[str] | None = None, exclude_paths: Sequence[str] | None = None,
|
|
104
|
+
include_subdomains: bool | None = None, respect_robots: bool | None = None,
|
|
105
|
+
js_render: bool | None = None, use_proxy: bool | None = None,
|
|
106
|
+
wait: bool = False, timeout: float = 1800) -> dict:
|
|
107
|
+
"""Start a crawl job (returns {"job_id", ...}). wait=True: block until it ends and return
|
|
108
|
+
the finished job; read the pages with crawl_pages(job_id)."""
|
|
109
|
+
body = _base.crawl_body(url, max_depth, limit, list(include_paths) if include_paths else None,
|
|
110
|
+
list(exclude_paths) if exclude_paths else None, include_subdomains,
|
|
111
|
+
respect_robots, js_render, use_proxy)
|
|
112
|
+
job = self.request("POST", "crawl", json=body, idempotent_key=_base.idempotency_key())
|
|
113
|
+
return self.wait_for_job(job["job_id"], timeout=timeout, raise_on_failure=False) if wait else job
|
|
114
|
+
|
|
115
|
+
def crawl_pages(self, job_id: str, *, only_ok: bool = False, page_size: int = 100) -> Iterator[dict]:
|
|
116
|
+
"""Every page of a crawl (url, title, markdown, status_code, ...), fetched page by page."""
|
|
117
|
+
offset = 0
|
|
118
|
+
while True:
|
|
119
|
+
r = self.request("GET", f"crawl/{job_id}/pages",
|
|
120
|
+
params={"offset": offset, "limit": page_size, "only_ok": str(only_ok).lower()})
|
|
121
|
+
pages = r.get("pages") or []
|
|
122
|
+
yield from pages
|
|
123
|
+
offset += len(pages)
|
|
124
|
+
if not pages or offset >= int(r.get("total") or 0):
|
|
125
|
+
return
|
|
126
|
+
|
|
127
|
+
# ── jobs ──────────────────────────────────────────────────────────────────────────────
|
|
128
|
+
def scrape_async(self, url: str, **kw) -> dict:
|
|
129
|
+
"""Queue a scrape; returns {"job_id", "status": "queued", ...}. See wait_for_job()."""
|
|
130
|
+
body = _base.scrape_body(url, kw.pop("format", None), kw.pop("js_render", None), kw.pop("use_proxy", None),
|
|
131
|
+
kw.pop("waf", None), None)
|
|
132
|
+
return self.request("POST", "async", json=body, idempotent_key=_base.idempotency_key())
|
|
133
|
+
|
|
134
|
+
def get_job(self, job_id: str) -> dict:
|
|
135
|
+
return self.request("GET", f"job/{job_id}")
|
|
136
|
+
|
|
137
|
+
def wait_for_job(self, job_id: str, *, timeout: float = 300, poll_interval: float = 1.0,
|
|
138
|
+
raise_on_failure: bool = True) -> dict:
|
|
139
|
+
"""Poll until the job is success / failure / cancelled. Raises JobFailedError (failure,
|
|
140
|
+
cancelled) unless raise_on_failure=False, and JobTimeoutError after `timeout` seconds."""
|
|
141
|
+
deadline = time.monotonic() + timeout
|
|
142
|
+
delay = poll_interval
|
|
143
|
+
while True:
|
|
144
|
+
job = self.get_job(job_id)
|
|
145
|
+
if job.get("status") in _base.TERMINAL:
|
|
146
|
+
if raise_on_failure and job["status"] != "success":
|
|
147
|
+
raise _base.job_failed(job)
|
|
148
|
+
return job
|
|
149
|
+
if time.monotonic() + delay > deadline:
|
|
150
|
+
raise _errors.JobTimeoutError(f"job {job_id} still {job.get('status')} after {timeout}s", job,
|
|
151
|
+
code="JOB_TIMEOUT")
|
|
152
|
+
time.sleep(delay)
|
|
153
|
+
delay = min(delay * 1.5, 10.0)
|
|
154
|
+
|
|
155
|
+
def cancel_job(self, job_id: str) -> dict:
|
|
156
|
+
"""Cancel a queued job, or stop a running crawl after its current page."""
|
|
157
|
+
return self.request("POST", f"job/{job_id}/cancel")
|
|
158
|
+
|
|
159
|
+
def jobs(self, *, limit: int = 50, offset: int = 0, status: str | None = None, type: str | None = None) -> list:
|
|
160
|
+
return self.request("GET", "jobs", params=_base._clean({"limit": limit, "offset": offset,
|
|
161
|
+
"status": status, "type": type}))
|
|
162
|
+
|
|
163
|
+
# ── account ───────────────────────────────────────────────────────────────────────────
|
|
164
|
+
def usage(self) -> dict:
|
|
165
|
+
return self.request("GET", "usage")
|
|
166
|
+
|
|
167
|
+
def rate_limit(self) -> dict:
|
|
168
|
+
return self.request("GET", "rate-limit")
|
scrapecore/_errors.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Errors raised by the client. Every API error carries the fields of the API's error envelope."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ScrapeCoreError(Exception):
|
|
8
|
+
"""Base class. `code` is stable (e.g. RATE_LIMITED); see https://scrapecore.com/docs/errors."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, message: str, *, code: str = "ERROR", status: int | None = None,
|
|
11
|
+
request_id: str | None = None, retryable: bool = False,
|
|
12
|
+
retry_after: float | None = None, details: dict | None = None, body: Any = None):
|
|
13
|
+
super().__init__(message)
|
|
14
|
+
self.message = message
|
|
15
|
+
self.code = code
|
|
16
|
+
self.status = status
|
|
17
|
+
self.request_id = request_id
|
|
18
|
+
self.retryable = retryable
|
|
19
|
+
self.retry_after = retry_after
|
|
20
|
+
self.details = details or {}
|
|
21
|
+
self.body = body
|
|
22
|
+
|
|
23
|
+
def __str__(self) -> str:
|
|
24
|
+
bits = [f"[{self.code}] {self.message}"]
|
|
25
|
+
if self.status:
|
|
26
|
+
bits.append(f"(HTTP {self.status})")
|
|
27
|
+
if self.request_id:
|
|
28
|
+
bits.append(f"request_id={self.request_id}")
|
|
29
|
+
return " ".join(bits)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class AuthenticationError(ScrapeCoreError):
|
|
33
|
+
"""The API key is missing, wrong or revoked (401)."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class InsufficientCreditsError(ScrapeCoreError):
|
|
37
|
+
"""Not enough credits left this month (402)."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class PermissionDeniedError(ScrapeCoreError):
|
|
41
|
+
"""The plan does not include this feature, or the account is suspended (403)."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class NotFoundError(ScrapeCoreError):
|
|
45
|
+
"""No such job / endpoint (404)."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class InvalidRequestError(ScrapeCoreError):
|
|
49
|
+
"""The request was rejected: a bad URL or parameter (400, 409, 413, 422)."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class RateLimitError(ScrapeCoreError):
|
|
53
|
+
"""Requests per minute/day, concurrency or queue limit reached (429). See `retry_after`."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class APIError(ScrapeCoreError):
|
|
57
|
+
"""The server or the target site failed (5xx). Usually worth retrying."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class APIConnectionError(ScrapeCoreError):
|
|
61
|
+
"""The API could not be reached (network error or timeout)."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class JobFailedError(ScrapeCoreError):
|
|
65
|
+
"""wait_for_job(): the job ended as failure or cancelled."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, message: str, job: dict, **kw):
|
|
68
|
+
super().__init__(message, **kw)
|
|
69
|
+
self.job = job
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class JobTimeoutError(ScrapeCoreError):
|
|
73
|
+
"""wait_for_job(): the job did not finish within the timeout (it keeps running)."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, message: str, job: dict, **kw):
|
|
76
|
+
super().__init__(message, **kw)
|
|
77
|
+
self.job = job
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_BY_STATUS = {401: AuthenticationError, 402: InsufficientCreditsError, 403: PermissionDeniedError,
|
|
81
|
+
404: NotFoundError, 429: RateLimitError}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def from_response(status: int, body: Any, headers) -> ScrapeCoreError:
|
|
85
|
+
err = body.get("error") if isinstance(body, dict) and isinstance(body.get("error"), dict) else {}
|
|
86
|
+
detail = body.get("detail") if isinstance(body, dict) else None
|
|
87
|
+
message = err.get("message") or (detail if isinstance(detail, str) else None) or f"HTTP {status}"
|
|
88
|
+
retry_after = err.get("retry_after_seconds")
|
|
89
|
+
if retry_after is None:
|
|
90
|
+
ra = headers.get("retry-after") if headers is not None else None
|
|
91
|
+
try:
|
|
92
|
+
retry_after = float(ra) if ra is not None else None
|
|
93
|
+
except ValueError:
|
|
94
|
+
retry_after = None
|
|
95
|
+
cls = _BY_STATUS.get(status) or (APIError if status >= 500 else InvalidRequestError)
|
|
96
|
+
return cls(message, code=err.get("code") or ("INTERNAL_ERROR" if status >= 500 else "INVALID_REQUEST"),
|
|
97
|
+
status=status, request_id=err.get("request_id") or (headers.get("x-request-id") if headers is not None else None),
|
|
98
|
+
retryable=bool(err.get("retryable", status in (408, 429) or status >= 500)),
|
|
99
|
+
retry_after=retry_after, details=err.get("details"), body=body)
|
scrapecore/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
scrapecore/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: scrapecore-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Official Python client for the ScrapeCore web scraping API
|
|
5
|
+
Project-URL: Homepage, https://scrapecore.com
|
|
6
|
+
Project-URL: Documentation, https://scrapecore.com/docs
|
|
7
|
+
Project-URL: Source, https://github.com/imrajeshkumar7492/scrapecore-integrations/tree/main/sdks/python
|
|
8
|
+
Project-URL: Issues, https://github.com/imrajeshkumar7492/scrapecore-integrations/issues
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Keywords: api,crawler,llm,markdown,rag,scraper,screenshot,web scraping
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: httpx<1,>=0.25
|
|
18
|
+
Provides-Extra: test
|
|
19
|
+
Requires-Dist: pytest-asyncio>=0.21; extra == 'test'
|
|
20
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# ScrapeCore Python SDK
|
|
24
|
+
|
|
25
|
+
The official Python client for the [ScrapeCore](https://scrapecore.com) API (v1). You can scrape
|
|
26
|
+
pages to HTML, Markdown or text, map a site, crawl it, take screenshots, and extract data with CSS
|
|
27
|
+
selectors.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install scrapecore-sdk
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from scrapecore import ScrapeCore
|
|
35
|
+
|
|
36
|
+
sc = ScrapeCore() # reads SCRAPECORE_API_KEY (or pass api_key="sk_live_...")
|
|
37
|
+
|
|
38
|
+
page = sc.scrape("https://example.com", format="markdown")
|
|
39
|
+
print(page["markdown"], page["credits_used"], page["credits_remaining"])
|
|
40
|
+
|
|
41
|
+
urls = sc.map("https://example.com", limit=500)["urls"]
|
|
42
|
+
|
|
43
|
+
job = sc.crawl("https://example.com/docs", max_depth=2, limit=100, include_paths=["^/docs"], wait=True)
|
|
44
|
+
for p in sc.crawl_pages(job["job_id"], only_ok=True):
|
|
45
|
+
print(p["url"], p["title"], len(p["markdown"]))
|
|
46
|
+
|
|
47
|
+
shot = sc.screenshot("https://example.com", full_page=True, device="mobile") # shot["image_base64"]
|
|
48
|
+
data = sc.extract("https://example.com", {"title": "h1", "links": "a"})
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Async:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from scrapecore import AsyncScrapeCore
|
|
55
|
+
|
|
56
|
+
async with AsyncScrapeCore() as sc:
|
|
57
|
+
page = await sc.scrape("https://example.com", format="markdown")
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Errors and retries
|
|
61
|
+
|
|
62
|
+
Every API error raises a subclass of `ScrapeCoreError`. Each error has these attributes: `code`
|
|
63
|
+
(stable, e.g. `RATE_LIMITED`), `status`, `message`, `request_id` (quote it to support),
|
|
64
|
+
`retryable` and `retry_after`.
|
|
65
|
+
|
|
66
|
+
| Exception | When |
|
|
67
|
+
|---|---|
|
|
68
|
+
| `AuthenticationError` | missing / wrong / revoked key (401) |
|
|
69
|
+
| `InsufficientCreditsError` | monthly credits used up (402) |
|
|
70
|
+
| `PermissionDeniedError` | feature not in your plan (403) |
|
|
71
|
+
| `InvalidRequestError` | bad URL or parameter (400/422) |
|
|
72
|
+
| `RateLimitError` | per-minute / daily / concurrency limit (429) |
|
|
73
|
+
| `APIError` | server or target-site failure (5xx) |
|
|
74
|
+
| `APIConnectionError` | network error or timeout |
|
|
75
|
+
|
|
76
|
+
The SDK automatically retries rate limits (except the daily limit), 5xx errors and network errors
|
|
77
|
+
(`max_retries=2`, honouring `Retry-After`). Job-creating calls send an `Idempotency-Key`, so a retry
|
|
78
|
+
never creates a second job. Failed scrapes are never charged.
|
|
79
|
+
|
|
80
|
+
## Responsible use
|
|
81
|
+
|
|
82
|
+
Use ScrapeCore only for content you are allowed to access. Crawls and maps respect `robots.txt` by
|
|
83
|
+
default. See the [acceptable use policy](https://scrapecore.com/acceptable-use).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
scrapecore/__init__.py,sha256=b4nlbg3lbev_TNvBUPFS-wpzWDquy4cvJG_Dp1ioOdY,787
|
|
2
|
+
scrapecore/_async_client.py,sha256=-uU_5QgLjfSjJzAvw_qAq-AZPzBNr3_f_9qRqO3M8kU,10156
|
|
3
|
+
scrapecore/_base.py,sha256=I4FgM3YdVfZdRvzPUOq6eJfzyrYVg8bWSfhn2qB68Ig,3724
|
|
4
|
+
scrapecore/_client.py,sha256=idBBizuxer3Nx_oJ5W4uSACRUml26bxuTBW5ydHyDoE,10121
|
|
5
|
+
scrapecore/_errors.py,sha256=2tGxDtzMgtnpxzHH7v_nkPtCBhszqcUxnlD-Yhn6cRo,3782
|
|
6
|
+
scrapecore/_version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
|
|
7
|
+
scrapecore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
scrapecore_sdk-0.1.0.dist-info/METADATA,sha256=inCkwxC5o741iOcCi7_GzuNsoAWwCFEG7fE0HMxnEow,3240
|
|
9
|
+
scrapecore_sdk-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
10
|
+
scrapecore_sdk-0.1.0.dist-info/RECORD,,
|