scrapecore-sdk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scrapecore/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """ScrapeCore Python SDK: web scraping, site maps, crawls and screenshots via the ScrapeCore API v1."""
2
+ from ._async_client import AsyncScrapeCore
3
+ from ._client import ScrapeCore
4
+ from ._errors import (APIConnectionError, APIError, AuthenticationError, InsufficientCreditsError,
5
+ InvalidRequestError, JobFailedError, JobTimeoutError, NotFoundError,
6
+ PermissionDeniedError, RateLimitError, ScrapeCoreError)
7
+ from ._version import __version__
8
+
9
+ __all__ = ["ScrapeCore", "AsyncScrapeCore", "ScrapeCoreError", "AuthenticationError", "InsufficientCreditsError",
10
+ "PermissionDeniedError", "NotFoundError", "InvalidRequestError", "RateLimitError", "APIError",
11
+ "APIConnectionError", "JobFailedError", "JobTimeoutError", "__version__"]
@@ -0,0 +1,161 @@
1
+ """Asynchronous client (asyncio). Same methods as ScrapeCore, awaitable."""
2
+ from __future__ import annotations
3
+
4
+ import asyncio
5
+ import time
6
+ from typing import Any, AsyncIterator, Sequence
7
+
8
+ import httpx
9
+
10
+ from . import _base, _errors
11
+
12
+
13
+ class AsyncScrapeCore:
14
+ """Async ScrapeCore client: `async with AsyncScrapeCore() as sc: page = await sc.scrape(url)`."""
15
+
16
+ def __init__(self, api_key: str | None = None, *, base_url: str | None = None, timeout: float = 120.0,
17
+ max_retries: int = 2, user_agent: str | None = None, http_client: httpx.AsyncClient | None = None):
18
+ self._s = _base.Settings(api_key, base_url, timeout, max_retries, user_agent)
19
+ self._http = http_client or httpx.AsyncClient(timeout=timeout)
20
+ self._own = http_client is None
21
+
22
+ # ── plumbing ──────────────────────────────────────────────────────────────────────────
23
+ async def close(self) -> None:
24
+ if self._own:
25
+ await self._http.aclose()
26
+
27
+ async def __aenter__(self) -> "AsyncScrapeCore":
28
+ return self
29
+
30
+ async def __aexit__(self, *exc) -> None:
31
+ await self.close()
32
+
33
+ async def request(self, method: str, path: str, *, json: Any = None, params: dict | None = None,
34
+ idempotent_key: str | None = None) -> Any:
35
+ """Any /api/v1 endpoint. Retries what is safe to retry."""
36
+ headers = dict(self._s.headers)
37
+ if idempotent_key:
38
+ headers["Idempotency-Key"] = idempotent_key
39
+ attempt = 0
40
+ while True:
41
+ try:
42
+ r = await self._http.request(method, self._s.url(path), json=json, params=params, headers=headers,
43
+ timeout=self._s.timeout)
44
+ return _base.parse(r.status_code, r.headers, r.content)
45
+ except _errors.ScrapeCoreError as e:
46
+ err = e
47
+ except httpx.TimeoutException as e:
48
+ err = _errors.APIConnectionError(f"timed out: {e}", code="TIMEOUT", retryable=True)
49
+ except httpx.HTTPError as e:
50
+ err = _errors.APIConnectionError(f"could not reach the API: {e}", code="CONNECTION_ERROR",
51
+ retryable=True)
52
+ if not _base.should_retry(err, attempt, self._s.max_retries):
53
+ raise err
54
+ await asyncio.sleep(_base.backoff(attempt, err))
55
+ attempt += 1
56
+
57
+ # ── scraping ──────────────────────────────────────────────────────────────────────────
58
+ async def scrape(self, url: str, *, format: str = "html", js_render: bool | None = None, use_proxy: bool | None = None,
59
+ waf: bool | None = None, max_age: int | None = None) -> dict:
60
+ """Scrape one page. format: "html", "markdown" or "text". max_age (seconds): a stored
61
+ result at most this old is fine (faster)."""
62
+ return await self.request("POST", "scrape", json=_base.scrape_body(url, format, js_render, use_proxy, waf, max_age))
63
+
64
+ async def markdown(self, url: str, **kw) -> str:
65
+ """The page as Markdown (just the text)."""
66
+ return (await self.scrape(url, format="markdown", **kw)).get("markdown") or ""
67
+
68
+ async def batch(self, urls: Sequence[str], *, js_render: bool | None = None, use_proxy: bool | None = None,
69
+ waf: bool | None = None) -> dict:
70
+ """Up to 50 URLs in one call (plan feature "batch")."""
71
+ return await self.request("POST", "batch", json=_base._clean({"urls": list(urls), "js_render": js_render,
72
+ "use_proxy": use_proxy, "waf": waf}))
73
+
74
+ async def screenshot(self, url: str, *, full_page: bool = False, wait: bool = True, timeout: float = 120, **options) -> dict:
75
+ """A screenshot (base64 image in result["image_base64"]). Options: device, width, height,
76
+ format ("png"/"jpeg"), selector, wait_for, dark_mode, block_ads, hide_cookie_banners, ..."""
77
+ body = {"url": url, "full_page": full_page, "sync": False, **options}
78
+ job = await self.request("POST", "screenshot", json=body, idempotent_key=_base.idempotency_key())
79
+ if "job_id" not in job or job.get("status") != "queued":
80
+ return job # ran directly
81
+ return (await self.wait_for_job(job["job_id"], timeout=timeout))["result"] if wait else job
82
+
83
+ async def extract(self, url: str, selectors: dict, *, js_render: bool | None = None) -> dict:
84
+ """CSS selectors -> values, e.g. {"title": "h1", "prices": ".price"}."""
85
+ return await self.request("POST", "extract", json=_base._clean({"url": url, "selectors": selectors,
86
+ "js_render": js_render, "sync": True}))
87
+
88
+ # ── site discovery and crawling ───────────────────────────────────────────────────────
89
+ async def map(self, url: str, *, limit: int | None = None, search: str | None = None,
90
+ include_subdomains: bool | None = None, respect_robots: bool | None = None) -> dict:
91
+ """Every URL of a site (sitemaps + links). Returns {"urls": [...], "count": n, ...}."""
92
+ return await self.request("POST", "map", json=_base.map_body(url, limit, search, include_subdomains, respect_robots))
93
+
94
+ async def crawl(self, url: str, *, max_depth: int | None = None, limit: int | None = None,
95
+ include_paths: Sequence[str] | None = None, exclude_paths: Sequence[str] | None = None,
96
+ include_subdomains: bool | None = None, respect_robots: bool | None = None,
97
+ js_render: bool | None = None, use_proxy: bool | None = None,
98
+ wait: bool = False, timeout: float = 1800) -> dict:
99
+ """Start a crawl job (returns {"job_id", ...}). wait=True: block until it ends and return
100
+ the finished job; read the pages with crawl_pages(job_id)."""
101
+ body = _base.crawl_body(url, max_depth, limit, list(include_paths) if include_paths else None,
102
+ list(exclude_paths) if exclude_paths else None, include_subdomains,
103
+ respect_robots, js_render, use_proxy)
104
+ job = await self.request("POST", "crawl", json=body, idempotent_key=_base.idempotency_key())
105
+ return await self.wait_for_job(job["job_id"], timeout=timeout, raise_on_failure=False) if wait else job
106
+
107
+ async def crawl_pages(self, job_id: str, *, only_ok: bool = False, page_size: int = 100) -> AsyncIterator[dict]:
108
+ """Every page of a crawl (url, title, markdown, status_code, ...), fetched page by page."""
109
+ offset = 0
110
+ while True:
111
+ r = await self.request("GET", f"crawl/{job_id}/pages",
112
+ params={"offset": offset, "limit": page_size, "only_ok": str(only_ok).lower()})
113
+ pages = r.get("pages") or []
114
+ for p in pages:
115
+ yield p
116
+ offset += len(pages)
117
+ if not pages or offset >= int(r.get("total") or 0):
118
+ return
119
+
120
+ # ── jobs ──────────────────────────────────────────────────────────────────────────────
121
+ async def scrape_async(self, url: str, **kw) -> dict:
122
+ """Queue a scrape; returns {"job_id", "status": "queued", ...}. See wait_for_job()."""
123
+ body = _base.scrape_body(url, kw.pop("format", None), kw.pop("js_render", None), kw.pop("use_proxy", None),
124
+ kw.pop("waf", None), None)
125
+ return await self.request("POST", "async", json=body, idempotent_key=_base.idempotency_key())
126
+
127
+ async def get_job(self, job_id: str) -> dict:
128
+ return await self.request("GET", f"job/{job_id}")
129
+
130
+ async def wait_for_job(self, job_id: str, *, timeout: float = 300, poll_interval: float = 1.0,
131
+ raise_on_failure: bool = True) -> dict:
132
+ """Poll until the job is success / failure / cancelled. Raises JobFailedError (failure,
133
+ cancelled) unless raise_on_failure=False, and JobTimeoutError after `timeout` seconds."""
134
+ deadline = time.monotonic() + timeout
135
+ delay = poll_interval
136
+ while True:
137
+ job = await self.get_job(job_id)
138
+ if job.get("status") in _base.TERMINAL:
139
+ if raise_on_failure and job["status"] != "success":
140
+ raise _base.job_failed(job)
141
+ return job
142
+ if time.monotonic() + delay > deadline:
143
+ raise _errors.JobTimeoutError(f"job {job_id} still {job.get('status')} after {timeout}s", job,
144
+ code="JOB_TIMEOUT")
145
+ await asyncio.sleep(delay)
146
+ delay = min(delay * 1.5, 10.0)
147
+
148
+ async def cancel_job(self, job_id: str) -> dict:
149
+ """Cancel a queued job, or stop a running crawl after its current page."""
150
+ return await self.request("POST", f"job/{job_id}/cancel")
151
+
152
+ async def jobs(self, *, limit: int = 50, offset: int = 0, status: str | None = None, type: str | None = None) -> list:
153
+ return await self.request("GET", "jobs", params=_base._clean({"limit": limit, "offset": offset,
154
+ "status": status, "type": type}))
155
+
156
+ # ── account ───────────────────────────────────────────────────────────────────────────
157
+ async def usage(self) -> dict:
158
+ return await self.request("GET", "usage")
159
+
160
+ async def rate_limit(self) -> dict:
161
+ return await self.request("GET", "rate-limit")
scrapecore/_base.py ADDED
@@ -0,0 +1,89 @@
1
+ """What the sync and async clients share: settings, request building, retry policy, parsing."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ import random
6
+ import uuid
7
+ from typing import Any
8
+
9
+ from . import _errors
10
+ from ._version import __version__
11
+
12
+ DEFAULT_BASE_URL = "https://api.scrapecore.com"
13
+ TERMINAL = ("success", "failure", "cancelled")
14
+
15
+
16
+ def _clean(d: dict) -> dict:
17
+ return {k: v for k, v in d.items() if v is not None}
18
+
19
+
20
+ class Settings:
21
+ def __init__(self, api_key: str | None, base_url: str | None, timeout: float, max_retries: int,
22
+ user_agent: str | None):
23
+ self.api_key = api_key or os.environ.get("SCRAPECORE_API_KEY")
24
+ if not self.api_key:
25
+ raise _errors.AuthenticationError(
26
+ "No API key: pass api_key=... or set SCRAPECORE_API_KEY (create one at "
27
+ "https://scrapecore.com/dashboard/api-keys)", code="INVALID_API_KEY")
28
+ self.base_url = (base_url or os.environ.get("SCRAPECORE_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
29
+ self.timeout = timeout
30
+ self.max_retries = max(0, int(max_retries))
31
+ ua = f"scrapecore-python/{__version__}"
32
+ self.headers = {"X-API-Key": self.api_key, "User-Agent": f"{user_agent} {ua}" if user_agent else ua,
33
+ "Accept": "application/json"}
34
+
35
+ def url(self, path: str) -> str:
36
+ return f"{self.base_url}/api/v1/{path.lstrip('/')}"
37
+
38
+
39
+ def backoff(attempt: int, err: _errors.ScrapeCoreError | None) -> float:
40
+ if err is not None and err.retry_after is not None:
41
+ return min(float(err.retry_after), 60.0)
42
+ return min(0.5 * (2 ** attempt), 8.0) * (0.75 + random.random() / 2)
43
+
44
+
45
+ def should_retry(err: _errors.ScrapeCoreError, attempt: int, max_retries: int) -> bool:
46
+ if attempt >= max_retries:
47
+ return False
48
+ if isinstance(err, _errors.APIConnectionError):
49
+ return True
50
+ # concurrency / per-minute / queue limits clear quickly; a daily limit does not
51
+ return err.retryable and err.code != "DAILY_LIMIT"
52
+
53
+
54
+ def parse(status: int, headers, content: bytes) -> Any:
55
+ import json
56
+ try:
57
+ body = json.loads(content) if content else None
58
+ except ValueError:
59
+ body = {"detail": content[:500].decode("utf-8", "replace")}
60
+ if status >= 400:
61
+ raise _errors.from_response(status, body, headers)
62
+ return body
63
+
64
+
65
+ def idempotency_key() -> str:
66
+ return str(uuid.uuid4())
67
+
68
+
69
+ # ── request bodies (one place, so both clients send the same thing) ─────────────────────────
70
+ def scrape_body(url, format, js_render, use_proxy, waf, max_age):
71
+ return _clean({"url": url, "format": format, "js_render": js_render, "use_proxy": use_proxy,
72
+ "waf": waf, "max_age": max_age})
73
+
74
+
75
+ def map_body(url, limit, search, include_subdomains, respect_robots):
76
+ return _clean({"url": url, "limit": limit, "search": search, "include_subdomains": include_subdomains,
77
+ "respect_robots": respect_robots})
78
+
79
+
80
+ def crawl_body(url, max_depth, limit, include_paths, exclude_paths, include_subdomains, respect_robots,
81
+ js_render, use_proxy):
82
+ return _clean({"url": url, "max_depth": max_depth, "limit": limit, "include_paths": include_paths,
83
+ "exclude_paths": exclude_paths, "include_subdomains": include_subdomains,
84
+ "respect_robots": respect_robots, "js_render": js_render, "use_proxy": use_proxy})
85
+
86
+
87
+ def job_failed(job: dict) -> _errors.JobFailedError:
88
+ return _errors.JobFailedError(f"job {job.get('job_id')} ended as {job.get('status')}: {job.get('error') or ''}".strip(),
89
+ job, code="JOB_" + str(job.get("status", "failed")).upper())
scrapecore/_client.py ADDED
@@ -0,0 +1,168 @@
1
+ """Synchronous client."""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+ from typing import Any, Iterator, Sequence
6
+
7
+ import httpx
8
+
9
+ from . import _base, _errors
10
+
11
+
12
+ class ScrapeCore:
13
+ """ScrapeCore API client.
14
+
15
+ from scrapecore import ScrapeCore
16
+ sc = ScrapeCore() # reads SCRAPECORE_API_KEY
17
+ page = sc.scrape("https://example.com", format="markdown")
18
+ print(page["markdown"])
19
+
20
+ Failed scrapes are not charged. Rate-limit and server errors are retried automatically
21
+ (max_retries, honouring Retry-After); every other error raises a ScrapeCoreError subclass.
22
+ """
23
+
24
+ def __init__(self, api_key: str | None = None, *, base_url: str | None = None, timeout: float = 120.0,
25
+ max_retries: int = 2, user_agent: str | None = None, http_client: httpx.Client | None = None):
26
+ self._s = _base.Settings(api_key, base_url, timeout, max_retries, user_agent)
27
+ self._http = http_client or httpx.Client(timeout=timeout)
28
+ self._own = http_client is None
29
+
30
+ # ── plumbing ──────────────────────────────────────────────────────────────────────────
31
+ def close(self) -> None:
32
+ if self._own:
33
+ self._http.close()
34
+
35
+ def __enter__(self) -> "ScrapeCore":
36
+ return self
37
+
38
+ def __exit__(self, *exc) -> None:
39
+ self.close()
40
+
41
+ def request(self, method: str, path: str, *, json: Any = None, params: dict | None = None,
42
+ idempotent_key: str | None = None) -> Any:
43
+ """Any /api/v1 endpoint. Retries what is safe to retry."""
44
+ headers = dict(self._s.headers)
45
+ if idempotent_key:
46
+ headers["Idempotency-Key"] = idempotent_key
47
+ attempt = 0
48
+ while True:
49
+ try:
50
+ r = self._http.request(method, self._s.url(path), json=json, params=params, headers=headers,
51
+ timeout=self._s.timeout)
52
+ return _base.parse(r.status_code, r.headers, r.content)
53
+ except _errors.ScrapeCoreError as e:
54
+ err = e
55
+ except httpx.TimeoutException as e:
56
+ err = _errors.APIConnectionError(f"timed out: {e}", code="TIMEOUT", retryable=True)
57
+ except httpx.HTTPError as e:
58
+ err = _errors.APIConnectionError(f"could not reach the API: {e}", code="CONNECTION_ERROR",
59
+ retryable=True)
60
+ if not _base.should_retry(err, attempt, self._s.max_retries):
61
+ raise err
62
+ time.sleep(_base.backoff(attempt, err))
63
+ attempt += 1
64
+
65
+ # ── scraping ──────────────────────────────────────────────────────────────────────────
66
+ def scrape(self, url: str, *, format: str = "html", js_render: bool | None = None, use_proxy: bool | None = None,
67
+ waf: bool | None = None, max_age: int | None = None) -> dict:
68
+ """Scrape one page. format: "html", "markdown" or "text". max_age (seconds): a stored
69
+ result at most this old is fine (faster)."""
70
+ return self.request("POST", "scrape", json=_base.scrape_body(url, format, js_render, use_proxy, waf, max_age))
71
+
72
+ def markdown(self, url: str, **kw) -> str:
73
+ """The page as Markdown (just the text)."""
74
+ return self.scrape(url, format="markdown", **kw).get("markdown") or ""
75
+
76
+ def batch(self, urls: Sequence[str], *, js_render: bool | None = None, use_proxy: bool | None = None,
77
+ waf: bool | None = None) -> dict:
78
+ """Up to 50 URLs in one call (plan feature "batch")."""
79
+ return self.request("POST", "batch", json=_base._clean({"urls": list(urls), "js_render": js_render,
80
+ "use_proxy": use_proxy, "waf": waf}))
81
+
82
+ def screenshot(self, url: str, *, full_page: bool = False, wait: bool = True, timeout: float = 120, **options) -> dict:
83
+ """A screenshot (base64 image in result["image_base64"]). Options: device, width, height,
84
+ format ("png"/"jpeg"), selector, wait_for, dark_mode, block_ads, hide_cookie_banners, ..."""
85
+ body = {"url": url, "full_page": full_page, "sync": False, **options}
86
+ job = self.request("POST", "screenshot", json=body, idempotent_key=_base.idempotency_key())
87
+ if "job_id" not in job or job.get("status") != "queued":
88
+ return job # ran directly
89
+ return self.wait_for_job(job["job_id"], timeout=timeout)["result"] if wait else job
90
+
91
+ def extract(self, url: str, selectors: dict, *, js_render: bool | None = None) -> dict:
92
+ """CSS selectors -> values, e.g. {"title": "h1", "prices": ".price"}."""
93
+ return self.request("POST", "extract", json=_base._clean({"url": url, "selectors": selectors,
94
+ "js_render": js_render, "sync": True}))
95
+
96
+ # ── site discovery and crawling ───────────────────────────────────────────────────────
97
+ def map(self, url: str, *, limit: int | None = None, search: str | None = None,
98
+ include_subdomains: bool | None = None, respect_robots: bool | None = None) -> dict:
99
+ """Every URL of a site (sitemaps + links). Returns {"urls": [...], "count": n, ...}."""
100
+ return self.request("POST", "map", json=_base.map_body(url, limit, search, include_subdomains, respect_robots))
101
+
102
+ def crawl(self, url: str, *, max_depth: int | None = None, limit: int | None = None,
103
+ include_paths: Sequence[str] | None = None, exclude_paths: Sequence[str] | None = None,
104
+ include_subdomains: bool | None = None, respect_robots: bool | None = None,
105
+ js_render: bool | None = None, use_proxy: bool | None = None,
106
+ wait: bool = False, timeout: float = 1800) -> dict:
107
+ """Start a crawl job (returns {"job_id", ...}). wait=True: block until it ends and return
108
+ the finished job; read the pages with crawl_pages(job_id)."""
109
+ body = _base.crawl_body(url, max_depth, limit, list(include_paths) if include_paths else None,
110
+ list(exclude_paths) if exclude_paths else None, include_subdomains,
111
+ respect_robots, js_render, use_proxy)
112
+ job = self.request("POST", "crawl", json=body, idempotent_key=_base.idempotency_key())
113
+ return self.wait_for_job(job["job_id"], timeout=timeout, raise_on_failure=False) if wait else job
114
+
115
+ def crawl_pages(self, job_id: str, *, only_ok: bool = False, page_size: int = 100) -> Iterator[dict]:
116
+ """Every page of a crawl (url, title, markdown, status_code, ...), fetched page by page."""
117
+ offset = 0
118
+ while True:
119
+ r = self.request("GET", f"crawl/{job_id}/pages",
120
+ params={"offset": offset, "limit": page_size, "only_ok": str(only_ok).lower()})
121
+ pages = r.get("pages") or []
122
+ yield from pages
123
+ offset += len(pages)
124
+ if not pages or offset >= int(r.get("total") or 0):
125
+ return
126
+
127
+ # ── jobs ──────────────────────────────────────────────────────────────────────────────
128
+ def scrape_async(self, url: str, **kw) -> dict:
129
+ """Queue a scrape; returns {"job_id", "status": "queued", ...}. See wait_for_job()."""
130
+ body = _base.scrape_body(url, kw.pop("format", None), kw.pop("js_render", None), kw.pop("use_proxy", None),
131
+ kw.pop("waf", None), None)
132
+ return self.request("POST", "async", json=body, idempotent_key=_base.idempotency_key())
133
+
134
+ def get_job(self, job_id: str) -> dict:
135
+ return self.request("GET", f"job/{job_id}")
136
+
137
+ def wait_for_job(self, job_id: str, *, timeout: float = 300, poll_interval: float = 1.0,
138
+ raise_on_failure: bool = True) -> dict:
139
+ """Poll until the job is success / failure / cancelled. Raises JobFailedError (failure,
140
+ cancelled) unless raise_on_failure=False, and JobTimeoutError after `timeout` seconds."""
141
+ deadline = time.monotonic() + timeout
142
+ delay = poll_interval
143
+ while True:
144
+ job = self.get_job(job_id)
145
+ if job.get("status") in _base.TERMINAL:
146
+ if raise_on_failure and job["status"] != "success":
147
+ raise _base.job_failed(job)
148
+ return job
149
+ if time.monotonic() + delay > deadline:
150
+ raise _errors.JobTimeoutError(f"job {job_id} still {job.get('status')} after {timeout}s", job,
151
+ code="JOB_TIMEOUT")
152
+ time.sleep(delay)
153
+ delay = min(delay * 1.5, 10.0)
154
+
155
+ def cancel_job(self, job_id: str) -> dict:
156
+ """Cancel a queued job, or stop a running crawl after its current page."""
157
+ return self.request("POST", f"job/{job_id}/cancel")
158
+
159
+ def jobs(self, *, limit: int = 50, offset: int = 0, status: str | None = None, type: str | None = None) -> list:
160
+ return self.request("GET", "jobs", params=_base._clean({"limit": limit, "offset": offset,
161
+ "status": status, "type": type}))
162
+
163
+ # ── account ───────────────────────────────────────────────────────────────────────────
164
+ def usage(self) -> dict:
165
+ return self.request("GET", "usage")
166
+
167
+ def rate_limit(self) -> dict:
168
+ return self.request("GET", "rate-limit")
scrapecore/_errors.py ADDED
@@ -0,0 +1,99 @@
1
+ """Errors raised by the client. Every API error carries the fields of the API's error envelope."""
2
+ from __future__ import annotations
3
+
4
+ from typing import Any
5
+
6
+
7
+ class ScrapeCoreError(Exception):
8
+ """Base class. `code` is stable (e.g. RATE_LIMITED); see https://scrapecore.com/docs/errors."""
9
+
10
+ def __init__(self, message: str, *, code: str = "ERROR", status: int | None = None,
11
+ request_id: str | None = None, retryable: bool = False,
12
+ retry_after: float | None = None, details: dict | None = None, body: Any = None):
13
+ super().__init__(message)
14
+ self.message = message
15
+ self.code = code
16
+ self.status = status
17
+ self.request_id = request_id
18
+ self.retryable = retryable
19
+ self.retry_after = retry_after
20
+ self.details = details or {}
21
+ self.body = body
22
+
23
+ def __str__(self) -> str:
24
+ bits = [f"[{self.code}] {self.message}"]
25
+ if self.status:
26
+ bits.append(f"(HTTP {self.status})")
27
+ if self.request_id:
28
+ bits.append(f"request_id={self.request_id}")
29
+ return " ".join(bits)
30
+
31
+
32
+ class AuthenticationError(ScrapeCoreError):
33
+ """The API key is missing, wrong or revoked (401)."""
34
+
35
+
36
+ class InsufficientCreditsError(ScrapeCoreError):
37
+ """Not enough credits left this month (402)."""
38
+
39
+
40
+ class PermissionDeniedError(ScrapeCoreError):
41
+ """The plan does not include this feature, or the account is suspended (403)."""
42
+
43
+
44
+ class NotFoundError(ScrapeCoreError):
45
+ """No such job / endpoint (404)."""
46
+
47
+
48
+ class InvalidRequestError(ScrapeCoreError):
49
+ """The request was rejected: a bad URL or parameter (400, 409, 413, 422)."""
50
+
51
+
52
+ class RateLimitError(ScrapeCoreError):
53
+ """Requests per minute/day, concurrency or queue limit reached (429). See `retry_after`."""
54
+
55
+
56
+ class APIError(ScrapeCoreError):
57
+ """The server or the target site failed (5xx). Usually worth retrying."""
58
+
59
+
60
+ class APIConnectionError(ScrapeCoreError):
61
+ """The API could not be reached (network error or timeout)."""
62
+
63
+
64
+ class JobFailedError(ScrapeCoreError):
65
+ """wait_for_job(): the job ended as failure or cancelled."""
66
+
67
+ def __init__(self, message: str, job: dict, **kw):
68
+ super().__init__(message, **kw)
69
+ self.job = job
70
+
71
+
72
+ class JobTimeoutError(ScrapeCoreError):
73
+ """wait_for_job(): the job did not finish within the timeout (it keeps running)."""
74
+
75
+ def __init__(self, message: str, job: dict, **kw):
76
+ super().__init__(message, **kw)
77
+ self.job = job
78
+
79
+
80
+ _BY_STATUS = {401: AuthenticationError, 402: InsufficientCreditsError, 403: PermissionDeniedError,
81
+ 404: NotFoundError, 429: RateLimitError}
82
+
83
+
84
+ def from_response(status: int, body: Any, headers) -> ScrapeCoreError:
85
+ err = body.get("error") if isinstance(body, dict) and isinstance(body.get("error"), dict) else {}
86
+ detail = body.get("detail") if isinstance(body, dict) else None
87
+ message = err.get("message") or (detail if isinstance(detail, str) else None) or f"HTTP {status}"
88
+ retry_after = err.get("retry_after_seconds")
89
+ if retry_after is None:
90
+ ra = headers.get("retry-after") if headers is not None else None
91
+ try:
92
+ retry_after = float(ra) if ra is not None else None
93
+ except ValueError:
94
+ retry_after = None
95
+ cls = _BY_STATUS.get(status) or (APIError if status >= 500 else InvalidRequestError)
96
+ return cls(message, code=err.get("code") or ("INTERNAL_ERROR" if status >= 500 else "INVALID_REQUEST"),
97
+ status=status, request_id=err.get("request_id") or (headers.get("x-request-id") if headers is not None else None),
98
+ retryable=bool(err.get("retryable", status in (408, 429) or status >= 500)),
99
+ retry_after=retry_after, details=err.get("details"), body=body)
scrapecore/_version.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
scrapecore/py.typed ADDED
File without changes
@@ -0,0 +1,83 @@
1
+ Metadata-Version: 2.5
2
+ Name: scrapecore-sdk
3
+ Version: 0.1.0
4
+ Summary: Official Python client for the ScrapeCore web scraping API
5
+ Project-URL: Homepage, https://scrapecore.com
6
+ Project-URL: Documentation, https://scrapecore.com/docs
7
+ Project-URL: Source, https://github.com/imrajeshkumar7492/scrapecore-integrations/tree/main/sdks/python
8
+ Project-URL: Issues, https://github.com/imrajeshkumar7492/scrapecore-integrations/issues
9
+ License-Expression: MIT
10
+ Keywords: api,crawler,llm,markdown,rag,scraper,screenshot,web scraping
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Internet :: WWW/HTTP
15
+ Classifier: Typing :: Typed
16
+ Requires-Python: >=3.10
17
+ Requires-Dist: httpx<1,>=0.25
18
+ Provides-Extra: test
19
+ Requires-Dist: pytest-asyncio>=0.21; extra == 'test'
20
+ Requires-Dist: pytest>=7; extra == 'test'
21
+ Description-Content-Type: text/markdown
22
+
23
+ # ScrapeCore Python SDK
24
+
25
+ The official Python client for the [ScrapeCore](https://scrapecore.com) API (v1). You can scrape
26
+ pages to HTML, Markdown or text, map a site, crawl it, take screenshots, and extract data with CSS
27
+ selectors.
28
+
29
+ ```bash
30
+ pip install scrapecore-sdk
31
+ ```
32
+
33
+ ```python
34
+ from scrapecore import ScrapeCore
35
+
36
+ sc = ScrapeCore() # reads SCRAPECORE_API_KEY (or pass api_key="sk_live_...")
37
+
38
+ page = sc.scrape("https://example.com", format="markdown")
39
+ print(page["markdown"], page["credits_used"], page["credits_remaining"])
40
+
41
+ urls = sc.map("https://example.com", limit=500)["urls"]
42
+
43
+ job = sc.crawl("https://example.com/docs", max_depth=2, limit=100, include_paths=["^/docs"], wait=True)
44
+ for p in sc.crawl_pages(job["job_id"], only_ok=True):
45
+ print(p["url"], p["title"], len(p["markdown"]))
46
+
47
+ shot = sc.screenshot("https://example.com", full_page=True, device="mobile") # shot["image_base64"]
48
+ data = sc.extract("https://example.com", {"title": "h1", "links": "a"})
49
+ ```
50
+
51
+ Async:
52
+
53
+ ```python
54
+ from scrapecore import AsyncScrapeCore
55
+
56
+ async with AsyncScrapeCore() as sc:
57
+ page = await sc.scrape("https://example.com", format="markdown")
58
+ ```
59
+
60
+ ## Errors and retries
61
+
62
+ Every API error raises a subclass of `ScrapeCoreError`. Each error has these attributes: `code`
63
+ (stable, e.g. `RATE_LIMITED`), `status`, `message`, `request_id` (quote it to support),
64
+ `retryable` and `retry_after`.
65
+
66
+ | Exception | When |
67
+ |---|---|
68
+ | `AuthenticationError` | missing / wrong / revoked key (401) |
69
+ | `InsufficientCreditsError` | monthly credits used up (402) |
70
+ | `PermissionDeniedError` | feature not in your plan (403) |
71
+ | `InvalidRequestError` | bad URL or parameter (400/422) |
72
+ | `RateLimitError` | per-minute / daily / concurrency limit (429) |
73
+ | `APIError` | server or target-site failure (5xx) |
74
+ | `APIConnectionError` | network error or timeout |
75
+
76
+ The SDK automatically retries rate limits (except the daily limit), 5xx errors and network errors
77
+ (`max_retries=2`, honouring `Retry-After`). Job-creating calls send an `Idempotency-Key`, so a retry
78
+ never creates a second job. Failed scrapes are never charged.
79
+
80
+ ## Responsible use
81
+
82
+ Use ScrapeCore only for content you are allowed to access. Crawls and maps respect `robots.txt` by
83
+ default. See the [acceptable use policy](https://scrapecore.com/acceptable-use).
@@ -0,0 +1,10 @@
1
+ scrapecore/__init__.py,sha256=b4nlbg3lbev_TNvBUPFS-wpzWDquy4cvJG_Dp1ioOdY,787
2
+ scrapecore/_async_client.py,sha256=-uU_5QgLjfSjJzAvw_qAq-AZPzBNr3_f_9qRqO3M8kU,10156
3
+ scrapecore/_base.py,sha256=I4FgM3YdVfZdRvzPUOq6eJfzyrYVg8bWSfhn2qB68Ig,3724
4
+ scrapecore/_client.py,sha256=idBBizuxer3Nx_oJ5W4uSACRUml26bxuTBW5ydHyDoE,10121
5
+ scrapecore/_errors.py,sha256=2tGxDtzMgtnpxzHH7v_nkPtCBhszqcUxnlD-Yhn6cRo,3782
6
+ scrapecore/_version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
7
+ scrapecore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ scrapecore_sdk-0.1.0.dist-info/METADATA,sha256=inCkwxC5o741iOcCi7_GzuNsoAWwCFEG7fE0HMxnEow,3240
9
+ scrapecore_sdk-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
10
+ scrapecore_sdk-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any