scrapecore-sdk 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/PKG-INFO +47 -3
- scrapecore_sdk-0.2.0/README.md +105 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/pyproject.toml +4 -1
- scrapecore_sdk-0.2.0/src/scrapecore/__main__.py +5 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/src/scrapecore/_async_client.py +119 -2
- scrapecore_sdk-0.2.0/src/scrapecore/_cli.py +402 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/src/scrapecore/_client.py +119 -2
- scrapecore_sdk-0.2.0/src/scrapecore/_version.py +1 -0
- scrapecore_sdk-0.2.0/tests/test_v02.py +169 -0
- scrapecore_sdk-0.1.0/README.md +0 -61
- scrapecore_sdk-0.1.0/src/scrapecore/_version.py +0 -1
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/.gitignore +0 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/src/scrapecore/__init__.py +0 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/src/scrapecore/_base.py +0 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/src/scrapecore/_errors.py +0 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/src/scrapecore/py.typed +0 -0
- {scrapecore_sdk-0.1.0 → scrapecore_sdk-0.2.0}/tests/test_client.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scrapecore-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Official Python client for the ScrapeCore web scraping API
|
|
5
5
|
Project-URL: Homepage, https://scrapecore.com
|
|
6
6
|
Project-URL: Documentation, https://scrapecore.com/docs
|
|
7
7
|
Project-URL: Source, https://github.com/imrajeshkumar7492/scrapecore-integrations/tree/main/sdks/python
|
|
8
8
|
Project-URL: Issues, https://github.com/imrajeshkumar7492/scrapecore-integrations/issues
|
|
9
9
|
License-Expression: MIT
|
|
10
|
-
Keywords: api,crawler,llm,markdown,rag,scraper,screenshot,web scraping
|
|
10
|
+
Keywords: api,cli,crawler,llm,markdown,rag,scraper,screenshot,web scraping
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -24,7 +24,8 @@ Description-Content-Type: text/markdown
|
|
|
24
24
|
|
|
25
25
|
The official Python client for the [ScrapeCore](https://scrapecore.com) API (v1). You can scrape
|
|
26
26
|
pages to HTML, Markdown or text, map a site, crawl it, take screenshots, and extract data with CSS
|
|
27
|
-
selectors.
|
|
27
|
+
selectors or with AI. You can also watch pages for changes. It includes a
|
|
28
|
+
`scrapecore` command-line tool.
|
|
28
29
|
|
|
29
30
|
```bash
|
|
30
31
|
pip install scrapecore-sdk
|
|
@@ -57,6 +58,49 @@ async with AsyncScrapeCore() as sc:
|
|
|
57
58
|
page = await sc.scrape("https://example.com", format="markdown")
|
|
58
59
|
```
|
|
59
60
|
|
|
61
|
+
## AI extraction, crawl exports, monitors
|
|
62
|
+
|
|
63
|
+
AI extraction uses your own AI provider key: OpenAI, Anthropic, Gemini, Mistral, Groq, DeepSeek, xAI,
|
|
64
|
+
Cohere, Azure OpenAI, OpenRouter or any OpenAI-compatible endpoint. Connect it in Dashboard → AI Extraction.
|
|
65
|
+
Your provider bills you for tokens; ScrapeCore does not.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
product = sc.ai_extract("https://shop.example/item", prompt="name, price and rating",
|
|
69
|
+
schema={"type": "object", "properties": {"name": {"type": "string"},
|
|
70
|
+
"price": {"type": "number"}}})["data"]
|
|
71
|
+
|
|
72
|
+
# many pages (or sitemap=..., site=...) as a background job, any connected provider
|
|
73
|
+
job = sc.ai_job(["https://shop.example/c/1", "https://shop.example/c/2"], instructions="every product with price",
|
|
74
|
+
schema={"name": "string", "price": "number"}, records="list", provider="openai", wait=True)
|
|
75
|
+
rows = sc.ai_job_records(job["job_id"])
|
|
76
|
+
open("products.xlsx", "wb").write(sc.ai_job_export(job["job_id"], "xlsx"))
|
|
77
|
+
|
|
78
|
+
llms_txt = sc.crawl_export(job["job_id"], "llms") # also "jsonl", "markdown", "chunks", "llms-full"
|
|
79
|
+
|
|
80
|
+
m = sc.create_monitor("https://competitor.example/pricing", interval_minutes=60, selector="#plans")
|
|
81
|
+
for change in sc.monitor_changes(m["id"]):
|
|
82
|
+
print(change["created_at"], change["summary"], change["diff"])
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Command line
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
scrapecore login # saves the key in ~/.config/scrapecore/config.json (mode 600)
|
|
89
|
+
scrapecore scrape https://example.com # Markdown to stdout
|
|
90
|
+
scrapecore map https://example.com --limit 200
|
|
91
|
+
scrapecore crawl https://docs.example.com --limit 50 --export llms -o llms.txt
|
|
92
|
+
scrapecore screenshot https://example.com --full-page -o page.png
|
|
93
|
+
scrapecore extract https://shop.example/item -p "name and price"
|
|
94
|
+
scrapecore ai https://shop.example/c/1 https://shop.example/c/2 -p "every product with price" --many -f xlsx -o out.xlsx
|
|
95
|
+
scrapecore monitor add https://competitor.example/pricing --every 60 --selector "#plans"
|
|
96
|
+
scrapecore monitor list
|
|
97
|
+
scrapecore usage
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Add `--json` to print the raw API response, or `-o FILE` to write the result to a file. The key
|
|
101
|
+
comes from `--api-key`, then `SCRAPECORE_API_KEY`, then the key saved by `scrapecore login`.
|
|
102
|
+
`python -m scrapecore` works too.
|
|
103
|
+
|
|
60
104
|
## Errors and retries
|
|
61
105
|
|
|
62
106
|
Every API error raises a subclass of `ScrapeCoreError`. Each error has these attributes: `code`
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# ScrapeCore Python SDK
|
|
2
|
+
|
|
3
|
+
The official Python client for the [ScrapeCore](https://scrapecore.com) API (v1). You can scrape
|
|
4
|
+
pages to HTML, Markdown or text, map a site, crawl it, take screenshots, and extract data with CSS
|
|
5
|
+
selectors or with AI. You can also watch pages for changes. It includes a
|
|
6
|
+
`scrapecore` command-line tool.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install scrapecore-sdk
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
from scrapecore import ScrapeCore
|
|
14
|
+
|
|
15
|
+
sc = ScrapeCore() # reads SCRAPECORE_API_KEY (or pass api_key="sk_live_...")
|
|
16
|
+
|
|
17
|
+
page = sc.scrape("https://example.com", format="markdown")
|
|
18
|
+
print(page["markdown"], page["credits_used"], page["credits_remaining"])
|
|
19
|
+
|
|
20
|
+
urls = sc.map("https://example.com", limit=500)["urls"]
|
|
21
|
+
|
|
22
|
+
job = sc.crawl("https://example.com/docs", max_depth=2, limit=100, include_paths=["^/docs"], wait=True)
|
|
23
|
+
for p in sc.crawl_pages(job["job_id"], only_ok=True):
|
|
24
|
+
print(p["url"], p["title"], len(p["markdown"]))
|
|
25
|
+
|
|
26
|
+
shot = sc.screenshot("https://example.com", full_page=True, device="mobile") # shot["image_base64"]
|
|
27
|
+
data = sc.extract("https://example.com", {"title": "h1", "links": "a"})
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Async:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from scrapecore import AsyncScrapeCore
|
|
34
|
+
|
|
35
|
+
async with AsyncScrapeCore() as sc:
|
|
36
|
+
page = await sc.scrape("https://example.com", format="markdown")
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## AI extraction, crawl exports, monitors
|
|
40
|
+
|
|
41
|
+
AI extraction uses your own AI provider key: OpenAI, Anthropic, Gemini, Mistral, Groq, DeepSeek, xAI,
|
|
42
|
+
Cohere, Azure OpenAI, OpenRouter or any OpenAI-compatible endpoint. Connect it in Dashboard → AI Extraction.
|
|
43
|
+
Your provider bills you for tokens; ScrapeCore does not.
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
product = sc.ai_extract("https://shop.example/item", prompt="name, price and rating",
|
|
47
|
+
schema={"type": "object", "properties": {"name": {"type": "string"},
|
|
48
|
+
"price": {"type": "number"}}})["data"]
|
|
49
|
+
|
|
50
|
+
# many pages (or sitemap=..., site=...) as a background job, any connected provider
|
|
51
|
+
job = sc.ai_job(["https://shop.example/c/1", "https://shop.example/c/2"], instructions="every product with price",
|
|
52
|
+
schema={"name": "string", "price": "number"}, records="list", provider="openai", wait=True)
|
|
53
|
+
rows = sc.ai_job_records(job["job_id"])
|
|
54
|
+
open("products.xlsx", "wb").write(sc.ai_job_export(job["job_id"], "xlsx"))
|
|
55
|
+
|
|
56
|
+
llms_txt = sc.crawl_export(job["job_id"], "llms") # also "jsonl", "markdown", "chunks", "llms-full"
|
|
57
|
+
|
|
58
|
+
m = sc.create_monitor("https://competitor.example/pricing", interval_minutes=60, selector="#plans")
|
|
59
|
+
for change in sc.monitor_changes(m["id"]):
|
|
60
|
+
print(change["created_at"], change["summary"], change["diff"])
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Command line
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
scrapecore login # saves the key in ~/.config/scrapecore/config.json (mode 600)
|
|
67
|
+
scrapecore scrape https://example.com # Markdown to stdout
|
|
68
|
+
scrapecore map https://example.com --limit 200
|
|
69
|
+
scrapecore crawl https://docs.example.com --limit 50 --export llms -o llms.txt
|
|
70
|
+
scrapecore screenshot https://example.com --full-page -o page.png
|
|
71
|
+
scrapecore extract https://shop.example/item -p "name and price"
|
|
72
|
+
scrapecore ai https://shop.example/c/1 https://shop.example/c/2 -p "every product with price" --many -f xlsx -o out.xlsx
|
|
73
|
+
scrapecore monitor add https://competitor.example/pricing --every 60 --selector "#plans"
|
|
74
|
+
scrapecore monitor list
|
|
75
|
+
scrapecore usage
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Add `--json` to print the raw API response, or `-o FILE` to write the result to a file. The key
|
|
79
|
+
comes from `--api-key`, then `SCRAPECORE_API_KEY`, then the key saved by `scrapecore login`.
|
|
80
|
+
`python -m scrapecore` works too.
|
|
81
|
+
|
|
82
|
+
## Errors and retries
|
|
83
|
+
|
|
84
|
+
Every API error raises a subclass of `ScrapeCoreError`. Each error has these attributes: `code`
|
|
85
|
+
(stable, e.g. `RATE_LIMITED`), `status`, `message`, `request_id` (quote it to support),
|
|
86
|
+
`retryable` and `retry_after`.
|
|
87
|
+
|
|
88
|
+
| Exception | When |
|
|
89
|
+
|---|---|
|
|
90
|
+
| `AuthenticationError` | missing / wrong / revoked key (401) |
|
|
91
|
+
| `InsufficientCreditsError` | monthly credits used up (402) |
|
|
92
|
+
| `PermissionDeniedError` | feature not in your plan (403) |
|
|
93
|
+
| `InvalidRequestError` | bad URL or parameter (400/422) |
|
|
94
|
+
| `RateLimitError` | per-minute / daily / concurrency limit (429) |
|
|
95
|
+
| `APIError` | server or target-site failure (5xx) |
|
|
96
|
+
| `APIConnectionError` | network error or timeout |
|
|
97
|
+
|
|
98
|
+
The SDK automatically retries rate limits (except the daily limit), 5xx errors and network errors
|
|
99
|
+
(`max_retries=2`, honouring `Retry-After`). Job-creating calls send an `Idempotency-Key`, so a retry
|
|
100
|
+
never creates a second job. Failed scrapes are never charged.
|
|
101
|
+
|
|
102
|
+
## Responsible use
|
|
103
|
+
|
|
104
|
+
Use ScrapeCore only for content you are allowed to access. Crawls and maps respect `robots.txt` by
|
|
105
|
+
default. See the [acceptable use policy](https://scrapecore.com/acceptable-use).
|
|
@@ -10,7 +10,7 @@ readme = "README.md"
|
|
|
10
10
|
license = "MIT"
|
|
11
11
|
requires-python = ">=3.10"
|
|
12
12
|
dependencies = ["httpx>=0.25,<1"]
|
|
13
|
-
keywords = ["web scraping", "scraper", "crawler", "markdown", "screenshot", "api", "llm", "rag"]
|
|
13
|
+
keywords = ["web scraping", "cli", "scraper", "crawler", "markdown", "screenshot", "api", "llm", "rag"]
|
|
14
14
|
classifiers = [
|
|
15
15
|
"Development Status :: 4 - Beta",
|
|
16
16
|
"Intended Audience :: Developers",
|
|
@@ -19,6 +19,9 @@ classifiers = [
|
|
|
19
19
|
"Typing :: Typed",
|
|
20
20
|
]
|
|
21
21
|
|
|
22
|
+
[project.scripts]
|
|
23
|
+
scrapecore = "scrapecore._cli:main"
|
|
24
|
+
|
|
22
25
|
[project.urls]
|
|
23
26
|
Homepage = "https://scrapecore.com"
|
|
24
27
|
Documentation = "https://scrapecore.com/docs"
|
|
@@ -31,8 +31,8 @@ class AsyncScrapeCore:
|
|
|
31
31
|
await self.close()
|
|
32
32
|
|
|
33
33
|
async def request(self, method: str, path: str, *, json: Any = None, params: dict | None = None,
|
|
34
|
-
idempotent_key: str | None = None) -> Any:
|
|
35
|
-
"""Any /api/v1 endpoint. Retries what is safe to retry."""
|
|
34
|
+
idempotent_key: str | None = None, raw: bool | str = False) -> Any:
|
|
35
|
+
"""Any /api/v1 endpoint. Retries what is safe to retry. raw=True returns the body as text."""
|
|
36
36
|
headers = dict(self._s.headers)
|
|
37
37
|
if idempotent_key:
|
|
38
38
|
headers["Idempotency-Key"] = idempotent_key
|
|
@@ -41,6 +41,8 @@ class AsyncScrapeCore:
|
|
|
41
41
|
try:
|
|
42
42
|
r = await self._http.request(method, self._s.url(path), json=json, params=params, headers=headers,
|
|
43
43
|
timeout=self._s.timeout)
|
|
44
|
+
if raw and r.status_code < 400:
|
|
45
|
+
return r.content if raw == "bytes" else r.text
|
|
44
46
|
return _base.parse(r.status_code, r.headers, r.content)
|
|
45
47
|
except _errors.ScrapeCoreError as e:
|
|
46
48
|
err = e
|
|
@@ -153,6 +155,121 @@ class AsyncScrapeCore:
|
|
|
153
155
|
return await self.request("GET", "jobs", params=_base._clean({"limit": limit, "offset": offset,
|
|
154
156
|
"status": status, "type": type}))
|
|
155
157
|
|
|
158
|
+
# ── AI extraction (your own AI provider key) ───────────────────────
|
|
159
|
+
async def ai_extract(self, url: str, *, prompt: str | None = None, schema: dict | None = None, model: str | None = None,
|
|
160
|
+
provider: str | None = None, records: str | None = None,
|
|
161
|
+
js_render: bool | None = None, use_proxy: bool | None = None) -> dict:
|
|
162
|
+
"""Structured data from ONE page with your own AI provider key (any provider connected in
|
|
163
|
+
Dashboard → AI Extraction: OpenAI, Anthropic, Gemini, Mistral, ...; default = your default one).
|
|
164
|
+
Give a prompt, a JSON schema, or both. records="list" returns every item. Returns {"data", ...}.
|
|
165
|
+
For many pages, sitemaps or whole sites use ai_job()."""
|
|
166
|
+
return await self.request("POST", "extract/ai", json=_base._clean({"url": url, "prompt": prompt, "schema": schema,
|
|
167
|
+
"model": model, "provider": provider, "records": records, "js_render": js_render,
|
|
168
|
+
"use_proxy": use_proxy}))
|
|
169
|
+
|
|
170
|
+
async def crawl_export(self, job_id: str, format: str = "jsonl", *, chunk_size: int | None = None,
|
|
171
|
+
overlap: int | None = None) -> str:
|
|
172
|
+
"""A crawl as one AI-ready document: "jsonl", "markdown", "chunks" (RAG-sized JSONL),
|
|
173
|
+
"llms" (llms.txt) or "llms-full" (llms-full.txt)."""
|
|
174
|
+
return await self.request("GET", f"crawl/{job_id}/export", raw=True,
|
|
175
|
+
params=_base._clean({"format": format, "chunk_size": chunk_size, "overlap": overlap}))
|
|
176
|
+
|
|
177
|
+
# ── page-change monitors ─────────────────────────────────────────────────────────────
|
|
178
|
+
async def monitors(self) -> dict:
|
|
179
|
+
"""{"monitors": [...], "limit": n, "intervals": [...]}"""
|
|
180
|
+
return await self.request("GET", "monitors")
|
|
181
|
+
|
|
182
|
+
async def create_monitor(self, url: str, *, interval_minutes: int = 1440, name: str | None = None,
|
|
183
|
+
selector: str | None = None, ignore_patterns: Sequence[str] | None = None,
|
|
184
|
+
min_change_chars: int | None = None, notify_email: bool | None = None,
|
|
185
|
+
js_render: bool | None = None, use_proxy: bool | None = None) -> dict:
|
|
186
|
+
"""Watch a page (optionally one CSS selector of it) and get alerted when it changes."""
|
|
187
|
+
return await self.request("POST", "monitors", json=_base._clean({
|
|
188
|
+
"url": url, "interval_minutes": interval_minutes, "name": name, "selector": selector,
|
|
189
|
+
"ignore_patterns": list(ignore_patterns) if ignore_patterns else None, "min_change_chars": min_change_chars,
|
|
190
|
+
"notify_email": notify_email, "js_render": js_render, "use_proxy": use_proxy}))
|
|
191
|
+
|
|
192
|
+
async def get_monitor(self, monitor_id: str) -> dict:
|
|
193
|
+
return await self.request("GET", f"monitors/{monitor_id}")
|
|
194
|
+
|
|
195
|
+
async def update_monitor(self, monitor_id: str, **changes) -> dict:
|
|
196
|
+
"""Change any setting; status="paused" / "active" pauses or resumes."""
|
|
197
|
+
return await self.request("PATCH", f"monitors/{monitor_id}", json=_base._clean(changes))
|
|
198
|
+
|
|
199
|
+
async def delete_monitor(self, monitor_id: str) -> dict:
|
|
200
|
+
return await self.request("DELETE", f"monitors/{monitor_id}")
|
|
201
|
+
|
|
202
|
+
async def check_monitor(self, monitor_id: str) -> dict:
|
|
203
|
+
"""Check now. Returns {"changed": bool, "summary"?, "monitor": {...}}."""
|
|
204
|
+
return await self.request("POST", f"monitors/{monitor_id}/check")
|
|
205
|
+
|
|
206
|
+
async def monitor_changes(self, monitor_id: str, *, limit: int = 20) -> list:
|
|
207
|
+
"""Detected changes, newest first, each with a unified "diff"."""
|
|
208
|
+
return (await self.request("GET", f"monitors/{monitor_id}/changes", params={"limit": limit}))["changes"]
|
|
209
|
+
|
|
210
|
+
# ── AI Extraction jobs (your own AI provider key; many URLs, sitemaps, whole sites) ─────
|
|
211
|
+
async def ai_providers(self) -> dict:
|
|
212
|
+
"""Your connected AI providers (keys masked) and the default one."""
|
|
213
|
+
return await self.request("GET", "ai/providers")
|
|
214
|
+
|
|
215
|
+
async def ai_job(self, urls: Sequence[str] | None = None, *, instructions: str = "", schema: dict | None = None,
|
|
216
|
+
fields: list | None = None, schema_id: str | None = None, records: str = "single",
|
|
217
|
+
provider: str | None = None, model: str | None = None, sitemap: str | None = None,
|
|
218
|
+
site: str | None = None, name: str | None = None, project: str | None = None,
|
|
219
|
+
wait: bool = False, timeout: float = 3600, **options) -> dict:
|
|
220
|
+
"""Start an AI extraction job. Give urls, or sitemap=..., or site=... (discover its pages).
|
|
221
|
+
records="list" extracts every item on each page. options: temperature, max_tokens,
|
|
222
|
+
missing_value, dedupe_on, js_render, use_proxy. wait=True blocks until the job ends."""
|
|
223
|
+
if sitemap or site:
|
|
224
|
+
source = {"type": "sitemap" if sitemap else "discover", "value": sitemap or site}
|
|
225
|
+
else:
|
|
226
|
+
source = {"type": "urls"}
|
|
227
|
+
body = _base._clean({"urls": list(urls or []), "source": source, "instructions": instructions,
|
|
228
|
+
"json_schema": schema, "fields": fields, "schema_id": schema_id, "records": records,
|
|
229
|
+
"provider": provider, "model": model, "name": name, "project": project,
|
|
230
|
+
"mode": "bulk" if urls and len(urls) > 1 else ("multipage" if site else "natural"),
|
|
231
|
+
"options": _base._clean(options) or None})
|
|
232
|
+
job = await self.request("POST", "ai/jobs", json=body, idempotent_key=_base.idempotency_key())
|
|
233
|
+
return (await self.wait_for_ai_job(job["job_id"], timeout=timeout)) if wait else job
|
|
234
|
+
|
|
235
|
+
async def get_ai_job(self, job_id: str) -> dict:
|
|
236
|
+
return await self.request("GET", f"ai/jobs/{job_id}")
|
|
237
|
+
|
|
238
|
+
async def wait_for_ai_job(self, job_id: str, *, timeout: float = 3600, poll_interval: float = 2.0) -> dict:
|
|
239
|
+
"""Poll until completed / partially_completed / failed / cancelled (never raises on failure)."""
|
|
240
|
+
deadline = time.monotonic() + timeout
|
|
241
|
+
while True:
|
|
242
|
+
job = await self.get_ai_job(job_id)
|
|
243
|
+
if job.get("status") in ("completed", "partially_completed", "failed", "cancelled"):
|
|
244
|
+
return job
|
|
245
|
+
if time.monotonic() + poll_interval > deadline:
|
|
246
|
+
raise _errors.JobTimeoutError(f"AI job {job_id} still {job.get('status')} after {timeout}s", job,
|
|
247
|
+
code="JOB_TIMEOUT")
|
|
248
|
+
await asyncio.sleep(poll_interval)
|
|
249
|
+
poll_interval = min(poll_interval * 1.3, 10.0)
|
|
250
|
+
|
|
251
|
+
async def ai_job_records(self, job_id: str, *, valid: str | None = None, page_size: int = 200) -> list:
|
|
252
|
+
"""Every record of a job (each: data, valid, errors, url, inferred). valid: "valid" | "invalid" | "missing"."""
|
|
253
|
+
out, offset = [], 0
|
|
254
|
+
while True:
|
|
255
|
+
r = await self.request("GET", f"ai/jobs/{job_id}/records",
|
|
256
|
+
params=_base._clean({"offset": offset, "limit": page_size, "valid": valid}))
|
|
257
|
+
out += r.get("records") or []
|
|
258
|
+
offset += len(r.get("records") or [])
|
|
259
|
+
if not r.get("records") or offset >= int(r.get("total") or 0):
|
|
260
|
+
return out
|
|
261
|
+
|
|
262
|
+
async def ai_job_export(self, job_id: str, format: str = "json") -> bytes:
|
|
263
|
+
"""The job's records as a file: "json", "csv" or "xlsx" (bytes)."""
|
|
264
|
+
return await self.request("GET", f"ai/jobs/{job_id}/export", params={"format": format}, raw="bytes")
|
|
265
|
+
|
|
266
|
+
async def cancel_ai_job(self, job_id: str) -> dict:
|
|
267
|
+
return await self.request("POST", f"ai/jobs/{job_id}/cancel")
|
|
268
|
+
|
|
269
|
+
async def retry_ai_job(self, job_id: str) -> dict:
|
|
270
|
+
"""Retry only the pages that failed."""
|
|
271
|
+
return await self.request("POST", f"ai/jobs/{job_id}/retry")
|
|
272
|
+
|
|
156
273
|
# ── account ───────────────────────────────────────────────────────────────────────────
|
|
157
274
|
async def usage(self) -> dict:
|
|
158
275
|
return await self.request("GET", "usage")
|
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
"""`scrapecore` command line: scrape, map, crawl, screenshot, AI extract, monitors, usage.
|
|
2
|
+
|
|
3
|
+
scrapecore login save an API key (~/.config/scrapecore/config.json, mode 600)
|
|
4
|
+
scrapecore scrape https://example.com the page as Markdown
|
|
5
|
+
scrapecore crawl https://docs.example.com --limit 50 --export llms -o llms.txt
|
|
6
|
+
|
|
7
|
+
The API key comes from --api-key, then SCRAPECORE_API_KEY, then the saved config.
|
|
8
|
+
Exit codes: 0 ok, 1 API or usage error, 2 bad arguments, 130 interrupted.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import base64
|
|
14
|
+
import getpass
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from ._errors import ScrapeCoreError
|
|
22
|
+
from ._version import __version__
|
|
23
|
+
|
|
24
|
+
CONFIG = Path(os.environ.get("SCRAPECORE_CONFIG") or
|
|
25
|
+
Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") / "scrapecore" / "config.json")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# ── config ──────────────────────────────────────────────────────────────────────────────
|
|
29
|
+
def load_config() -> dict:
|
|
30
|
+
try:
|
|
31
|
+
return json.loads(CONFIG.read_text())
|
|
32
|
+
except (OSError, ValueError):
|
|
33
|
+
return {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def save_config(cfg: dict) -> None:
|
|
37
|
+
CONFIG.parent.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
tmp = CONFIG.with_suffix(".tmp")
|
|
39
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
40
|
+
with os.fdopen(fd, "w") as f:
|
|
41
|
+
json.dump(cfg, f, indent=2)
|
|
42
|
+
os.replace(tmp, CONFIG)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def client(args):
|
|
46
|
+
from ._client import ScrapeCore
|
|
47
|
+
cfg = load_config()
|
|
48
|
+
key = args.api_key or os.environ.get("SCRAPECORE_API_KEY") or cfg.get("api_key")
|
|
49
|
+
if not key:
|
|
50
|
+
die("No API key. Run `scrapecore login`, set SCRAPECORE_API_KEY, or pass --api-key.")
|
|
51
|
+
return ScrapeCore(key, base_url=args.base_url or os.environ.get("SCRAPECORE_BASE_URL") or cfg.get("base_url"),
|
|
52
|
+
user_agent=f"scrapecore-cli/{__version__}")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ── output ──────────────────────────────────────────────────────────────────────────────
|
|
56
|
+
def die(msg: str, code: int = 1):
|
|
57
|
+
print(f"error: {msg}", file=sys.stderr)
|
|
58
|
+
sys.exit(code)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def info(msg: str):
|
|
62
|
+
print(msg, file=sys.stderr)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def out(args, data: Any, text: str | None = None):
|
|
66
|
+
"""--json prints the API response; otherwise `text` (or pretty JSON). -o writes to a file."""
|
|
67
|
+
body = json.dumps(data, indent=2, ensure_ascii=False) if (args.json or text is None) else text
|
|
68
|
+
dest = getattr(args, "output", None)
|
|
69
|
+
if dest:
|
|
70
|
+
Path(dest).write_text(body if body.endswith("\n") else body + "\n", encoding="utf-8")
|
|
71
|
+
info(f"saved {dest}")
|
|
72
|
+
else:
|
|
73
|
+
print(body)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def table(rows: list[list], headers: list[str]) -> str:
|
|
77
|
+
rows = [[("" if c is None else str(c)) for c in r] for r in rows]
|
|
78
|
+
w = [max(len(h), *(len(r[i]) for r in rows)) if rows else len(h) for i, h in enumerate(headers)]
|
|
79
|
+
line = lambda r: " ".join(c.ljust(w[i]) for i, c in enumerate(r)).rstrip() # noqa: E731
|
|
80
|
+
return "\n".join([line(headers), line(["-" * x for x in w]), *map(line, rows)])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ── commands ────────────────────────────────────────────────────────────────────────────
|
|
84
|
+
def cmd_login(args):
|
|
85
|
+
key = args.api_key or os.environ.get("SCRAPECORE_API_KEY")
|
|
86
|
+
if not key:
|
|
87
|
+
info("Create a key at https://scrapecore.com/dashboard/api-keys")
|
|
88
|
+
key = getpass.getpass("API key: ").strip()
|
|
89
|
+
if not key:
|
|
90
|
+
die("no key entered", 2)
|
|
91
|
+
from ._client import ScrapeCore
|
|
92
|
+
base = args.base_url or os.environ.get("SCRAPECORE_BASE_URL")
|
|
93
|
+
with ScrapeCore(key, base_url=base, user_agent=f"scrapecore-cli/{__version__}") as sc:
|
|
94
|
+
u = sc.usage() # proves the key works
|
|
95
|
+
cfg = load_config()
|
|
96
|
+
cfg["api_key"] = key
|
|
97
|
+
if base:
|
|
98
|
+
cfg["base_url"] = base
|
|
99
|
+
save_config(cfg)
|
|
100
|
+
left = (u.get("credits_limit") or 0) - (u.get("credits_used") or 0) if "credits_limit" in u else None
|
|
101
|
+
info(f"Logged in. Key saved to {CONFIG}" + (f" · {left:,} credits left" if isinstance(left, int) else ""))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def cmd_logout(args):
|
|
105
|
+
cfg = load_config()
|
|
106
|
+
if cfg.pop("api_key", None):
|
|
107
|
+
save_config(cfg)
|
|
108
|
+
info("Logged out (saved key removed).")
|
|
109
|
+
else:
|
|
110
|
+
info("No saved key.")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def cmd_scrape(args):
|
|
114
|
+
with client(args) as sc:
|
|
115
|
+
r = sc.scrape(args.url, format=args.format, js_render=args.js or None, use_proxy=args.proxy or None,
|
|
116
|
+
max_age=args.max_age)
|
|
117
|
+
out(args, r, r.get(args.format) or r.get("html") or r.get("text") or "")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def cmd_map(args):
|
|
121
|
+
with client(args) as sc:
|
|
122
|
+
r = sc.map(args.url, limit=args.limit, search=args.search, include_subdomains=args.subdomains or None)
|
|
123
|
+
out(args, r, "\n".join(r.get("urls") or []))
|
|
124
|
+
info(f"{r.get('count', len(r.get('urls') or []))} URL(s)")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def cmd_crawl(args):
|
|
128
|
+
with client(args) as sc:
|
|
129
|
+
job = sc.crawl(args.url, max_depth=args.depth, limit=args.limit, include_paths=args.include,
|
|
130
|
+
exclude_paths=args.exclude, js_render=args.js or None, use_proxy=args.proxy or None)
|
|
131
|
+
jid = job["job_id"]
|
|
132
|
+
info(f"crawl {jid} queued (up to {job.get('max_credits', '?')} credits)")
|
|
133
|
+
if args.no_wait:
|
|
134
|
+
return out(args, job, jid)
|
|
135
|
+
done = sc.wait_for_job(jid, timeout=args.timeout, raise_on_failure=False)
|
|
136
|
+
r = done.get("result") or {}
|
|
137
|
+
info(f"{done['status']}: {r.get('pages_ok', 0)} page(s) ok, {r.get('pages_failed', 0)} failed, "
|
|
138
|
+
f"{r.get('credits_used', 0)} credits")
|
|
139
|
+
if done["status"] != "success" and not r.get("pages_ok"):
|
|
140
|
+
die(done.get("error") or r.get("error") or "crawl failed")
|
|
141
|
+
out(args, None, sc.crawl_export(jid, args.export, chunk_size=args.chunk_size)) if not args.json \
|
|
142
|
+
else out(args, done)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def cmd_export(args):
|
|
146
|
+
with client(args) as sc:
|
|
147
|
+
out(args, None, sc.crawl_export(args.job_id, args.format, chunk_size=args.chunk_size, overlap=args.overlap))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def cmd_screenshot(args):
|
|
151
|
+
opts = {k: v for k, v in {"device": args.device, "format": args.image_format}.items() if v}
|
|
152
|
+
with client(args) as sc:
|
|
153
|
+
r = sc.screenshot(args.url, full_page=args.full_page, **opts)
|
|
154
|
+
if args.json:
|
|
155
|
+
return out(args, r)
|
|
156
|
+
img = r.get("image_base64")
|
|
157
|
+
if not img:
|
|
158
|
+
die(r.get("error") or "no image returned")
|
|
159
|
+
dest = args.output or f"screenshot.{args.image_format or 'png'}"
|
|
160
|
+
Path(dest).write_bytes(base64.b64decode(img))
|
|
161
|
+
info(f"saved {dest}")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def cmd_extract(args):
|
|
165
|
+
with client(args) as sc:
|
|
166
|
+
if args.selectors:
|
|
167
|
+
try:
|
|
168
|
+
sel = json.loads(args.selectors)
|
|
169
|
+
except ValueError:
|
|
170
|
+
die('--selectors must be JSON, e.g. \'{"title": "h1"}\'', 2)
|
|
171
|
+
r = sc.extract(args.url, sel, js_render=args.js or None)
|
|
172
|
+
return out(args, r, json.dumps(r.get("data", r), indent=2, ensure_ascii=False))
|
|
173
|
+
schema = None
|
|
174
|
+
if args.schema:
|
|
175
|
+
try:
|
|
176
|
+
schema = json.loads(Path(args.schema).read_text() if os.path.exists(args.schema) else args.schema)
|
|
177
|
+
except ValueError:
|
|
178
|
+
die("--schema must be a JSON file or a JSON string", 2)
|
|
179
|
+
if not (args.prompt or schema):
|
|
180
|
+
die("give --prompt and/or --schema (AI extraction), or --selectors (CSS)", 2)
|
|
181
|
+
r = sc.ai_extract(args.url, prompt=args.prompt, schema=schema, model=args.model, js_render=args.js or None)
|
|
182
|
+
out(args, r, json.dumps(r.get("data"), indent=2, ensure_ascii=False))
|
|
183
|
+
if r.get("usage") and not args.json:
|
|
184
|
+
info(f"{r.get('model')} · {r['usage'].get('input_tokens', 0)} in / {r['usage'].get('output_tokens', 0)} out "
|
|
185
|
+
f"tokens (your Anthropic key) · {r.get('credits_used', 0)} credits")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def cmd_ai(args):
|
|
189
|
+
"""AI Extraction job: many URLs / a sitemap / a whole site, with your own AI provider key."""
|
|
190
|
+
urls = list(args.urls or [])
|
|
191
|
+
if args.file:
|
|
192
|
+
import re
|
|
193
|
+
text = Path(args.file).read_text(encoding="utf-8", errors="ignore")
|
|
194
|
+
urls += re.findall(r"https?://[^\s,;\"'<>]+", text)
|
|
195
|
+
schema = None
|
|
196
|
+
if args.schema:
|
|
197
|
+
try:
|
|
198
|
+
schema = json.loads(Path(args.schema).read_text() if os.path.exists(args.schema) else args.schema)
|
|
199
|
+
except ValueError:
|
|
200
|
+
die("--schema must be a JSON file or a JSON string", 2)
|
|
201
|
+
if not (urls or args.sitemap or args.site):
|
|
202
|
+
die("give URLs, --file, --sitemap or --site", 2)
|
|
203
|
+
if not (args.prompt or schema):
|
|
204
|
+
die("give --prompt and/or --schema", 2)
|
|
205
|
+
with client(args) as sc:
|
|
206
|
+
job = sc.ai_job(urls or None, instructions=args.prompt or "", schema=schema, sitemap=args.sitemap, site=args.site,
|
|
207
|
+
records="list" if args.many else "single", provider=args.provider, model=args.model,
|
|
208
|
+
name=args.name, project=args.project)
|
|
209
|
+
jid = job["job_id"]
|
|
210
|
+
info(f"AI job {jid} queued: {job.get('provider')} / {job.get('model')}")
|
|
211
|
+
if args.no_wait:
|
|
212
|
+
return out(args, job, jid)
|
|
213
|
+
last = None
|
|
214
|
+
import time as _t
|
|
215
|
+
while True:
|
|
216
|
+
j = sc.get_ai_job(jid)
|
|
217
|
+
t = j.get("totals") or {}
|
|
218
|
+
line = f"{j['status']}: {t.get('ok', 0) + t.get('failed', 0)}/{t.get('urls', 0)} pages, {t.get('records', 0)} records"
|
|
219
|
+
if line != last:
|
|
220
|
+
info(line)
|
|
221
|
+
last = line
|
|
222
|
+
if j["status"] in ("completed", "partially_completed", "failed", "cancelled"):
|
|
223
|
+
break
|
|
224
|
+
_t.sleep(3)
|
|
225
|
+
if j["status"] == "failed" and not t.get("records"):
|
|
226
|
+
die(j.get("error") or "the job failed")
|
|
227
|
+
if args.json:
|
|
228
|
+
return out(args, sc.ai_job_records(jid))
|
|
229
|
+
fmt = args.format
|
|
230
|
+
data = sc.ai_job_export(jid, fmt)
|
|
231
|
+
if fmt == "xlsx" and not args.output:
|
|
232
|
+
args.output = f"{jid}.xlsx"
|
|
233
|
+
if args.output:
|
|
234
|
+
Path(args.output).write_bytes(data)
|
|
235
|
+
info(f"saved {args.output}")
|
|
236
|
+
else:
|
|
237
|
+
sys.stdout.write(data.decode("utf-8-sig"))
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def cmd_monitor(args):
|
|
243
|
+
with client(args) as sc:
|
|
244
|
+
a = args.action
|
|
245
|
+
if a == "list":
|
|
246
|
+
r = sc.monitors()
|
|
247
|
+
rows = [[m["id"], m["status"], m["interval_minutes"], m["changes"], m.get("last_changed_at") or "", m["url"]]
|
|
248
|
+
for m in r["monitors"]]
|
|
249
|
+
return out(args, r, table(rows, ["ID", "STATUS", "EVERY(MIN)", "CHANGES", "LAST CHANGE", "URL"])
|
|
250
|
+
+ f"\n{len(rows)} / {r['limit']} monitors")
|
|
251
|
+
if a == "add":
|
|
252
|
+
if not args.target:
|
|
253
|
+
die("usage: scrapecore monitor add URL", 2)
|
|
254
|
+
m = sc.create_monitor(args.target, interval_minutes=args.every, selector=args.selector, name=args.name)
|
|
255
|
+
return out(args, m, f"monitor {m['id']} created: first check sets the baseline")
|
|
256
|
+
if not args.target:
|
|
257
|
+
die(f"usage: scrapecore monitor {a} MONITOR_ID", 2)
|
|
258
|
+
if a == "check":
|
|
259
|
+
r = sc.check_monitor(args.target)
|
|
260
|
+
msg = (f"changed: {r.get('summary')}" if r.get("changed") else "baseline saved" if r.get("baseline")
|
|
261
|
+
else "no change" if r.get("ok") else f"check failed: {r.get('error')}")
|
|
262
|
+
return out(args, r, msg)
|
|
263
|
+
if a == "changes":
|
|
264
|
+
ch = sc.monitor_changes(args.target, limit=args.limit or 5)
|
|
265
|
+
return out(args, ch, "\n\n".join(f"# {c['created_at']} {c['summary']}\n{c['diff']}" for c in ch) or "no changes yet")
|
|
266
|
+
if a in ("pause", "resume"):
|
|
267
|
+
m = sc.update_monitor(args.target, status="paused" if a == "pause" else "active")
|
|
268
|
+
return out(args, m, f"monitor {m['id']} {m['status']}")
|
|
269
|
+
if a == "rm":
|
|
270
|
+
sc.delete_monitor(args.target)
|
|
271
|
+
return out(args, {"ok": True}, f"monitor {args.target} deleted")
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def cmd_jobs(args):
|
|
275
|
+
with client(args) as sc:
|
|
276
|
+
if args.job_id:
|
|
277
|
+
j = sc.get_job(args.job_id)
|
|
278
|
+
return out(args, j)
|
|
279
|
+
r = sc.jobs(limit=args.limit or 20, status=args.status, type=args.type)
|
|
280
|
+
rows = r if isinstance(r, list) else r.get("jobs", [])
|
|
281
|
+
out(args, r, table([[j.get("job_id"), j.get("type"), j.get("status"), j.get("created_at"), j.get("url")] for j in rows],
|
|
282
|
+
["JOB", "TYPE", "STATUS", "CREATED", "URL"]))
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def cmd_usage(args):
|
|
286
|
+
with client(args) as sc:
|
|
287
|
+
u = sc.usage()
|
|
288
|
+
lines = [f"{k.replace('_', ' ')}: {v:,}" if isinstance(v, int) else f"{k.replace('_', ' ')}: {v}"
|
|
289
|
+
for k, v in u.items() if not isinstance(v, (dict, list))]
|
|
290
|
+
out(args, u, "\n".join(lines))
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# ── parser ──────────────────────────────────────────────────────────────────────────────
|
|
294
|
+
def parser() -> argparse.ArgumentParser:
|
|
295
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
296
|
+
common.add_argument("--api-key", help="API key (default: SCRAPECORE_API_KEY or `scrapecore login`)")
|
|
297
|
+
common.add_argument("--base-url", help=argparse.SUPPRESS)
|
|
298
|
+
common.add_argument("--json", action="store_true", help="print the raw API response as JSON")
|
|
299
|
+
common.add_argument("-o", "--output", help="write the result to a file")
|
|
300
|
+
|
|
301
|
+
p = argparse.ArgumentParser(prog="scrapecore", description="ScrapeCore from the command line.",
|
|
302
|
+
epilog="Docs: https://scrapecore.com/docs/cli")
|
|
303
|
+
p.add_argument("-V", "--version", action="version", version=f"scrapecore {__version__}")
|
|
304
|
+
sub = p.add_subparsers(dest="cmd", metavar="COMMAND")
|
|
305
|
+
|
|
306
|
+
def add(name, fn, help_):
|
|
307
|
+
sp = sub.add_parser(name, parents=[common], help=help_, description=help_)
|
|
308
|
+
sp.set_defaults(fn=fn)
|
|
309
|
+
return sp
|
|
310
|
+
|
|
311
|
+
add("login", cmd_login, "save an API key for later commands")
|
|
312
|
+
add("logout", cmd_logout, "remove the saved API key")
|
|
313
|
+
s = add("scrape", cmd_scrape, "scrape one page")
|
|
314
|
+
s.add_argument("url")
|
|
315
|
+
s.add_argument("-f", "--format", choices=["markdown", "html", "text"], default="markdown")
|
|
316
|
+
s.add_argument("--js", action="store_true", help="render JavaScript")
|
|
317
|
+
s.add_argument("--proxy", action="store_true", help="use a proxy")
|
|
318
|
+
s.add_argument("--max-age", type=int, help="accept a stored result up to this many seconds old")
|
|
319
|
+
s = add("map", cmd_map, "list every URL of a site")
|
|
320
|
+
s.add_argument("url")
|
|
321
|
+
s.add_argument("--limit", type=int)
|
|
322
|
+
s.add_argument("--search", help="only URLs containing this text")
|
|
323
|
+
s.add_argument("--subdomains", action="store_true")
|
|
324
|
+
s = add("crawl", cmd_crawl, "crawl a site and print it as an AI-ready export")
|
|
325
|
+
s.add_argument("url")
|
|
326
|
+
s.add_argument("--depth", type=int, help="link depth (default 2)")
|
|
327
|
+
s.add_argument("--limit", type=int, help="max pages (default 50)")
|
|
328
|
+
s.add_argument("--include", action="append", help="path pattern to include (repeatable)")
|
|
329
|
+
s.add_argument("--exclude", action="append", help="path pattern to exclude (repeatable)")
|
|
330
|
+
s.add_argument("--js", action="store_true")
|
|
331
|
+
s.add_argument("--proxy", action="store_true")
|
|
332
|
+
s.add_argument("--export", choices=["jsonl", "markdown", "chunks", "llms", "llms-full"], default="markdown")
|
|
333
|
+
s.add_argument("--chunk-size", type=int)
|
|
334
|
+
s.add_argument("--timeout", type=float, default=1800)
|
|
335
|
+
s.add_argument("--no-wait", action="store_true", help="print the job id and exit")
|
|
336
|
+
s = add("export", cmd_export, "export a finished crawl")
|
|
337
|
+
s.add_argument("job_id")
|
|
338
|
+
s.add_argument("-f", "--format", choices=["jsonl", "markdown", "chunks", "llms", "llms-full"], default="jsonl")
|
|
339
|
+
s.add_argument("--chunk-size", type=int)
|
|
340
|
+
s.add_argument("--overlap", type=int)
|
|
341
|
+
s = add("screenshot", cmd_screenshot, "save a screenshot")
|
|
342
|
+
s.add_argument("url")
|
|
343
|
+
s.add_argument("--full-page", action="store_true")
|
|
344
|
+
s.add_argument("--device", help="desktop, desktop_hd, laptop, tablet or mobile")
|
|
345
|
+
s.add_argument("--image-format", choices=["png", "jpeg"])
|
|
346
|
+
s = add("extract", cmd_extract, "structured data: AI (--prompt/--schema, your Anthropic key) or CSS (--selectors)")
|
|
347
|
+
s.add_argument("url")
|
|
348
|
+
s.add_argument("-p", "--prompt")
|
|
349
|
+
s.add_argument("--schema", help="JSON schema (file path or JSON string)")
|
|
350
|
+
s.add_argument("--model")
|
|
351
|
+
s.add_argument("--selectors", help='CSS selectors as JSON, e.g. \'{"title": "h1"}\'')
|
|
352
|
+
s.add_argument("--js", action="store_true")
|
|
353
|
+
s = add("ai", cmd_ai, "AI extraction job over many pages (your own AI provider key)")
|
|
354
|
+
s.add_argument("urls", nargs="*", help="page URLs")
|
|
355
|
+
s.add_argument("-p", "--prompt", help="what to extract, in plain words")
|
|
356
|
+
s.add_argument("--schema", help="JSON schema or {field: type} map (file path or JSON string)")
|
|
357
|
+
s.add_argument("--file", help="CSV/TXT file with URLs")
|
|
358
|
+
s.add_argument("--sitemap", help="a sitemap URL")
|
|
359
|
+
s.add_argument("--site", help="discover and process a whole site")
|
|
360
|
+
s.add_argument("--many", action="store_true", help="many records per page (listings)")
|
|
361
|
+
s.add_argument("--provider", help="openai, anthropic, gemini, mistral, ... (default: your default)")
|
|
362
|
+
s.add_argument("--model")
|
|
363
|
+
s.add_argument("--name")
|
|
364
|
+
s.add_argument("--project")
|
|
365
|
+
s.add_argument("-f", "--format", choices=["json", "csv", "xlsx"], default="json")
|
|
366
|
+
s.add_argument("--no-wait", action="store_true")
|
|
367
|
+
s = add("monitor", cmd_monitor, "page-change monitors: list | add URL | check ID | changes ID | pause ID | resume ID | rm ID")
|
|
368
|
+
s.add_argument("action", choices=["list", "add", "check", "changes", "pause", "resume", "rm"])
|
|
369
|
+
s.add_argument("target", nargs="?", help="URL (add) or monitor id")
|
|
370
|
+
s.add_argument("--every", type=int, default=1440, help="minutes between checks (15, 60, 360, 1440, 10080)")
|
|
371
|
+
s.add_argument("--selector", help="only watch this CSS selector")
|
|
372
|
+
s.add_argument("--name")
|
|
373
|
+
s.add_argument("--limit", type=int)
|
|
374
|
+
s = add("jobs", cmd_jobs, "list async jobs, or show one")
|
|
375
|
+
s.add_argument("job_id", nargs="?")
|
|
376
|
+
s.add_argument("--limit", type=int)
|
|
377
|
+
s.add_argument("--status")
|
|
378
|
+
s.add_argument("--type")
|
|
379
|
+
add("usage", cmd_usage, "credits and limits")
|
|
380
|
+
return p
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def main(argv: list[str] | None = None) -> int:
|
|
384
|
+
p = parser()
|
|
385
|
+
args = p.parse_args(argv)
|
|
386
|
+
if not getattr(args, "fn", None):
|
|
387
|
+
p.print_help()
|
|
388
|
+
return 2
|
|
389
|
+
try:
|
|
390
|
+
args.fn(args)
|
|
391
|
+
except ScrapeCoreError as e:
|
|
392
|
+
code = getattr(e, "code", None)
|
|
393
|
+
die(f"{e}" + (f" [{code}]" if code and code not in str(e) else ""))
|
|
394
|
+
except KeyboardInterrupt:
|
|
395
|
+
return 130
|
|
396
|
+
except BrokenPipeError:
|
|
397
|
+
return 0
|
|
398
|
+
return 0
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
if __name__ == "__main__":
|
|
402
|
+
sys.exit(main())
|
|
@@ -39,8 +39,8 @@ class ScrapeCore:
|
|
|
39
39
|
self.close()
|
|
40
40
|
|
|
41
41
|
def request(self, method: str, path: str, *, json: Any = None, params: dict | None = None,
|
|
42
|
-
idempotent_key: str | None = None) -> Any:
|
|
43
|
-
"""Any /api/v1 endpoint. Retries what is safe to retry."""
|
|
42
|
+
idempotent_key: str | None = None, raw: bool | str = False) -> Any:
|
|
43
|
+
"""Any /api/v1 endpoint. Retries what is safe to retry. raw=True returns the body as text."""
|
|
44
44
|
headers = dict(self._s.headers)
|
|
45
45
|
if idempotent_key:
|
|
46
46
|
headers["Idempotency-Key"] = idempotent_key
|
|
@@ -49,6 +49,8 @@ class ScrapeCore:
|
|
|
49
49
|
try:
|
|
50
50
|
r = self._http.request(method, self._s.url(path), json=json, params=params, headers=headers,
|
|
51
51
|
timeout=self._s.timeout)
|
|
52
|
+
if raw and r.status_code < 400:
|
|
53
|
+
return r.content if raw == "bytes" else r.text
|
|
52
54
|
return _base.parse(r.status_code, r.headers, r.content)
|
|
53
55
|
except _errors.ScrapeCoreError as e:
|
|
54
56
|
err = e
|
|
@@ -160,6 +162,121 @@ class ScrapeCore:
|
|
|
160
162
|
return self.request("GET", "jobs", params=_base._clean({"limit": limit, "offset": offset,
|
|
161
163
|
"status": status, "type": type}))
|
|
162
164
|
|
|
165
|
+
# ── AI extraction (your own AI provider key) ───────────────────────
|
|
166
|
+
def ai_extract(self, url: str, *, prompt: str | None = None, schema: dict | None = None, model: str | None = None,
|
|
167
|
+
provider: str | None = None, records: str | None = None,
|
|
168
|
+
js_render: bool | None = None, use_proxy: bool | None = None) -> dict:
|
|
169
|
+
"""Structured data from ONE page with your own AI provider key (any provider connected in
|
|
170
|
+
Dashboard → AI Extraction: OpenAI, Anthropic, Gemini, Mistral, ...; default = your default one).
|
|
171
|
+
Give a prompt, a JSON schema, or both. records="list" returns every item. Returns {"data", ...}.
|
|
172
|
+
For many pages, sitemaps or whole sites use ai_job()."""
|
|
173
|
+
return self.request("POST", "extract/ai", json=_base._clean({"url": url, "prompt": prompt, "schema": schema,
|
|
174
|
+
"model": model, "provider": provider, "records": records, "js_render": js_render,
|
|
175
|
+
"use_proxy": use_proxy}))
|
|
176
|
+
|
|
177
|
+
def crawl_export(self, job_id: str, format: str = "jsonl", *, chunk_size: int | None = None,
|
|
178
|
+
overlap: int | None = None) -> str:
|
|
179
|
+
"""A crawl as one AI-ready document: "jsonl", "markdown", "chunks" (RAG-sized JSONL),
|
|
180
|
+
"llms" (llms.txt) or "llms-full" (llms-full.txt)."""
|
|
181
|
+
return self.request("GET", f"crawl/{job_id}/export", raw=True,
|
|
182
|
+
params=_base._clean({"format": format, "chunk_size": chunk_size, "overlap": overlap}))
|
|
183
|
+
|
|
184
|
+
# ── page-change monitors ─────────────────────────────────────────────────────────────
|
|
185
|
+
def monitors(self) -> dict:
|
|
186
|
+
"""{"monitors": [...], "limit": n, "intervals": [...]}"""
|
|
187
|
+
return self.request("GET", "monitors")
|
|
188
|
+
|
|
189
|
+
def create_monitor(self, url: str, *, interval_minutes: int = 1440, name: str | None = None,
|
|
190
|
+
selector: str | None = None, ignore_patterns: Sequence[str] | None = None,
|
|
191
|
+
min_change_chars: int | None = None, notify_email: bool | None = None,
|
|
192
|
+
js_render: bool | None = None, use_proxy: bool | None = None) -> dict:
|
|
193
|
+
"""Watch a page (optionally one CSS selector of it) and get alerted when it changes."""
|
|
194
|
+
return self.request("POST", "monitors", json=_base._clean({
|
|
195
|
+
"url": url, "interval_minutes": interval_minutes, "name": name, "selector": selector,
|
|
196
|
+
"ignore_patterns": list(ignore_patterns) if ignore_patterns else None, "min_change_chars": min_change_chars,
|
|
197
|
+
"notify_email": notify_email, "js_render": js_render, "use_proxy": use_proxy}))
|
|
198
|
+
|
|
199
|
+
def get_monitor(self, monitor_id: str) -> dict:
|
|
200
|
+
return self.request("GET", f"monitors/{monitor_id}")
|
|
201
|
+
|
|
202
|
+
def update_monitor(self, monitor_id: str, **changes) -> dict:
|
|
203
|
+
"""Change any setting; status="paused" / "active" pauses or resumes."""
|
|
204
|
+
return self.request("PATCH", f"monitors/{monitor_id}", json=_base._clean(changes))
|
|
205
|
+
|
|
206
|
+
def delete_monitor(self, monitor_id: str) -> dict:
|
|
207
|
+
return self.request("DELETE", f"monitors/{monitor_id}")
|
|
208
|
+
|
|
209
|
+
def check_monitor(self, monitor_id: str) -> dict:
|
|
210
|
+
"""Check now. Returns {"changed": bool, "summary"?, "monitor": {...}}."""
|
|
211
|
+
return self.request("POST", f"monitors/{monitor_id}/check")
|
|
212
|
+
|
|
213
|
+
def monitor_changes(self, monitor_id: str, *, limit: int = 20) -> list:
|
|
214
|
+
"""Detected changes, newest first, each with a unified "diff"."""
|
|
215
|
+
return (self.request("GET", f"monitors/{monitor_id}/changes", params={"limit": limit}))["changes"]
|
|
216
|
+
|
|
217
|
+
# ── AI Extraction jobs (your own AI provider key; many URLs, sitemaps, whole sites) ─────
|
|
218
|
+
def ai_providers(self) -> dict:
|
|
219
|
+
"""Your connected AI providers (keys masked) and the default one."""
|
|
220
|
+
return self.request("GET", "ai/providers")
|
|
221
|
+
|
|
222
|
+
def ai_job(self, urls: Sequence[str] | None = None, *, instructions: str = "", schema: dict | None = None,
|
|
223
|
+
fields: list | None = None, schema_id: str | None = None, records: str = "single",
|
|
224
|
+
provider: str | None = None, model: str | None = None, sitemap: str | None = None,
|
|
225
|
+
site: str | None = None, name: str | None = None, project: str | None = None,
|
|
226
|
+
wait: bool = False, timeout: float = 3600, **options) -> dict:
|
|
227
|
+
"""Start an AI extraction job. Give urls, or sitemap=..., or site=... (discover its pages).
|
|
228
|
+
records="list" extracts every item on each page. options: temperature, max_tokens,
|
|
229
|
+
missing_value, dedupe_on, js_render, use_proxy. wait=True blocks until the job ends."""
|
|
230
|
+
if sitemap or site:
|
|
231
|
+
source = {"type": "sitemap" if sitemap else "discover", "value": sitemap or site}
|
|
232
|
+
else:
|
|
233
|
+
source = {"type": "urls"}
|
|
234
|
+
body = _base._clean({"urls": list(urls or []), "source": source, "instructions": instructions,
|
|
235
|
+
"json_schema": schema, "fields": fields, "schema_id": schema_id, "records": records,
|
|
236
|
+
"provider": provider, "model": model, "name": name, "project": project,
|
|
237
|
+
"mode": "bulk" if urls and len(urls) > 1 else ("multipage" if site else "natural"),
|
|
238
|
+
"options": _base._clean(options) or None})
|
|
239
|
+
job = self.request("POST", "ai/jobs", json=body, idempotent_key=_base.idempotency_key())
|
|
240
|
+
return (self.wait_for_ai_job(job["job_id"], timeout=timeout)) if wait else job
|
|
241
|
+
|
|
242
|
+
def get_ai_job(self, job_id: str) -> dict:
|
|
243
|
+
return self.request("GET", f"ai/jobs/{job_id}")
|
|
244
|
+
|
|
245
|
+
def wait_for_ai_job(self, job_id: str, *, timeout: float = 3600, poll_interval: float = 2.0) -> dict:
|
|
246
|
+
"""Poll until completed / partially_completed / failed / cancelled (never raises on failure)."""
|
|
247
|
+
deadline = time.monotonic() + timeout
|
|
248
|
+
while True:
|
|
249
|
+
job = self.get_ai_job(job_id)
|
|
250
|
+
if job.get("status") in ("completed", "partially_completed", "failed", "cancelled"):
|
|
251
|
+
return job
|
|
252
|
+
if time.monotonic() + poll_interval > deadline:
|
|
253
|
+
raise _errors.JobTimeoutError(f"AI job {job_id} still {job.get('status')} after {timeout}s", job,
|
|
254
|
+
code="JOB_TIMEOUT")
|
|
255
|
+
time.sleep(poll_interval)
|
|
256
|
+
poll_interval = min(poll_interval * 1.3, 10.0)
|
|
257
|
+
|
|
258
|
+
def ai_job_records(self, job_id: str, *, valid: str | None = None, page_size: int = 200) -> list:
|
|
259
|
+
"""Every record of a job (each: data, valid, errors, url, inferred). valid: "valid" | "invalid" | "missing"."""
|
|
260
|
+
out, offset = [], 0
|
|
261
|
+
while True:
|
|
262
|
+
r = self.request("GET", f"ai/jobs/{job_id}/records",
|
|
263
|
+
params=_base._clean({"offset": offset, "limit": page_size, "valid": valid}))
|
|
264
|
+
out += r.get("records") or []
|
|
265
|
+
offset += len(r.get("records") or [])
|
|
266
|
+
if not r.get("records") or offset >= int(r.get("total") or 0):
|
|
267
|
+
return out
|
|
268
|
+
|
|
269
|
+
def ai_job_export(self, job_id: str, format: str = "json") -> bytes:
|
|
270
|
+
"""The job's records as a file: "json", "csv" or "xlsx" (bytes)."""
|
|
271
|
+
return self.request("GET", f"ai/jobs/{job_id}/export", params={"format": format}, raw="bytes")
|
|
272
|
+
|
|
273
|
+
def cancel_ai_job(self, job_id: str) -> dict:
|
|
274
|
+
return self.request("POST", f"ai/jobs/{job_id}/cancel")
|
|
275
|
+
|
|
276
|
+
def retry_ai_job(self, job_id: str) -> dict:
|
|
277
|
+
"""Retry only the pages that failed."""
|
|
278
|
+
return self.request("POST", f"ai/jobs/{job_id}/retry")
|
|
279
|
+
|
|
163
280
|
# ── account ───────────────────────────────────────────────────────────────────────────
|
|
164
281
|
def usage(self) -> dict:
|
|
165
282
|
return self.request("GET", "usage")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""0.2: AI extraction, crawl exports, monitors, and the `scrapecore` CLI."""
|
|
2
|
+
import base64
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from scrapecore import AsyncScrapeCore, ScrapeCore, _cli
|
|
9
|
+
|
|
10
|
+
KEY = "sk_test_" + "x" * 32
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def mk(handler, cls=ScrapeCore):
|
|
14
|
+
http = (httpx.AsyncClient if cls is AsyncScrapeCore else httpx.Client)(transport=httpx.MockTransport(handler))
|
|
15
|
+
return cls(KEY, base_url="https://api.test", http_client=http)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_ai_extract_export_monitors_paths():
|
|
19
|
+
seen = []
|
|
20
|
+
|
|
21
|
+
def h(req):
|
|
22
|
+
seen.append((req.method, req.url.path, req.url.query.decode(), json.loads(req.content) if req.content else None))
|
|
23
|
+
if req.url.path.endswith("/export"):
|
|
24
|
+
return httpx.Response(200, text="# Site\n\n> summary\n", headers={"content-type": "text/plain"})
|
|
25
|
+
if req.url.path.endswith("/changes"):
|
|
26
|
+
return httpx.Response(200, json={"changes": [{"diff": "-a\n+b"}]})
|
|
27
|
+
return httpx.Response(200, json={"ok": True})
|
|
28
|
+
sc = mk(h)
|
|
29
|
+
sc.ai_extract("https://x.example", prompt="price", schema={"type": "object"})
|
|
30
|
+
assert sc.crawl_export("j1", "llms") == "# Site\n\n> summary\n"
|
|
31
|
+
sc.create_monitor("https://x.example", interval_minutes=60, selector="#p")
|
|
32
|
+
sc.update_monitor("m1", status="paused")
|
|
33
|
+
assert sc.monitor_changes("m1", limit=2) == [{"diff": "-a\n+b"}]
|
|
34
|
+
assert seen[0][:2] == ("POST", "/api/v1/extract/ai") and seen[0][3] == {"url": "https://x.example", "prompt": "price",
|
|
35
|
+
"schema": {"type": "object"}}
|
|
36
|
+
assert seen[1][:3] == ("GET", "/api/v1/crawl/j1/export", "format=llms")
|
|
37
|
+
assert seen[2][3] == {"url": "https://x.example", "interval_minutes": 60, "selector": "#p"}
|
|
38
|
+
assert seen[3][:2] == ("PATCH", "/api/v1/monitors/m1") and seen[3][3] == {"status": "paused"}
|
|
39
|
+
assert seen[4][2] == "limit=2"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
async def test_async_parity():
|
|
43
|
+
async def h(req):
|
|
44
|
+
return httpx.Response(200, text="a\nb\n") if req.url.path.endswith("/export") else httpx.Response(200, json={"monitors": []})
|
|
45
|
+
sc = mk(h, AsyncScrapeCore)
|
|
46
|
+
assert await sc.crawl_export("j", "chunks", chunk_size=500) == "a\nb\n"
|
|
47
|
+
assert await sc.monitors() == {"monitors": []}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@pytest.fixture()
|
|
51
|
+
def cli(monkeypatch, tmp_path):
|
|
52
|
+
monkeypatch.setattr(_cli, "CONFIG", tmp_path / "cfg" / "config.json")
|
|
53
|
+
monkeypatch.delenv("SCRAPECORE_API_KEY", raising=False)
|
|
54
|
+
calls = []
|
|
55
|
+
routes = {}
|
|
56
|
+
|
|
57
|
+
def handler(req):
|
|
58
|
+
calls.append((req.method, req.url.path, req.headers.get("x-api-key"), req.headers.get("user-agent")))
|
|
59
|
+
for (m, suffix), resp in routes.items():
|
|
60
|
+
if req.method == m and req.url.path.endswith(suffix):
|
|
61
|
+
return resp(req) if callable(resp) else resp
|
|
62
|
+
return httpx.Response(404, json={"error": {"code": "NOT_FOUND", "message": "no route", "status": 404}})
|
|
63
|
+
real = ScrapeCore.__init__
|
|
64
|
+
|
|
65
|
+
def init(self, api_key=None, **kw):
|
|
66
|
+
kw["http_client"] = httpx.Client(transport=httpx.MockTransport(handler))
|
|
67
|
+
kw["base_url"] = "https://api.test"
|
|
68
|
+
real(self, api_key, **kw)
|
|
69
|
+
monkeypatch.setattr(ScrapeCore, "__init__", init)
|
|
70
|
+
return calls, routes
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_login_saves_key_privately_and_is_used(cli, monkeypatch, capsys):
|
|
74
|
+
calls, routes = cli
|
|
75
|
+
routes[("GET", "/usage")] = httpx.Response(200, json={"credits_used": 10, "credits_limit": 1000})
|
|
76
|
+
routes[("POST", "/scrape")] = httpx.Response(200, json={"markdown": "# Example"})
|
|
77
|
+
monkeypatch.setattr(_cli.getpass, "getpass", lambda prompt: KEY)
|
|
78
|
+
assert _cli.main(["login"]) == 0
|
|
79
|
+
cfg = _cli.CONFIG
|
|
80
|
+
assert json.loads(cfg.read_text())["api_key"] == KEY and (cfg.stat().st_mode & 0o777) == 0o600
|
|
81
|
+
assert "990 credits left" in capsys.readouterr().err
|
|
82
|
+
assert _cli.main(["scrape", "https://example.com"]) == 0
|
|
83
|
+
assert capsys.readouterr().out.strip() == "# Example"
|
|
84
|
+
assert calls[-1][2] == KEY and calls[-1][3].startswith("scrapecore-cli/")
|
|
85
|
+
assert _cli.main(["logout"]) == 0 and "api_key" not in json.loads(cfg.read_text())
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_no_key_is_a_clear_error(cli, capsys):
|
|
89
|
+
with pytest.raises(SystemExit) as e:
|
|
90
|
+
_cli.main(["scrape", "https://example.com"])
|
|
91
|
+
assert e.value.code == 1 and "scrapecore login" in capsys.readouterr().err
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_api_error_exit_code(cli, monkeypatch, capsys):
|
|
95
|
+
calls, routes = cli
|
|
96
|
+
monkeypatch.setenv("SCRAPECORE_API_KEY", KEY)
|
|
97
|
+
routes[("POST", "/scrape")] = httpx.Response(402, json={"error": {"code": "INSUFFICIENT_CREDITS",
|
|
98
|
+
"message": "Out of credits", "status": 402}})
|
|
99
|
+
with pytest.raises(SystemExit) as e:
|
|
100
|
+
_cli.main(["scrape", "https://example.com"])
|
|
101
|
+
assert e.value.code == 1 and "Out of credits" in capsys.readouterr().err
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_crawl_waits_and_exports(cli, monkeypatch, capsys, tmp_path):
|
|
105
|
+
calls, routes = cli
|
|
106
|
+
monkeypatch.setenv("SCRAPECORE_API_KEY", KEY)
|
|
107
|
+
monkeypatch.setattr("scrapecore._client.time.sleep", lambda s: None)
|
|
108
|
+
routes[("POST", "/crawl")] = httpx.Response(200, json={"job_id": "j9", "status": "queued", "max_credits": 50})
|
|
109
|
+
routes[("GET", "/job/j9")] = httpx.Response(200, json={"status": "success", "result": {"pages_ok": 3, "pages_failed": 0,
|
|
110
|
+
"credits_used": 3}})
|
|
111
|
+
routes[("GET", "/crawl/j9/export")] = httpx.Response(200, text="# llms\n")
|
|
112
|
+
out = tmp_path / "llms.txt"
|
|
113
|
+
assert _cli.main(["crawl", "https://docs.example", "--limit", "5", "--export", "llms", "-o", str(out)]) == 0
|
|
114
|
+
assert out.read_text() == "# llms\n" and "3 page(s) ok" in capsys.readouterr().err
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_screenshot_writes_png(cli, monkeypatch, tmp_path):
|
|
118
|
+
calls, routes = cli
|
|
119
|
+
monkeypatch.setenv("SCRAPECORE_API_KEY", KEY)
|
|
120
|
+
png = b"\x89PNG\r\n\x1a\nfake"
|
|
121
|
+
routes[("POST", "/screenshot")] = httpx.Response(200, json={"image_base64": base64.b64encode(png).decode()})
|
|
122
|
+
dest = tmp_path / "s.png"
|
|
123
|
+
assert _cli.main(["screenshot", "https://example.com", "-o", str(dest)]) == 0
|
|
124
|
+
assert dest.read_bytes() == png
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_monitor_list_table_and_json(cli, monkeypatch, capsys):
|
|
128
|
+
calls, routes = cli
|
|
129
|
+
monkeypatch.setenv("SCRAPECORE_API_KEY", KEY)
|
|
130
|
+
data = {"monitors": [{"id": "m1", "status": "active", "interval_minutes": 60, "changes": 2, "last_changed_at": None,
|
|
131
|
+
"url": "https://x.example"}], "limit": 10}
|
|
132
|
+
routes[("GET", "/monitors")] = httpx.Response(200, json=data)
|
|
133
|
+
assert _cli.main(["monitor", "list"]) == 0
|
|
134
|
+
text = capsys.readouterr().out
|
|
135
|
+
assert "m1" in text and "1 / 10 monitors" in text
|
|
136
|
+
assert _cli.main(["monitor", "list", "--json"]) == 0
|
|
137
|
+
assert json.loads(capsys.readouterr().out) == data
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_extract_needs_prompt_schema_or_selectors(cli, monkeypatch):
|
|
141
|
+
monkeypatch.setenv("SCRAPECORE_API_KEY", KEY)
|
|
142
|
+
with pytest.raises(SystemExit) as e:
|
|
143
|
+
_cli.main(["extract", "https://x.example"])
|
|
144
|
+
assert e.value.code == 2
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_ai_job_methods_and_cli(cli, monkeypatch, capsys, tmp_path):
|
|
148
|
+
calls, routes = cli
|
|
149
|
+
monkeypatch.setenv("SCRAPECORE_API_KEY", KEY)
|
|
150
|
+
seen = {}
|
|
151
|
+
|
|
152
|
+
def create(req):
|
|
153
|
+
seen["body"] = json.loads(req.content)
|
|
154
|
+
return httpx.Response(200, json={"job_id": "aij_1", "status": "queued", "provider": "openai", "model": "gpt-4.1"})
|
|
155
|
+
routes[("POST", "/ai/jobs")] = create
|
|
156
|
+
routes[("GET", "/ai/jobs/aij_1")] = httpx.Response(200, json={"job_id": "aij_1", "status": "completed",
|
|
157
|
+
"totals": {"ok": 2, "failed": 0, "urls": 2, "records": 3}})
|
|
158
|
+
routes[("GET", "/ai/jobs/aij_1/export")] = httpx.Response(200, content=b"name,price\nMug,12\n")
|
|
159
|
+
dest = tmp_path / "out.csv"
|
|
160
|
+
assert _cli.main(["ai", "https://a.example/1", "https://a.example/2", "-p", "products", "--many", "-f", "csv",
|
|
161
|
+
"--provider", "openai", "-o", str(dest)]) == 0
|
|
162
|
+
assert dest.read_bytes() == b"name,price\nMug,12\n"
|
|
163
|
+
b = seen["body"]
|
|
164
|
+
assert b["urls"] == ["https://a.example/1", "https://a.example/2"] and b["records"] == "list" and b["mode"] == "bulk"
|
|
165
|
+
assert b["provider"] == "openai" and b["instructions"] == "products"
|
|
166
|
+
assert "completed: 2/2 pages, 3 records" in capsys.readouterr().err
|
|
167
|
+
with pytest.raises(SystemExit) as e:
|
|
168
|
+
_cli.main(["ai", "-p", "x"])
|
|
169
|
+
assert e.value.code == 2
|
scrapecore_sdk-0.1.0/README.md
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
# ScrapeCore Python SDK
|
|
2
|
-
|
|
3
|
-
The official Python client for the [ScrapeCore](https://scrapecore.com) API (v1). You can scrape
|
|
4
|
-
pages to HTML, Markdown or text, map a site, crawl it, take screenshots, and extract data with CSS
|
|
5
|
-
selectors.
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
pip install scrapecore-sdk
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
```python
|
|
12
|
-
from scrapecore import ScrapeCore
|
|
13
|
-
|
|
14
|
-
sc = ScrapeCore() # reads SCRAPECORE_API_KEY (or pass api_key="sk_live_...")
|
|
15
|
-
|
|
16
|
-
page = sc.scrape("https://example.com", format="markdown")
|
|
17
|
-
print(page["markdown"], page["credits_used"], page["credits_remaining"])
|
|
18
|
-
|
|
19
|
-
urls = sc.map("https://example.com", limit=500)["urls"]
|
|
20
|
-
|
|
21
|
-
job = sc.crawl("https://example.com/docs", max_depth=2, limit=100, include_paths=["^/docs"], wait=True)
|
|
22
|
-
for p in sc.crawl_pages(job["job_id"], only_ok=True):
|
|
23
|
-
print(p["url"], p["title"], len(p["markdown"]))
|
|
24
|
-
|
|
25
|
-
shot = sc.screenshot("https://example.com", full_page=True, device="mobile") # shot["image_base64"]
|
|
26
|
-
data = sc.extract("https://example.com", {"title": "h1", "links": "a"})
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
Async:
|
|
30
|
-
|
|
31
|
-
```python
|
|
32
|
-
from scrapecore import AsyncScrapeCore
|
|
33
|
-
|
|
34
|
-
async with AsyncScrapeCore() as sc:
|
|
35
|
-
page = await sc.scrape("https://example.com", format="markdown")
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
## Errors and retries
|
|
39
|
-
|
|
40
|
-
Every API error raises a subclass of `ScrapeCoreError`. Each error has these attributes: `code`
|
|
41
|
-
(stable, e.g. `RATE_LIMITED`), `status`, `message`, `request_id` (quote it to support),
|
|
42
|
-
`retryable` and `retry_after`.
|
|
43
|
-
|
|
44
|
-
| Exception | When |
|
|
45
|
-
|---|---|
|
|
46
|
-
| `AuthenticationError` | missing / wrong / revoked key (401) |
|
|
47
|
-
| `InsufficientCreditsError` | monthly credits used up (402) |
|
|
48
|
-
| `PermissionDeniedError` | feature not in your plan (403) |
|
|
49
|
-
| `InvalidRequestError` | bad URL or parameter (400/422) |
|
|
50
|
-
| `RateLimitError` | per-minute / daily / concurrency limit (429) |
|
|
51
|
-
| `APIError` | server or target-site failure (5xx) |
|
|
52
|
-
| `APIConnectionError` | network error or timeout |
|
|
53
|
-
|
|
54
|
-
The SDK automatically retries rate limits (except the daily limit), 5xx errors and network errors
|
|
55
|
-
(`max_retries=2`, honouring `Retry-After`). Job-creating calls send an `Idempotency-Key`, so a retry
|
|
56
|
-
never creates a second job. Failed scrapes are never charged.
|
|
57
|
-
|
|
58
|
-
## Responsible use
|
|
59
|
-
|
|
60
|
-
Use ScrapeCore only for content you are allowed to access. Crawls and maps respect `robots.txt` by
|
|
61
|
-
default. See the [acceptable use policy](https://scrapecore.com/acceptable-use).
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|