webextract-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webextract/__init__.py +20 -0
- webextract/_client.py +279 -0
- webextract/_errors.py +82 -0
- webextract/py.typed +0 -0
- webextract_sdk-0.1.0.dist-info/METADATA +96 -0
- webextract_sdk-0.1.0.dist-info/RECORD +8 -0
- webextract_sdk-0.1.0.dist-info/WHEEL +4 -0
- webextract_sdk-0.1.0.dist-info/licenses/LICENSE +21 -0
webextract/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""WebExtract: web page to clean Markdown + fixed-schema JSON for AI agents (https://webextract.dev)."""
|
|
2
|
+
from ._client import (
|
|
3
|
+
DEFAULT_BASE_URL, ENV_API_KEY, AsyncWebExtract, ResponseMeta, WebExtract, WebExtractResponse, __version__, error_from_response,
|
|
4
|
+
)
|
|
5
|
+
from ._errors import (
|
|
6
|
+
PAYMENT_HINTS, TARGET_FETCH_CODES, AuthenticationError, ConnectionError, InvalidRequestError, PaymentRequiredError,
|
|
7
|
+
RateLimitError, TargetFetchError, WebExtractError,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
TOOL_DESCRIPTIONS = {
|
|
11
|
+
"extract_url": "Fetch a public web page (robots.txt respected) and return clean Markdown plus fixed-schema JSON: article metadata, product/price (with price_detail when available; a missing price is never reported as 0), and tables. Paid per call (USDC on Base and 6 other chains via x402, or prepaid credits); free tier 50 calls/day per IP.",
|
|
12
|
+
"page_markdown": "Fetch a public web page (robots.txt respected) and return clean Markdown only. Cheaper than extract_url. Use it when you only need the text.",
|
|
13
|
+
"url": "Public http(s) URL of the page to read",
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"WebExtract", "AsyncWebExtract", "ENV_API_KEY", "WebExtractResponse", "ResponseMeta", "DEFAULT_BASE_URL", "__version__",
|
|
18
|
+
"error_from_response", "WebExtractError", "PaymentRequiredError", "TargetFetchError", "InvalidRequestError",
|
|
19
|
+
"AuthenticationError", "RateLimitError", "ConnectionError", "PAYMENT_HINTS", "TARGET_FETCH_CODES", "TOOL_DESCRIPTIONS",
|
|
20
|
+
]
|
webextract/_client.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"""Sync and async clients for https://api.webextract.dev. The package never handles private keys:
|
|
2
|
+
x402 signing happens inside the ``payment_client`` you pass (e.g. ``x402HttpxClient``)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Any, Dict, Generic, Literal, Optional, TypeVar
|
|
10
|
+
from urllib.parse import urlsplit
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
from ._errors import (
|
|
15
|
+
AuthenticationError, ConnectionError, InvalidRequestError, PaymentRequiredError, RateLimitError,
|
|
16
|
+
TargetFetchError, TARGET_FETCH_CODES, WebExtractError,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__version__ = "0.1.0"
|
|
20
|
+
ENV_API_KEY = "WEBEXTRACT_API_KEY"
|
|
21
|
+
DEFAULT_BASE_URL = "https://api.webextract.dev"
|
|
22
|
+
Billing = Literal["key", "free", "x402", "x402-testnet"]
|
|
23
|
+
T = TypeVar("T")
|
|
24
|
+
_PROTECTED = {"authorization", "x-billing", "payment-signature", "x-payment"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class ResponseMeta:
|
|
29
|
+
status: int
|
|
30
|
+
billing: str
|
|
31
|
+
free_remaining: Optional[int]
|
|
32
|
+
credits_charged: Optional[float]
|
|
33
|
+
credits_remaining: Optional[float]
|
|
34
|
+
sandbox: bool
|
|
35
|
+
payment_response: Optional[Dict[str, Any]]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class WebExtractResponse(Generic[T]):
|
|
40
|
+
data: T
|
|
41
|
+
meta: ResponseMeta
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _num(v: Optional[str]) -> Optional[float]:
|
|
45
|
+
if v is None or not v.strip():
|
|
46
|
+
return None
|
|
47
|
+
try:
|
|
48
|
+
return float(v)
|
|
49
|
+
except ValueError:
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _b64json(v: Optional[str]) -> Optional[Dict[str, Any]]:
|
|
54
|
+
if not v:
|
|
55
|
+
return None
|
|
56
|
+
try:
|
|
57
|
+
o = json.loads(base64.b64decode(v))
|
|
58
|
+
return o if isinstance(o, dict) else None
|
|
59
|
+
except Exception:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _check_url(url: str) -> str:
|
|
64
|
+
if not isinstance(url, str) or not url.strip():
|
|
65
|
+
raise InvalidRequestError("url is required", status=0, code="missing_url", charged=False)
|
|
66
|
+
u = url.strip()
|
|
67
|
+
parts = urlsplit(u)
|
|
68
|
+
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
69
|
+
raise InvalidRequestError("url must be an absolute http(s) URL", status=0, code="invalid_url", charged=False)
|
|
70
|
+
return u
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def error_from_response(res: Any) -> WebExtractError:
|
|
74
|
+
"""Map an error response (httpx or requests) to a WebExtractError subclass."""
|
|
75
|
+
status = res.status_code
|
|
76
|
+
try:
|
|
77
|
+
body = res.json()
|
|
78
|
+
except Exception:
|
|
79
|
+
body = None
|
|
80
|
+
if not isinstance(body, dict):
|
|
81
|
+
body = None
|
|
82
|
+
code = (body or {}).get("error") if isinstance((body or {}).get("error"), str) else f"http_{status}"
|
|
83
|
+
message = (body or {}).get("message") or (res.text[:200] if body is None and res.text else f"HTTP {status}")
|
|
84
|
+
charged = (body or {}).get("charged") if isinstance((body or {}).get("charged"), bool) else False
|
|
85
|
+
if status == 402:
|
|
86
|
+
reason = res.headers.get("x-payment-reason") or (body or {}).get("reason") or ("no_credits" if code == "no_credits" else "x402_required")
|
|
87
|
+
pr = _b64json(res.headers.get("payment-required"))
|
|
88
|
+
accepts = pr.get("accepts") if isinstance(pr, dict) and isinstance(pr.get("accepts"), list) else []
|
|
89
|
+
networks = list(dict.fromkeys(a.get("network") for a in accepts if isinstance(a, dict) and a.get("network")))
|
|
90
|
+
if not networks:
|
|
91
|
+
networks = list(((body or {}).get("x402") or {}).get("networks") or [])
|
|
92
|
+
fr = _num(res.headers.get("x-free-remaining"))
|
|
93
|
+
return PaymentRequiredError(
|
|
94
|
+
message, status=status, code="payment_required" if code == "http_402" else code, charged=False,
|
|
95
|
+
retryable=reason == "settlement_failed", body=body, reason=reason, payment_required=pr, networks=networks,
|
|
96
|
+
buy_url=((body or {}).get("stripe") or {}).get("buy_url") or (body or {}).get("buy"),
|
|
97
|
+
free_remaining=int(fr) if fr is not None else None)
|
|
98
|
+
if code in TARGET_FETCH_CODES or status == 451:
|
|
99
|
+
return TargetFetchError(message, status=status, code=code, charged=False, retryable=code == "fetch_failed", body=body)
|
|
100
|
+
if status == 400:
|
|
101
|
+
return InvalidRequestError(message, status=status, code=code, charged=False, body=body)
|
|
102
|
+
if status == 401:
|
|
103
|
+
return AuthenticationError(message, status=status, code=code, charged=False, body=body)
|
|
104
|
+
if status == 429:
|
|
105
|
+
return RateLimitError(message, status=status, code=code, charged=False, retryable=True, body=body,
|
|
106
|
+
retry_after=_num(res.headers.get("retry-after")))
|
|
107
|
+
return WebExtractError(message, status=status, code=code, charged=charged, retryable=status >= 500, body=body)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class _Base:
|
|
111
|
+
def __init__(self, *, api_key: Optional[str], billing: Optional[Billing], payment_client: Any,
|
|
112
|
+
http_client: Any, base_url: Optional[str], timeout: float,
|
|
113
|
+
headers: Optional[Dict[str, str]], settlement_retries: int = 1) -> None:
|
|
114
|
+
if api_key is not None and (not isinstance(api_key, str) or not api_key.strip()):
|
|
115
|
+
raise TypeError("api_key must be a non-empty string")
|
|
116
|
+
if api_key is None and payment_client is None and billing in (None, "key"):
|
|
117
|
+
# Fallback only: an explicit api_key wins; with payment_client or a non-key billing the env var is ignored.
|
|
118
|
+
env_key = os.environ.get(ENV_API_KEY, "").strip()
|
|
119
|
+
api_key = env_key or None
|
|
120
|
+
if api_key and payment_client is not None:
|
|
121
|
+
raise TypeError("pass either api_key or payment_client, not both (a Bearer key always wins on the server)")
|
|
122
|
+
self.billing: str = billing or ("key" if api_key else "x402" if payment_client is not None else "free")
|
|
123
|
+
if self.billing == "key" and not api_key:
|
|
124
|
+
raise TypeError(f'billing "key" needs api_key (or the {ENV_API_KEY} environment variable)')
|
|
125
|
+
if self.billing in ("x402", "x402-testnet") and payment_client is None:
|
|
126
|
+
raise TypeError(f'billing "{self.billing}" needs payment_client (e.g. x402HttpxClient from the x402 SDK)')
|
|
127
|
+
if self.billing != "key" and api_key:
|
|
128
|
+
raise TypeError(f'billing "{self.billing}" cannot be combined with api_key')
|
|
129
|
+
if self.billing not in ("key", "free", "x402", "x402-testnet"):
|
|
130
|
+
raise TypeError(f"unknown billing {self.billing!r}")
|
|
131
|
+
self._api_key = api_key
|
|
132
|
+
self.base_url = (base_url or DEFAULT_BASE_URL).rstrip("/")
|
|
133
|
+
self.timeout = timeout
|
|
134
|
+
if not isinstance(settlement_retries, int) or settlement_retries < 0:
|
|
135
|
+
raise TypeError("settlement_retries must be an int >= 0")
|
|
136
|
+
self.settlement_retries = settlement_retries
|
|
137
|
+
self._extra = {k: v for k, v in (headers or {}).items() if k.lower() not in _PROTECTED}
|
|
138
|
+
self._transport_client = payment_client if payment_client is not None else http_client
|
|
139
|
+
|
|
140
|
+
def __repr__(self) -> str: # never show the key
|
|
141
|
+
return f"{type(self).__name__}(base_url={self.base_url!r}, billing={self.billing!r})"
|
|
142
|
+
|
|
143
|
+
def _headers(self, billing_header: bool = True) -> Dict[str, str]:
|
|
144
|
+
h = {"accept": "application/json", "user-agent": f"webextract-python/{__version__}", **self._extra}
|
|
145
|
+
if self._api_key:
|
|
146
|
+
h["authorization"] = f"Bearer {self._api_key}"
|
|
147
|
+
if billing_header and self.billing in ("free", "x402-testnet"):
|
|
148
|
+
h["x-billing"] = self.billing
|
|
149
|
+
return h
|
|
150
|
+
|
|
151
|
+
def _should_retry(self, e: WebExtractError, attempt: int) -> bool:
|
|
152
|
+
# settlement_failed = payment verified but not settled; the API says "not charged, retry with a new payment".
|
|
153
|
+
# Only this reason is retried automatically; the payment client signs a fresh payload on the new request.
|
|
154
|
+
return isinstance(e, PaymentRequiredError) and e.reason == "settlement_failed" and attempt < self.settlement_retries
|
|
155
|
+
|
|
156
|
+
def _parse(self, res: Any) -> WebExtractResponse[Dict[str, Any]]:
|
|
157
|
+
if 200 <= res.status_code < 300:
|
|
158
|
+
try:
|
|
159
|
+
data = res.json()
|
|
160
|
+
except Exception:
|
|
161
|
+
data = None
|
|
162
|
+
if not isinstance(data, dict):
|
|
163
|
+
raise WebExtractError("response was not JSON", status=res.status_code, code="invalid_response", body=res.text[:500])
|
|
164
|
+
fr = _num(res.headers.get("x-free-remaining"))
|
|
165
|
+
return WebExtractResponse(data=data, meta=ResponseMeta(
|
|
166
|
+
status=res.status_code, billing=self.billing, free_remaining=int(fr) if fr is not None else None,
|
|
167
|
+
credits_charged=_num(res.headers.get("x-credits-charged")),
|
|
168
|
+
credits_remaining=_num(res.headers.get("x-credits-remaining")),
|
|
169
|
+
sandbox=res.headers.get("x-sandbox") == "true",
|
|
170
|
+
payment_response=_b64json(res.headers.get("payment-response"))))
|
|
171
|
+
raise error_from_response(res)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _wrap_exc(e: Exception) -> ConnectionError:
|
|
175
|
+
timed_out = isinstance(e, httpx.TimeoutException) or "Timeout" in type(e).__name__
|
|
176
|
+
msg = "request timed out" if timed_out else f"request failed before a response: {type(e).__name__}: {e}"
|
|
177
|
+
err = ConnectionError(msg, status=0, code="timeout" if timed_out else "connection_error", charged=None, retryable=True)
|
|
178
|
+
err.__cause__ = e
|
|
179
|
+
return err
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class WebExtract(_Base):
|
|
183
|
+
"""Synchronous client. ``payment_client`` may be any object with a requests/httpx-style
|
|
184
|
+
``get(url, params=, headers=, timeout=)``, e.g. the ``requests.Session`` returned by
|
|
185
|
+
``x402.http.clients.x402_requests(x402_client)`` (the x402 SDK has no sync httpx client).
|
|
186
|
+
|
|
187
|
+
>>> WebExtract().markdown("https://example.com").data["markdown"] # free tier (X-Billing: free)
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
def __init__(self, *, api_key: Optional[str] = None, billing: Optional[Billing] = None,
|
|
191
|
+
payment_client: Any = None, http_client: Optional[httpx.Client] = None,
|
|
192
|
+
base_url: Optional[str] = None, timeout: float = 60.0, headers: Optional[Dict[str, str]] = None,
|
|
193
|
+
settlement_retries: int = 1) -> None:
|
|
194
|
+
super().__init__(api_key=api_key, billing=billing, payment_client=payment_client, http_client=http_client,
|
|
195
|
+
base_url=base_url, timeout=timeout, headers=headers, settlement_retries=settlement_retries)
|
|
196
|
+
self._owned = self._transport_client is None
|
|
197
|
+
self._http: httpx.Client = self._transport_client or httpx.Client(timeout=timeout)
|
|
198
|
+
|
|
199
|
+
def extract(self, url: str) -> WebExtractResponse[Dict[str, Any]]:
|
|
200
|
+
"""GET /v1/extract: Markdown + fixed-schema JSON. $0.005 or 1 credit."""
|
|
201
|
+
return self._get("/v1/extract", {"url": _check_url(url)})
|
|
202
|
+
|
|
203
|
+
def markdown(self, url: str) -> WebExtractResponse[Dict[str, Any]]:
|
|
204
|
+
"""GET /v1/markdown: Markdown only. $0.002 or 0.4 credit."""
|
|
205
|
+
return self._get("/v1/markdown", {"url": _check_url(url)})
|
|
206
|
+
|
|
207
|
+
def balance(self) -> Dict[str, Any]:
|
|
208
|
+
"""GET /v1/balance: credits, or free calls left today for this IP. Never charged."""
|
|
209
|
+
return self._get("/v1/balance", None, billing_header=False).data
|
|
210
|
+
|
|
211
|
+
def _get(self, path: str, params: Optional[Dict[str, str]], billing_header: bool = True) -> WebExtractResponse[Dict[str, Any]]:
|
|
212
|
+
attempt = 0
|
|
213
|
+
while True:
|
|
214
|
+
try:
|
|
215
|
+
res = self._http.get(self.base_url + path, params=params, headers=self._headers(billing_header), timeout=self.timeout)
|
|
216
|
+
except Exception as e:
|
|
217
|
+
raise _wrap_exc(e) from e
|
|
218
|
+
try:
|
|
219
|
+
return self._parse(res)
|
|
220
|
+
except WebExtractError as e:
|
|
221
|
+
if not self._should_retry(e, attempt):
|
|
222
|
+
raise
|
|
223
|
+
attempt += 1
|
|
224
|
+
|
|
225
|
+
def close(self) -> None:
|
|
226
|
+
if self._owned:
|
|
227
|
+
self._http.close()
|
|
228
|
+
|
|
229
|
+
def __enter__(self) -> "WebExtract":
|
|
230
|
+
return self
|
|
231
|
+
|
|
232
|
+
def __exit__(self, *a: Any) -> None:
|
|
233
|
+
self.close()
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class AsyncWebExtract(_Base):
|
|
237
|
+
"""Async client. For x402 pass ``payment_client=x402HttpxClient(x402_client)`` from the official x402 SDK."""
|
|
238
|
+
|
|
239
|
+
def __init__(self, *, api_key: Optional[str] = None, billing: Optional[Billing] = None,
|
|
240
|
+
payment_client: Optional[httpx.AsyncClient] = None, http_client: Optional[httpx.AsyncClient] = None,
|
|
241
|
+
base_url: Optional[str] = None, timeout: float = 60.0, headers: Optional[Dict[str, str]] = None,
|
|
242
|
+
settlement_retries: int = 1) -> None:
|
|
243
|
+
super().__init__(api_key=api_key, billing=billing, payment_client=payment_client, http_client=http_client,
|
|
244
|
+
base_url=base_url, timeout=timeout, headers=headers, settlement_retries=settlement_retries)
|
|
245
|
+
self._owned = self._transport_client is None
|
|
246
|
+
self._http: httpx.AsyncClient = self._transport_client or httpx.AsyncClient(timeout=timeout)
|
|
247
|
+
|
|
248
|
+
async def extract(self, url: str) -> WebExtractResponse[Dict[str, Any]]:
|
|
249
|
+
return await self._get("/v1/extract", {"url": _check_url(url)})
|
|
250
|
+
|
|
251
|
+
async def markdown(self, url: str) -> WebExtractResponse[Dict[str, Any]]:
|
|
252
|
+
return await self._get("/v1/markdown", {"url": _check_url(url)})
|
|
253
|
+
|
|
254
|
+
async def balance(self) -> Dict[str, Any]:
|
|
255
|
+
return (await self._get("/v1/balance", None, billing_header=False)).data
|
|
256
|
+
|
|
257
|
+
async def _get(self, path: str, params: Optional[Dict[str, str]], billing_header: bool = True) -> WebExtractResponse[Dict[str, Any]]:
|
|
258
|
+
attempt = 0
|
|
259
|
+
while True:
|
|
260
|
+
try:
|
|
261
|
+
res = await self._http.get(self.base_url + path, params=params, headers=self._headers(billing_header), timeout=self.timeout)
|
|
262
|
+
except Exception as e:
|
|
263
|
+
raise _wrap_exc(e) from e
|
|
264
|
+
try:
|
|
265
|
+
return self._parse(res)
|
|
266
|
+
except WebExtractError as e:
|
|
267
|
+
if not self._should_retry(e, attempt):
|
|
268
|
+
raise
|
|
269
|
+
attempt += 1
|
|
270
|
+
|
|
271
|
+
async def aclose(self) -> None:
|
|
272
|
+
if self._owned:
|
|
273
|
+
await self._http.aclose()
|
|
274
|
+
|
|
275
|
+
async def __aenter__(self) -> "AsyncWebExtract":
|
|
276
|
+
return self
|
|
277
|
+
|
|
278
|
+
async def __aexit__(self, *a: Any) -> None:
|
|
279
|
+
await self.aclose()
|
webextract/_errors.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Error classes. Every error raised by the client is a WebExtractError (or subclass)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any, Dict, List, Optional
|
|
5
|
+
|
|
6
|
+
TARGET_FETCH_CODES = frozenset({
|
|
7
|
+
"upstream_error", "blocked_by_site", "robots_disallowed", "dns_failed", "fetch_failed", "too_large", "unsupported_type",
|
|
8
|
+
})
|
|
9
|
+
|
|
10
|
+
PAYMENT_HINTS: Dict[str, str] = {
|
|
11
|
+
"x402_required": 'No API key, payment or free opt-in. Use billing="free" (50 calls/day per IP), an api_key (https://webextract.dev/#pricing), or a payment_client from the x402 SDK.',
|
|
12
|
+
"x402_requested": "x402 requested. Pass a payment_client (e.g. x402HttpxClient) so one of the PAYMENT-REQUIRED options can be signed.",
|
|
13
|
+
"free_limit_reached": "Free daily limit (50 calls/day per IP, shared with MCP) is used up. Buy credits at https://webextract.dev/#pricing or pay per call via x402 (USDC on Base and 6 other chains).",
|
|
14
|
+
"invalid_payment": "The x402 payment was rejected. Check the signer, network and USDC balance, then retry with a new payment.",
|
|
15
|
+
"unsupported_network": "The payment used a network/asset this endpoint does not accept. See `networks`.",
|
|
16
|
+
"settlement_failed": "Payment verified but settlement did not complete. Not charged. Retry with a freshly signed payment.",
|
|
17
|
+
"no_credits": "API key has no credits left. Top up at https://webextract.dev/#pricing.",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class WebExtractError(Exception):
|
|
22
|
+
"""Base error. ``charged`` is False whenever the API says so (failed fetches are never billed)."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, message: str, *, status: int, code: str, charged: Optional[bool] = None,
|
|
25
|
+
retryable: bool = False, body: Any = None) -> None:
|
|
26
|
+
super().__init__(message)
|
|
27
|
+
self.message = message
|
|
28
|
+
self.status = status
|
|
29
|
+
self.code = code
|
|
30
|
+
self.charged = charged
|
|
31
|
+
self.retryable = retryable
|
|
32
|
+
self.body = body
|
|
33
|
+
|
|
34
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
35
|
+
"""Compact, model-friendly description (used by the LangChain tools)."""
|
|
36
|
+
return {"error": self.code, "status": self.status, "message": self.message,
|
|
37
|
+
"charged": self.charged, "retryable": self.retryable}
|
|
38
|
+
|
|
39
|
+
def __repr__(self) -> str:
|
|
40
|
+
return f"{type(self).__name__}(status={self.status}, code={self.code!r}, message={self.message!r})"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class PaymentRequiredError(WebExtractError):
|
|
44
|
+
"""HTTP 402. ``reason`` explains why; ``payment_required`` is the decoded PAYMENT-REQUIRED header."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, message: str, *, reason: str, payment_required: Optional[Dict[str, Any]],
|
|
47
|
+
networks: List[str], buy_url: Optional[str], free_remaining: Optional[int], **kw: Any) -> None:
|
|
48
|
+
super().__init__(message, **kw)
|
|
49
|
+
self.reason = reason
|
|
50
|
+
self.payment_required = payment_required
|
|
51
|
+
self.networks = networks
|
|
52
|
+
self.buy_url = buy_url
|
|
53
|
+
self.free_remaining = free_remaining
|
|
54
|
+
self.hint = PAYMENT_HINTS.get(reason, "Payment required.")
|
|
55
|
+
|
|
56
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
57
|
+
return {**super().to_dict(), "reason": self.reason, "hint": self.hint, "networks": self.networks,
|
|
58
|
+
"buy_url": self.buy_url, "free_remaining": self.free_remaining}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class TargetFetchError(WebExtractError):
|
|
62
|
+
"""The target page could not be fetched (robots.txt, HTTP error, blocked, DNS, timeout, size, type). Never charged."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class InvalidRequestError(WebExtractError):
|
|
66
|
+
"""400: missing/invalid url, private host, invalid billing mode. Never charged."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class AuthenticationError(WebExtractError):
|
|
70
|
+
"""401: unknown API key."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class RateLimitError(WebExtractError):
|
|
74
|
+
"""429: rate limited (e.g. sandbox per-IP limits)."""
|
|
75
|
+
|
|
76
|
+
def __init__(self, message: str, *, retry_after: Optional[float], **kw: Any) -> None:
|
|
77
|
+
super().__init__(message, **kw)
|
|
78
|
+
self.retry_after = retry_after
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class ConnectionError(WebExtractError): # noqa: A001 - deliberate, namespaced as webextract.ConnectionError
|
|
82
|
+
"""No HTTP response (network error, timeout, or the payment client raised before sending)."""
|
webextract/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: webextract-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Thin client for the WebExtract API: web page to clean Markdown + fixed-schema JSON for AI agents. Prepaid key, free tier, or pay per call with x402.
|
|
5
|
+
Project-URL: Homepage, https://webextract.dev
|
|
6
|
+
Project-URL: Documentation, https://webextract.dev/docs
|
|
7
|
+
Project-URL: Source, https://github.com/nyx464664816-stack/webextract
|
|
8
|
+
Project-URL: Issues, https://github.com/nyx464664816-stack/webextract/issues
|
|
9
|
+
Author: TAINCHENG, L.L.C.
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ai-agents,extraction,llm,markdown,mcp,web-scraping,webextract,x402
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Requires-Dist: httpx<1,>=0.27
|
|
19
|
+
Provides-Extra: x402
|
|
20
|
+
Requires-Dist: x402[evm,httpx]<3,>=2.25; extra == 'x402'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# webextract-sdk (Python)
|
|
24
|
+
|
|
25
|
+
Thin client for the [WebExtract](https://webextract.dev) API: a public web page in, clean Markdown plus fixed-schema JSON out (article metadata, product and price, tables). It's built for AI agents.
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install webextract-sdk # core (httpx only)
|
|
29
|
+
pip install "webextract-sdk[x402]" # + the official x402 SDK, for pay-per-call
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The PyPI name is `webextract-sdk`; the import name is `webextract` (`from webextract import WebExtract`).
|
|
33
|
+
|
|
34
|
+
## Billing: pick one
|
|
35
|
+
|
|
36
|
+
| Mode | How | Cost |
|
|
37
|
+
|---|---|---|
|
|
38
|
+
| Free tier (default) | `WebExtract()` sends `X-Billing: free` (when `WEBEXTRACT_API_KEY` is not set) | 50 calls/day per IP (shared with MCP) |
|
|
39
|
+
| Prepaid credits | `WebExtract(api_key="lx_...")` or env `WEBEXTRACT_API_KEY` → `Authorization: Bearer` | `/v1/extract` 1 credit, `/v1/markdown` 0.4 credit. Buy at https://webextract.dev/#pricing |
|
|
40
|
+
| x402 pay per call | `AsyncWebExtract(payment_client=x402HttpxClient(...))` | `/v1/extract` $0.005, `/v1/markdown` $0.002 in USDC on Base and 6 other chains |
|
|
41
|
+
| x402 sandbox | add `billing="x402-testnet"` | Base Sepolia **test** USDC only. Nothing real is charged |
|
|
42
|
+
|
|
43
|
+
**This package never handles private keys.** With x402, signing happens inside the client you build with the official `x402` SDK.
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from webextract import WebExtract, PaymentRequiredError, TargetFetchError
|
|
47
|
+
|
|
48
|
+
with WebExtract() as we: # free tier
|
|
49
|
+
r = we.extract("https://web-scraping.dev/product/1")
|
|
50
|
+
print(r.data["product"], r.meta.free_remaining)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### x402 (async, official SDK)
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import asyncio, os
|
|
57
|
+
from eth_account import Account
|
|
58
|
+
from x402 import x402Client
|
|
59
|
+
from x402.http.clients import x402HttpxClient
|
|
60
|
+
from x402.mechanisms.evm.exact import ExactEvmScheme
|
|
61
|
+
from webextract import AsyncWebExtract
|
|
62
|
+
|
|
63
|
+
async def main():
|
|
64
|
+
signer = Account.from_key(os.environ["X402_PRIVATE_KEY"]) # your key, your process
|
|
65
|
+
xc = x402Client(); xc.register("eip155:8453", ExactEvmScheme(signer=signer))
|
|
66
|
+
async with x402HttpxClient(xc, timeout=60) as pay:
|
|
67
|
+
we = AsyncWebExtract(payment_client=pay) # billing="x402"
|
|
68
|
+
r = await we.markdown("https://example.com")
|
|
69
|
+
print(r.meta.payment_response) # settlement receipt (tx hash)
|
|
70
|
+
asyncio.run(main())
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
**`WEBEXTRACT_API_KEY`**: if `api_key` isn't passed, the client reads this environment variable. An explicit `api_key` always wins. The variable is ignored when `payment_client` is given or `billing` is anything other than `"key"`, so it never causes the "api_key + payment_client" error.
|
|
74
|
+
|
|
75
|
+
**`settlement_retries`** (default `1`): a 402 `settlement_failed` (payment verified, not settled, **not charged**) is retried once, and the payment client signs a fresh payment. Other errors are never retried.
|
|
76
|
+
|
|
77
|
+
For a sync client, pass the `requests.Session` from `x402.http.clients.x402_requests(xc)` as `payment_client=`. The x402 SDK has no sync httpx client.
|
|
78
|
+
|
|
79
|
+
## Errors
|
|
80
|
+
|
|
81
|
+
Every error is a `WebExtractError` with `status`, `code`, `message`, `charged`, `retryable`, `body`.
|
|
82
|
+
|
|
83
|
+
| Class | When | Notes |
|
|
84
|
+
|---|---|---|
|
|
85
|
+
| `PaymentRequiredError` | HTTP 402 | `.reason` is one of `x402_required`, `x402_requested`, `free_limit_reached`, `invalid_payment`, `unsupported_network`, `settlement_failed` (retryable), `no_credits`. Also has `.networks`, `.payment_required` (decoded header), `.buy_url`, `.free_remaining`, `.hint` |
|
|
86
|
+
| `TargetFetchError` | target page failed: `upstream_error`, `blocked_by_site` (422), `robots_disallowed` (451), `dns_failed`, `fetch_failed`, `too_large`, `unsupported_type` | never charged |
|
|
87
|
+
| `InvalidRequestError` | 400 (`missing_url`, `invalid_url`, `forbidden_host`, ...) or client-side URL check | never charged |
|
|
88
|
+
| `AuthenticationError` | 401 `invalid_api_key` | |
|
|
89
|
+
| `RateLimitError` | 429 (sandbox limits) | `.retry_after` |
|
|
90
|
+
| `ConnectionError` | no response: network error, timeout, or the payment client raised | `code` is `timeout` or `connection_error` |
|
|
91
|
+
|
|
92
|
+
## Response fields
|
|
93
|
+
|
|
94
|
+
The responses are the API JSON as-is (`schema_version` "1.0"). Fields added on 2026-10-21 (`content_source`, `product.price_detail`, `product_unresolved`) may be missing on older deployments, so read them with `.get()`. A missing price is never reported as `0`. `price_detail.unresolved.reason` says why it's missing.
|
|
95
|
+
|
|
96
|
+
License: MIT © TAINCHENG, L.L.C.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
webextract/__init__.py,sha256=X0YKasEcjbuAb76U645SI-HsOvdqaga_vmSALmFWgzk,1430
|
|
2
|
+
webextract/_client.py,sha256=kZgJcBmkpqDaBr__QIc3g98c-r_VYgg6OL3oEZwpsqU,13767
|
|
3
|
+
webextract/_errors.py,sha256=pH7hqJRRbQt4P_9bbTDJpaD-t9n-oQfv6B7EaLbTEZQ,4037
|
|
4
|
+
webextract/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
webextract_sdk-0.1.0.dist-info/METADATA,sha256=vSPuraeK5vrR_1MUKthyI6_LeEBgwQ-xaTFOUBsbrNk,5308
|
|
6
|
+
webextract_sdk-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
7
|
+
webextract_sdk-0.1.0.dist-info/licenses/LICENSE,sha256=hXkDs6G0olq1DFC3W5VBplfPxLDpKDauCGXAhOEOdUM,1074
|
|
8
|
+
webextract_sdk-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 TAINCHENG, L.L.C.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|