readfetch-client 0.1.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- readfetch_client/__init__.py +51 -0
- readfetch_client/client.py +277 -0
- readfetch_client/config.py +142 -0
- readfetch_client/endpoints.py +77 -0
- readfetch_client/feedback_validation.py +23 -0
- readfetch_client/http_json.py +43 -0
- readfetch_client/payment.py +22 -0
- readfetch_client-0.1.11.dist-info/METADATA +93 -0
- readfetch_client-0.1.11.dist-info/RECORD +10 -0
- readfetch_client-0.1.11.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""readfetch x402 REST client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"buyer_address",
|
|
9
|
+
"call",
|
|
10
|
+
"call_with_meta",
|
|
11
|
+
"pay",
|
|
12
|
+
"payment_config",
|
|
13
|
+
"payment_from_response",
|
|
14
|
+
"probe",
|
|
15
|
+
"probe_with_meta",
|
|
16
|
+
"quote",
|
|
17
|
+
"readfetch_batch",
|
|
18
|
+
"readfetch_crawl",
|
|
19
|
+
"readfetch_crawl_status",
|
|
20
|
+
"readfetch_diff",
|
|
21
|
+
"readfetch_fetch",
|
|
22
|
+
"readfetch_map",
|
|
23
|
+
"readfetch_pdf",
|
|
24
|
+
"readfetch_residential",
|
|
25
|
+
"readfetch_scrape",
|
|
26
|
+
"readfetch_search",
|
|
27
|
+
"readfetch_section",
|
|
28
|
+
"readfetch_screenshot",
|
|
29
|
+
"readfetch_text",
|
|
30
|
+
"readfetch_url",
|
|
31
|
+
"request_id_from_response",
|
|
32
|
+
"submit_claim",
|
|
33
|
+
"submit_feedback",
|
|
34
|
+
"x402_http_client",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def __getattr__(name: str) -> Any:
|
|
39
|
+
if name not in __all__:
|
|
40
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
41
|
+
if name in ("payment_config", "buyer_address", "readfetch_url"):
|
|
42
|
+
from readfetch_client import config as cfg
|
|
43
|
+
|
|
44
|
+
return getattr(cfg, name)
|
|
45
|
+
if name == "payment_from_response":
|
|
46
|
+
from readfetch_client import payment as pay_mod
|
|
47
|
+
|
|
48
|
+
return pay_mod.payment_from_response
|
|
49
|
+
from readfetch_client import client as impl
|
|
50
|
+
|
|
51
|
+
return getattr(impl, name)
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
from eth_account import Account
|
|
10
|
+
from x402 import x402Client
|
|
11
|
+
from x402.http.clients import x402HttpxClient
|
|
12
|
+
from x402.mechanisms.evm import EthAccountSigner
|
|
13
|
+
from x402.mechanisms.evm.exact.register import register_exact_evm_client
|
|
14
|
+
|
|
15
|
+
from readfetch_client.config import max_payment_usd, readfetch_base, readfetch_url, require_account_hex, x402_network
|
|
16
|
+
from readfetch_client.endpoints import BY_NAME, endpoint_path
|
|
17
|
+
from readfetch_client.feedback_validation import normalize_feedback
|
|
18
|
+
from readfetch_client.http_json import assert_response_size, json_object_from_response
|
|
19
|
+
from readfetch_client.payment import payment_from_response
|
|
20
|
+
|
|
21
|
+
_TIMEOUT = 120.0
|
|
22
|
+
|
|
23
|
+
_x402_clients: dict[tuple[str, str, str], x402Client] = {}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def request_id_from_response(response: httpx.Response) -> str | None:
|
|
27
|
+
return response.headers.get("x-request-id") or response.headers.get("X-Request-Id")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _load_private_key(env: Mapping[str, str] | None = None) -> str:
|
|
31
|
+
try:
|
|
32
|
+
return require_account_hex(env)
|
|
33
|
+
except ValueError as error:
|
|
34
|
+
raise RuntimeError(str(error)) from error
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _x402_client(env: Mapping[str, str] | None = None) -> x402Client:
|
|
38
|
+
env = os.environ if env is None else env
|
|
39
|
+
hex_key = _load_private_key(env)
|
|
40
|
+
fingerprint = hashlib.sha256(hex_key.encode()).hexdigest()
|
|
41
|
+
network = x402_network(env)
|
|
42
|
+
cap = max_payment_usd(env)
|
|
43
|
+
cache_key = (fingerprint, network, cap)
|
|
44
|
+
cached = _x402_clients.get(cache_key)
|
|
45
|
+
if cached is not None:
|
|
46
|
+
return cached
|
|
47
|
+
client = x402Client().set_spend_controls(
|
|
48
|
+
{"max_amount_per_payment": f"${cap}"}
|
|
49
|
+
)
|
|
50
|
+
account = Account.from_key(hex_key)
|
|
51
|
+
register_exact_evm_client(client, EthAccountSigner(account), networks=network)
|
|
52
|
+
_x402_clients[cache_key] = client
|
|
53
|
+
return client
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
async def _request_json(
|
|
57
|
+
*,
|
|
58
|
+
method: str,
|
|
59
|
+
url: str,
|
|
60
|
+
body: dict[str, Any] | None,
|
|
61
|
+
paid: bool,
|
|
62
|
+
accept: tuple[int, ...],
|
|
63
|
+
env: Mapping[str, str],
|
|
64
|
+
) -> tuple[dict[str, Any], str | None, httpx.Response]:
|
|
65
|
+
headers = {"content-type": "application/json"} if method != "GET" else {}
|
|
66
|
+
if paid:
|
|
67
|
+
async with x402HttpxClient(_x402_client(env)) as http:
|
|
68
|
+
if method == "GET":
|
|
69
|
+
response = await http.get(url, headers=headers, timeout=_TIMEOUT)
|
|
70
|
+
else:
|
|
71
|
+
response = await http.post(
|
|
72
|
+
url, json=body or {}, headers=headers, timeout=_TIMEOUT
|
|
73
|
+
)
|
|
74
|
+
await response.aread()
|
|
75
|
+
else:
|
|
76
|
+
async with httpx.AsyncClient(timeout=_TIMEOUT) as http:
|
|
77
|
+
if method == "GET":
|
|
78
|
+
response = await http.get(url, headers=headers)
|
|
79
|
+
else:
|
|
80
|
+
response = await http.post(url, json=body or {}, headers=headers)
|
|
81
|
+
await response.aread()
|
|
82
|
+
if response.status_code not in accept:
|
|
83
|
+
assert_response_size(response)
|
|
84
|
+
raise RuntimeError(
|
|
85
|
+
f"{response.status_code}: {(response.text or '')[:400]}"
|
|
86
|
+
)
|
|
87
|
+
data: dict[str, Any] = json_object_from_response(response)
|
|
88
|
+
return data, request_id_from_response(response), response
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def x402_http_client(env: Mapping[str, str] | None = None) -> x402HttpxClient:
|
|
92
|
+
"""Paid `httpx` client — signs x402 on 402 and retries (custom URLs)."""
|
|
93
|
+
return x402HttpxClient(_x402_client(env))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def call(
|
|
97
|
+
name: str,
|
|
98
|
+
body: dict[str, Any] | None = None,
|
|
99
|
+
*,
|
|
100
|
+
path_params: dict[str, str] | None = None,
|
|
101
|
+
env: Mapping[str, str] | None = None,
|
|
102
|
+
) -> dict[str, Any]:
|
|
103
|
+
meta = await call_with_meta(
|
|
104
|
+
name, body, path_params=path_params, env=env
|
|
105
|
+
)
|
|
106
|
+
return meta["data"]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def call_with_meta(
|
|
110
|
+
name: str,
|
|
111
|
+
body: dict[str, Any] | None = None,
|
|
112
|
+
*,
|
|
113
|
+
path_params: dict[str, str] | None = None,
|
|
114
|
+
env: Mapping[str, str] | None = None,
|
|
115
|
+
) -> dict[str, Any]:
|
|
116
|
+
spec = BY_NAME.get(name)
|
|
117
|
+
if spec is None:
|
|
118
|
+
raise ValueError(f"unknown endpoint `{name}`")
|
|
119
|
+
env = os.environ if env is None else env
|
|
120
|
+
path = endpoint_path(name, path_params)
|
|
121
|
+
url = readfetch_url(env, path)
|
|
122
|
+
accept = spec.accept if spec.accept else (200,)
|
|
123
|
+
data, rid, response = await _request_json(
|
|
124
|
+
method=spec.method,
|
|
125
|
+
url=url,
|
|
126
|
+
body=body,
|
|
127
|
+
paid=not spec.free,
|
|
128
|
+
accept=accept,
|
|
129
|
+
env=env,
|
|
130
|
+
)
|
|
131
|
+
return {
|
|
132
|
+
"data": data,
|
|
133
|
+
"request_id": rid,
|
|
134
|
+
"payment": payment_from_response(response),
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def quote(
|
|
139
|
+
name: str,
|
|
140
|
+
body: dict[str, Any] | None = None,
|
|
141
|
+
*,
|
|
142
|
+
path_params: dict[str, str] | None = None,
|
|
143
|
+
env: Mapping[str, str] | None = None,
|
|
144
|
+
) -> dict[str, Any]:
|
|
145
|
+
"""402 quote — no wallet. Paid catalog routes only."""
|
|
146
|
+
spec = BY_NAME.get(name)
|
|
147
|
+
if spec is None:
|
|
148
|
+
raise ValueError(f"unknown endpoint `{name}`")
|
|
149
|
+
if spec.free:
|
|
150
|
+
raise ValueError(f"`{name}` is free — use call() instead of quote()")
|
|
151
|
+
env = os.environ if env is None else env
|
|
152
|
+
path = endpoint_path(name, path_params)
|
|
153
|
+
url = readfetch_url(env, path)
|
|
154
|
+
headers = {"content-type": "application/json"} if spec.method != "GET" else {}
|
|
155
|
+
async with httpx.AsyncClient(timeout=_TIMEOUT) as http:
|
|
156
|
+
if spec.method == "GET":
|
|
157
|
+
response = await http.get(url, headers=headers)
|
|
158
|
+
else:
|
|
159
|
+
response = await http.post(url, json=body or {}, headers=headers)
|
|
160
|
+
await response.aread()
|
|
161
|
+
if response.status_code != 402:
|
|
162
|
+
assert_response_size(response)
|
|
163
|
+
raise RuntimeError(
|
|
164
|
+
f"expected 402 quote, got {response.status_code}: "
|
|
165
|
+
f"{(response.text or '')[:400]}"
|
|
166
|
+
)
|
|
167
|
+
return {
|
|
168
|
+
"status": 402,
|
|
169
|
+
"body": json_object_from_response(response),
|
|
170
|
+
"payment_required": payment_from_response(response)["payment_required"],
|
|
171
|
+
"request_id": request_id_from_response(response),
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
async def pay(
|
|
176
|
+
name: str,
|
|
177
|
+
body: dict[str, Any] | None = None,
|
|
178
|
+
*,
|
|
179
|
+
path_params: dict[str, str] | None = None,
|
|
180
|
+
env: Mapping[str, str] | None = None,
|
|
181
|
+
) -> dict[str, Any]:
|
|
182
|
+
"""Sign x402 and complete a paid call (alias for `call`)."""
|
|
183
|
+
return await call(name, body, path_params=path_params, env=env)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
async def probe(url: str, *, env: Mapping[str, str] | None = None) -> dict:
|
|
187
|
+
data, _ = await probe_with_meta(url, env=env)
|
|
188
|
+
return data
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
async def probe_with_meta(
|
|
192
|
+
url: str, *, env: Mapping[str, str] | None = None
|
|
193
|
+
) -> tuple[dict, str | None]:
|
|
194
|
+
meta = await call_with_meta("probe", {"url": url}, env=env)
|
|
195
|
+
return meta["data"], meta["request_id"]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
async def submit_feedback(
|
|
199
|
+
request_id: str,
|
|
200
|
+
category: str,
|
|
201
|
+
message: str,
|
|
202
|
+
*,
|
|
203
|
+
env: Mapping[str, str] | None = None,
|
|
204
|
+
) -> dict:
|
|
205
|
+
rid, cat, msg = normalize_feedback(request_id, category, message)
|
|
206
|
+
return await call(
|
|
207
|
+
"feedback",
|
|
208
|
+
{"request_id": rid, "category": cat, "message": msg},
|
|
209
|
+
env=env,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
async def submit_claim(
|
|
214
|
+
claim: dict[str, Any], *, env: Mapping[str, str] | None = None
|
|
215
|
+
) -> dict:
|
|
216
|
+
return await call("claims", claim, env=env)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
async def readfetch_text(
|
|
220
|
+
url: str | dict[str, Any], *, env: Mapping[str, str] | None = None
|
|
221
|
+
) -> dict:
|
|
222
|
+
body = {"url": url} if isinstance(url, str) else url
|
|
223
|
+
return await call("text", body, env=env)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
async def readfetch_map(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
227
|
+
return await call("map", body, env=env)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
async def readfetch_scrape(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
231
|
+
return await call("scrape", body, env=env)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
async def readfetch_residential(
|
|
235
|
+
body: dict[str, Any], *, env: Mapping[str, str] | None = None
|
|
236
|
+
) -> dict:
|
|
237
|
+
return await call("residential", body, env=env)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
async def readfetch_fetch(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
241
|
+
return await call("fetch", body, env=env)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
async def readfetch_batch(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
245
|
+
return await call("batch", body, env=env)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
async def readfetch_pdf(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
249
|
+
return await call("pdf", body, env=env)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
async def readfetch_section(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
253
|
+
return await call("section", body, env=env)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
async def readfetch_diff(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
257
|
+
return await call("diff", body, env=env)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
async def readfetch_screenshot(
|
|
261
|
+
body: dict[str, Any], *, env: Mapping[str, str] | None = None
|
|
262
|
+
) -> dict:
|
|
263
|
+
return await call("screenshot", body, env=env)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
async def readfetch_search(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
267
|
+
return await call("search", body, env=env)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
async def readfetch_crawl(body: dict[str, Any], *, env: Mapping[str, str] | None = None) -> dict:
|
|
271
|
+
return await call("crawl", body, env=env)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
async def readfetch_crawl_status(
|
|
275
|
+
job_id: str, *, env: Mapping[str, str] | None = None
|
|
276
|
+
) -> dict:
|
|
277
|
+
return await call("crawl_status", None, path_params={"id": job_id}, env=env)
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
|
|
9
|
+
_PRODUCTION_HOSTS = frozenset({"readfetch.com", "www.readfetch.com"})
|
|
10
|
+
_DEV_HOSTS = frozenset({"localhost", "127.0.0.1", "[::1]", "::1"})
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def is_dev_host(hostname: str) -> bool:
|
|
14
|
+
return _normalize_hostname(hostname) in _DEV_HOSTS
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_MAX_PAYMENT_RE = re.compile(r"^\d+(\.\d{1,6})?$")
|
|
18
|
+
_HEX_KEY_RE = re.compile(r"^0x[0-9a-fA-F]{64}$")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _normalize_hostname(hostname: str) -> str:
|
|
22
|
+
return hostname.rstrip(".").lower()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _format_origin(scheme: str, parsed) -> str:
|
|
26
|
+
host = parsed.hostname
|
|
27
|
+
if not host:
|
|
28
|
+
raise ValueError("READFETCH_BASE_URL must include a hostname")
|
|
29
|
+
host = _normalize_hostname(host)
|
|
30
|
+
if parsed.port:
|
|
31
|
+
return f"{scheme}://{host}:{parsed.port}"
|
|
32
|
+
return f"{scheme}://{host}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def readfetch_base(env: Mapping[str, str] | None = None) -> str:
|
|
36
|
+
env = os.environ if env is None else env
|
|
37
|
+
raw = env.get("READFETCH_BASE_URL", "https://readfetch.com").rstrip("/")
|
|
38
|
+
parsed = urlparse(raw)
|
|
39
|
+
if parsed.username or parsed.password:
|
|
40
|
+
raise ValueError("READFETCH_BASE_URL must not include userinfo")
|
|
41
|
+
if parsed.scheme not in ("http", "https"):
|
|
42
|
+
raise ValueError("READFETCH_BASE_URL must use http or https")
|
|
43
|
+
host = _normalize_hostname(parsed.hostname or "")
|
|
44
|
+
if not host:
|
|
45
|
+
raise ValueError("READFETCH_BASE_URL must include a hostname")
|
|
46
|
+
if host in _PRODUCTION_HOSTS:
|
|
47
|
+
if parsed.scheme != "https":
|
|
48
|
+
raise ValueError("READFETCH_BASE_URL must use https for readfetch.com")
|
|
49
|
+
path = parsed.path or ""
|
|
50
|
+
if path not in ("", "/"):
|
|
51
|
+
raise ValueError(
|
|
52
|
+
"READFETCH_BASE_URL must not include a path on readfetch.com"
|
|
53
|
+
)
|
|
54
|
+
if parsed.port is not None and parsed.port != 443:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"READFETCH_BASE_URL must use default https port for readfetch.com"
|
|
57
|
+
)
|
|
58
|
+
return _format_origin("https", parsed)
|
|
59
|
+
if env.get("READFETCH_DEV") == "1" and is_dev_host(host):
|
|
60
|
+
scheme = parsed.scheme or "http"
|
|
61
|
+
return _format_origin(scheme, parsed)
|
|
62
|
+
raise ValueError(
|
|
63
|
+
"READFETCH_BASE_URL must be https://readfetch.com "
|
|
64
|
+
"(or localhost with READFETCH_DEV=1)"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def readfetch_url(env: Mapping[str, str] | None, path: str) -> str:
|
|
69
|
+
base = readfetch_base(env)
|
|
70
|
+
if not path.startswith("/"):
|
|
71
|
+
raise ValueError("endpoint path must start with /")
|
|
72
|
+
url = f"{base}{path}"
|
|
73
|
+
if urlparse(url).netloc != urlparse(base).netloc:
|
|
74
|
+
raise ValueError("request URL origin must match READFETCH_BASE_URL")
|
|
75
|
+
return url
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def max_payment_usd(env: Mapping[str, str] | None = None) -> str:
|
|
79
|
+
env = os.environ if env is None else env
|
|
80
|
+
raw = (env.get("READFETCH_MAX_PAYMENT_USD") or "1").strip()
|
|
81
|
+
if not _MAX_PAYMENT_RE.fullmatch(raw):
|
|
82
|
+
raise ValueError("READFETCH_MAX_PAYMENT_USD must be a decimal amount")
|
|
83
|
+
amount = float(raw)
|
|
84
|
+
if amount <= 0 or amount > 100:
|
|
85
|
+
raise ValueError("READFETCH_MAX_PAYMENT_USD must be between 0 and 100")
|
|
86
|
+
return raw
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def x402_network(env: Mapping[str, str] | None = None) -> str:
|
|
90
|
+
env = os.environ if env is None else env
|
|
91
|
+
base = readfetch_base(env)
|
|
92
|
+
host = _normalize_hostname(urlparse(base).hostname or "")
|
|
93
|
+
if env.get("READFETCH_DEV") == "1" and is_dev_host(host):
|
|
94
|
+
return "eip155:84532"
|
|
95
|
+
return "eip155:8453"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def load_account_hex(env: Mapping[str, str] | None = None) -> str | None:
|
|
99
|
+
"""Return normalized key hex, or None if unset or invalid (no raise)."""
|
|
100
|
+
env = os.environ if env is None else env
|
|
101
|
+
raw = (env.get("EVM_PRIVATE_KEY") or env.get("X402_BUYER_KEY") or "").strip()
|
|
102
|
+
if not raw:
|
|
103
|
+
return None
|
|
104
|
+
hex_key = raw if raw.startswith("0x") else f"0x{raw}"
|
|
105
|
+
if not _HEX_KEY_RE.fullmatch(hex_key):
|
|
106
|
+
return None
|
|
107
|
+
return hex_key.lower()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def require_account_hex(env: Mapping[str, str] | None = None) -> str:
|
|
111
|
+
"""Strict key load for paid calls — raises if missing or malformed."""
|
|
112
|
+
hex_key = load_account_hex(env)
|
|
113
|
+
if hex_key is None:
|
|
114
|
+
env = os.environ if env is None else env
|
|
115
|
+
if not (env.get("EVM_PRIVATE_KEY") or env.get("X402_BUYER_KEY") or "").strip():
|
|
116
|
+
raise ValueError("Set EVM_PRIVATE_KEY (Base wallet with USDC)")
|
|
117
|
+
raise ValueError("EVM_PRIVATE_KEY must be 64 hex bytes (optional 0x prefix)")
|
|
118
|
+
return hex_key
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def buyer_address(env: Mapping[str, str] | None = None) -> str | None:
|
|
122
|
+
hex_key = load_account_hex(env)
|
|
123
|
+
if not hex_key:
|
|
124
|
+
return None
|
|
125
|
+
try:
|
|
126
|
+
from eth_account import Account
|
|
127
|
+
|
|
128
|
+
return Account.from_key(hex_key).address
|
|
129
|
+
except (ValueError, TypeError):
|
|
130
|
+
return None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def payment_config(env: Mapping[str, str] | None = None) -> dict[str, object]:
|
|
134
|
+
env = os.environ if env is None else env
|
|
135
|
+
address = buyer_address(env)
|
|
136
|
+
return {
|
|
137
|
+
"base_url": readfetch_base(env),
|
|
138
|
+
"network": x402_network(env),
|
|
139
|
+
"max_payment_usd": max_payment_usd(env),
|
|
140
|
+
"buyer_address": address,
|
|
141
|
+
"signing_ready": address is not None,
|
|
142
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Published routes — keep in sync with catalog.json (published + integrator)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Literal
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
Method = Literal["GET", "POST"]
|
|
11
|
+
_PATH_PARAM = re.compile(r"\{(\w+)\}")
|
|
12
|
+
MAX_PATH_PARAM_LEN = 512
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class EndpointSpec:
|
|
17
|
+
name: str
|
|
18
|
+
method: Method
|
|
19
|
+
path: str
|
|
20
|
+
free: bool
|
|
21
|
+
accept: tuple[int, ...] = ()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
CATALOG_ENDPOINTS: tuple[EndpointSpec, ...] = (
|
|
25
|
+
EndpointSpec("text", "POST", "/v1/text", False),
|
|
26
|
+
EndpointSpec("map", "POST", "/v1/map", False),
|
|
27
|
+
EndpointSpec("scrape", "POST", "/v1/scrape", False),
|
|
28
|
+
EndpointSpec("residential", "POST", "/v1/scrape/residential", False),
|
|
29
|
+
EndpointSpec("fetch", "POST", "/v1/fetch", False),
|
|
30
|
+
EndpointSpec("batch", "POST", "/v1/batch", False),
|
|
31
|
+
EndpointSpec("pdf", "POST", "/v1/pdf", False),
|
|
32
|
+
EndpointSpec("section", "POST", "/v1/section", False),
|
|
33
|
+
EndpointSpec("diff", "POST", "/v1/diff", False),
|
|
34
|
+
EndpointSpec("screenshot", "POST", "/v1/screenshot", False),
|
|
35
|
+
EndpointSpec("search", "POST", "/v1/search", False),
|
|
36
|
+
EndpointSpec("crawl", "POST", "/v1/crawl", False),
|
|
37
|
+
EndpointSpec("crawl_status", "GET", "/v1/crawl/{id}", True),
|
|
38
|
+
EndpointSpec("probe", "POST", "/v1/probe", True),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
INTEGRATOR_ENDPOINTS: tuple[EndpointSpec, ...] = (
|
|
42
|
+
EndpointSpec("feedback", "POST", "/v1/feedback", True, (202,)),
|
|
43
|
+
EndpointSpec("claims", "POST", "/v1/claims", True),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
BY_NAME: dict[str, EndpointSpec] = {
|
|
47
|
+
spec.name: spec for spec in (*CATALOG_ENDPOINTS, *INTEGRATOR_ENDPOINTS)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def endpoint_path(name: str, path_params: dict[str, str] | None = None) -> str:
|
|
52
|
+
spec = BY_NAME.get(name)
|
|
53
|
+
if spec is None:
|
|
54
|
+
known = ", ".join(sorted(BY_NAME))
|
|
55
|
+
raise ValueError(f"unknown endpoint `{name}`; known: {known}")
|
|
56
|
+
path_params = path_params or {}
|
|
57
|
+
out = spec.path
|
|
58
|
+
for key in _PATH_PARAM.findall(spec.path):
|
|
59
|
+
raw = path_params.get(key)
|
|
60
|
+
if raw is None:
|
|
61
|
+
raise ValueError(f"`{name}` requires path_params {key}")
|
|
62
|
+
value = str(raw).strip()
|
|
63
|
+
if not value:
|
|
64
|
+
raise ValueError(f"`{name}` requires path_params {key}")
|
|
65
|
+
if len(value) > MAX_PATH_PARAM_LEN:
|
|
66
|
+
raise ValueError(
|
|
67
|
+
f"`{name}` path_params.{key} exceeds {MAX_PATH_PARAM_LEN} characters"
|
|
68
|
+
)
|
|
69
|
+
out = out.replace(f"{{{key}}}", quote(value, safe=""))
|
|
70
|
+
if "{" in out:
|
|
71
|
+
raise ValueError(f"`{name}` has unresolved path template")
|
|
72
|
+
return out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def export_name(name: str) -> str:
|
|
76
|
+
parts = name.split("_")
|
|
77
|
+
return "readfetch" + "".join(part[:1].upper() + part[1:] for part in parts)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
FEEDBACK_CATEGORIES = frozenset({"integration", "docs", "site", "other"})
|
|
4
|
+
MAX_FEEDBACK_REQUEST_ID_LEN = 256
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def normalize_feedback(
|
|
8
|
+
request_id: str, category: str, message: str
|
|
9
|
+
) -> tuple[str, str, str]:
|
|
10
|
+
rid = (request_id or "").strip()
|
|
11
|
+
cat = (category or "").strip()
|
|
12
|
+
msg = (message or "").strip()
|
|
13
|
+
if not rid:
|
|
14
|
+
raise ValueError("request_id is required")
|
|
15
|
+
if len(rid) > MAX_FEEDBACK_REQUEST_ID_LEN:
|
|
16
|
+
raise ValueError(
|
|
17
|
+
f"request_id exceeds {MAX_FEEDBACK_REQUEST_ID_LEN} characters"
|
|
18
|
+
)
|
|
19
|
+
if cat not in FEEDBACK_CATEGORIES:
|
|
20
|
+
raise ValueError("category must be integration, docs, site, or other")
|
|
21
|
+
if not msg or len(msg) > 500:
|
|
22
|
+
raise ValueError("message is required (max 500 characters)")
|
|
23
|
+
return rid, cat, msg
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
MAX_JSON_RESPONSE_BYTES = 16 * 1024 * 1024
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def assert_response_size(response: httpx.Response) -> None:
|
|
11
|
+
raw = response.headers.get("content-length")
|
|
12
|
+
if raw:
|
|
13
|
+
try:
|
|
14
|
+
n = int(raw)
|
|
15
|
+
except ValueError:
|
|
16
|
+
n = -1
|
|
17
|
+
if n > MAX_JSON_RESPONSE_BYTES:
|
|
18
|
+
raise RuntimeError(
|
|
19
|
+
f"{response.status_code}: content-length {n} exceeds "
|
|
20
|
+
f"{MAX_JSON_RESPONSE_BYTES} bytes"
|
|
21
|
+
)
|
|
22
|
+
if response.content and len(response.content) > MAX_JSON_RESPONSE_BYTES:
|
|
23
|
+
raise RuntimeError(
|
|
24
|
+
f"{response.status_code}: response body exceeds "
|
|
25
|
+
f"{MAX_JSON_RESPONSE_BYTES} bytes"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def json_object_from_response(response: httpx.Response) -> dict[str, Any]:
|
|
30
|
+
assert_response_size(response)
|
|
31
|
+
if not response.content:
|
|
32
|
+
return {}
|
|
33
|
+
try:
|
|
34
|
+
parsed = response.json()
|
|
35
|
+
except Exception as error:
|
|
36
|
+
raise RuntimeError(
|
|
37
|
+
f"{response.status_code}: {(response.text or '')[:400]} (invalid JSON)"
|
|
38
|
+
) from error
|
|
39
|
+
if not isinstance(parsed, dict):
|
|
40
|
+
raise RuntimeError(
|
|
41
|
+
f"{response.status_code}: expected JSON object, got {type(parsed).__name__}"
|
|
42
|
+
)
|
|
43
|
+
return parsed
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def payment_from_response(response: httpx.Response) -> dict[str, str | None]:
|
|
7
|
+
"""Headers after a successful paid call (200) or on 402 quote."""
|
|
8
|
+
return {
|
|
9
|
+
"receipt": response.headers.get("receipt") or response.headers.get("RECEIPT"),
|
|
10
|
+
"payment_response": (
|
|
11
|
+
response.headers.get("payment-response")
|
|
12
|
+
or response.headers.get("PAYMENT-RESPONSE")
|
|
13
|
+
or response.headers.get("x-payment-response")
|
|
14
|
+
or response.headers.get("X-PAYMENT-RESPONSE")
|
|
15
|
+
),
|
|
16
|
+
"payment_required": (
|
|
17
|
+
response.headers.get("payment-required")
|
|
18
|
+
or response.headers.get("PAYMENT-REQUIRED")
|
|
19
|
+
),
|
|
20
|
+
"credit_applied": response.headers.get("x-credit-applied")
|
|
21
|
+
or response.headers.get("X-CREDIT-APPLIED"),
|
|
22
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: readfetch-client
|
|
3
|
+
Version: 0.1.11
|
|
4
|
+
Summary: x402 REST client for readfetch.com (probe, text, …)
|
|
5
|
+
Project-URL: Homepage, https://readfetch.com
|
|
6
|
+
Project-URL: Repository, https://github.com/readfetch/client-py
|
|
7
|
+
License: MIT
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Requires-Dist: eth-account>=0.13
|
|
10
|
+
Requires-Dist: httpx>=0.27
|
|
11
|
+
Requires-Dist: x402[evm,httpx]>=2.0
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# readfetch-client
|
|
15
|
+
|
|
16
|
+
Python x402 REST client for [readfetch.com](https://readfetch.com) — all published catalog routes plus feedback and claims.
|
|
17
|
+
|
|
18
|
+
Checkout: `tools/client-py`. GitHub: `./scripts/publish-client-py.sh`.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install readfetch-client
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Use
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
export EVM_PRIVATE_KEY=0x… # paid routes only
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
import asyncio
|
|
34
|
+
from readfetch_client import (
|
|
35
|
+
call,
|
|
36
|
+
probe_with_meta,
|
|
37
|
+
readfetch_text,
|
|
38
|
+
readfetch_map,
|
|
39
|
+
readfetch_crawl_status,
|
|
40
|
+
submit_feedback,
|
|
41
|
+
submit_claim,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
async def main():
|
|
45
|
+
check, request_id = await probe_with_meta("https://example.com")
|
|
46
|
+
if not check.get("challenge"):
|
|
47
|
+
await readfetch_text("https://example.com")
|
|
48
|
+
await readfetch_map({"url": "https://example.com", "max_urls": 20})
|
|
49
|
+
job = await call("crawl", {"url": "https://example.com", "max_pages": 3})
|
|
50
|
+
await readfetch_crawl_status(job["id"])
|
|
51
|
+
if request_id:
|
|
52
|
+
await submit_feedback(request_id, "integration", "…")
|
|
53
|
+
|
|
54
|
+
asyncio.run(main())
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Named helpers: `readfetch_text`, `readfetch_map`, `readfetch_scrape`, `readfetch_residential`, `readfetch_fetch`, `readfetch_batch`, `readfetch_pdf`, `readfetch_section`, `readfetch_diff`, `readfetch_screenshot`, `readfetch_search`, `readfetch_crawl`, `readfetch_crawl_status`. Generic: `await call("search", {"query": "…"})`.
|
|
58
|
+
|
|
59
|
+
## Payment (x402)
|
|
60
|
+
|
|
61
|
+
Paid routes use **`x402` + `x402HttpxClient`**: on **402**, the client signs and retries. No key is required for quotes.
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from readfetch_client import (
|
|
65
|
+
payment_config,
|
|
66
|
+
quote,
|
|
67
|
+
call_with_meta,
|
|
68
|
+
x402_http_client,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
print(payment_config()) # base_url, network, max_payment_usd, buyer_address, signing_ready
|
|
72
|
+
|
|
73
|
+
q = await quote("text", {"url": "https://example.com"}) # 402 + payment_required
|
|
74
|
+
|
|
75
|
+
meta = await call_with_meta("text", {"url": "https://example.com"})
|
|
76
|
+
# meta["data"], meta["request_id"], meta["payment"] (receipt, payment_response, …)
|
|
77
|
+
|
|
78
|
+
async with x402_http_client() as http:
|
|
79
|
+
...
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`pay(name, body)` aliases `call()` for paid endpoints.
|
|
83
|
+
|
|
84
|
+
## Environment
|
|
85
|
+
|
|
86
|
+
| Variable | Default | Purpose |
|
|
87
|
+
|----------|---------|---------|
|
|
88
|
+
| `EVM_PRIVATE_KEY` / `X402_BUYER_KEY` | — | Required before paid calls |
|
|
89
|
+
| `READFETCH_BASE_URL` | `https://readfetch.com` | Production host only unless dev |
|
|
90
|
+
| `READFETCH_MAX_PAYMENT_USD` | `1` | Per-payment spend cap |
|
|
91
|
+
| `READFETCH_DEV=1` | off | Allow local base URL for integration tests |
|
|
92
|
+
|
|
93
|
+
MCP: [@readfetch/mcp-x402-proxy](https://github.com/readfetch/mcp-x402-proxy). Feedback: [docs#feedback](https://readfetch.com/docs#feedback).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
readfetch_client/__init__.py,sha256=fKvq2YvObqgpF6SsYYhKVqyIggjtZqWngtnpsXoiYgA,1231
|
|
2
|
+
readfetch_client/client.py,sha256=3mjgJSH7VD1znZKn4p4M89R2rB9Ok2bVXBweXLDkSPU,9104
|
|
3
|
+
readfetch_client/config.py,sha256=vYnZLN-x2AHfUoH4qRyX4HqJQBlyF7M-N6pSPbs_aC8,5129
|
|
4
|
+
readfetch_client/endpoints.py,sha256=YBNOqSVpBuQ58CRWBdDge8g-uGNRAY01u8DG8BVUDj4,2695
|
|
5
|
+
readfetch_client/feedback_validation.py,sha256=zorLvLX0AfHtu83z_knpfwtjw4XyRWACQlWDIs0UfeY,823
|
|
6
|
+
readfetch_client/http_json.py,sha256=31wamfukcPWH5sFIM1ATUpYZQCulva6Y3PFAcO42Zk0,1303
|
|
7
|
+
readfetch_client/payment.py,sha256=_PQspdrRIuyoBDBgJy25tDbcx_hjVdr4ahL1_Kf3x7M,841
|
|
8
|
+
readfetch_client-0.1.11.dist-info/METADATA,sha256=8NtMdVotn_vls4H98y3S-yjWHAuQOvH5Bxl92YkFEkc,2972
|
|
9
|
+
readfetch_client-0.1.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
10
|
+
readfetch_client-0.1.11.dist-info/RECORD,,
|