crawlutron-sdk 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlutron_sdk/__init__.py +107 -0
- crawlutron_sdk/client.py +496 -0
- crawlutron_sdk/errors.py +169 -0
- crawlutron_sdk/models.py +482 -0
- crawlutron_sdk/py.typed +0 -0
- crawlutron_sdk-0.5.0.dist-info/METADATA +29 -0
- crawlutron_sdk-0.5.0.dist-info/RECORD +8 -0
- crawlutron_sdk-0.5.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Lightweight typed HTTP client SDK for a running crawlutron service. ``crawlutron_sdk`` is a thin facade
|
|
2
|
+
over the service's wire contract. The server is the sole enforcement chokepoint; this package reimplements
|
|
3
|
+
no admission, policy, robots, metering, or extraction logic.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from .client import CrawlutronClient, CrawlutronClientSync
|
|
9
|
+
from .errors import (
|
|
10
|
+
CrawlutronAcceptedError,
|
|
11
|
+
CrawlutronBadGatewayError,
|
|
12
|
+
CrawlutronBadRequestError,
|
|
13
|
+
CrawlutronClientClosedRequestError,
|
|
14
|
+
CrawlutronClientError,
|
|
15
|
+
CrawlutronConflictError,
|
|
16
|
+
CrawlutronError,
|
|
17
|
+
CrawlutronForbiddenError,
|
|
18
|
+
CrawlutronInternalServerError,
|
|
19
|
+
CrawlutronNotFoundError,
|
|
20
|
+
CrawlutronRequestTooLargeError,
|
|
21
|
+
CrawlutronServerError,
|
|
22
|
+
CrawlutronServiceUnavailableError,
|
|
23
|
+
CrawlutronTooManyRequestsError,
|
|
24
|
+
CrawlutronTransportError,
|
|
25
|
+
CrawlutronUnauthorizedError,
|
|
26
|
+
CrawlutronUnprocessableEntityError,
|
|
27
|
+
error_from_response,
|
|
28
|
+
)
|
|
29
|
+
from .models import (
|
|
30
|
+
AdmitRequest,
|
|
31
|
+
AdmitResponse,
|
|
32
|
+
AgentHintsResponse,
|
|
33
|
+
CapabilitiesResponse,
|
|
34
|
+
CrawlAccepted,
|
|
35
|
+
CrawlRequest,
|
|
36
|
+
CrawlResponse,
|
|
37
|
+
ErrorEnvelope,
|
|
38
|
+
HealthResponse,
|
|
39
|
+
HostPolicySuggestion,
|
|
40
|
+
JobControlRequest,
|
|
41
|
+
JobCreateRequest,
|
|
42
|
+
JobCreateResponse,
|
|
43
|
+
JobDeletedResponse,
|
|
44
|
+
JobListResponse,
|
|
45
|
+
JobStatusResponse,
|
|
46
|
+
MapRequest,
|
|
47
|
+
PolicySuggestionsResponse,
|
|
48
|
+
ReadinessResponse,
|
|
49
|
+
RenderRequest,
|
|
50
|
+
SearchRequest,
|
|
51
|
+
SearchResponse,
|
|
52
|
+
SearchResultItem,
|
|
53
|
+
SiteMapResponse,
|
|
54
|
+
WatchCreateRequest,
|
|
55
|
+
WatchListResponse,
|
|
56
|
+
WatchState,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
__all__ = [
|
|
60
|
+
"CrawlutronAcceptedError",
|
|
61
|
+
"CrawlutronBadGatewayError",
|
|
62
|
+
"CrawlutronBadRequestError",
|
|
63
|
+
"CrawlutronClient",
|
|
64
|
+
"CrawlutronClientClosedRequestError",
|
|
65
|
+
"CrawlutronClientError",
|
|
66
|
+
"CrawlutronClientSync",
|
|
67
|
+
"CrawlutronConflictError",
|
|
68
|
+
"CrawlutronError",
|
|
69
|
+
"CrawlutronForbiddenError",
|
|
70
|
+
"CrawlutronInternalServerError",
|
|
71
|
+
"CrawlutronNotFoundError",
|
|
72
|
+
"CrawlutronRequestTooLargeError",
|
|
73
|
+
"CrawlutronServerError",
|
|
74
|
+
"CrawlutronServiceUnavailableError",
|
|
75
|
+
"CrawlutronTooManyRequestsError",
|
|
76
|
+
"CrawlutronTransportError",
|
|
77
|
+
"CrawlutronUnauthorizedError",
|
|
78
|
+
"CrawlutronUnprocessableEntityError",
|
|
79
|
+
"WatchState",
|
|
80
|
+
"WatchListResponse",
|
|
81
|
+
"WatchCreateRequest",
|
|
82
|
+
"SearchResultItem",
|
|
83
|
+
"SiteMapResponse",
|
|
84
|
+
"SearchResponse",
|
|
85
|
+
"SearchRequest",
|
|
86
|
+
"ReadinessResponse",
|
|
87
|
+
"PolicySuggestionsResponse",
|
|
88
|
+
"RenderRequest",
|
|
89
|
+
"MapRequest",
|
|
90
|
+
"JobStatusResponse",
|
|
91
|
+
"JobListResponse",
|
|
92
|
+
"JobDeletedResponse",
|
|
93
|
+
"JobCreateResponse",
|
|
94
|
+
"JobCreateRequest",
|
|
95
|
+
"JobControlRequest",
|
|
96
|
+
"HealthResponse",
|
|
97
|
+
"HostPolicySuggestion",
|
|
98
|
+
"ErrorEnvelope",
|
|
99
|
+
"CrawlResponse",
|
|
100
|
+
"CrawlRequest",
|
|
101
|
+
"CrawlAccepted",
|
|
102
|
+
"CapabilitiesResponse",
|
|
103
|
+
"AgentHintsResponse",
|
|
104
|
+
"AdmitResponse",
|
|
105
|
+
"AdmitRequest",
|
|
106
|
+
"error_from_response",
|
|
107
|
+
]
|
crawlutron_sdk/client.py
ADDED
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
"""Typed async HTTP facade over a running crawlutron service."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Awaitable, Callable
|
|
8
|
+
from typing import Literal, Never, TypedDict, TypeVar
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
from pydantic import BaseModel
|
|
12
|
+
|
|
13
|
+
from .errors import CrawlutronAcceptedError, CrawlutronTransportError, error_from_response
|
|
14
|
+
from .models import (
|
|
15
|
+
AdmitRequest,
|
|
16
|
+
AdmitResponse,
|
|
17
|
+
AgentHintsResponse,
|
|
18
|
+
CapabilitiesResponse,
|
|
19
|
+
CrawlAccepted,
|
|
20
|
+
CrawlRequest,
|
|
21
|
+
CrawlResponse,
|
|
22
|
+
HealthResponse,
|
|
23
|
+
JobControlRequest,
|
|
24
|
+
JobCreateRequest,
|
|
25
|
+
JobCreateResponse,
|
|
26
|
+
JobListResponse,
|
|
27
|
+
JobStatusResponse,
|
|
28
|
+
MapRequest,
|
|
29
|
+
PolicySuggestionsResponse,
|
|
30
|
+
ReadinessResponse,
|
|
31
|
+
RenderRequest,
|
|
32
|
+
SearchRequest,
|
|
33
|
+
SearchResponse,
|
|
34
|
+
SiteMapResponse,
|
|
35
|
+
WatchCreateRequest,
|
|
36
|
+
WatchListResponse,
|
|
37
|
+
WatchState,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
_DEFAULT_BASE_URL = "http://127.0.0.1:8765"
|
|
41
|
+
_DEFAULT_TIMEOUT = 300.0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def _poll_result(
|
|
45
|
+
request_id: str,
|
|
46
|
+
fetch: Callable[[], Awaitable[CrawlResponse]],
|
|
47
|
+
deadline: float | None,
|
|
48
|
+
poll_interval: float,
|
|
49
|
+
timeout_factory: Callable[[str], CrawlutronTransportError],
|
|
50
|
+
) -> CrawlResponse:
|
|
51
|
+
while True:
|
|
52
|
+
try:
|
|
53
|
+
return await fetch()
|
|
54
|
+
except CrawlutronAcceptedError:
|
|
55
|
+
pass
|
|
56
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
57
|
+
raise timeout_factory(request_id)
|
|
58
|
+
await asyncio.sleep(poll_interval)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
_ModelT = TypeVar("_ModelT", bound=BaseModel)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class _ClientKwargs(TypedDict):
|
|
65
|
+
base_url: str
|
|
66
|
+
timeout: float | None
|
|
67
|
+
headers: dict[str, str]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _str_key_dict(data: object) -> dict[str, object]:
|
|
71
|
+
if not isinstance(data, dict):
|
|
72
|
+
return {}
|
|
73
|
+
return {str(key): value for key, value in data.items()}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _json_body(response: httpx.Response) -> dict[str, object]:
|
|
77
|
+
if not response.content:
|
|
78
|
+
return {}
|
|
79
|
+
return _str_key_dict(response.json())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _payload_dict(payload: object) -> dict[str, object]:
|
|
83
|
+
dump = getattr(payload, "model_dump", None)
|
|
84
|
+
if callable(dump):
|
|
85
|
+
return _str_key_dict(dump())
|
|
86
|
+
return {}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _job_list_query(limit: int | None, offset: int) -> str:
|
|
90
|
+
parts = [f"limit={limit}"] if limit is not None else []
|
|
91
|
+
parts += [f"offset={offset}"] if offset else []
|
|
92
|
+
return f"?{'&'.join(parts)}" if parts else ""
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class _ClientBase:
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
*,
|
|
99
|
+
base_url: str = _DEFAULT_BASE_URL,
|
|
100
|
+
api_key: str | None = None,
|
|
101
|
+
timeout: float | None = _DEFAULT_TIMEOUT,
|
|
102
|
+
transport: httpx.BaseTransport | httpx.AsyncBaseTransport | None = None,
|
|
103
|
+
) -> None:
|
|
104
|
+
self._base_url = base_url.rstrip("/")
|
|
105
|
+
self._api_key = api_key
|
|
106
|
+
self._timeout = timeout
|
|
107
|
+
self._transport = transport
|
|
108
|
+
self._client: httpx.Client | httpx.AsyncClient | None = None
|
|
109
|
+
|
|
110
|
+
def _headers(self) -> dict[str, str]:
|
|
111
|
+
headers: dict[str, str] = {}
|
|
112
|
+
if self._api_key is not None:
|
|
113
|
+
headers["X-API-Key"] = self._api_key
|
|
114
|
+
return headers
|
|
115
|
+
|
|
116
|
+
def _client_kwargs(self) -> _ClientKwargs:
|
|
117
|
+
return {
|
|
118
|
+
"base_url": self._base_url,
|
|
119
|
+
"timeout": self._timeout,
|
|
120
|
+
"headers": self._headers(),
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
def _raise_for_status(
|
|
124
|
+
self,
|
|
125
|
+
status_code: int,
|
|
126
|
+
body: dict[str, object],
|
|
127
|
+
*,
|
|
128
|
+
accepted_if_pending: bool = False,
|
|
129
|
+
) -> None:
|
|
130
|
+
"""Raise typed errors; with ``accepted_if_pending`` also refuse a 202."""
|
|
131
|
+
if accepted_if_pending and status_code == 202:
|
|
132
|
+
raise error_from_response(status_code, body)
|
|
133
|
+
if status_code < 400:
|
|
134
|
+
return
|
|
135
|
+
raise error_from_response(status_code, body)
|
|
136
|
+
|
|
137
|
+
def _check(self, response: httpx.Response) -> None:
|
|
138
|
+
"""Raise typed errors for a response with no body to return."""
|
|
139
|
+
self._raise_for_status(response.status_code, _json_body(response))
|
|
140
|
+
|
|
141
|
+
def _finish(
|
|
142
|
+
self,
|
|
143
|
+
response: httpx.Response,
|
|
144
|
+
model: type[_ModelT],
|
|
145
|
+
*,
|
|
146
|
+
accepted_if_pending: bool = False,
|
|
147
|
+
) -> _ModelT:
|
|
148
|
+
"""Validate one response body against ``model`` after status checking."""
|
|
149
|
+
body = _json_body(response)
|
|
150
|
+
self._raise_for_status(response.status_code, body, accepted_if_pending=accepted_if_pending)
|
|
151
|
+
return model.model_validate(body)
|
|
152
|
+
|
|
153
|
+
def _transport_error(self, error: httpx.HTTPError) -> CrawlutronTransportError:
|
|
154
|
+
return CrawlutronTransportError(str(error), status_code=None)
|
|
155
|
+
|
|
156
|
+
def _transport_failure(self, error: httpx.HTTPError) -> Never:
|
|
157
|
+
raise self._transport_error(error) from error
|
|
158
|
+
|
|
159
|
+
def _make_timeout(self, request_id: str, max_wait: float | None = None) -> CrawlutronTransportError:
|
|
160
|
+
bound = f" within {max_wait}s" if max_wait is not None else ""
|
|
161
|
+
return CrawlutronTransportError(
|
|
162
|
+
f"crawl {request_id} did not reach a terminal state{bound}", status_code=None
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
def _make_wait_deadline(self, max_wait: float | None) -> float | None:
|
|
166
|
+
if max_wait is not None:
|
|
167
|
+
return time.monotonic() + max_wait
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class CrawlutronClient(_ClientBase):
|
|
172
|
+
_client: httpx.AsyncClient | None = None
|
|
173
|
+
|
|
174
|
+
async def __aenter__(self) -> CrawlutronClient:
|
|
175
|
+
return self
|
|
176
|
+
|
|
177
|
+
async def __aexit__(self, *args: object) -> None:
|
|
178
|
+
await self.aclose()
|
|
179
|
+
|
|
180
|
+
async def _get_client(self) -> httpx.AsyncClient:
|
|
181
|
+
client = self._client
|
|
182
|
+
if client is None or not isinstance(client, httpx.AsyncClient):
|
|
183
|
+
transport = self._transport
|
|
184
|
+
if transport is not None and not isinstance(transport, httpx.AsyncBaseTransport):
|
|
185
|
+
raise CrawlutronTransportError(
|
|
186
|
+
"sync-only transport cannot drive an async client",
|
|
187
|
+
status_code=None,
|
|
188
|
+
)
|
|
189
|
+
self._client = httpx.AsyncClient(**self._client_kwargs(), transport=transport)
|
|
190
|
+
client = self._client
|
|
191
|
+
if not isinstance(client, httpx.AsyncClient):
|
|
192
|
+
raise CrawlutronTransportError("unexpected http client type", status_code=None)
|
|
193
|
+
return client
|
|
194
|
+
|
|
195
|
+
async def aclose(self) -> None:
|
|
196
|
+
client = self._client
|
|
197
|
+
if client is None:
|
|
198
|
+
return
|
|
199
|
+
if not isinstance(client, httpx.AsyncClient):
|
|
200
|
+
raise CrawlutronTransportError("unexpected http client type", status_code=None)
|
|
201
|
+
await client.aclose()
|
|
202
|
+
self._client = None
|
|
203
|
+
|
|
204
|
+
async def _request(
|
|
205
|
+
self,
|
|
206
|
+
method: str,
|
|
207
|
+
path: str,
|
|
208
|
+
*,
|
|
209
|
+
json: dict[str, object] | None = None,
|
|
210
|
+
) -> httpx.Response:
|
|
211
|
+
client = await self._get_client()
|
|
212
|
+
try:
|
|
213
|
+
return await client.request(method, path, json=json)
|
|
214
|
+
except httpx.HTTPError as error:
|
|
215
|
+
raise self._transport_error(error) from error
|
|
216
|
+
|
|
217
|
+
async def crawl(self, payload: CrawlRequest) -> CrawlResponse:
|
|
218
|
+
"""Synchronous crawl door: returns the unified crawl response."""
|
|
219
|
+
response = await self._request("POST", "/v1/crawl", json=_payload_dict(payload))
|
|
220
|
+
return self._finish(response, CrawlResponse, accepted_if_pending=True)
|
|
221
|
+
|
|
222
|
+
async def crawl_async(self, payload: CrawlRequest) -> CrawlAccepted:
|
|
223
|
+
"""Async crawl door: returns the accepted handle (request_id, status_url)."""
|
|
224
|
+
request = payload.model_copy(update={"async_mode": True})
|
|
225
|
+
response = await self._request("POST", "/v1/crawl", json=_payload_dict(request))
|
|
226
|
+
return self._finish(response, CrawlAccepted)
|
|
227
|
+
|
|
228
|
+
async def get_crawl_result(self, request_id: str) -> CrawlResponse:
|
|
229
|
+
"""Retrieve a completed crawl; raises if still accepted (202) or gone (404)."""
|
|
230
|
+
response = await self._request("GET", f"/v1/crawl/{request_id}")
|
|
231
|
+
return self._finish(response, CrawlResponse, accepted_if_pending=True)
|
|
232
|
+
|
|
233
|
+
async def wait_for(
|
|
234
|
+
self, request_id: str, *, poll_interval: float = 1.0, max_wait: float | None = None
|
|
235
|
+
) -> CrawlResponse:
|
|
236
|
+
"""Poll ``get_crawl_result`` until the crawl reaches a terminal state."""
|
|
237
|
+
return await _poll_result(
|
|
238
|
+
request_id,
|
|
239
|
+
lambda: self.get_crawl_result(request_id),
|
|
240
|
+
self._make_wait_deadline(max_wait),
|
|
241
|
+
poll_interval,
|
|
242
|
+
lambda rid: self._make_timeout(rid, max_wait),
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
async def crawl_and_wait(self, payload: CrawlRequest) -> CrawlResponse:
|
|
246
|
+
"""Crawl asynchronously and wait for the unified response."""
|
|
247
|
+
accepted = await self.crawl_async(payload)
|
|
248
|
+
return await self.wait_for(accepted.request_id)
|
|
249
|
+
|
|
250
|
+
async def admit(self, payload: AdmitRequest) -> AdmitResponse:
|
|
251
|
+
"""Pre-flight admission verdict for one URL."""
|
|
252
|
+
response = await self._request("POST", "/v1/admit", json=_payload_dict(payload))
|
|
253
|
+
return self._finish(response, AdmitResponse)
|
|
254
|
+
|
|
255
|
+
async def map_snapshot(self, payload: MapRequest) -> SiteMapResponse:
|
|
256
|
+
"""Discover the URL frontier for a root URL (POST /v1/map)."""
|
|
257
|
+
response = await self._request("POST", "/v1/map", json=_payload_dict(payload))
|
|
258
|
+
return self._finish(response, SiteMapResponse)
|
|
259
|
+
|
|
260
|
+
async def render(self, payload: RenderRequest) -> CrawlResponse:
|
|
261
|
+
"""Browser-render one or more pages (POST /v1/render)."""
|
|
262
|
+
response = await self._request("POST", "/v1/render", json=_payload_dict(payload))
|
|
263
|
+
return self._finish(response, CrawlResponse)
|
|
264
|
+
|
|
265
|
+
async def policy_suggestions(self) -> PolicySuggestionsResponse:
|
|
266
|
+
"""Propose candidate host policy rules (GET /v1/policy-suggestions)."""
|
|
267
|
+
response = await self._request("GET", "/v1/policy-suggestions")
|
|
268
|
+
return self._finish(response, PolicySuggestionsResponse)
|
|
269
|
+
|
|
270
|
+
async def create_job(self, payload: JobCreateRequest) -> JobCreateResponse:
|
|
271
|
+
"""Create and start a resumable whole-site crawl job."""
|
|
272
|
+
response = await self._request("POST", "/v1/jobs", json=_payload_dict(payload))
|
|
273
|
+
return self._finish(response, JobCreateResponse)
|
|
274
|
+
|
|
275
|
+
async def list_jobs(self, limit: int | None = None, offset: int = 0) -> JobListResponse:
|
|
276
|
+
"""List the caller's crawl jobs; ``limit``/``offset`` page the result."""
|
|
277
|
+
response = await self._request("GET", f"/v1/jobs{_job_list_query(limit, offset)}")
|
|
278
|
+
return self._finish(response, JobListResponse)
|
|
279
|
+
|
|
280
|
+
async def get_job(self, job_id: str) -> JobStatusResponse:
|
|
281
|
+
"""Get one crawl job's state and results."""
|
|
282
|
+
response = await self._request("GET", f"/v1/jobs/{job_id}")
|
|
283
|
+
return self._finish(response, JobStatusResponse)
|
|
284
|
+
|
|
285
|
+
async def control_job(
|
|
286
|
+
self, job_id: str, action: Literal["pause", "resume", "cancel"]
|
|
287
|
+
) -> JobStatusResponse:
|
|
288
|
+
"""Pause/resume/cancel a crawl job."""
|
|
289
|
+
payload = JobControlRequest(action=action)
|
|
290
|
+
response = await self._request("POST", f"/v1/jobs/{job_id}/control", json=_payload_dict(payload))
|
|
291
|
+
return self._finish(response, JobStatusResponse)
|
|
292
|
+
|
|
293
|
+
async def delete_job(self, job_id: str) -> None:
|
|
294
|
+
"""Cancel (if running) and delete a crawl job."""
|
|
295
|
+
response = await self._request("DELETE", f"/v1/jobs/{job_id}")
|
|
296
|
+
self._check(response)
|
|
297
|
+
|
|
298
|
+
async def search(self, payload: SearchRequest) -> SearchResponse:
|
|
299
|
+
"""Run a web search against the configured provider."""
|
|
300
|
+
response = await self._request("POST", "/v1/search", json=_payload_dict(payload))
|
|
301
|
+
return self._finish(response, SearchResponse)
|
|
302
|
+
|
|
303
|
+
async def create_watch(self, payload: WatchCreateRequest) -> WatchState:
|
|
304
|
+
"""Register a URL revalidation watch."""
|
|
305
|
+
response = await self._request("POST", "/v1/watches", json=_payload_dict(payload))
|
|
306
|
+
return self._finish(response, WatchState)
|
|
307
|
+
|
|
308
|
+
async def list_watches(self) -> WatchListResponse:
|
|
309
|
+
"""List the caller's registered watches."""
|
|
310
|
+
response = await self._request("GET", "/v1/watches")
|
|
311
|
+
return self._finish(response, WatchListResponse)
|
|
312
|
+
|
|
313
|
+
async def get_watch(self, watch_id: str) -> WatchState:
|
|
314
|
+
"""Get one watch's revalidation state."""
|
|
315
|
+
response = await self._request("GET", f"/v1/watches/{watch_id}")
|
|
316
|
+
return self._finish(response, WatchState)
|
|
317
|
+
|
|
318
|
+
async def delete_watch(self, watch_id: str) -> None:
|
|
319
|
+
"""Stop checking a watch's URL."""
|
|
320
|
+
response = await self._request("DELETE", f"/v1/watches/{watch_id}")
|
|
321
|
+
self._check(response)
|
|
322
|
+
|
|
323
|
+
async def capabilities(self) -> CapabilitiesResponse:
|
|
324
|
+
"""Service capabilities discovery (GET /v1/capabilities)."""
|
|
325
|
+
response = await self._request("GET", "/v1/capabilities")
|
|
326
|
+
return self._finish(response, CapabilitiesResponse)
|
|
327
|
+
|
|
328
|
+
async def agent_hints(self) -> AgentHintsResponse:
|
|
329
|
+
"""Agent guidance (GET /v1/agent-hints)."""
|
|
330
|
+
response = await self._request("GET", "/v1/agent-hints")
|
|
331
|
+
return self._finish(response, AgentHintsResponse)
|
|
332
|
+
|
|
333
|
+
async def health(self) -> HealthResponse:
|
|
334
|
+
"""Lightweight liveness (GET /health)."""
|
|
335
|
+
response = await self._request("GET", "/health")
|
|
336
|
+
return self._finish(response, HealthResponse)
|
|
337
|
+
|
|
338
|
+
async def ready(self) -> ReadinessResponse:
|
|
339
|
+
"""Crawl capacity/readiness (GET /ready)."""
|
|
340
|
+
response = await self._request("GET", "/ready")
|
|
341
|
+
return self._finish(response, ReadinessResponse)
|
|
342
|
+
|
|
343
|
+
def sync(self) -> CrawlutronClientSync:
|
|
344
|
+
"""A sync client sharing this client's configuration."""
|
|
345
|
+
return CrawlutronClientSync(
|
|
346
|
+
base_url=self._base_url,
|
|
347
|
+
api_key=self._api_key,
|
|
348
|
+
timeout=self._timeout,
|
|
349
|
+
transport=self._transport,
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class CrawlutronClientSync(_ClientBase):
|
|
354
|
+
_client: httpx.Client | None = None
|
|
355
|
+
|
|
356
|
+
def __enter__(self) -> CrawlutronClientSync:
|
|
357
|
+
return self
|
|
358
|
+
|
|
359
|
+
def __exit__(self, *args: object) -> None:
|
|
360
|
+
self.close()
|
|
361
|
+
|
|
362
|
+
def _get_client(self) -> httpx.Client:
|
|
363
|
+
client = self._client
|
|
364
|
+
if not isinstance(client, httpx.Client):
|
|
365
|
+
transport = self._transport
|
|
366
|
+
if transport is not None and not isinstance(transport, httpx.BaseTransport):
|
|
367
|
+
raise CrawlutronTransportError(
|
|
368
|
+
"async-only transport cannot drive a sync client; pass an httpx.BaseTransport",
|
|
369
|
+
status_code=None,
|
|
370
|
+
)
|
|
371
|
+
self._client = httpx.Client(**self._client_kwargs(), transport=transport)
|
|
372
|
+
client = self._client
|
|
373
|
+
return client
|
|
374
|
+
|
|
375
|
+
def close(self) -> None:
|
|
376
|
+
if self._client is None:
|
|
377
|
+
return
|
|
378
|
+
client = self._client
|
|
379
|
+
if not isinstance(client, httpx.Client):
|
|
380
|
+
raise CrawlutronTransportError("unexpected http client type", status_code=None)
|
|
381
|
+
client.close()
|
|
382
|
+
self._client = None
|
|
383
|
+
|
|
384
|
+
def _request(self, method: str, path: str, *, json: dict[str, object] | None = None) -> httpx.Response:
|
|
385
|
+
client = self._get_client()
|
|
386
|
+
try:
|
|
387
|
+
return client.request(method, path, json=json)
|
|
388
|
+
except httpx.HTTPError as error:
|
|
389
|
+
self._transport_failure(error)
|
|
390
|
+
|
|
391
|
+
def crawl(self, payload: CrawlRequest) -> CrawlResponse:
|
|
392
|
+
response = self._request("POST", "/v1/crawl", json=_payload_dict(payload))
|
|
393
|
+
return self._finish(response, CrawlResponse, accepted_if_pending=True)
|
|
394
|
+
|
|
395
|
+
def crawl_async(self, payload: CrawlRequest) -> CrawlAccepted:
|
|
396
|
+
request = payload.model_copy(update={"async_mode": True})
|
|
397
|
+
response = self._request("POST", "/v1/crawl", json=_payload_dict(request))
|
|
398
|
+
return self._finish(response, CrawlAccepted)
|
|
399
|
+
|
|
400
|
+
def get_crawl_result(self, request_id: str) -> CrawlResponse:
|
|
401
|
+
"""Retrieve a completed crawl; raises if still accepted (202) or gone (404)."""
|
|
402
|
+
response = self._request("GET", f"/v1/crawl/{request_id}")
|
|
403
|
+
return self._finish(response, CrawlResponse, accepted_if_pending=True)
|
|
404
|
+
|
|
405
|
+
def wait_for(
|
|
406
|
+
self, request_id: str, *, max_wait: float | None = None, poll_interval: float = 1.0
|
|
407
|
+
) -> CrawlResponse:
|
|
408
|
+
"""Poll ``get_crawl_result`` until the crawl reaches a terminal state."""
|
|
409
|
+
deadline = self._make_wait_deadline(max_wait)
|
|
410
|
+
while True:
|
|
411
|
+
try:
|
|
412
|
+
return self.get_crawl_result(request_id)
|
|
413
|
+
except CrawlutronAcceptedError:
|
|
414
|
+
pass
|
|
415
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
416
|
+
raise self._make_timeout(request_id, max_wait)
|
|
417
|
+
time.sleep(poll_interval)
|
|
418
|
+
|
|
419
|
+
def admit(self, payload: AdmitRequest) -> AdmitResponse:
|
|
420
|
+
response = self._request("POST", "/v1/admit", json=_payload_dict(payload))
|
|
421
|
+
return self._finish(response, AdmitResponse)
|
|
422
|
+
|
|
423
|
+
def map_snapshot(self, payload: MapRequest) -> SiteMapResponse:
|
|
424
|
+
response = self._request("POST", "/v1/map", json=_payload_dict(payload))
|
|
425
|
+
return self._finish(response, SiteMapResponse)
|
|
426
|
+
|
|
427
|
+
def render(self, payload: RenderRequest) -> CrawlResponse:
|
|
428
|
+
response = self._request("POST", "/v1/render", json=_payload_dict(payload))
|
|
429
|
+
return self._finish(response, CrawlResponse)
|
|
430
|
+
|
|
431
|
+
def policy_suggestions(self) -> PolicySuggestionsResponse:
|
|
432
|
+
response = self._request("GET", "/v1/policy-suggestions")
|
|
433
|
+
return self._finish(response, PolicySuggestionsResponse)
|
|
434
|
+
|
|
435
|
+
def create_job(self, payload: JobCreateRequest) -> JobCreateResponse:
|
|
436
|
+
response = self._request("POST", "/v1/jobs", json=_payload_dict(payload))
|
|
437
|
+
return self._finish(response, JobCreateResponse)
|
|
438
|
+
|
|
439
|
+
def list_jobs(self, limit: int | None = None, offset: int = 0) -> JobListResponse:
|
|
440
|
+
response = self._request("GET", f"/v1/jobs{_job_list_query(limit, offset)}")
|
|
441
|
+
return self._finish(response, JobListResponse)
|
|
442
|
+
|
|
443
|
+
def get_job(self, job_id: str) -> JobStatusResponse:
|
|
444
|
+
response = self._request("GET", f"/v1/jobs/{job_id}")
|
|
445
|
+
return self._finish(response, JobStatusResponse)
|
|
446
|
+
|
|
447
|
+
def control_job(self, job_id: str, action: Literal["pause", "resume", "cancel"]) -> JobStatusResponse:
|
|
448
|
+
payload = JobControlRequest(action=action)
|
|
449
|
+
response = self._request("POST", f"/v1/jobs/{job_id}/control", json=_payload_dict(payload))
|
|
450
|
+
return self._finish(response, JobStatusResponse)
|
|
451
|
+
|
|
452
|
+
def delete_job(self, job_id: str) -> None:
|
|
453
|
+
response = self._request("DELETE", f"/v1/jobs/{job_id}")
|
|
454
|
+
self._check(response)
|
|
455
|
+
|
|
456
|
+
def search(self, payload: SearchRequest) -> SearchResponse:
|
|
457
|
+
response = self._request("POST", "/v1/search", json=_payload_dict(payload))
|
|
458
|
+
return self._finish(response, SearchResponse)
|
|
459
|
+
|
|
460
|
+
def create_watch(self, payload: WatchCreateRequest) -> WatchState:
|
|
461
|
+
response = self._request("POST", "/v1/watches", json=_payload_dict(payload))
|
|
462
|
+
return self._finish(response, WatchState)
|
|
463
|
+
|
|
464
|
+
def list_watches(self) -> WatchListResponse:
|
|
465
|
+
response = self._request("GET", "/v1/watches")
|
|
466
|
+
return self._finish(response, WatchListResponse)
|
|
467
|
+
|
|
468
|
+
def get_watch(self, watch_id: str) -> WatchState:
|
|
469
|
+
response = self._request("GET", f"/v1/watches/{watch_id}")
|
|
470
|
+
return self._finish(response, WatchState)
|
|
471
|
+
|
|
472
|
+
def delete_watch(self, watch_id: str) -> None:
|
|
473
|
+
response = self._request("DELETE", f"/v1/watches/{watch_id}")
|
|
474
|
+
self._check(response)
|
|
475
|
+
|
|
476
|
+
def agent_hints(self) -> AgentHintsResponse:
|
|
477
|
+
response = self._request("GET", "/v1/agent-hints")
|
|
478
|
+
return self._finish(response, AgentHintsResponse)
|
|
479
|
+
|
|
480
|
+
def capabilities(self) -> CapabilitiesResponse:
|
|
481
|
+
response = self._request("GET", "/v1/capabilities")
|
|
482
|
+
return self._finish(response, CapabilitiesResponse)
|
|
483
|
+
|
|
484
|
+
def health(self) -> HealthResponse:
|
|
485
|
+
response = self._request("GET", "/health")
|
|
486
|
+
return self._finish(response, HealthResponse)
|
|
487
|
+
|
|
488
|
+
def ready(self) -> ReadinessResponse:
|
|
489
|
+
response = self._request("GET", "/ready")
|
|
490
|
+
return self._finish(response, ReadinessResponse)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
__all__ = [
|
|
494
|
+
"CrawlutronClient",
|
|
495
|
+
"CrawlutronClientSync",
|
|
496
|
+
]
|
crawlutron_sdk/errors.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Typed error hierarchy for the crawlutron service facade.
|
|
2
|
+
|
|
3
|
+
Every service refusal carries a wire envelope ``{code, message, details}`` and an HTTP status. The
|
|
4
|
+
hierarchy mirrors the status families the service emits so callers can catch by family or by exact status.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CrawlutronError(Exception):
|
|
11
|
+
"""Base error: the service envelope code/message/details plus HTTP status."""
|
|
12
|
+
|
|
13
|
+
status_code: int | None = None
|
|
14
|
+
code: str = "crawlutron_error"
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
message: str,
|
|
19
|
+
*,
|
|
20
|
+
code: str | None = None,
|
|
21
|
+
details: dict[str, object] | None = None,
|
|
22
|
+
status_code: int | None = None,
|
|
23
|
+
) -> None:
|
|
24
|
+
self.message = message
|
|
25
|
+
self.code = code if code is not None else self.code
|
|
26
|
+
self.details = details or {}
|
|
27
|
+
self.status_code = status_code if status_code is not None else self.status_code
|
|
28
|
+
super().__init__(message)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class CrawlutronClientError(CrawlutronError):
|
|
32
|
+
"""4xx caller error."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class CrawlutronUnauthorizedError(CrawlutronClientError):
|
|
36
|
+
status_code = 401
|
|
37
|
+
code = "unauthorized"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class CrawlutronForbiddenError(CrawlutronClientError):
|
|
41
|
+
status_code = 403
|
|
42
|
+
code = "forbidden"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class CrawlutronNotFoundError(CrawlutronClientError):
|
|
46
|
+
status_code = 404
|
|
47
|
+
code = "not_found"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class CrawlutronBadRequestError(CrawlutronClientError):
|
|
51
|
+
status_code = 400
|
|
52
|
+
code = "bad_request"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class CrawlutronConflictError(CrawlutronClientError):
|
|
56
|
+
status_code = 409
|
|
57
|
+
code = "invalid_state"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class CrawlutronClientClosedRequestError(CrawlutronClientError):
|
|
61
|
+
"""499: the service saw the caller hang up. Envelope-construction only.
|
|
62
|
+
|
|
63
|
+
A connected SDK caller sees httpx TransportError; the service raises 499
|
|
64
|
+
only after detecting the disconnect, so this class never crosses live HTTP."""
|
|
65
|
+
|
|
66
|
+
status_code = 499
|
|
67
|
+
code = "client_disconnected"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class CrawlutronRequestTooLargeError(CrawlutronClientError):
|
|
71
|
+
status_code = 413
|
|
72
|
+
code = "request_too_large"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class CrawlutronTooManyRequestsError(CrawlutronClientError):
|
|
76
|
+
status_code = 429
|
|
77
|
+
code = "too_many_requests"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class CrawlutronUnprocessableEntityError(CrawlutronClientError):
|
|
81
|
+
status_code = 422
|
|
82
|
+
code = "validation_error"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class CrawlutronAcceptedError(CrawlutronClientError):
|
|
86
|
+
status_code = 202
|
|
87
|
+
code = "accepted"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class CrawlutronServerError(CrawlutronError):
|
|
91
|
+
"""5xx service error."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class CrawlutronBadGatewayError(CrawlutronServerError):
|
|
95
|
+
status_code = 502
|
|
96
|
+
code = "bad_gateway"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class CrawlutronServiceUnavailableError(CrawlutronServerError):
|
|
100
|
+
status_code = 503
|
|
101
|
+
code = "service_unavailable"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class CrawlutronInternalServerError(CrawlutronServerError):
|
|
105
|
+
status_code = 500
|
|
106
|
+
code = "internal_server_error"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class CrawlutronTransportError(CrawlutronError):
|
|
110
|
+
"""Network/transport failure; no service HTTP status applies."""
|
|
111
|
+
|
|
112
|
+
status_code = None
|
|
113
|
+
code = "transport_error"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
_STATUS_TO_ERROR: dict[int, type[CrawlutronError]] = {
|
|
117
|
+
202: CrawlutronAcceptedError,
|
|
118
|
+
400: CrawlutronBadRequestError,
|
|
119
|
+
401: CrawlutronUnauthorizedError,
|
|
120
|
+
403: CrawlutronForbiddenError,
|
|
121
|
+
404: CrawlutronNotFoundError,
|
|
122
|
+
409: CrawlutronConflictError,
|
|
123
|
+
413: CrawlutronRequestTooLargeError,
|
|
124
|
+
422: CrawlutronUnprocessableEntityError,
|
|
125
|
+
429: CrawlutronTooManyRequestsError,
|
|
126
|
+
499: CrawlutronClientClosedRequestError,
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
_SERVER_ERRORS: dict[int, type[CrawlutronServerError]] = {
|
|
130
|
+
500: CrawlutronInternalServerError,
|
|
131
|
+
502: CrawlutronBadGatewayError,
|
|
132
|
+
503: CrawlutronServiceUnavailableError,
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def error_from_response(status_code: int, envelope: dict[str, object]) -> CrawlutronError:
|
|
137
|
+
"""Build the matching typed error from a service error response."""
|
|
138
|
+
code = str(envelope.get("code", "crawlutron_error"))
|
|
139
|
+
message = str(envelope.get("message", "Request failed"))
|
|
140
|
+
details = envelope.get("details")
|
|
141
|
+
if not isinstance(details, dict):
|
|
142
|
+
details = {}
|
|
143
|
+
if 500 <= status_code < 600:
|
|
144
|
+
klass: type[CrawlutronError] = _SERVER_ERRORS.get(status_code, CrawlutronInternalServerError)
|
|
145
|
+
else:
|
|
146
|
+
klass = _STATUS_TO_ERROR.get(status_code, CrawlutronError)
|
|
147
|
+
return klass(message=message, code=code, details=details, status_code=status_code)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
__all__ = [
|
|
151
|
+
"CrawlutronAcceptedError",
|
|
152
|
+
"CrawlutronBadGatewayError",
|
|
153
|
+
"CrawlutronBadRequestError",
|
|
154
|
+
"CrawlutronClientClosedRequestError",
|
|
155
|
+
"CrawlutronClientError",
|
|
156
|
+
"CrawlutronConflictError",
|
|
157
|
+
"CrawlutronError",
|
|
158
|
+
"CrawlutronForbiddenError",
|
|
159
|
+
"CrawlutronInternalServerError",
|
|
160
|
+
"CrawlutronNotFoundError",
|
|
161
|
+
"CrawlutronRequestTooLargeError",
|
|
162
|
+
"CrawlutronServerError",
|
|
163
|
+
"CrawlutronServiceUnavailableError",
|
|
164
|
+
"CrawlutronTooManyRequestsError",
|
|
165
|
+
"CrawlutronTransportError",
|
|
166
|
+
"CrawlutronUnauthorizedError",
|
|
167
|
+
"CrawlutronUnprocessableEntityError",
|
|
168
|
+
"error_from_response",
|
|
169
|
+
]
|
crawlutron_sdk/models.py
ADDED
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
"""SDK-owned pydantic v2 wire mirrors of the service contract: request models forbid extra fields
|
|
2
|
+
(WatchCreateRequest allows), responses tolerate them. The SDK never imports crawlutron and is shape-pinned
|
|
3
|
+
against the served OpenAPI -- drift reds.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from typing import Annotated, Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class _UrlSource(BaseModel):
|
|
14
|
+
"""Shared exactly-one-of url/urls base for request models."""
|
|
15
|
+
|
|
16
|
+
url: str | None = None
|
|
17
|
+
urls: list[str] | None = None
|
|
18
|
+
|
|
19
|
+
@model_validator(mode="after")
|
|
20
|
+
def _require_exactly_one_url_source(self) -> _UrlSource:
|
|
21
|
+
has_url = self.url is not None
|
|
22
|
+
has_urls = self.urls is not None
|
|
23
|
+
if has_url == has_urls:
|
|
24
|
+
raise ValueError("supply exactly one of 'url' or 'urls'")
|
|
25
|
+
if has_urls and not self.urls:
|
|
26
|
+
raise ValueError("'urls' must contain at least one URL")
|
|
27
|
+
return self
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CrawlRequest(_UrlSource):
|
|
31
|
+
"""Request body for ``POST /v1/crawl`` (one URL source plus a policy)."""
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(extra="forbid")
|
|
34
|
+
|
|
35
|
+
policy: dict[str, object] = Field(
|
|
36
|
+
default_factory=dict,
|
|
37
|
+
description="Resolved caller-provided policy/execution plan.",
|
|
38
|
+
)
|
|
39
|
+
validators: dict[str, object] = Field(
|
|
40
|
+
default_factory=dict,
|
|
41
|
+
description="Optional map of submitted URL -> cached validators.",
|
|
42
|
+
)
|
|
43
|
+
async_mode: bool = Field(
|
|
44
|
+
default=False,
|
|
45
|
+
description="When true, accept immediately (202) and run in the background.",
|
|
46
|
+
)
|
|
47
|
+
webhook_url: Annotated[
|
|
48
|
+
str | None,
|
|
49
|
+
Field(description="Optional callback URL (https:// required)."),
|
|
50
|
+
] = None
|
|
51
|
+
output_format: Annotated[
|
|
52
|
+
Literal["json", "jsonl", "ndjson", "markdown", "csv"] | None,
|
|
53
|
+
Field(description="Output format; json is the default."),
|
|
54
|
+
] = None
|
|
55
|
+
changed_since: Annotated[
|
|
56
|
+
float | None,
|
|
57
|
+
Field(
|
|
58
|
+
ge=0,
|
|
59
|
+
allow_inf_nan=False,
|
|
60
|
+
description="Epoch seconds; when set, the caller requests delta classification relative to "
|
|
61
|
+
"that time.",
|
|
62
|
+
),
|
|
63
|
+
] = None
|
|
64
|
+
baseline_request_id: Annotated[
|
|
65
|
+
str | None,
|
|
66
|
+
Field(
|
|
67
|
+
description="Prior crawl/job/watch request id to use as the delta baseline.",
|
|
68
|
+
),
|
|
69
|
+
] = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class CrawlResponse(BaseModel):
|
|
73
|
+
"""The unified crawl response envelope (200 from ``POST /v1/crawl``)."""
|
|
74
|
+
|
|
75
|
+
model_config = ConfigDict(extra="allow")
|
|
76
|
+
|
|
77
|
+
request_id: str
|
|
78
|
+
submitted_count: int
|
|
79
|
+
policy_schema_version: str
|
|
80
|
+
success: bool
|
|
81
|
+
results: list[dict[str, object]] = Field(default_factory=list)
|
|
82
|
+
delta_summary: Annotated[
|
|
83
|
+
dict[str, object] | None,
|
|
84
|
+
Field(
|
|
85
|
+
description="Per-result delta accounting, populated when delta crawl mode is requested. "
|
|
86
|
+
"Carries degraded_baseline=true when the baseline row was missing/expired/foreign, so "
|
|
87
|
+
"freshness-classified results are distinguishable from baseline-lost ones.",
|
|
88
|
+
),
|
|
89
|
+
] = None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class CrawlAccepted(BaseModel):
|
|
93
|
+
"""The async-accepted body (202 from ``POST /v1/crawl``)."""
|
|
94
|
+
|
|
95
|
+
model_config = ConfigDict(extra="allow")
|
|
96
|
+
|
|
97
|
+
status: str
|
|
98
|
+
request_id: str
|
|
99
|
+
status_url: str
|
|
100
|
+
webhook_url: Annotated[str | None, Field(default=None)] = None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class AdmitRequest(BaseModel):
|
|
104
|
+
"""Request body for ``POST /v1/admit`` (one URL to judge pre-flight)."""
|
|
105
|
+
|
|
106
|
+
model_config = ConfigDict(extra="forbid")
|
|
107
|
+
|
|
108
|
+
url: str = Field(..., description="The raw URL to admit or refuse.")
|
|
109
|
+
check_robots: bool = Field(
|
|
110
|
+
default=False,
|
|
111
|
+
description="Opt-in robots.txt check (the only I/O path).",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class AdmitResponse(BaseModel):
|
|
116
|
+
"""The admission verdict with the per-check reason table."""
|
|
117
|
+
|
|
118
|
+
model_config = ConfigDict(extra="allow")
|
|
119
|
+
|
|
120
|
+
admitted: bool
|
|
121
|
+
classification: Annotated[
|
|
122
|
+
Literal[
|
|
123
|
+
"invalid_url",
|
|
124
|
+
"unsupported_scheme",
|
|
125
|
+
"userinfo_refused",
|
|
126
|
+
"ipvfuture_refused",
|
|
127
|
+
"invalid_port",
|
|
128
|
+
"invalid_idn_host",
|
|
129
|
+
"host_denied",
|
|
130
|
+
"host_not_allowed",
|
|
131
|
+
"private_network_target",
|
|
132
|
+
"robots_disallowed",
|
|
133
|
+
]
|
|
134
|
+
| None,
|
|
135
|
+
Field(default=None),
|
|
136
|
+
] = None
|
|
137
|
+
canonical_url: Annotated[str | None, Field(default=None)] = None
|
|
138
|
+
checks: dict[str, str | None] = Field(default_factory=dict)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class MapRequest(BaseModel):
|
|
142
|
+
"""Request body for ``POST /v1/map`` (discover a root URL's frontier)."""
|
|
143
|
+
|
|
144
|
+
model_config = ConfigDict(extra="forbid")
|
|
145
|
+
|
|
146
|
+
url: str = Field(..., description="Root public HTTP(S) URL to map.")
|
|
147
|
+
policy: dict[str, object] = Field(
|
|
148
|
+
default_factory=dict,
|
|
149
|
+
description="Optional crawl policy for the mapping crawl.",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class SiteMapResponse(BaseModel):
|
|
154
|
+
"""The de-duplicated URL frontier for a mapping crawl (200 from ``POST /v1/map``)."""
|
|
155
|
+
|
|
156
|
+
model_config = ConfigDict(extra="allow")
|
|
157
|
+
|
|
158
|
+
url: str | None = None
|
|
159
|
+
links: list[str] = Field(default_factory=list)
|
|
160
|
+
sitemaps: list[str] = Field(default_factory=list)
|
|
161
|
+
feeds: list[str] = Field(default_factory=list)
|
|
162
|
+
all_urls: list[str] = Field(default_factory=list)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class RenderRequest(_UrlSource):
|
|
166
|
+
"""Request body for ``POST /v1/render`` (backward-compatible browser render)."""
|
|
167
|
+
|
|
168
|
+
model_config = ConfigDict(extra="forbid")
|
|
169
|
+
|
|
170
|
+
options: dict[str, object] | None = None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class JobCreateRequest(BaseModel):
|
|
174
|
+
"""Request body for ``POST /v1/jobs`` (resumable whole-site crawl job)."""
|
|
175
|
+
|
|
176
|
+
model_config = ConfigDict(extra="forbid")
|
|
177
|
+
|
|
178
|
+
root_url: str = Field(..., description="Root public HTTP(S) URL; stored canonically at depth 0.")
|
|
179
|
+
policy: dict[str, object] = Field(
|
|
180
|
+
default_factory=dict,
|
|
181
|
+
description="Crawl policy (strategies, limits, scope, extraction).",
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class JobCreateResponse(BaseModel):
|
|
186
|
+
"""Response from creating a crawl job."""
|
|
187
|
+
|
|
188
|
+
model_config = ConfigDict(extra="allow")
|
|
189
|
+
|
|
190
|
+
job_id: str
|
|
191
|
+
status: str
|
|
192
|
+
root_url: str
|
|
193
|
+
created_at: float
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class JobStatusResponse(BaseModel):
|
|
197
|
+
"""Status of a crawl job."""
|
|
198
|
+
|
|
199
|
+
model_config = ConfigDict(extra="allow")
|
|
200
|
+
|
|
201
|
+
job_id: str
|
|
202
|
+
root_url: str
|
|
203
|
+
owner: str = ""
|
|
204
|
+
status: str
|
|
205
|
+
kind: str
|
|
206
|
+
created_at: float
|
|
207
|
+
updated_at: float
|
|
208
|
+
frontier_size: int
|
|
209
|
+
visited_size: int
|
|
210
|
+
results_count: int
|
|
211
|
+
cap_truncated: bool = False
|
|
212
|
+
stats: dict[str, int] = Field(default_factory=dict)
|
|
213
|
+
error: Annotated[str | None, Field(default=None)] = None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class JobListResponse(BaseModel):
|
|
217
|
+
"""A page of crawl jobs for the caller, with the caller's total row count."""
|
|
218
|
+
|
|
219
|
+
model_config = ConfigDict(extra="allow")
|
|
220
|
+
|
|
221
|
+
jobs: list[JobStatusResponse] = Field(default_factory=list)
|
|
222
|
+
total: int = 0
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class JobControlRequest(BaseModel):
|
|
226
|
+
"""Control a crawl job (pause/resume/cancel)."""
|
|
227
|
+
|
|
228
|
+
model_config = ConfigDict(extra="forbid")
|
|
229
|
+
|
|
230
|
+
action: Literal["pause", "resume", "cancel"] = Field(..., description="One of: pause, resume, cancel.")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
class JobDeletedResponse(BaseModel):
|
|
234
|
+
"""The delete outcome as a typed body."""
|
|
235
|
+
|
|
236
|
+
model_config = ConfigDict(extra="allow")
|
|
237
|
+
|
|
238
|
+
deleted: bool
|
|
239
|
+
job_id: str
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class SearchRequest(BaseModel):
|
|
243
|
+
"""Request body for ``POST /v1/search`` (web search query)."""
|
|
244
|
+
|
|
245
|
+
model_config = ConfigDict(extra="forbid")
|
|
246
|
+
|
|
247
|
+
query: str = Field(..., min_length=1, description="Web search query.")
|
|
248
|
+
limit: Annotated[
|
|
249
|
+
int | None,
|
|
250
|
+
Field(
|
|
251
|
+
default=None,
|
|
252
|
+
ge=1,
|
|
253
|
+
le=100,
|
|
254
|
+
description="Maximum results to return; the server clamps this to its "
|
|
255
|
+
"CRAWLUTRON_SEARCH_LIMIT_MAX (drift-guarded against the server model).",
|
|
256
|
+
),
|
|
257
|
+
] = None
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class SearchResultItem(BaseModel):
|
|
261
|
+
"""One SERP-style search result."""
|
|
262
|
+
|
|
263
|
+
model_config = ConfigDict(extra="allow")
|
|
264
|
+
|
|
265
|
+
url: str
|
|
266
|
+
title: str
|
|
267
|
+
description: str = ""
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class SearchResponse(BaseModel):
|
|
271
|
+
"""Response for ``POST /v1/search``."""
|
|
272
|
+
|
|
273
|
+
model_config = ConfigDict(extra="allow")
|
|
274
|
+
|
|
275
|
+
success: bool = True
|
|
276
|
+
query: str
|
|
277
|
+
provider: str
|
|
278
|
+
limit: int
|
|
279
|
+
data: list[SearchResultItem] = Field(default_factory=list)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class WatchCreateRequest(BaseModel):
|
|
283
|
+
"""A watch registration (POST /v1/watches)."""
|
|
284
|
+
|
|
285
|
+
model_config = ConfigDict(extra="allow")
|
|
286
|
+
|
|
287
|
+
url: str = Field(..., description="Public HTTP(S) URL to revalidate.")
|
|
288
|
+
interval_seconds: int = Field(..., gt=0, description="Seconds between revalidation checks.")
|
|
289
|
+
webhook_url: Annotated[str | None, Field(default=None)] = None
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
class WatchState(BaseModel):
|
|
293
|
+
"""One row of the watches table, as shared on the wire."""
|
|
294
|
+
|
|
295
|
+
model_config = ConfigDict(extra="allow")
|
|
296
|
+
|
|
297
|
+
watch_id: str
|
|
298
|
+
owner: str = ""
|
|
299
|
+
url: str
|
|
300
|
+
webhook_url: Annotated[str | None, Field(default=None)] = None
|
|
301
|
+
interval_seconds: int
|
|
302
|
+
status: Literal["active", "deactivated"] = "active"
|
|
303
|
+
validators: dict[str, object] = Field(default_factory=dict)
|
|
304
|
+
next_check_at: float
|
|
305
|
+
last_checked_at: Annotated[float | None, Field(default=None)] = None
|
|
306
|
+
last_outcome: Annotated[
|
|
307
|
+
Literal[
|
|
308
|
+
"rendered",
|
|
309
|
+
"captcha_detected",
|
|
310
|
+
"bot_challenge_detected",
|
|
311
|
+
"access_denied",
|
|
312
|
+
"login_required",
|
|
313
|
+
"paywall_detected",
|
|
314
|
+
"rate_limited",
|
|
315
|
+
"cookie_consent_required",
|
|
316
|
+
"javascript_required",
|
|
317
|
+
"unsupported_browser",
|
|
318
|
+
"geolocation_blocked",
|
|
319
|
+
"waf_challenge_detected",
|
|
320
|
+
"not_found",
|
|
321
|
+
"site_error",
|
|
322
|
+
"gone",
|
|
323
|
+
"binary_no_text",
|
|
324
|
+
"robots_disallowed",
|
|
325
|
+
"navigation_timeout",
|
|
326
|
+
"browser_crash",
|
|
327
|
+
"network_error",
|
|
328
|
+
"connection_error",
|
|
329
|
+
"http_status_error",
|
|
330
|
+
"url_budget_exceeded",
|
|
331
|
+
"browser_pool_exhausted",
|
|
332
|
+
"circuit_breaker_open",
|
|
333
|
+
"admission_stalled",
|
|
334
|
+
"payload_too_large",
|
|
335
|
+
"too_many_redirects",
|
|
336
|
+
"http_error",
|
|
337
|
+
"value_error",
|
|
338
|
+
"tls_error",
|
|
339
|
+
"dns_error",
|
|
340
|
+
"invalid_url",
|
|
341
|
+
"unsupported_scheme",
|
|
342
|
+
"private_network_target",
|
|
343
|
+
"host_denied",
|
|
344
|
+
"host_not_allowed",
|
|
345
|
+
"userinfo_refused",
|
|
346
|
+
"unknown_error",
|
|
347
|
+
"not_modified",
|
|
348
|
+
]
|
|
349
|
+
| None,
|
|
350
|
+
Field(default=None),
|
|
351
|
+
] = None
|
|
352
|
+
last_changed_at: Annotated[float | None, Field(default=None)] = None
|
|
353
|
+
consecutive_failures: int = 0
|
|
354
|
+
created_at: float
|
|
355
|
+
updated_at: float
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class WatchListResponse(BaseModel):
|
|
359
|
+
"""Every registered watch, each with the state of its last check."""
|
|
360
|
+
|
|
361
|
+
model_config = ConfigDict(extra="allow")
|
|
362
|
+
|
|
363
|
+
watches: list[WatchState] = Field(default_factory=list)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
class CapabilitiesResponse(BaseModel):
|
|
367
|
+
"""The /v1/capabilities discovery payload."""
|
|
368
|
+
|
|
369
|
+
model_config = ConfigDict(extra="allow")
|
|
370
|
+
|
|
371
|
+
service: str
|
|
372
|
+
version: str
|
|
373
|
+
policy_schema_version: str
|
|
374
|
+
supported_strategies: list[str] = Field(default_factory=list)
|
|
375
|
+
supported_extractions: list[str] = Field(default_factory=list)
|
|
376
|
+
supported_classifications: list[str] = Field(default_factory=list)
|
|
377
|
+
output_fields: list[str] = Field(default_factory=list)
|
|
378
|
+
default_limits: dict[str, object] = Field(default_factory=dict)
|
|
379
|
+
features: dict[str, bool] = Field(default_factory=dict)
|
|
380
|
+
auth: dict[str, str] = Field(default_factory=dict)
|
|
381
|
+
crawl_timing: dict[str, object] = Field(default_factory=dict)
|
|
382
|
+
search: dict[str, object] = Field(default_factory=dict)
|
|
383
|
+
mcp: dict[str, object] = Field(default_factory=dict)
|
|
384
|
+
openapi_url: str = ""
|
|
385
|
+
swagger_ui_url: str = ""
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
class AgentHintsResponse(BaseModel):
|
|
389
|
+
"""The /v1/agent-hints guidance payload."""
|
|
390
|
+
|
|
391
|
+
model_config = ConfigDict(extra="allow")
|
|
392
|
+
|
|
393
|
+
service: str
|
|
394
|
+
strategies: dict[str, str] = Field(default_factory=dict)
|
|
395
|
+
extractions: dict[str, str] = Field(default_factory=dict)
|
|
396
|
+
guidance: list[str] = Field(default_factory=list)
|
|
397
|
+
example_policies: dict[str, object] = Field(default_factory=dict)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class HostPolicySuggestion(BaseModel):
|
|
401
|
+
"""One host's candidate rule, with the evidence that proposed it."""
|
|
402
|
+
|
|
403
|
+
model_config = ConfigDict(extra="allow")
|
|
404
|
+
|
|
405
|
+
host: str
|
|
406
|
+
rule: str
|
|
407
|
+
blocked_attempts: int
|
|
408
|
+
observed_attempts: int
|
|
409
|
+
classifications: list[str] = Field(default_factory=list)
|
|
410
|
+
signatures: list[str] = Field(default_factory=list)
|
|
411
|
+
evidence: list[dict[str, object]] = Field(default_factory=list)
|
|
412
|
+
rationale: str
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
class PolicySuggestionsResponse(BaseModel):
|
|
416
|
+
"""Candidate host policy rules drawn from retained crawl records."""
|
|
417
|
+
|
|
418
|
+
model_config = ConfigDict(extra="allow")
|
|
419
|
+
|
|
420
|
+
records_examined: int
|
|
421
|
+
records_skipped: int = 0
|
|
422
|
+
suggestions: list[HostPolicySuggestion] = Field(default_factory=list)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
class HealthResponse(BaseModel):
|
|
426
|
+
"""The /health liveness response."""
|
|
427
|
+
|
|
428
|
+
model_config = ConfigDict(extra="allow")
|
|
429
|
+
|
|
430
|
+
status: str = "ok"
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
class ReadinessResponse(BaseModel):
|
|
434
|
+
"""The /ready crawl-capacity response."""
|
|
435
|
+
|
|
436
|
+
model_config = ConfigDict(extra="allow")
|
|
437
|
+
|
|
438
|
+
status: str
|
|
439
|
+
active_requests: int
|
|
440
|
+
max_active_requests: int
|
|
441
|
+
saturated: bool
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class ErrorEnvelope(BaseModel):
|
|
445
|
+
"""The service error envelope: {code, message, details}."""
|
|
446
|
+
|
|
447
|
+
model_config = ConfigDict(extra="allow")
|
|
448
|
+
|
|
449
|
+
code: str
|
|
450
|
+
message: str
|
|
451
|
+
details: dict[str, object] = Field(default_factory=dict)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
__all__ = [
|
|
455
|
+
"AdmitRequest",
|
|
456
|
+
"AdmitResponse",
|
|
457
|
+
"AgentHintsResponse",
|
|
458
|
+
"CapabilitiesResponse",
|
|
459
|
+
"CrawlAccepted",
|
|
460
|
+
"CrawlRequest",
|
|
461
|
+
"CrawlResponse",
|
|
462
|
+
"ErrorEnvelope",
|
|
463
|
+
"HealthResponse",
|
|
464
|
+
"HostPolicySuggestion",
|
|
465
|
+
"JobControlRequest",
|
|
466
|
+
"JobCreateRequest",
|
|
467
|
+
"JobCreateResponse",
|
|
468
|
+
"JobDeletedResponse",
|
|
469
|
+
"JobListResponse",
|
|
470
|
+
"JobStatusResponse",
|
|
471
|
+
"MapRequest",
|
|
472
|
+
"PolicySuggestionsResponse",
|
|
473
|
+
"ReadinessResponse",
|
|
474
|
+
"RenderRequest",
|
|
475
|
+
"SearchRequest",
|
|
476
|
+
"SearchResponse",
|
|
477
|
+
"SearchResultItem",
|
|
478
|
+
"SiteMapResponse",
|
|
479
|
+
"WatchCreateRequest",
|
|
480
|
+
"WatchListResponse",
|
|
481
|
+
"WatchState",
|
|
482
|
+
]
|
crawlutron_sdk/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: crawlutron-sdk
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Python client for the Crawlutron execution service
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.14
|
|
7
|
+
Requires-Dist: httpx
|
|
8
|
+
Requires-Dist: pydantic
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# crawlutron-sdk
|
|
12
|
+
|
|
13
|
+
The Python client for a crawlutron service: async and sync clients, typed models,
|
|
14
|
+
and error mapping.
|
|
15
|
+
|
|
16
|
+
The SDK is deliberately a thin, curated client. It mirrors the served OpenAPI
|
|
17
|
+
schema by hand rather than importing `crawlutron`, so it stays installable beside
|
|
18
|
+
any application and carries no crawler dependency.
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from crawlutron_sdk import AsyncClient
|
|
22
|
+
|
|
23
|
+
async with AsyncClient("https://crawl.example.com", api_key="...") as client:
|
|
24
|
+
capabilities = await client.capabilities()
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## License
|
|
28
|
+
|
|
29
|
+
MIT. See the repository's `LICENSE`.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
crawlutron_sdk/__init__.py,sha256=RFemgWft_HDQxKt8dya-ZqlyGxEvpk3Ec6YAmIQ35zw,2872
|
|
2
|
+
crawlutron_sdk/client.py,sha256=yWXF5Vsc_h-cTcLGIlNiy2axfzpTMwWDQ7QDbnRGBd8,19821
|
|
3
|
+
crawlutron_sdk/errors.py,sha256=cLkh4d-FLQU9dQn4fziWXAwqTEGi4MjvQh3wH_Dff7A,4876
|
|
4
|
+
crawlutron_sdk/models.py,sha256=2lZ2CoFb9Fqx4A9tugGBIUMp3bVmCnPHjw82Q9Qxwp8,14012
|
|
5
|
+
crawlutron_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
crawlutron_sdk-0.5.0.dist-info/METADATA,sha256=S-9tHMMwv9GAUKTMY1Tf2FIRCAh62MRiM8X9p8GDN90,812
|
|
7
|
+
crawlutron_sdk-0.5.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
8
|
+
crawlutron_sdk-0.5.0.dist-info/RECORD,,
|