crawlutron-sdk 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,38 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .pytest_cache/
4
+ report/
5
+ # Build artifacts from `python -m build` / editable installs (#26).
6
+ dist/
7
+ build/
8
+ # Both spellings: a linked worktree reaches the venv through a `.venv` symlink,
9
+ # which the trailing-slash form does not match -- and a committed symlink into
10
+ # one checkout's venv is worse than no venv at all.
11
+ .venv/
12
+ .venv
13
+ .dynamic-task-ledger.json
14
+ *.egg-info/
15
+ .cache/
16
+ .mypy_cache/
17
+ .ruff_cache/
18
+ .gate_fanout/
19
+ .coverage
20
+ htmlcov/
21
+ # Session-analysis residue written by the cortexkit tooling. It is ephemeral
22
+ # per-session output, not project content, and its own managed block already
23
+ # declares `magic-context/` untracked -- the historian XML under it was
24
+ # committed anyway, which is how a directory full of transcript digests ended
25
+ # up in the tree this service is extracted from.
26
+ .cortexkit/
27
+ logs/
28
+ cache/
29
+ .env
30
+ tmp/
31
+ *.log
32
+ .DS_Store
33
+
34
+ .coverage.*
35
+ .opencode
36
+ .hardstand/
37
+
38
+
@@ -0,0 +1,29 @@
1
+ Metadata-Version: 2.5
2
+ Name: crawlutron-sdk
3
+ Version: 0.5.0
4
+ Summary: Python client for the Crawlutron execution service
5
+ License-Expression: MIT
6
+ Requires-Python: >=3.14
7
+ Requires-Dist: httpx
8
+ Requires-Dist: pydantic
9
+ Description-Content-Type: text/markdown
10
+
11
+ # crawlutron-sdk
12
+
13
+ The Python client for a crawlutron service: async and sync clients, typed models,
14
+ and error mapping.
15
+
16
+ The SDK is deliberately a thin, curated client. It mirrors the served OpenAPI
17
+ schema by hand rather than importing `crawlutron`, so it stays installable beside
18
+ any application and carries no crawler dependency.
19
+
20
+ ```python
21
+ from crawlutron_sdk import AsyncClient
22
+
23
+ async with AsyncClient("https://crawl.example.com", api_key="...") as client:
24
+ capabilities = await client.capabilities()
25
+ ```
26
+
27
+ ## License
28
+
29
+ MIT. See the repository's `LICENSE`.
@@ -0,0 +1,19 @@
1
+ # crawlutron-sdk
2
+
3
+ The Python client for a crawlutron service: async and sync clients, typed models,
4
+ and error mapping.
5
+
6
+ The SDK is deliberately a thin, curated client. It mirrors the served OpenAPI
7
+ schema by hand rather than importing `crawlutron`, so it stays installable beside
8
+ any application and carries no crawler dependency.
9
+
10
+ ```python
11
+ from crawlutron_sdk import AsyncClient
12
+
13
+ async with AsyncClient("https://crawl.example.com", api_key="...") as client:
14
+ capabilities = await client.capabilities()
15
+ ```
16
+
17
+ ## License
18
+
19
+ MIT. See the repository's `LICENSE`.
@@ -0,0 +1,18 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "crawlutron-sdk"
7
+ version = "0.5.0"
8
+ description = "Python client for the Crawlutron execution service"
9
+ readme = "README.md"
10
+ requires-python = ">=3.14"
11
+ license = "MIT"
12
+ dependencies = [
13
+ "httpx",
14
+ "pydantic",
15
+ ]
16
+
17
+ [tool.hatch.build.targets.wheel]
18
+ packages = ["src/crawlutron_sdk"]
@@ -0,0 +1,107 @@
1
+ """Lightweight typed HTTP client SDK for a running crawlutron service. ``crawlutron_sdk`` is a thin facade
2
+ over the service's wire contract. The server is the sole enforcement chokepoint; this package reimplements
3
+ no admission, policy, robots, metering, or extraction logic.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from .client import CrawlutronClient, CrawlutronClientSync
9
+ from .errors import (
10
+ CrawlutronAcceptedError,
11
+ CrawlutronBadGatewayError,
12
+ CrawlutronBadRequestError,
13
+ CrawlutronClientClosedRequestError,
14
+ CrawlutronClientError,
15
+ CrawlutronConflictError,
16
+ CrawlutronError,
17
+ CrawlutronForbiddenError,
18
+ CrawlutronInternalServerError,
19
+ CrawlutronNotFoundError,
20
+ CrawlutronRequestTooLargeError,
21
+ CrawlutronServerError,
22
+ CrawlutronServiceUnavailableError,
23
+ CrawlutronTooManyRequestsError,
24
+ CrawlutronTransportError,
25
+ CrawlutronUnauthorizedError,
26
+ CrawlutronUnprocessableEntityError,
27
+ error_from_response,
28
+ )
29
+ from .models import (
30
+ AdmitRequest,
31
+ AdmitResponse,
32
+ AgentHintsResponse,
33
+ CapabilitiesResponse,
34
+ CrawlAccepted,
35
+ CrawlRequest,
36
+ CrawlResponse,
37
+ ErrorEnvelope,
38
+ HealthResponse,
39
+ HostPolicySuggestion,
40
+ JobControlRequest,
41
+ JobCreateRequest,
42
+ JobCreateResponse,
43
+ JobDeletedResponse,
44
+ JobListResponse,
45
+ JobStatusResponse,
46
+ MapRequest,
47
+ PolicySuggestionsResponse,
48
+ ReadinessResponse,
49
+ RenderRequest,
50
+ SearchRequest,
51
+ SearchResponse,
52
+ SearchResultItem,
53
+ SiteMapResponse,
54
+ WatchCreateRequest,
55
+ WatchListResponse,
56
+ WatchState,
57
+ )
58
+
59
+ __all__ = [
60
+ "CrawlutronAcceptedError",
61
+ "CrawlutronBadGatewayError",
62
+ "CrawlutronBadRequestError",
63
+ "CrawlutronClient",
64
+ "CrawlutronClientClosedRequestError",
65
+ "CrawlutronClientError",
66
+ "CrawlutronClientSync",
67
+ "CrawlutronConflictError",
68
+ "CrawlutronError",
69
+ "CrawlutronForbiddenError",
70
+ "CrawlutronInternalServerError",
71
+ "CrawlutronNotFoundError",
72
+ "CrawlutronRequestTooLargeError",
73
+ "CrawlutronServerError",
74
+ "CrawlutronServiceUnavailableError",
75
+ "CrawlutronTooManyRequestsError",
76
+ "CrawlutronTransportError",
77
+ "CrawlutronUnauthorizedError",
78
+ "CrawlutronUnprocessableEntityError",
79
+ "WatchState",
80
+ "WatchListResponse",
81
+ "WatchCreateRequest",
82
+ "SearchResultItem",
83
+ "SiteMapResponse",
84
+ "SearchResponse",
85
+ "SearchRequest",
86
+ "ReadinessResponse",
87
+ "PolicySuggestionsResponse",
88
+ "RenderRequest",
89
+ "MapRequest",
90
+ "JobStatusResponse",
91
+ "JobListResponse",
92
+ "JobDeletedResponse",
93
+ "JobCreateResponse",
94
+ "JobCreateRequest",
95
+ "JobControlRequest",
96
+ "HealthResponse",
97
+ "HostPolicySuggestion",
98
+ "ErrorEnvelope",
99
+ "CrawlResponse",
100
+ "CrawlRequest",
101
+ "CrawlAccepted",
102
+ "CapabilitiesResponse",
103
+ "AgentHintsResponse",
104
+ "AdmitResponse",
105
+ "AdmitRequest",
106
+ "error_from_response",
107
+ ]
@@ -0,0 +1,496 @@
1
+ """Typed async HTTP facade over a running crawlutron service."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import time
7
+ from collections.abc import Awaitable, Callable
8
+ from typing import Literal, Never, TypedDict, TypeVar
9
+
10
+ import httpx
11
+ from pydantic import BaseModel
12
+
13
+ from .errors import CrawlutronAcceptedError, CrawlutronTransportError, error_from_response
14
+ from .models import (
15
+ AdmitRequest,
16
+ AdmitResponse,
17
+ AgentHintsResponse,
18
+ CapabilitiesResponse,
19
+ CrawlAccepted,
20
+ CrawlRequest,
21
+ CrawlResponse,
22
+ HealthResponse,
23
+ JobControlRequest,
24
+ JobCreateRequest,
25
+ JobCreateResponse,
26
+ JobListResponse,
27
+ JobStatusResponse,
28
+ MapRequest,
29
+ PolicySuggestionsResponse,
30
+ ReadinessResponse,
31
+ RenderRequest,
32
+ SearchRequest,
33
+ SearchResponse,
34
+ SiteMapResponse,
35
+ WatchCreateRequest,
36
+ WatchListResponse,
37
+ WatchState,
38
+ )
39
+
40
+ _DEFAULT_BASE_URL = "http://127.0.0.1:8765"
41
+ _DEFAULT_TIMEOUT = 300.0
42
+
43
+
44
+ async def _poll_result(
45
+ request_id: str,
46
+ fetch: Callable[[], Awaitable[CrawlResponse]],
47
+ deadline: float | None,
48
+ poll_interval: float,
49
+ timeout_factory: Callable[[str], CrawlutronTransportError],
50
+ ) -> CrawlResponse:
51
+ while True:
52
+ try:
53
+ return await fetch()
54
+ except CrawlutronAcceptedError:
55
+ pass
56
+ if deadline is not None and time.monotonic() >= deadline:
57
+ raise timeout_factory(request_id)
58
+ await asyncio.sleep(poll_interval)
59
+
60
+
61
+ _ModelT = TypeVar("_ModelT", bound=BaseModel)
62
+
63
+
64
+ class _ClientKwargs(TypedDict):
65
+ base_url: str
66
+ timeout: float | None
67
+ headers: dict[str, str]
68
+
69
+
70
+ def _str_key_dict(data: object) -> dict[str, object]:
71
+ if not isinstance(data, dict):
72
+ return {}
73
+ return {str(key): value for key, value in data.items()}
74
+
75
+
76
+ def _json_body(response: httpx.Response) -> dict[str, object]:
77
+ if not response.content:
78
+ return {}
79
+ return _str_key_dict(response.json())
80
+
81
+
82
+ def _payload_dict(payload: object) -> dict[str, object]:
83
+ dump = getattr(payload, "model_dump", None)
84
+ if callable(dump):
85
+ return _str_key_dict(dump())
86
+ return {}
87
+
88
+
89
+ def _job_list_query(limit: int | None, offset: int) -> str:
90
+ parts = [f"limit={limit}"] if limit is not None else []
91
+ parts += [f"offset={offset}"] if offset else []
92
+ return f"?{'&'.join(parts)}" if parts else ""
93
+
94
+
95
+ class _ClientBase:
96
+ def __init__(
97
+ self,
98
+ *,
99
+ base_url: str = _DEFAULT_BASE_URL,
100
+ api_key: str | None = None,
101
+ timeout: float | None = _DEFAULT_TIMEOUT,
102
+ transport: httpx.BaseTransport | httpx.AsyncBaseTransport | None = None,
103
+ ) -> None:
104
+ self._base_url = base_url.rstrip("/")
105
+ self._api_key = api_key
106
+ self._timeout = timeout
107
+ self._transport = transport
108
+ self._client: httpx.Client | httpx.AsyncClient | None = None
109
+
110
+ def _headers(self) -> dict[str, str]:
111
+ headers: dict[str, str] = {}
112
+ if self._api_key is not None:
113
+ headers["X-API-Key"] = self._api_key
114
+ return headers
115
+
116
+ def _client_kwargs(self) -> _ClientKwargs:
117
+ return {
118
+ "base_url": self._base_url,
119
+ "timeout": self._timeout,
120
+ "headers": self._headers(),
121
+ }
122
+
123
+ def _raise_for_status(
124
+ self,
125
+ status_code: int,
126
+ body: dict[str, object],
127
+ *,
128
+ accepted_if_pending: bool = False,
129
+ ) -> None:
130
+ """Raise typed errors; with ``accepted_if_pending`` also refuse a 202."""
131
+ if accepted_if_pending and status_code == 202:
132
+ raise error_from_response(status_code, body)
133
+ if status_code < 400:
134
+ return
135
+ raise error_from_response(status_code, body)
136
+
137
+ def _check(self, response: httpx.Response) -> None:
138
+ """Raise typed errors for a response with no body to return."""
139
+ self._raise_for_status(response.status_code, _json_body(response))
140
+
141
+ def _finish(
142
+ self,
143
+ response: httpx.Response,
144
+ model: type[_ModelT],
145
+ *,
146
+ accepted_if_pending: bool = False,
147
+ ) -> _ModelT:
148
+ """Validate one response body against ``model`` after status checking."""
149
+ body = _json_body(response)
150
+ self._raise_for_status(response.status_code, body, accepted_if_pending=accepted_if_pending)
151
+ return model.model_validate(body)
152
+
153
+ def _transport_error(self, error: httpx.HTTPError) -> CrawlutronTransportError:
154
+ return CrawlutronTransportError(str(error), status_code=None)
155
+
156
+ def _transport_failure(self, error: httpx.HTTPError) -> Never:
157
+ raise self._transport_error(error) from error
158
+
159
+ def _make_timeout(self, request_id: str, max_wait: float | None = None) -> CrawlutronTransportError:
160
+ bound = f" within {max_wait}s" if max_wait is not None else ""
161
+ return CrawlutronTransportError(
162
+ f"crawl {request_id} did not reach a terminal state{bound}", status_code=None
163
+ )
164
+
165
+ def _make_wait_deadline(self, max_wait: float | None) -> float | None:
166
+ if max_wait is not None:
167
+ return time.monotonic() + max_wait
168
+ return None
169
+
170
+
171
+ class CrawlutronClient(_ClientBase):
172
+ _client: httpx.AsyncClient | None = None
173
+
174
+ async def __aenter__(self) -> CrawlutronClient:
175
+ return self
176
+
177
+ async def __aexit__(self, *args: object) -> None:
178
+ await self.aclose()
179
+
180
+ async def _get_client(self) -> httpx.AsyncClient:
181
+ client = self._client
182
+ if client is None or not isinstance(client, httpx.AsyncClient):
183
+ transport = self._transport
184
+ if transport is not None and not isinstance(transport, httpx.AsyncBaseTransport):
185
+ raise CrawlutronTransportError(
186
+ "sync-only transport cannot drive an async client",
187
+ status_code=None,
188
+ )
189
+ self._client = httpx.AsyncClient(**self._client_kwargs(), transport=transport)
190
+ client = self._client
191
+ if not isinstance(client, httpx.AsyncClient):
192
+ raise CrawlutronTransportError("unexpected http client type", status_code=None)
193
+ return client
194
+
195
+ async def aclose(self) -> None:
196
+ client = self._client
197
+ if client is None:
198
+ return
199
+ if not isinstance(client, httpx.AsyncClient):
200
+ raise CrawlutronTransportError("unexpected http client type", status_code=None)
201
+ await client.aclose()
202
+ self._client = None
203
+
204
+ async def _request(
205
+ self,
206
+ method: str,
207
+ path: str,
208
+ *,
209
+ json: dict[str, object] | None = None,
210
+ ) -> httpx.Response:
211
+ client = await self._get_client()
212
+ try:
213
+ return await client.request(method, path, json=json)
214
+ except httpx.HTTPError as error:
215
+ raise self._transport_error(error) from error
216
+
217
+ async def crawl(self, payload: CrawlRequest) -> CrawlResponse:
218
+ """Synchronous crawl door: returns the unified crawl response."""
219
+ response = await self._request("POST", "/v1/crawl", json=_payload_dict(payload))
220
+ return self._finish(response, CrawlResponse, accepted_if_pending=True)
221
+
222
+ async def crawl_async(self, payload: CrawlRequest) -> CrawlAccepted:
223
+ """Async crawl door: returns the accepted handle (request_id, status_url)."""
224
+ request = payload.model_copy(update={"async_mode": True})
225
+ response = await self._request("POST", "/v1/crawl", json=_payload_dict(request))
226
+ return self._finish(response, CrawlAccepted)
227
+
228
+ async def get_crawl_result(self, request_id: str) -> CrawlResponse:
229
+ """Retrieve a completed crawl; raises if still accepted (202) or gone (404)."""
230
+ response = await self._request("GET", f"/v1/crawl/{request_id}")
231
+ return self._finish(response, CrawlResponse, accepted_if_pending=True)
232
+
233
+ async def wait_for(
234
+ self, request_id: str, *, poll_interval: float = 1.0, max_wait: float | None = None
235
+ ) -> CrawlResponse:
236
+ """Poll ``get_crawl_result`` until the crawl reaches a terminal state."""
237
+ return await _poll_result(
238
+ request_id,
239
+ lambda: self.get_crawl_result(request_id),
240
+ self._make_wait_deadline(max_wait),
241
+ poll_interval,
242
+ lambda rid: self._make_timeout(rid, max_wait),
243
+ )
244
+
245
+ async def crawl_and_wait(self, payload: CrawlRequest) -> CrawlResponse:
246
+ """Crawl asynchronously and wait for the unified response."""
247
+ accepted = await self.crawl_async(payload)
248
+ return await self.wait_for(accepted.request_id)
249
+
250
+ async def admit(self, payload: AdmitRequest) -> AdmitResponse:
251
+ """Pre-flight admission verdict for one URL."""
252
+ response = await self._request("POST", "/v1/admit", json=_payload_dict(payload))
253
+ return self._finish(response, AdmitResponse)
254
+
255
+ async def map_snapshot(self, payload: MapRequest) -> SiteMapResponse:
256
+ """Discover the URL frontier for a root URL (POST /v1/map)."""
257
+ response = await self._request("POST", "/v1/map", json=_payload_dict(payload))
258
+ return self._finish(response, SiteMapResponse)
259
+
260
+ async def render(self, payload: RenderRequest) -> CrawlResponse:
261
+ """Browser-render one or more pages (POST /v1/render)."""
262
+ response = await self._request("POST", "/v1/render", json=_payload_dict(payload))
263
+ return self._finish(response, CrawlResponse)
264
+
265
+ async def policy_suggestions(self) -> PolicySuggestionsResponse:
266
+ """Propose candidate host policy rules (GET /v1/policy-suggestions)."""
267
+ response = await self._request("GET", "/v1/policy-suggestions")
268
+ return self._finish(response, PolicySuggestionsResponse)
269
+
270
+ async def create_job(self, payload: JobCreateRequest) -> JobCreateResponse:
271
+ """Create and start a resumable whole-site crawl job."""
272
+ response = await self._request("POST", "/v1/jobs", json=_payload_dict(payload))
273
+ return self._finish(response, JobCreateResponse)
274
+
275
+ async def list_jobs(self, limit: int | None = None, offset: int = 0) -> JobListResponse:
276
+ """List the caller's crawl jobs; ``limit``/``offset`` page the result."""
277
+ response = await self._request("GET", f"/v1/jobs{_job_list_query(limit, offset)}")
278
+ return self._finish(response, JobListResponse)
279
+
280
+ async def get_job(self, job_id: str) -> JobStatusResponse:
281
+ """Get one crawl job's state and results."""
282
+ response = await self._request("GET", f"/v1/jobs/{job_id}")
283
+ return self._finish(response, JobStatusResponse)
284
+
285
+ async def control_job(
286
+ self, job_id: str, action: Literal["pause", "resume", "cancel"]
287
+ ) -> JobStatusResponse:
288
+ """Pause/resume/cancel a crawl job."""
289
+ payload = JobControlRequest(action=action)
290
+ response = await self._request("POST", f"/v1/jobs/{job_id}/control", json=_payload_dict(payload))
291
+ return self._finish(response, JobStatusResponse)
292
+
293
+ async def delete_job(self, job_id: str) -> None:
294
+ """Cancel (if running) and delete a crawl job."""
295
+ response = await self._request("DELETE", f"/v1/jobs/{job_id}")
296
+ self._check(response)
297
+
298
+ async def search(self, payload: SearchRequest) -> SearchResponse:
299
+ """Run a web search against the configured provider."""
300
+ response = await self._request("POST", "/v1/search", json=_payload_dict(payload))
301
+ return self._finish(response, SearchResponse)
302
+
303
+ async def create_watch(self, payload: WatchCreateRequest) -> WatchState:
304
+ """Register a URL revalidation watch."""
305
+ response = await self._request("POST", "/v1/watches", json=_payload_dict(payload))
306
+ return self._finish(response, WatchState)
307
+
308
+ async def list_watches(self) -> WatchListResponse:
309
+ """List the caller's registered watches."""
310
+ response = await self._request("GET", "/v1/watches")
311
+ return self._finish(response, WatchListResponse)
312
+
313
+ async def get_watch(self, watch_id: str) -> WatchState:
314
+ """Get one watch's revalidation state."""
315
+ response = await self._request("GET", f"/v1/watches/{watch_id}")
316
+ return self._finish(response, WatchState)
317
+
318
+ async def delete_watch(self, watch_id: str) -> None:
319
+ """Stop checking a watch's URL."""
320
+ response = await self._request("DELETE", f"/v1/watches/{watch_id}")
321
+ self._check(response)
322
+
323
+ async def capabilities(self) -> CapabilitiesResponse:
324
+ """Service capabilities discovery (GET /v1/capabilities)."""
325
+ response = await self._request("GET", "/v1/capabilities")
326
+ return self._finish(response, CapabilitiesResponse)
327
+
328
+ async def agent_hints(self) -> AgentHintsResponse:
329
+ """Agent guidance (GET /v1/agent-hints)."""
330
+ response = await self._request("GET", "/v1/agent-hints")
331
+ return self._finish(response, AgentHintsResponse)
332
+
333
+ async def health(self) -> HealthResponse:
334
+ """Lightweight liveness (GET /health)."""
335
+ response = await self._request("GET", "/health")
336
+ return self._finish(response, HealthResponse)
337
+
338
+ async def ready(self) -> ReadinessResponse:
339
+ """Crawl capacity/readiness (GET /ready)."""
340
+ response = await self._request("GET", "/ready")
341
+ return self._finish(response, ReadinessResponse)
342
+
343
+ def sync(self) -> CrawlutronClientSync:
344
+ """A sync client sharing this client's configuration."""
345
+ return CrawlutronClientSync(
346
+ base_url=self._base_url,
347
+ api_key=self._api_key,
348
+ timeout=self._timeout,
349
+ transport=self._transport,
350
+ )
351
+
352
+
353
+ class CrawlutronClientSync(_ClientBase):
354
+ _client: httpx.Client | None = None
355
+
356
+ def __enter__(self) -> CrawlutronClientSync:
357
+ return self
358
+
359
+ def __exit__(self, *args: object) -> None:
360
+ self.close()
361
+
362
+ def _get_client(self) -> httpx.Client:
363
+ client = self._client
364
+ if not isinstance(client, httpx.Client):
365
+ transport = self._transport
366
+ if transport is not None and not isinstance(transport, httpx.BaseTransport):
367
+ raise CrawlutronTransportError(
368
+ "async-only transport cannot drive a sync client; pass an httpx.BaseTransport",
369
+ status_code=None,
370
+ )
371
+ self._client = httpx.Client(**self._client_kwargs(), transport=transport)
372
+ client = self._client
373
+ return client
374
+
375
+ def close(self) -> None:
376
+ if self._client is None:
377
+ return
378
+ client = self._client
379
+ if not isinstance(client, httpx.Client):
380
+ raise CrawlutronTransportError("unexpected http client type", status_code=None)
381
+ client.close()
382
+ self._client = None
383
+
384
+ def _request(self, method: str, path: str, *, json: dict[str, object] | None = None) -> httpx.Response:
385
+ client = self._get_client()
386
+ try:
387
+ return client.request(method, path, json=json)
388
+ except httpx.HTTPError as error:
389
+ self._transport_failure(error)
390
+
391
+ def crawl(self, payload: CrawlRequest) -> CrawlResponse:
392
+ response = self._request("POST", "/v1/crawl", json=_payload_dict(payload))
393
+ return self._finish(response, CrawlResponse, accepted_if_pending=True)
394
+
395
+ def crawl_async(self, payload: CrawlRequest) -> CrawlAccepted:
396
+ request = payload.model_copy(update={"async_mode": True})
397
+ response = self._request("POST", "/v1/crawl", json=_payload_dict(request))
398
+ return self._finish(response, CrawlAccepted)
399
+
400
+ def get_crawl_result(self, request_id: str) -> CrawlResponse:
401
+ """Retrieve a completed crawl; raises if still accepted (202) or gone (404)."""
402
+ response = self._request("GET", f"/v1/crawl/{request_id}")
403
+ return self._finish(response, CrawlResponse, accepted_if_pending=True)
404
+
405
+ def wait_for(
406
+ self, request_id: str, *, max_wait: float | None = None, poll_interval: float = 1.0
407
+ ) -> CrawlResponse:
408
+ """Poll ``get_crawl_result`` until the crawl reaches a terminal state."""
409
+ deadline = self._make_wait_deadline(max_wait)
410
+ while True:
411
+ try:
412
+ return self.get_crawl_result(request_id)
413
+ except CrawlutronAcceptedError:
414
+ pass
415
+ if deadline is not None and time.monotonic() >= deadline:
416
+ raise self._make_timeout(request_id, max_wait)
417
+ time.sleep(poll_interval)
418
+
419
+ def admit(self, payload: AdmitRequest) -> AdmitResponse:
420
+ response = self._request("POST", "/v1/admit", json=_payload_dict(payload))
421
+ return self._finish(response, AdmitResponse)
422
+
423
+ def map_snapshot(self, payload: MapRequest) -> SiteMapResponse:
424
+ response = self._request("POST", "/v1/map", json=_payload_dict(payload))
425
+ return self._finish(response, SiteMapResponse)
426
+
427
+ def render(self, payload: RenderRequest) -> CrawlResponse:
428
+ response = self._request("POST", "/v1/render", json=_payload_dict(payload))
429
+ return self._finish(response, CrawlResponse)
430
+
431
+ def policy_suggestions(self) -> PolicySuggestionsResponse:
432
+ response = self._request("GET", "/v1/policy-suggestions")
433
+ return self._finish(response, PolicySuggestionsResponse)
434
+
435
+ def create_job(self, payload: JobCreateRequest) -> JobCreateResponse:
436
+ response = self._request("POST", "/v1/jobs", json=_payload_dict(payload))
437
+ return self._finish(response, JobCreateResponse)
438
+
439
+ def list_jobs(self, limit: int | None = None, offset: int = 0) -> JobListResponse:
440
+ response = self._request("GET", f"/v1/jobs{_job_list_query(limit, offset)}")
441
+ return self._finish(response, JobListResponse)
442
+
443
+ def get_job(self, job_id: str) -> JobStatusResponse:
444
+ response = self._request("GET", f"/v1/jobs/{job_id}")
445
+ return self._finish(response, JobStatusResponse)
446
+
447
+ def control_job(self, job_id: str, action: Literal["pause", "resume", "cancel"]) -> JobStatusResponse:
448
+ payload = JobControlRequest(action=action)
449
+ response = self._request("POST", f"/v1/jobs/{job_id}/control", json=_payload_dict(payload))
450
+ return self._finish(response, JobStatusResponse)
451
+
452
+ def delete_job(self, job_id: str) -> None:
453
+ response = self._request("DELETE", f"/v1/jobs/{job_id}")
454
+ self._check(response)
455
+
456
+ def search(self, payload: SearchRequest) -> SearchResponse:
457
+ response = self._request("POST", "/v1/search", json=_payload_dict(payload))
458
+ return self._finish(response, SearchResponse)
459
+
460
+ def create_watch(self, payload: WatchCreateRequest) -> WatchState:
461
+ response = self._request("POST", "/v1/watches", json=_payload_dict(payload))
462
+ return self._finish(response, WatchState)
463
+
464
+ def list_watches(self) -> WatchListResponse:
465
+ response = self._request("GET", "/v1/watches")
466
+ return self._finish(response, WatchListResponse)
467
+
468
+ def get_watch(self, watch_id: str) -> WatchState:
469
+ response = self._request("GET", f"/v1/watches/{watch_id}")
470
+ return self._finish(response, WatchState)
471
+
472
+ def delete_watch(self, watch_id: str) -> None:
473
+ response = self._request("DELETE", f"/v1/watches/{watch_id}")
474
+ self._check(response)
475
+
476
+ def agent_hints(self) -> AgentHintsResponse:
477
+ response = self._request("GET", "/v1/agent-hints")
478
+ return self._finish(response, AgentHintsResponse)
479
+
480
+ def capabilities(self) -> CapabilitiesResponse:
481
+ response = self._request("GET", "/v1/capabilities")
482
+ return self._finish(response, CapabilitiesResponse)
483
+
484
+ def health(self) -> HealthResponse:
485
+ response = self._request("GET", "/health")
486
+ return self._finish(response, HealthResponse)
487
+
488
+ def ready(self) -> ReadinessResponse:
489
+ response = self._request("GET", "/ready")
490
+ return self._finish(response, ReadinessResponse)
491
+
492
+
493
+ __all__ = [
494
+ "CrawlutronClient",
495
+ "CrawlutronClientSync",
496
+ ]
@@ -0,0 +1,169 @@
1
+ """Typed error hierarchy for the crawlutron service facade.
2
+
3
+ Every service refusal carries a wire envelope ``{code, message, details}`` and an HTTP status. The
4
+ hierarchy mirrors the status families the service emits so callers can catch by family or by exact status.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+
10
+ class CrawlutronError(Exception):
11
+ """Base error: the service envelope code/message/details plus HTTP status."""
12
+
13
+ status_code: int | None = None
14
+ code: str = "crawlutron_error"
15
+
16
+ def __init__(
17
+ self,
18
+ message: str,
19
+ *,
20
+ code: str | None = None,
21
+ details: dict[str, object] | None = None,
22
+ status_code: int | None = None,
23
+ ) -> None:
24
+ self.message = message
25
+ self.code = code if code is not None else self.code
26
+ self.details = details or {}
27
+ self.status_code = status_code if status_code is not None else self.status_code
28
+ super().__init__(message)
29
+
30
+
31
+ class CrawlutronClientError(CrawlutronError):
32
+ """4xx caller error."""
33
+
34
+
35
+ class CrawlutronUnauthorizedError(CrawlutronClientError):
36
+ status_code = 401
37
+ code = "unauthorized"
38
+
39
+
40
+ class CrawlutronForbiddenError(CrawlutronClientError):
41
+ status_code = 403
42
+ code = "forbidden"
43
+
44
+
45
+ class CrawlutronNotFoundError(CrawlutronClientError):
46
+ status_code = 404
47
+ code = "not_found"
48
+
49
+
50
+ class CrawlutronBadRequestError(CrawlutronClientError):
51
+ status_code = 400
52
+ code = "bad_request"
53
+
54
+
55
+ class CrawlutronConflictError(CrawlutronClientError):
56
+ status_code = 409
57
+ code = "invalid_state"
58
+
59
+
60
+ class CrawlutronClientClosedRequestError(CrawlutronClientError):
61
+ """499: the service saw the caller hang up. Envelope-construction only.
62
+
63
+ A connected SDK caller sees httpx TransportError; the service raises 499
64
+ only after detecting the disconnect, so this class never crosses live HTTP."""
65
+
66
+ status_code = 499
67
+ code = "client_disconnected"
68
+
69
+
70
+ class CrawlutronRequestTooLargeError(CrawlutronClientError):
71
+ status_code = 413
72
+ code = "request_too_large"
73
+
74
+
75
+ class CrawlutronTooManyRequestsError(CrawlutronClientError):
76
+ status_code = 429
77
+ code = "too_many_requests"
78
+
79
+
80
+ class CrawlutronUnprocessableEntityError(CrawlutronClientError):
81
+ status_code = 422
82
+ code = "validation_error"
83
+
84
+
85
+ class CrawlutronAcceptedError(CrawlutronClientError):
86
+ status_code = 202
87
+ code = "accepted"
88
+
89
+
90
+ class CrawlutronServerError(CrawlutronError):
91
+ """5xx service error."""
92
+
93
+
94
+ class CrawlutronBadGatewayError(CrawlutronServerError):
95
+ status_code = 502
96
+ code = "bad_gateway"
97
+
98
+
99
+ class CrawlutronServiceUnavailableError(CrawlutronServerError):
100
+ status_code = 503
101
+ code = "service_unavailable"
102
+
103
+
104
+ class CrawlutronInternalServerError(CrawlutronServerError):
105
+ status_code = 500
106
+ code = "internal_server_error"
107
+
108
+
109
+ class CrawlutronTransportError(CrawlutronError):
110
+ """Network/transport failure; no service HTTP status applies."""
111
+
112
+ status_code = None
113
+ code = "transport_error"
114
+
115
+
116
+ _STATUS_TO_ERROR: dict[int, type[CrawlutronError]] = {
117
+ 202: CrawlutronAcceptedError,
118
+ 400: CrawlutronBadRequestError,
119
+ 401: CrawlutronUnauthorizedError,
120
+ 403: CrawlutronForbiddenError,
121
+ 404: CrawlutronNotFoundError,
122
+ 409: CrawlutronConflictError,
123
+ 413: CrawlutronRequestTooLargeError,
124
+ 422: CrawlutronUnprocessableEntityError,
125
+ 429: CrawlutronTooManyRequestsError,
126
+ 499: CrawlutronClientClosedRequestError,
127
+ }
128
+
129
+ _SERVER_ERRORS: dict[int, type[CrawlutronServerError]] = {
130
+ 500: CrawlutronInternalServerError,
131
+ 502: CrawlutronBadGatewayError,
132
+ 503: CrawlutronServiceUnavailableError,
133
+ }
134
+
135
+
136
+ def error_from_response(status_code: int, envelope: dict[str, object]) -> CrawlutronError:
137
+ """Build the matching typed error from a service error response."""
138
+ code = str(envelope.get("code", "crawlutron_error"))
139
+ message = str(envelope.get("message", "Request failed"))
140
+ details = envelope.get("details")
141
+ if not isinstance(details, dict):
142
+ details = {}
143
+ if 500 <= status_code < 600:
144
+ klass: type[CrawlutronError] = _SERVER_ERRORS.get(status_code, CrawlutronInternalServerError)
145
+ else:
146
+ klass = _STATUS_TO_ERROR.get(status_code, CrawlutronError)
147
+ return klass(message=message, code=code, details=details, status_code=status_code)
148
+
149
+
150
+ __all__ = [
151
+ "CrawlutronAcceptedError",
152
+ "CrawlutronBadGatewayError",
153
+ "CrawlutronBadRequestError",
154
+ "CrawlutronClientClosedRequestError",
155
+ "CrawlutronClientError",
156
+ "CrawlutronConflictError",
157
+ "CrawlutronError",
158
+ "CrawlutronForbiddenError",
159
+ "CrawlutronInternalServerError",
160
+ "CrawlutronNotFoundError",
161
+ "CrawlutronRequestTooLargeError",
162
+ "CrawlutronServerError",
163
+ "CrawlutronServiceUnavailableError",
164
+ "CrawlutronTooManyRequestsError",
165
+ "CrawlutronTransportError",
166
+ "CrawlutronUnauthorizedError",
167
+ "CrawlutronUnprocessableEntityError",
168
+ "error_from_response",
169
+ ]
@@ -0,0 +1,482 @@
1
+ """SDK-owned pydantic v2 wire mirrors of the service contract: request models forbid extra fields
2
+ (WatchCreateRequest allows), responses tolerate them. The SDK never imports crawlutron and is shape-pinned
3
+ against the served OpenAPI -- drift reds.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from typing import Annotated, Literal
9
+
10
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
11
+
12
+
13
+ class _UrlSource(BaseModel):
14
+ """Shared exactly-one-of url/urls base for request models."""
15
+
16
+ url: str | None = None
17
+ urls: list[str] | None = None
18
+
19
+ @model_validator(mode="after")
20
+ def _require_exactly_one_url_source(self) -> _UrlSource:
21
+ has_url = self.url is not None
22
+ has_urls = self.urls is not None
23
+ if has_url == has_urls:
24
+ raise ValueError("supply exactly one of 'url' or 'urls'")
25
+ if has_urls and not self.urls:
26
+ raise ValueError("'urls' must contain at least one URL")
27
+ return self
28
+
29
+
30
+ class CrawlRequest(_UrlSource):
31
+ """Request body for ``POST /v1/crawl`` (one URL source plus a policy)."""
32
+
33
+ model_config = ConfigDict(extra="forbid")
34
+
35
+ policy: dict[str, object] = Field(
36
+ default_factory=dict,
37
+ description="Resolved caller-provided policy/execution plan.",
38
+ )
39
+ validators: dict[str, object] = Field(
40
+ default_factory=dict,
41
+ description="Optional map of submitted URL -> cached validators.",
42
+ )
43
+ async_mode: bool = Field(
44
+ default=False,
45
+ description="When true, accept immediately (202) and run in the background.",
46
+ )
47
+ webhook_url: Annotated[
48
+ str | None,
49
+ Field(description="Optional callback URL (https:// required)."),
50
+ ] = None
51
+ output_format: Annotated[
52
+ Literal["json", "jsonl", "ndjson", "markdown", "csv"] | None,
53
+ Field(description="Output format; json is the default."),
54
+ ] = None
55
+ changed_since: Annotated[
56
+ float | None,
57
+ Field(
58
+ ge=0,
59
+ allow_inf_nan=False,
60
+ description="Epoch seconds; when set, the caller requests delta classification relative to "
61
+ "that time.",
62
+ ),
63
+ ] = None
64
+ baseline_request_id: Annotated[
65
+ str | None,
66
+ Field(
67
+ description="Prior crawl/job/watch request id to use as the delta baseline.",
68
+ ),
69
+ ] = None
70
+
71
+
72
+ class CrawlResponse(BaseModel):
73
+ """The unified crawl response envelope (200 from ``POST /v1/crawl``)."""
74
+
75
+ model_config = ConfigDict(extra="allow")
76
+
77
+ request_id: str
78
+ submitted_count: int
79
+ policy_schema_version: str
80
+ success: bool
81
+ results: list[dict[str, object]] = Field(default_factory=list)
82
+ delta_summary: Annotated[
83
+ dict[str, object] | None,
84
+ Field(
85
+ description="Per-result delta accounting, populated when delta crawl mode is requested. "
86
+ "Carries degraded_baseline=true when the baseline row was missing/expired/foreign, so "
87
+ "freshness-classified results are distinguishable from baseline-lost ones.",
88
+ ),
89
+ ] = None
90
+
91
+
92
+ class CrawlAccepted(BaseModel):
93
+ """The async-accepted body (202 from ``POST /v1/crawl``)."""
94
+
95
+ model_config = ConfigDict(extra="allow")
96
+
97
+ status: str
98
+ request_id: str
99
+ status_url: str
100
+ webhook_url: Annotated[str | None, Field(default=None)] = None
101
+
102
+
103
+ class AdmitRequest(BaseModel):
104
+ """Request body for ``POST /v1/admit`` (one URL to judge pre-flight)."""
105
+
106
+ model_config = ConfigDict(extra="forbid")
107
+
108
+ url: str = Field(..., description="The raw URL to admit or refuse.")
109
+ check_robots: bool = Field(
110
+ default=False,
111
+ description="Opt-in robots.txt check (the only I/O path).",
112
+ )
113
+
114
+
115
+ class AdmitResponse(BaseModel):
116
+ """The admission verdict with the per-check reason table."""
117
+
118
+ model_config = ConfigDict(extra="allow")
119
+
120
+ admitted: bool
121
+ classification: Annotated[
122
+ Literal[
123
+ "invalid_url",
124
+ "unsupported_scheme",
125
+ "userinfo_refused",
126
+ "ipvfuture_refused",
127
+ "invalid_port",
128
+ "invalid_idn_host",
129
+ "host_denied",
130
+ "host_not_allowed",
131
+ "private_network_target",
132
+ "robots_disallowed",
133
+ ]
134
+ | None,
135
+ Field(default=None),
136
+ ] = None
137
+ canonical_url: Annotated[str | None, Field(default=None)] = None
138
+ checks: dict[str, str | None] = Field(default_factory=dict)
139
+
140
+
141
+ class MapRequest(BaseModel):
142
+ """Request body for ``POST /v1/map`` (discover a root URL's frontier)."""
143
+
144
+ model_config = ConfigDict(extra="forbid")
145
+
146
+ url: str = Field(..., description="Root public HTTP(S) URL to map.")
147
+ policy: dict[str, object] = Field(
148
+ default_factory=dict,
149
+ description="Optional crawl policy for the mapping crawl.",
150
+ )
151
+
152
+
153
+ class SiteMapResponse(BaseModel):
154
+ """The de-duplicated URL frontier for a mapping crawl (200 from ``POST /v1/map``)."""
155
+
156
+ model_config = ConfigDict(extra="allow")
157
+
158
+ url: str | None = None
159
+ links: list[str] = Field(default_factory=list)
160
+ sitemaps: list[str] = Field(default_factory=list)
161
+ feeds: list[str] = Field(default_factory=list)
162
+ all_urls: list[str] = Field(default_factory=list)
163
+
164
+
165
+ class RenderRequest(_UrlSource):
166
+ """Request body for ``POST /v1/render`` (backward-compatible browser render)."""
167
+
168
+ model_config = ConfigDict(extra="forbid")
169
+
170
+ options: dict[str, object] | None = None
171
+
172
+
173
+ class JobCreateRequest(BaseModel):
174
+ """Request body for ``POST /v1/jobs`` (resumable whole-site crawl job)."""
175
+
176
+ model_config = ConfigDict(extra="forbid")
177
+
178
+ root_url: str = Field(..., description="Root public HTTP(S) URL; stored canonically at depth 0.")
179
+ policy: dict[str, object] = Field(
180
+ default_factory=dict,
181
+ description="Crawl policy (strategies, limits, scope, extraction).",
182
+ )
183
+
184
+
185
+ class JobCreateResponse(BaseModel):
186
+ """Response from creating a crawl job."""
187
+
188
+ model_config = ConfigDict(extra="allow")
189
+
190
+ job_id: str
191
+ status: str
192
+ root_url: str
193
+ created_at: float
194
+
195
+
196
+ class JobStatusResponse(BaseModel):
197
+ """Status of a crawl job."""
198
+
199
+ model_config = ConfigDict(extra="allow")
200
+
201
+ job_id: str
202
+ root_url: str
203
+ owner: str = ""
204
+ status: str
205
+ kind: str
206
+ created_at: float
207
+ updated_at: float
208
+ frontier_size: int
209
+ visited_size: int
210
+ results_count: int
211
+ cap_truncated: bool = False
212
+ stats: dict[str, int] = Field(default_factory=dict)
213
+ error: Annotated[str | None, Field(default=None)] = None
214
+
215
+
216
+ class JobListResponse(BaseModel):
217
+ """A page of crawl jobs for the caller, with the caller's total row count."""
218
+
219
+ model_config = ConfigDict(extra="allow")
220
+
221
+ jobs: list[JobStatusResponse] = Field(default_factory=list)
222
+ total: int = 0
223
+
224
+
225
+ class JobControlRequest(BaseModel):
226
+ """Control a crawl job (pause/resume/cancel)."""
227
+
228
+ model_config = ConfigDict(extra="forbid")
229
+
230
+ action: Literal["pause", "resume", "cancel"] = Field(..., description="One of: pause, resume, cancel.")
231
+
232
+
233
+ class JobDeletedResponse(BaseModel):
234
+ """The delete outcome as a typed body."""
235
+
236
+ model_config = ConfigDict(extra="allow")
237
+
238
+ deleted: bool
239
+ job_id: str
240
+
241
+
242
+ class SearchRequest(BaseModel):
243
+ """Request body for ``POST /v1/search`` (web search query)."""
244
+
245
+ model_config = ConfigDict(extra="forbid")
246
+
247
+ query: str = Field(..., min_length=1, description="Web search query.")
248
+ limit: Annotated[
249
+ int | None,
250
+ Field(
251
+ default=None,
252
+ ge=1,
253
+ le=100,
254
+ description="Maximum results to return; the server clamps this to its "
255
+ "CRAWLUTRON_SEARCH_LIMIT_MAX (drift-guarded against the server model).",
256
+ ),
257
+ ] = None
258
+
259
+
260
+ class SearchResultItem(BaseModel):
261
+ """One SERP-style search result."""
262
+
263
+ model_config = ConfigDict(extra="allow")
264
+
265
+ url: str
266
+ title: str
267
+ description: str = ""
268
+
269
+
270
+ class SearchResponse(BaseModel):
271
+ """Response for ``POST /v1/search``."""
272
+
273
+ model_config = ConfigDict(extra="allow")
274
+
275
+ success: bool = True
276
+ query: str
277
+ provider: str
278
+ limit: int
279
+ data: list[SearchResultItem] = Field(default_factory=list)
280
+
281
+
282
+ class WatchCreateRequest(BaseModel):
283
+ """A watch registration (POST /v1/watches)."""
284
+
285
+ model_config = ConfigDict(extra="allow")
286
+
287
+ url: str = Field(..., description="Public HTTP(S) URL to revalidate.")
288
+ interval_seconds: int = Field(..., gt=0, description="Seconds between revalidation checks.")
289
+ webhook_url: Annotated[str | None, Field(default=None)] = None
290
+
291
+
292
+ class WatchState(BaseModel):
293
+ """One row of the watches table, as shared on the wire."""
294
+
295
+ model_config = ConfigDict(extra="allow")
296
+
297
+ watch_id: str
298
+ owner: str = ""
299
+ url: str
300
+ webhook_url: Annotated[str | None, Field(default=None)] = None
301
+ interval_seconds: int
302
+ status: Literal["active", "deactivated"] = "active"
303
+ validators: dict[str, object] = Field(default_factory=dict)
304
+ next_check_at: float
305
+ last_checked_at: Annotated[float | None, Field(default=None)] = None
306
+ last_outcome: Annotated[
307
+ Literal[
308
+ "rendered",
309
+ "captcha_detected",
310
+ "bot_challenge_detected",
311
+ "access_denied",
312
+ "login_required",
313
+ "paywall_detected",
314
+ "rate_limited",
315
+ "cookie_consent_required",
316
+ "javascript_required",
317
+ "unsupported_browser",
318
+ "geolocation_blocked",
319
+ "waf_challenge_detected",
320
+ "not_found",
321
+ "site_error",
322
+ "gone",
323
+ "binary_no_text",
324
+ "robots_disallowed",
325
+ "navigation_timeout",
326
+ "browser_crash",
327
+ "network_error",
328
+ "connection_error",
329
+ "http_status_error",
330
+ "url_budget_exceeded",
331
+ "browser_pool_exhausted",
332
+ "circuit_breaker_open",
333
+ "admission_stalled",
334
+ "payload_too_large",
335
+ "too_many_redirects",
336
+ "http_error",
337
+ "value_error",
338
+ "tls_error",
339
+ "dns_error",
340
+ "invalid_url",
341
+ "unsupported_scheme",
342
+ "private_network_target",
343
+ "host_denied",
344
+ "host_not_allowed",
345
+ "userinfo_refused",
346
+ "unknown_error",
347
+ "not_modified",
348
+ ]
349
+ | None,
350
+ Field(default=None),
351
+ ] = None
352
+ last_changed_at: Annotated[float | None, Field(default=None)] = None
353
+ consecutive_failures: int = 0
354
+ created_at: float
355
+ updated_at: float
356
+
357
+
358
+ class WatchListResponse(BaseModel):
359
+ """Every registered watch, each with the state of its last check."""
360
+
361
+ model_config = ConfigDict(extra="allow")
362
+
363
+ watches: list[WatchState] = Field(default_factory=list)
364
+
365
+
366
+ class CapabilitiesResponse(BaseModel):
367
+ """The /v1/capabilities discovery payload."""
368
+
369
+ model_config = ConfigDict(extra="allow")
370
+
371
+ service: str
372
+ version: str
373
+ policy_schema_version: str
374
+ supported_strategies: list[str] = Field(default_factory=list)
375
+ supported_extractions: list[str] = Field(default_factory=list)
376
+ supported_classifications: list[str] = Field(default_factory=list)
377
+ output_fields: list[str] = Field(default_factory=list)
378
+ default_limits: dict[str, object] = Field(default_factory=dict)
379
+ features: dict[str, bool] = Field(default_factory=dict)
380
+ auth: dict[str, str] = Field(default_factory=dict)
381
+ crawl_timing: dict[str, object] = Field(default_factory=dict)
382
+ search: dict[str, object] = Field(default_factory=dict)
383
+ mcp: dict[str, object] = Field(default_factory=dict)
384
+ openapi_url: str = ""
385
+ swagger_ui_url: str = ""
386
+
387
+
388
+ class AgentHintsResponse(BaseModel):
389
+ """The /v1/agent-hints guidance payload."""
390
+
391
+ model_config = ConfigDict(extra="allow")
392
+
393
+ service: str
394
+ strategies: dict[str, str] = Field(default_factory=dict)
395
+ extractions: dict[str, str] = Field(default_factory=dict)
396
+ guidance: list[str] = Field(default_factory=list)
397
+ example_policies: dict[str, object] = Field(default_factory=dict)
398
+
399
+
400
+ class HostPolicySuggestion(BaseModel):
401
+ """One host's candidate rule, with the evidence that proposed it."""
402
+
403
+ model_config = ConfigDict(extra="allow")
404
+
405
+ host: str
406
+ rule: str
407
+ blocked_attempts: int
408
+ observed_attempts: int
409
+ classifications: list[str] = Field(default_factory=list)
410
+ signatures: list[str] = Field(default_factory=list)
411
+ evidence: list[dict[str, object]] = Field(default_factory=list)
412
+ rationale: str
413
+
414
+
415
+ class PolicySuggestionsResponse(BaseModel):
416
+ """Candidate host policy rules drawn from retained crawl records."""
417
+
418
+ model_config = ConfigDict(extra="allow")
419
+
420
+ records_examined: int
421
+ records_skipped: int = 0
422
+ suggestions: list[HostPolicySuggestion] = Field(default_factory=list)
423
+
424
+
425
+ class HealthResponse(BaseModel):
426
+ """The /health liveness response."""
427
+
428
+ model_config = ConfigDict(extra="allow")
429
+
430
+ status: str = "ok"
431
+
432
+
433
+ class ReadinessResponse(BaseModel):
434
+ """The /ready crawl-capacity response."""
435
+
436
+ model_config = ConfigDict(extra="allow")
437
+
438
+ status: str
439
+ active_requests: int
440
+ max_active_requests: int
441
+ saturated: bool
442
+
443
+
444
+ class ErrorEnvelope(BaseModel):
445
+ """The service error envelope: {code, message, details}."""
446
+
447
+ model_config = ConfigDict(extra="allow")
448
+
449
+ code: str
450
+ message: str
451
+ details: dict[str, object] = Field(default_factory=dict)
452
+
453
+
454
+ __all__ = [
455
+ "AdmitRequest",
456
+ "AdmitResponse",
457
+ "AgentHintsResponse",
458
+ "CapabilitiesResponse",
459
+ "CrawlAccepted",
460
+ "CrawlRequest",
461
+ "CrawlResponse",
462
+ "ErrorEnvelope",
463
+ "HealthResponse",
464
+ "HostPolicySuggestion",
465
+ "JobControlRequest",
466
+ "JobCreateRequest",
467
+ "JobCreateResponse",
468
+ "JobDeletedResponse",
469
+ "JobListResponse",
470
+ "JobStatusResponse",
471
+ "MapRequest",
472
+ "PolicySuggestionsResponse",
473
+ "ReadinessResponse",
474
+ "RenderRequest",
475
+ "SearchRequest",
476
+ "SearchResponse",
477
+ "SearchResultItem",
478
+ "SiteMapResponse",
479
+ "WatchCreateRequest",
480
+ "WatchListResponse",
481
+ "WatchState",
482
+ ]
File without changes