scoutline 0.17.7 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -26
- package/dist/capabilities/quota.d.ts.map +1 -1
- package/dist/capabilities/quota.js +4 -1
- package/dist/capabilities/quota.js.map +1 -1
- package/dist/capabilities/reader.d.ts +8 -0
- package/dist/capabilities/reader.d.ts.map +1 -1
- package/dist/capabilities/reader.js.map +1 -1
- package/dist/commands/crawl.js +3 -3
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +10 -9
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +65 -19
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/map.js +3 -3
- package/dist/commands/quota.d.ts.map +1 -1
- package/dist/commands/quota.js +1 -2
- package/dist/commands/quota.js.map +1 -1
- package/dist/commands/read.d.ts.map +1 -1
- package/dist/commands/read.js +13 -5
- package/dist/commands/read.js.map +1 -1
- package/dist/commands/research.d.ts.map +1 -1
- package/dist/commands/research.js +7 -6
- package/dist/commands/research.js.map +1 -1
- package/dist/commands/search.d.ts.map +1 -1
- package/dist/commands/search.js +2 -3
- package/dist/commands/search.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -10
- package/dist/index.js.map +1 -1
- package/dist/lib/quota-mapping.d.ts +23 -2
- package/dist/lib/quota-mapping.d.ts.map +1 -1
- package/dist/lib/quota-mapping.js +47 -2
- package/dist/lib/quota-mapping.js.map +1 -1
- package/dist/lib/redact.d.ts +9 -1
- package/dist/lib/redact.d.ts.map +1 -1
- package/dist/lib/redact.js +21 -1
- package/dist/lib/redact.js.map +1 -1
- package/dist/providers/linkup/adapter.d.ts +92 -0
- package/dist/providers/linkup/adapter.d.ts.map +1 -0
- package/dist/providers/linkup/adapter.js +747 -0
- package/dist/providers/linkup/adapter.js.map +1 -0
- package/dist/providers/linkup/client.d.ts +157 -0
- package/dist/providers/linkup/client.d.ts.map +1 -0
- package/dist/providers/linkup/client.js +371 -0
- package/dist/providers/linkup/client.js.map +1 -0
- package/dist/providers/linkup/credentials.d.ts +41 -0
- package/dist/providers/linkup/credentials.d.ts.map +1 -0
- package/dist/providers/linkup/credentials.js +66 -0
- package/dist/providers/linkup/credentials.js.map +1 -0
- package/dist/providers/linkup/diagnostics.d.ts +39 -0
- package/dist/providers/linkup/diagnostics.d.ts.map +1 -0
- package/dist/providers/linkup/diagnostics.js +49 -0
- package/dist/providers/linkup/diagnostics.js.map +1 -0
- package/dist/providers/linkup/quota.d.ts +55 -0
- package/dist/providers/linkup/quota.d.ts.map +1 -0
- package/dist/providers/linkup/quota.js +100 -0
- package/dist/providers/linkup/quota.js.map +1 -0
- package/dist/providers/registry.d.ts.map +1 -1
- package/dist/providers/registry.js +6 -0
- package/dist/providers/registry.js.map +1 -1
- package/dist/providers/spider/adapter.d.ts +130 -0
- package/dist/providers/spider/adapter.d.ts.map +1 -0
- package/dist/providers/spider/adapter.js +683 -0
- package/dist/providers/spider/adapter.js.map +1 -0
- package/dist/providers/spider/client.d.ts +122 -0
- package/dist/providers/spider/client.d.ts.map +1 -0
- package/dist/providers/spider/client.js +267 -0
- package/dist/providers/spider/client.js.map +1 -0
- package/dist/providers/spider/credentials.d.ts +31 -0
- package/dist/providers/spider/credentials.d.ts.map +1 -0
- package/dist/providers/spider/credentials.js +52 -0
- package/dist/providers/spider/credentials.js.map +1 -0
- package/dist/providers/spider/diagnostics.d.ts +33 -0
- package/dist/providers/spider/diagnostics.d.ts.map +1 -0
- package/dist/providers/spider/diagnostics.js +55 -0
- package/dist/providers/spider/diagnostics.js.map +1 -0
- package/dist/providers/spider/quota.d.ts +61 -0
- package/dist/providers/spider/quota.d.ts.map +1 -0
- package/dist/providers/spider/quota.js +90 -0
- package/dist/providers/spider/quota.js.map +1 -0
- package/dist/providers/types.d.ts +1 -1
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/providers/types.js +3 -0
- package/dist/providers/types.js.map +1 -1
- package/dist/providers/you/adapter.d.ts +92 -0
- package/dist/providers/you/adapter.d.ts.map +1 -0
- package/dist/providers/you/adapter.js +469 -0
- package/dist/providers/you/adapter.js.map +1 -0
- package/dist/providers/you/client.d.ts +94 -0
- package/dist/providers/you/client.d.ts.map +1 -0
- package/dist/providers/you/client.js +242 -0
- package/dist/providers/you/client.js.map +1 -0
- package/dist/providers/you/credentials.d.ts +40 -0
- package/dist/providers/you/credentials.d.ts.map +1 -0
- package/dist/providers/you/credentials.js +61 -0
- package/dist/providers/you/credentials.js.map +1 -0
- package/dist/providers/you/diagnostics.d.ts +43 -0
- package/dist/providers/you/diagnostics.d.ts.map +1 -0
- package/dist/providers/you/diagnostics.js +60 -0
- package/dist/providers/you/diagnostics.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spider.cloud Adapter — Search, Reader, Crawl, and Map Capabilities.
|
|
3
|
+
*
|
|
4
|
+
* Owns credentials, transport lifecycle, Provider field mapping, and
|
|
5
|
+
* failure normalization. Clones the Firecrawl adapter's capability
|
|
6
|
+
* structure (the locked analog adapter) with Spider-specific wire
|
|
7
|
+
* differences: the search term rides the `search` body field (not
|
|
8
|
+
* `query`), `domain` maps to `whitelist`, `location` maps to
|
|
9
|
+
* `country_code`, and a non-general `topic` is a query keyword (Z.AI /
|
|
10
|
+
* MiniMax precedent). `type` is Brave-only and rejected in `validate`
|
|
11
|
+
* before any transport call so the option-level fallback contract can
|
|
12
|
+
* continue past Spider to a capable Provider. The Reader POSTs the
|
|
13
|
+
* locked four-field `/scrape` body and rejects every request control
|
|
14
|
+
* with no Spider-native wire equivalent instead of accept-and-drop.
|
|
15
|
+
* Crawl and Map are SYNCHRONOUS one-shot POSTs (`/crawl`, `/links`)
|
|
16
|
+
* returning the final JSON array — no async job file, no poll loop —
|
|
17
|
+
* with the locked 200-status crawl filter and link deduplication.
|
|
18
|
+
* Quota reads GET /data/credits into a single unknown-limit "credits"
|
|
19
|
+
* category (remaining only — no fabricated percent/limit), and the
|
|
20
|
+
* Diagnostics probe rides the same free credits endpoint.
|
|
21
|
+
*
|
|
22
|
+
* The descriptor and capability interfaces below are declared locally
|
|
23
|
+
* with the `"spider"` literal; they become assignable to the shared
|
|
24
|
+
* `ProviderDescriptor` / `SearchCapability` contracts once `"spider"`
|
|
25
|
+
* joins `PROVIDER_IDS` during registry wiring.
|
|
26
|
+
*/
|
|
27
|
+
import crypto from "node:crypto";
|
|
28
|
+
import { decodeReaderFetchResult } from "../../capabilities/reader.js";
|
|
29
|
+
import { decodeCrawlResult } from "../../capabilities/crawl.js";
|
|
30
|
+
import { decodeMapResult } from "../../capabilities/map.js";
|
|
31
|
+
import { ApiError, AuthError, ConfigurationError, NetworkError, QuotaError, TimeoutError, UnsupportedOptionError, ValidationError, } from "../../lib/errors.js";
|
|
32
|
+
import { applySearchTopic } from "../../lib/search-topic.js";
|
|
33
|
+
import { fetchSpiderCrawl, fetchSpiderLinks, fetchSpiderScrape, fetchSpiderSearch, } from "./client.js";
|
|
34
|
+
import { getSpiderApiKey, requireSpiderApiKey } from "./credentials.js";
|
|
35
|
+
import { createSpiderQuotaCapability } from "./quota.js";
|
|
36
|
+
import { createSpiderDiagnosticsCapability } from "./diagnostics.js";
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// Credential + shape helpers
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
/**
|
|
41
|
+
* Full lowercase SHA-256 hex digest of the active credential. Cache code
|
|
42
|
+
* consumes the digest directly and never re-hashes it.
|
|
43
|
+
*/
|
|
44
|
+
function credentialFingerprint(apiKey) {
|
|
45
|
+
return crypto.createHash("sha256").update(apiKey, "utf8").digest("hex");
|
|
46
|
+
}
|
|
47
|
+
function isPlainObject(value) {
|
|
48
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
49
|
+
}
|
|
50
|
+
function assertHttpUrl(url, capability) {
|
|
51
|
+
if (typeof url !== "string" || url.length === 0) {
|
|
52
|
+
throw new ValidationError(`Spider ${capability} URL must be a non-empty string`);
|
|
53
|
+
}
|
|
54
|
+
if (!/^https?:\/\//.test(url)) {
|
|
55
|
+
throw new ValidationError("URL must start with http:// or https://");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// Request mapping
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
/**
|
|
62
|
+
* Map `controls.recency` to the Google-style `tbs` filter Spider.cloud
|
|
63
|
+
* accepts. `noLimit` (and any unknown value) omits `tbs` entirely.
|
|
64
|
+
*/
|
|
65
|
+
function mapRecencyToTbs(recency) {
|
|
66
|
+
switch (recency) {
|
|
67
|
+
case "oneDay":
|
|
68
|
+
return "qdr:d";
|
|
69
|
+
case "oneWeek":
|
|
70
|
+
return "qdr:w";
|
|
71
|
+
case "oneMonth":
|
|
72
|
+
return "qdr:m";
|
|
73
|
+
case "oneYear":
|
|
74
|
+
return "qdr:y";
|
|
75
|
+
case "noLimit":
|
|
76
|
+
return undefined;
|
|
77
|
+
default:
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Map Provider-neutral `SearchControls` to Spider-native search params.
|
|
83
|
+
*
|
|
84
|
+
* domain -> whitelist: [domain]
|
|
85
|
+
* recency -> tbs (qdr:*)
|
|
86
|
+
* location -> country_code (lowercased)
|
|
87
|
+
* contentSize -> no branch: the canonical payload always requests
|
|
88
|
+
* `return_format: "markdown"` with `metadata: true`
|
|
89
|
+
* (SCHEMA §1), which is exactly what `high` observes;
|
|
90
|
+
* `medium` sees the same default format on the body.
|
|
91
|
+
* topic -> keyword appended to the search field (never a wire
|
|
92
|
+
* parameter)
|
|
93
|
+
*/
|
|
94
|
+
function mapSearchControls(request) {
|
|
95
|
+
const controls = request.controls;
|
|
96
|
+
const params = {
|
|
97
|
+
search: applySearchTopic(request.query, controls?.topic),
|
|
98
|
+
return_format: "markdown",
|
|
99
|
+
metadata: true,
|
|
100
|
+
};
|
|
101
|
+
if (controls?.domain) {
|
|
102
|
+
params.whitelist = [controls.domain];
|
|
103
|
+
}
|
|
104
|
+
if (controls?.recency) {
|
|
105
|
+
const tbs = mapRecencyToTbs(controls.recency);
|
|
106
|
+
if (tbs)
|
|
107
|
+
params.tbs = tbs;
|
|
108
|
+
}
|
|
109
|
+
if (controls?.location) {
|
|
110
|
+
params.country_code = controls.location.toLowerCase();
|
|
111
|
+
}
|
|
112
|
+
return params;
|
|
113
|
+
}
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
// Response normalization
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
/**
|
|
118
|
+
* Normalize a raw Spider.cloud search response into `SearchSource[]`.
|
|
119
|
+
*
|
|
120
|
+
* The endpoint returns a flat array of page objects; a documented
|
|
121
|
+
* wrapper (`{ content: [...] }`) is also accepted. Per-item mapping
|
|
122
|
+
* (SCHEMA §1): `metadata.title` (empty string when absent) -> title,
|
|
123
|
+
* `url` -> url, `metadata.description || content || ""` -> summary, and
|
|
124
|
+
* every row is attributed to `"spider.cloud"`. Any malformed shape is a
|
|
125
|
+
* retryable `ApiError` 500.
|
|
126
|
+
*/
|
|
127
|
+
function normalizeSpiderSearchResults(raw) {
|
|
128
|
+
const results = Array.isArray(raw)
|
|
129
|
+
? raw
|
|
130
|
+
: isPlainObject(raw) && Array.isArray(raw.content)
|
|
131
|
+
? raw.content
|
|
132
|
+
: undefined;
|
|
133
|
+
if (results === undefined) {
|
|
134
|
+
throw new ApiError("Spider search returned a malformed response", 500);
|
|
135
|
+
}
|
|
136
|
+
const out = [];
|
|
137
|
+
for (const entry of results) {
|
|
138
|
+
if (!isPlainObject(entry) || typeof entry.url !== "string") {
|
|
139
|
+
throw new ApiError("Spider search returned a malformed response", 500);
|
|
140
|
+
}
|
|
141
|
+
const metadata = isPlainObject(entry.metadata) ? entry.metadata : undefined;
|
|
142
|
+
const title = typeof metadata?.title === "string" ? metadata.title : "";
|
|
143
|
+
const description = typeof metadata?.description === "string" ? metadata.description : undefined;
|
|
144
|
+
const content = typeof entry.content === "string" ? entry.content : undefined;
|
|
145
|
+
const result = {
|
|
146
|
+
title,
|
|
147
|
+
url: entry.url,
|
|
148
|
+
summary: description || content || "",
|
|
149
|
+
source: "spider.cloud",
|
|
150
|
+
};
|
|
151
|
+
out.push(result);
|
|
152
|
+
}
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
155
|
+
// ---------------------------------------------------------------------------
|
|
156
|
+
// Reader response normalization
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
/**
|
|
159
|
+
* Normalize a raw Spider.cloud /scrape response into a
|
|
160
|
+
* `ReaderFetchResult`.
|
|
161
|
+
*
|
|
162
|
+
* The endpoint returns an array of page objects (one for the requested
|
|
163
|
+
* URL); a documented wrapper (`{ content: [...] }`) is also accepted.
|
|
164
|
+
* Per-item mapping: `content` -> content (non-empty, else `ApiError` —
|
|
165
|
+
* an empty fetch is an error, not a degenerate result), `url` ->
|
|
166
|
+
* finalUrl (request URL fallback), `metadata.title` -> title (null when
|
|
167
|
+
* absent/blank). The provider-derived `metadata` and `external` blobs
|
|
168
|
+
* are preserved verbatim on the `ReaderFetchResult` when the page
|
|
169
|
+
* carries them. Any malformed shape is a retryable `ApiError` 500.
|
|
170
|
+
*/
|
|
171
|
+
function normalizeSpiderScrapeResult(raw, request) {
|
|
172
|
+
const results = Array.isArray(raw)
|
|
173
|
+
? raw
|
|
174
|
+
: isPlainObject(raw) && Array.isArray(raw.content)
|
|
175
|
+
? raw.content
|
|
176
|
+
: undefined;
|
|
177
|
+
if (results === undefined || results.length === 0) {
|
|
178
|
+
throw new ApiError("Spider scrape returned a malformed response", 500);
|
|
179
|
+
}
|
|
180
|
+
const page = results[0];
|
|
181
|
+
if (!isPlainObject(page)) {
|
|
182
|
+
throw new ApiError("Spider scrape returned a malformed response", 500);
|
|
183
|
+
}
|
|
184
|
+
const content = typeof page.content === "string" ? page.content : undefined;
|
|
185
|
+
if (content === undefined || content.length === 0) {
|
|
186
|
+
throw new ApiError("Spider scrape returned no content", 500);
|
|
187
|
+
}
|
|
188
|
+
const finalUrl = typeof page.url === "string" && page.url.length > 0 ? page.url : request.url;
|
|
189
|
+
const metadata = isPlainObject(page.metadata) ? page.metadata : undefined;
|
|
190
|
+
const rawTitle = typeof metadata?.title === "string" ? metadata.title : undefined;
|
|
191
|
+
const title = rawTitle !== undefined && rawTitle.trim().length > 0 ? rawTitle : null;
|
|
192
|
+
return {
|
|
193
|
+
schemaVersion: 1,
|
|
194
|
+
url: request.url,
|
|
195
|
+
finalUrl,
|
|
196
|
+
title,
|
|
197
|
+
content,
|
|
198
|
+
contentFormat: request.format ?? "markdown",
|
|
199
|
+
...(metadata !== undefined ? { metadata } : {}),
|
|
200
|
+
...(page.external !== undefined ? { external: page.external } : {}),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
// ---------------------------------------------------------------------------
|
|
204
|
+
// Crawl / map response normalization
|
|
205
|
+
// ---------------------------------------------------------------------------
|
|
206
|
+
/**
|
|
207
|
+
* Resolve the Spider response envelope: a bare JSON array, or the
|
|
208
|
+
* documented `{ content: [...] }` wrapper. `undefined` means the payload
|
|
209
|
+
* is neither — a malformed response.
|
|
210
|
+
*/
|
|
211
|
+
function resolveSpiderEnvelope(raw) {
|
|
212
|
+
if (Array.isArray(raw))
|
|
213
|
+
return raw;
|
|
214
|
+
if (isPlainObject(raw) && Array.isArray(raw.content))
|
|
215
|
+
return raw.content;
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Normalize a raw Spider.cloud /crawl response into a `CrawlResult`.
|
|
220
|
+
*
|
|
221
|
+
* The endpoint is synchronous (locked contract): the response IS the
|
|
222
|
+
* final array of crawled pages — no job id, no poll. The locked filter
|
|
223
|
+
* keeps only pages with `status === 200` AND a non-empty `content`
|
|
224
|
+
* (SCHEMA §3); everything else (404s, error rows, empty bodies) is
|
|
225
|
+
* dropped, not an error. Per-page mapping: `url` -> url, `content` ->
|
|
226
|
+
* content, `contentFormat` defaults to `"markdown"` (Firecrawl
|
|
227
|
+
* precedent). Any non-object entry is a retryable `ApiError` 500.
|
|
228
|
+
*/
|
|
229
|
+
function normalizeSpiderCrawlResult(raw, request) {
|
|
230
|
+
const results = resolveSpiderEnvelope(raw);
|
|
231
|
+
if (results === undefined) {
|
|
232
|
+
throw new ApiError("Spider crawl returned a malformed response", 500);
|
|
233
|
+
}
|
|
234
|
+
const contentFormat = request.format ?? "markdown";
|
|
235
|
+
const pages = [];
|
|
236
|
+
for (const entry of results) {
|
|
237
|
+
if (!isPlainObject(entry)) {
|
|
238
|
+
throw new ApiError("Spider crawl returned a malformed response", 500);
|
|
239
|
+
}
|
|
240
|
+
if (entry.status !== 200)
|
|
241
|
+
continue;
|
|
242
|
+
const content = typeof entry.content === "string" ? entry.content : undefined;
|
|
243
|
+
if (content === undefined || content.length === 0)
|
|
244
|
+
continue;
|
|
245
|
+
if (typeof entry.url !== "string" || entry.url.length === 0) {
|
|
246
|
+
throw new ApiError("Spider crawl returned a malformed response", 500);
|
|
247
|
+
}
|
|
248
|
+
pages.push({ url: entry.url, content, contentFormat });
|
|
249
|
+
}
|
|
250
|
+
return {
|
|
251
|
+
schemaVersion: 1,
|
|
252
|
+
baseUrl: request.url,
|
|
253
|
+
pages,
|
|
254
|
+
totalPages: pages.length,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Normalize a raw Spider.cloud /links response into a `MapResult`.
|
|
259
|
+
*
|
|
260
|
+
* Two accepted item shapes: the documented `{ url, status, links: [...] }`
|
|
261
|
+
* row (SCHEMA §4 — each `links[]` string is a discovered URL) and the
|
|
262
|
+
* flat `{ url }` row the API also returns for single-page maps. Every
|
|
263
|
+
* row carries its own `url`, and that URL is part of the map — a
|
|
264
|
+
* links row must not drop the mapped page itself (a site map that
|
|
265
|
+
* omits its root URL is wrong). All URLs are deduplicated through a
|
|
266
|
+
* `Set` (insertion order preserved); `totalUrls` is the deduplicated
|
|
267
|
+
* count. Any non-object entry, missing/invalid row `url`,
|
|
268
|
+
* present-but-non-array `links` (neither accepted shape), or
|
|
269
|
+
* non-string link is a retryable `ApiError` 500.
|
|
270
|
+
*/
|
|
271
|
+
function normalizeSpiderLinksResult(raw, request) {
|
|
272
|
+
const results = resolveSpiderEnvelope(raw);
|
|
273
|
+
if (results === undefined) {
|
|
274
|
+
throw new ApiError("Spider links returned a malformed response", 500);
|
|
275
|
+
}
|
|
276
|
+
const urls = new Set();
|
|
277
|
+
for (const entry of results) {
|
|
278
|
+
if (!isPlainObject(entry)) {
|
|
279
|
+
throw new ApiError("Spider links returned a malformed response", 500);
|
|
280
|
+
}
|
|
281
|
+
// The row's own URL first: the mapped page belongs in the map even
|
|
282
|
+
// when its links array is present (the documented row shape carries
|
|
283
|
+
// both). Deduplication absorbs a url that also appears in links.
|
|
284
|
+
if (typeof entry.url !== "string" || entry.url.length === 0) {
|
|
285
|
+
throw new ApiError("Spider links returned a malformed response", 500);
|
|
286
|
+
}
|
|
287
|
+
urls.add(entry.url);
|
|
288
|
+
if (entry.links !== undefined && !Array.isArray(entry.links)) {
|
|
289
|
+
throw new ApiError("Spider links returned a malformed response", 500);
|
|
290
|
+
}
|
|
291
|
+
if (Array.isArray(entry.links)) {
|
|
292
|
+
for (const link of entry.links) {
|
|
293
|
+
if (typeof link !== "string" || link.length === 0) {
|
|
294
|
+
throw new ApiError("Spider links returned a malformed response", 500);
|
|
295
|
+
}
|
|
296
|
+
urls.add(link);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
const unique = [...urls];
|
|
301
|
+
return {
|
|
302
|
+
schemaVersion: 1,
|
|
303
|
+
baseUrl: request.url,
|
|
304
|
+
urls: unique,
|
|
305
|
+
totalUrls: unique.length,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
// ---------------------------------------------------------------------------
|
|
309
|
+
// Failure normalization
|
|
310
|
+
// ---------------------------------------------------------------------------
|
|
311
|
+
function inferStatusCode(known) {
|
|
312
|
+
if (typeof known === "number" && Number.isFinite(known))
|
|
313
|
+
return known;
|
|
314
|
+
return 500;
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Status-keyed outward message for rewrapped Spider ApiErrors. Curated
|
|
318
|
+
* constants only — a raw Provider body embedded upstream never survives.
|
|
319
|
+
*/
|
|
320
|
+
function spiderApiErrorMessage(statusCode) {
|
|
321
|
+
if (statusCode === 429)
|
|
322
|
+
return "Spider rate limit exceeded";
|
|
323
|
+
return "Spider request failed";
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Normalize a Provider failure with sanitized messages. Raw response
|
|
327
|
+
* bodies never cross the adapter boundary. Mirrors the Firecrawl
|
|
328
|
+
* adapter's `normalizeFirecrawlError`.
|
|
329
|
+
*/
|
|
330
|
+
function normalizeSpiderError(error) {
|
|
331
|
+
// QuotaError pass-through — terminal retry guarantee preserved.
|
|
332
|
+
if (error instanceof QuotaError)
|
|
333
|
+
return error;
|
|
334
|
+
// Configuration/option/validation errors carry clean, human-authored
|
|
335
|
+
// messages and are safe to surface verbatim.
|
|
336
|
+
if (error instanceof ValidationError ||
|
|
337
|
+
error instanceof UnsupportedOptionError ||
|
|
338
|
+
error instanceof ConfigurationError) {
|
|
339
|
+
return error;
|
|
340
|
+
}
|
|
341
|
+
// Re-wrap typed transport errors with sanitized messages so a raw
|
|
342
|
+
// Provider response body embedded upstream never survives.
|
|
343
|
+
if (error instanceof AuthError) {
|
|
344
|
+
return new AuthError("Spider authentication failed", "SPIDER_API_KEY");
|
|
345
|
+
}
|
|
346
|
+
if (error instanceof NetworkError) {
|
|
347
|
+
return new NetworkError("Spider network error");
|
|
348
|
+
}
|
|
349
|
+
if (error instanceof TimeoutError) {
|
|
350
|
+
return new TimeoutError(error.durationMs);
|
|
351
|
+
}
|
|
352
|
+
if (error instanceof ApiError) {
|
|
353
|
+
const statusCode = inferStatusCode(error.statusCode);
|
|
354
|
+
return new ApiError(spiderApiErrorMessage(statusCode), statusCode);
|
|
355
|
+
}
|
|
356
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
357
|
+
const lower = message.toLowerCase();
|
|
358
|
+
if (lower.includes("401") ||
|
|
359
|
+
lower.includes("403") ||
|
|
360
|
+
lower.includes("unauthorized") ||
|
|
361
|
+
lower.includes("forbidden")) {
|
|
362
|
+
return new AuthError("Spider authentication failed");
|
|
363
|
+
}
|
|
364
|
+
if (lower.includes("timeout") || lower.includes("timed out") || lower.includes("etimedout")) {
|
|
365
|
+
return new TimeoutError(30000);
|
|
366
|
+
}
|
|
367
|
+
if (lower.includes("econnrefused") ||
|
|
368
|
+
lower.includes("econnreset") ||
|
|
369
|
+
lower.includes("network") ||
|
|
370
|
+
lower.includes("enotfound") ||
|
|
371
|
+
lower.includes("fetch failed")) {
|
|
372
|
+
return new NetworkError("Spider network error");
|
|
373
|
+
}
|
|
374
|
+
if (lower.includes("429") || lower.includes("rate limit")) {
|
|
375
|
+
return new ApiError("Spider rate limit exceeded", 429);
|
|
376
|
+
}
|
|
377
|
+
return new ApiError("Spider request failed", 500);
|
|
378
|
+
}
|
|
379
|
+
function createSpiderSearchCapability(options) {
|
|
380
|
+
const { env, transport } = options;
|
|
381
|
+
const capability = {
|
|
382
|
+
validate(request) {
|
|
383
|
+
if (!request || typeof request.query !== "string" || request.query.trim() === "") {
|
|
384
|
+
throw new ValidationError("Search query must contain at least one non-whitespace character");
|
|
385
|
+
}
|
|
386
|
+
// Spider.cloud supports domain, recency, contentSize, location, and
|
|
387
|
+
// topic. `type` is Brave-only — reject before any transport call so
|
|
388
|
+
// the option-level fallback contract can continue past Spider to a
|
|
389
|
+
// Provider that supports it.
|
|
390
|
+
if (request.controls?.type !== undefined) {
|
|
391
|
+
throw new UnsupportedOptionError("spider", "search", "type");
|
|
392
|
+
}
|
|
393
|
+
},
|
|
394
|
+
cacheIdentity(request) {
|
|
395
|
+
const apiKey = getSpiderApiKey(env) ?? "";
|
|
396
|
+
const identityRequest = {
|
|
397
|
+
query: request.query,
|
|
398
|
+
};
|
|
399
|
+
if (request.controls) {
|
|
400
|
+
identityRequest.controls = request.controls;
|
|
401
|
+
}
|
|
402
|
+
return {
|
|
403
|
+
provider: "spider",
|
|
404
|
+
capability: "search",
|
|
405
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
406
|
+
request: identityRequest,
|
|
407
|
+
};
|
|
408
|
+
},
|
|
409
|
+
async invoke(request) {
|
|
410
|
+
capability.validate(request);
|
|
411
|
+
const apiKey = requireSpiderApiKey(env);
|
|
412
|
+
try {
|
|
413
|
+
const params = mapSearchControls(request);
|
|
414
|
+
const raw = await fetchSpiderSearch(apiKey, params, transport);
|
|
415
|
+
return normalizeSpiderSearchResults(raw);
|
|
416
|
+
}
|
|
417
|
+
catch (error) {
|
|
418
|
+
throw normalizeSpiderError(error);
|
|
419
|
+
}
|
|
420
|
+
},
|
|
421
|
+
};
|
|
422
|
+
return capability;
|
|
423
|
+
}
|
|
424
|
+
// ---------------------------------------------------------------------------
|
|
425
|
+
// Reader Capability
|
|
426
|
+
// ---------------------------------------------------------------------------
|
|
427
|
+
/** Z.AI-only reader options that Spider.cloud has no native equivalent for. */
|
|
428
|
+
const UNSUPPORTED_READER_OPTIONS = [
|
|
429
|
+
"withLinksSummary",
|
|
430
|
+
"noGfm",
|
|
431
|
+
"keepImgDataUrl",
|
|
432
|
+
"withImagesSummary",
|
|
433
|
+
];
|
|
434
|
+
/**
|
|
435
|
+
* Reject reader request controls the locked Spider `/scrape` body does
|
|
436
|
+
* not document. Accept-and-drop is banned: a control is either
|
|
437
|
+
* wire-consumed or rejected here, before any transport call.
|
|
438
|
+
*/
|
|
439
|
+
function assertNoUnsupportedReaderOptions(request) {
|
|
440
|
+
for (const key of UNSUPPORTED_READER_OPTIONS) {
|
|
441
|
+
// Only reject when the user explicitly enabled the option (`true`);
|
|
442
|
+
// the read command handler sets `false` when the flag is absent.
|
|
443
|
+
if (request[key] === true) {
|
|
444
|
+
throw new UnsupportedOptionError("spider", "reader", key);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
// No native Spider `/scrape` image or timeout field exists — reject
|
|
448
|
+
// rather than silently discard the intent. `retainImages` is set only
|
|
449
|
+
// when an images flag is passed, so `!== undefined` is exact.
|
|
450
|
+
if (request.retainImages !== undefined) {
|
|
451
|
+
throw new UnsupportedOptionError("spider", "reader", "retainImages");
|
|
452
|
+
}
|
|
453
|
+
if (request.timeout !== undefined) {
|
|
454
|
+
throw new UnsupportedOptionError("spider", "reader", "timeout");
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
function createSpiderReaderCapability(options) {
|
|
458
|
+
const { env, transport } = options;
|
|
459
|
+
const fetch = {
|
|
460
|
+
kind: "reader-fetch",
|
|
461
|
+
validate(request) {
|
|
462
|
+
assertHttpUrl(request.url, "reader");
|
|
463
|
+
assertNoUnsupportedReaderOptions(request);
|
|
464
|
+
},
|
|
465
|
+
cacheIdentity(request) {
|
|
466
|
+
const apiKey = getSpiderApiKey(env) ?? "";
|
|
467
|
+
return {
|
|
468
|
+
provider: "spider",
|
|
469
|
+
capability: "reader",
|
|
470
|
+
operation: "reader-fetch",
|
|
471
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
472
|
+
request,
|
|
473
|
+
legacyCandidates: [],
|
|
474
|
+
};
|
|
475
|
+
},
|
|
476
|
+
decodeCached(value) {
|
|
477
|
+
return decodeReaderFetchResult(value);
|
|
478
|
+
},
|
|
479
|
+
async invoke(request, signal) {
|
|
480
|
+
fetch.validate(request);
|
|
481
|
+
const apiKey = requireSpiderApiKey(env);
|
|
482
|
+
try {
|
|
483
|
+
const contentFormat = request.format ?? "markdown";
|
|
484
|
+
const params = {
|
|
485
|
+
url: request.url,
|
|
486
|
+
return_format: contentFormat,
|
|
487
|
+
filter_output_main_only: true,
|
|
488
|
+
stealth: true,
|
|
489
|
+
};
|
|
490
|
+
const raw = await fetchSpiderScrape(apiKey, params, transport, signal);
|
|
491
|
+
return normalizeSpiderScrapeResult(raw, request);
|
|
492
|
+
}
|
|
493
|
+
catch (error) {
|
|
494
|
+
throw normalizeSpiderError(error);
|
|
495
|
+
}
|
|
496
|
+
},
|
|
497
|
+
};
|
|
498
|
+
return { fetch };
|
|
499
|
+
}
|
|
500
|
+
// ---------------------------------------------------------------------------
|
|
501
|
+
// Crawl Capability
|
|
502
|
+
// ---------------------------------------------------------------------------
|
|
503
|
+
/**
|
|
504
|
+
* Crawl request controls the locked Spider `/crawl` body does not
|
|
505
|
+
* document. Accept-and-drop is banned: a control is either
|
|
506
|
+
* wire-consumed (`limit`, `depth`, `format`) or rejected here, before
|
|
507
|
+
* any transport call. `breadth`/`selectPaths`/`excludePaths`/
|
|
508
|
+
* `instructions`/`contentSize`/`timeout` have no Spider-native wire
|
|
509
|
+
* equivalent (SCHEMA §3 documents `url`, `limit`, `depth`,
|
|
510
|
+
* `return_format` — plus provider-side fields the neutral request
|
|
511
|
+
* cannot express).
|
|
512
|
+
*/
|
|
513
|
+
const UNSUPPORTED_CRAWL_OPTIONS = [
|
|
514
|
+
"breadth",
|
|
515
|
+
"selectPaths",
|
|
516
|
+
"excludePaths",
|
|
517
|
+
"instructions",
|
|
518
|
+
"contentSize",
|
|
519
|
+
"timeout",
|
|
520
|
+
];
|
|
521
|
+
function createSpiderCrawlCapability(options) {
|
|
522
|
+
const { env, transport } = options;
|
|
523
|
+
const fetch = {
|
|
524
|
+
kind: "crawl-fetch",
|
|
525
|
+
validate(request) {
|
|
526
|
+
assertHttpUrl(request.url, "crawl");
|
|
527
|
+
// The crawl endpoint is synchronous: no async-job state file, no
|
|
528
|
+
// poll loop, no request-scoped timeout field on the wire.
|
|
529
|
+
if (request.depth !== undefined) {
|
|
530
|
+
if (!Number.isInteger(request.depth) || request.depth < 1 || request.depth > 5) {
|
|
531
|
+
throw new ValidationError("Crawl depth must be an integer between 1 and 5");
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
if (request.limit !== undefined) {
|
|
535
|
+
if (!Number.isInteger(request.limit) || request.limit < 1) {
|
|
536
|
+
throw new ValidationError("Crawl limit must be a positive integer");
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
for (const key of UNSUPPORTED_CRAWL_OPTIONS) {
|
|
540
|
+
if (request[key] !== undefined) {
|
|
541
|
+
throw new UnsupportedOptionError("spider", "crawl", key);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
},
|
|
545
|
+
cacheIdentity(request) {
|
|
546
|
+
const apiKey = getSpiderApiKey(env) ?? "";
|
|
547
|
+
return {
|
|
548
|
+
provider: "spider",
|
|
549
|
+
capability: "crawl",
|
|
550
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
551
|
+
request,
|
|
552
|
+
};
|
|
553
|
+
},
|
|
554
|
+
decodeCached(value) {
|
|
555
|
+
return decodeCrawlResult(value);
|
|
556
|
+
},
|
|
557
|
+
async invoke(request) {
|
|
558
|
+
fetch.validate(request);
|
|
559
|
+
const apiKey = requireSpiderApiKey(env);
|
|
560
|
+
try {
|
|
561
|
+
const params = {
|
|
562
|
+
url: request.url,
|
|
563
|
+
return_format: request.format ?? "markdown",
|
|
564
|
+
};
|
|
565
|
+
if (request.limit !== undefined) {
|
|
566
|
+
params.limit = request.limit;
|
|
567
|
+
}
|
|
568
|
+
if (request.depth !== undefined) {
|
|
569
|
+
params.depth = request.depth;
|
|
570
|
+
}
|
|
571
|
+
// One-shot synchronous POST — the response is the final page
|
|
572
|
+
// array (locked contract; never a job id to poll).
|
|
573
|
+
const raw = await fetchSpiderCrawl(apiKey, params, transport);
|
|
574
|
+
return normalizeSpiderCrawlResult(raw, request);
|
|
575
|
+
}
|
|
576
|
+
catch (error) {
|
|
577
|
+
throw normalizeSpiderError(error);
|
|
578
|
+
}
|
|
579
|
+
},
|
|
580
|
+
};
|
|
581
|
+
return { fetch };
|
|
582
|
+
}
|
|
583
|
+
// ---------------------------------------------------------------------------
|
|
584
|
+
// Map Capability
|
|
585
|
+
// ---------------------------------------------------------------------------
|
|
586
|
+
/**
|
|
587
|
+
* Map request controls the locked Spider `/links` body does not
|
|
588
|
+
* document. Accept-and-drop is banned: a control is either
|
|
589
|
+
* wire-consumed (`limit`) or rejected here, before any transport call.
|
|
590
|
+
* SCHEMA §4 documents exactly `url` + `limit`; `depth`, `breadth`,
|
|
591
|
+
* `selectPaths`, `excludePaths`, and `instructions` have no
|
|
592
|
+
* Spider-native wire equivalent.
|
|
593
|
+
*/
|
|
594
|
+
const UNSUPPORTED_MAP_OPTIONS = [
|
|
595
|
+
"depth",
|
|
596
|
+
"breadth",
|
|
597
|
+
"selectPaths",
|
|
598
|
+
"excludePaths",
|
|
599
|
+
"instructions",
|
|
600
|
+
];
|
|
601
|
+
function createSpiderMapCapability(options) {
|
|
602
|
+
const { env, transport } = options;
|
|
603
|
+
const fetch = {
|
|
604
|
+
kind: "map-fetch",
|
|
605
|
+
validate(request) {
|
|
606
|
+
assertHttpUrl(request.url, "map");
|
|
607
|
+
if (request.limit !== undefined) {
|
|
608
|
+
if (!Number.isInteger(request.limit) || request.limit < 1) {
|
|
609
|
+
throw new ValidationError("Map limit must be a positive integer");
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
for (const key of UNSUPPORTED_MAP_OPTIONS) {
|
|
613
|
+
if (request[key] !== undefined) {
|
|
614
|
+
throw new UnsupportedOptionError("spider", "map", key);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
},
|
|
618
|
+
cacheIdentity(request) {
|
|
619
|
+
const apiKey = getSpiderApiKey(env) ?? "";
|
|
620
|
+
return {
|
|
621
|
+
provider: "spider",
|
|
622
|
+
capability: "map",
|
|
623
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
624
|
+
request,
|
|
625
|
+
};
|
|
626
|
+
},
|
|
627
|
+
decodeCached(value) {
|
|
628
|
+
return decodeMapResult(value);
|
|
629
|
+
},
|
|
630
|
+
async invoke(request) {
|
|
631
|
+
fetch.validate(request);
|
|
632
|
+
const apiKey = requireSpiderApiKey(env);
|
|
633
|
+
try {
|
|
634
|
+
const params = { url: request.url };
|
|
635
|
+
if (request.limit !== undefined) {
|
|
636
|
+
params.limit = request.limit;
|
|
637
|
+
}
|
|
638
|
+
const raw = await fetchSpiderLinks(apiKey, params, transport);
|
|
639
|
+
return normalizeSpiderLinksResult(raw, request);
|
|
640
|
+
}
|
|
641
|
+
catch (error) {
|
|
642
|
+
throw normalizeSpiderError(error);
|
|
643
|
+
}
|
|
644
|
+
},
|
|
645
|
+
};
|
|
646
|
+
return { fetch };
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Build the Spider.cloud Provider Descriptor. Advertises the full
|
|
650
|
+
* Spider.cloud capability set (Search, Reader, Crawl, Map, Quota,
|
|
651
|
+
* Diagnostics). `create()` is side-effect-free; the transport is
|
|
652
|
+
* invoked per Capability call.
|
|
653
|
+
*/
|
|
654
|
+
export function createSpiderDescriptor(dependencies) {
|
|
655
|
+
const transport = dependencies?.transport;
|
|
656
|
+
return {
|
|
657
|
+
id: "spider",
|
|
658
|
+
isConfigured(env) {
|
|
659
|
+
return getSpiderApiKey(env) !== undefined;
|
|
660
|
+
},
|
|
661
|
+
capabilities() {
|
|
662
|
+
return new Set([
|
|
663
|
+
"search",
|
|
664
|
+
"reader",
|
|
665
|
+
"crawl",
|
|
666
|
+
"map",
|
|
667
|
+
"quota",
|
|
668
|
+
"diagnostics",
|
|
669
|
+
]);
|
|
670
|
+
},
|
|
671
|
+
create(context) {
|
|
672
|
+
const search = createSpiderSearchCapability({ env: context.env, transport });
|
|
673
|
+
const reader = createSpiderReaderCapability({ env: context.env, transport });
|
|
674
|
+
const crawl = createSpiderCrawlCapability({ env: context.env, transport });
|
|
675
|
+
const map = createSpiderMapCapability({ env: context.env, transport });
|
|
676
|
+
const quota = createSpiderQuotaCapability({ env: context.env, transport });
|
|
677
|
+
const diagnostics = createSpiderDiagnosticsCapability({ env: context.env, transport });
|
|
678
|
+
return { id: "spider", search, reader, crawl, map, quota, diagnostics };
|
|
679
|
+
},
|
|
680
|
+
credentialEnvVars: ["SPIDER_API_KEY"],
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
//# sourceMappingURL=adapter.js.map
|