scoutline 0.17.7 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -26
- package/dist/capabilities/quota.d.ts.map +1 -1
- package/dist/capabilities/quota.js +4 -1
- package/dist/capabilities/quota.js.map +1 -1
- package/dist/capabilities/reader.d.ts +8 -0
- package/dist/capabilities/reader.d.ts.map +1 -1
- package/dist/capabilities/reader.js.map +1 -1
- package/dist/commands/crawl.js +3 -3
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +10 -9
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +65 -19
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/map.js +3 -3
- package/dist/commands/quota.d.ts.map +1 -1
- package/dist/commands/quota.js +1 -2
- package/dist/commands/quota.js.map +1 -1
- package/dist/commands/read.d.ts.map +1 -1
- package/dist/commands/read.js +13 -5
- package/dist/commands/read.js.map +1 -1
- package/dist/commands/research.d.ts.map +1 -1
- package/dist/commands/research.js +7 -6
- package/dist/commands/research.js.map +1 -1
- package/dist/commands/search.d.ts.map +1 -1
- package/dist/commands/search.js +2 -3
- package/dist/commands/search.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -10
- package/dist/index.js.map +1 -1
- package/dist/lib/quota-mapping.d.ts +23 -2
- package/dist/lib/quota-mapping.d.ts.map +1 -1
- package/dist/lib/quota-mapping.js +47 -2
- package/dist/lib/quota-mapping.js.map +1 -1
- package/dist/lib/redact.d.ts +9 -1
- package/dist/lib/redact.d.ts.map +1 -1
- package/dist/lib/redact.js +21 -1
- package/dist/lib/redact.js.map +1 -1
- package/dist/providers/linkup/adapter.d.ts +92 -0
- package/dist/providers/linkup/adapter.d.ts.map +1 -0
- package/dist/providers/linkup/adapter.js +747 -0
- package/dist/providers/linkup/adapter.js.map +1 -0
- package/dist/providers/linkup/client.d.ts +157 -0
- package/dist/providers/linkup/client.d.ts.map +1 -0
- package/dist/providers/linkup/client.js +371 -0
- package/dist/providers/linkup/client.js.map +1 -0
- package/dist/providers/linkup/credentials.d.ts +41 -0
- package/dist/providers/linkup/credentials.d.ts.map +1 -0
- package/dist/providers/linkup/credentials.js +66 -0
- package/dist/providers/linkup/credentials.js.map +1 -0
- package/dist/providers/linkup/diagnostics.d.ts +39 -0
- package/dist/providers/linkup/diagnostics.d.ts.map +1 -0
- package/dist/providers/linkup/diagnostics.js +49 -0
- package/dist/providers/linkup/diagnostics.js.map +1 -0
- package/dist/providers/linkup/quota.d.ts +55 -0
- package/dist/providers/linkup/quota.d.ts.map +1 -0
- package/dist/providers/linkup/quota.js +100 -0
- package/dist/providers/linkup/quota.js.map +1 -0
- package/dist/providers/registry.d.ts.map +1 -1
- package/dist/providers/registry.js +6 -0
- package/dist/providers/registry.js.map +1 -1
- package/dist/providers/spider/adapter.d.ts +130 -0
- package/dist/providers/spider/adapter.d.ts.map +1 -0
- package/dist/providers/spider/adapter.js +683 -0
- package/dist/providers/spider/adapter.js.map +1 -0
- package/dist/providers/spider/client.d.ts +122 -0
- package/dist/providers/spider/client.d.ts.map +1 -0
- package/dist/providers/spider/client.js +267 -0
- package/dist/providers/spider/client.js.map +1 -0
- package/dist/providers/spider/credentials.d.ts +31 -0
- package/dist/providers/spider/credentials.d.ts.map +1 -0
- package/dist/providers/spider/credentials.js +52 -0
- package/dist/providers/spider/credentials.js.map +1 -0
- package/dist/providers/spider/diagnostics.d.ts +33 -0
- package/dist/providers/spider/diagnostics.d.ts.map +1 -0
- package/dist/providers/spider/diagnostics.js +55 -0
- package/dist/providers/spider/diagnostics.js.map +1 -0
- package/dist/providers/spider/quota.d.ts +61 -0
- package/dist/providers/spider/quota.d.ts.map +1 -0
- package/dist/providers/spider/quota.js +90 -0
- package/dist/providers/spider/quota.js.map +1 -0
- package/dist/providers/types.d.ts +1 -1
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/providers/types.js +3 -0
- package/dist/providers/types.js.map +1 -1
- package/dist/providers/you/adapter.d.ts +92 -0
- package/dist/providers/you/adapter.d.ts.map +1 -0
- package/dist/providers/you/adapter.js +469 -0
- package/dist/providers/you/adapter.js.map +1 -0
- package/dist/providers/you/client.d.ts +94 -0
- package/dist/providers/you/client.d.ts.map +1 -0
- package/dist/providers/you/client.js +242 -0
- package/dist/providers/you/client.js.map +1 -0
- package/dist/providers/you/credentials.d.ts +40 -0
- package/dist/providers/you/credentials.d.ts.map +1 -0
- package/dist/providers/you/credentials.js +61 -0
- package/dist/providers/you/credentials.js.map +1 -0
- package/dist/providers/you/diagnostics.d.ts +43 -0
- package/dist/providers/you/diagnostics.d.ts.map +1 -0
- package/dist/providers/you/diagnostics.js +60 -0
- package/dist/providers/you/diagnostics.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,747 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Linkup Provider Adapter.
|
|
3
|
+
*
|
|
4
|
+
* Implements the Linkup Provider Descriptor with the Search capability
|
|
5
|
+
* on top of the direct-HTTP transport (`./client.ts`). The Adapter owns
|
|
6
|
+
* credentials, transport lifecycle, Provider field mapping, and failure
|
|
7
|
+
* normalization; shared execution owns cache and retry policy.
|
|
8
|
+
*
|
|
9
|
+
* Structurally cloned from `providers/tavily/adapter.ts` (Tavily/Parallel
|
|
10
|
+
* research-poll analog, IMPLEMENTATION-CONTRACT analog-adapter table).
|
|
11
|
+
*
|
|
12
|
+
* Boundary rules (ARCHITECTURE.md §2):
|
|
13
|
+
* - May import capability types, normalized errors, Provider identity
|
|
14
|
+
* types, and the Adapter-local credential and transport Modules.
|
|
15
|
+
* - Must NOT import command presentation, output mode, or another
|
|
16
|
+
* Provider's Adapter.
|
|
17
|
+
*
|
|
18
|
+
* Field mapping (Linkup wire → normalized):
|
|
19
|
+
* Search results[].name -> title
|
|
20
|
+
* Search results[].url -> url
|
|
21
|
+
* Search results[].content -> summary (falls back to snippet)
|
|
22
|
+
* Search results[].favicon -> (dropped)
|
|
23
|
+
* Search results[].type -> (dropped)
|
|
24
|
+
*
|
|
25
|
+
* Control mapping (SearchControls → Linkup-native API params):
|
|
26
|
+
* domain -> includeDomains: [domain]
|
|
27
|
+
* recency -> fromDate/toDate ISO date window (oneDay/oneWeek/
|
|
28
|
+
* oneMonth/oneYear; noLimit omits dates)
|
|
29
|
+
* contentSize -> depth (high→"deep", medium/absent→"standard")
|
|
30
|
+
* topic -> keyword appended to q
|
|
31
|
+
* location -> locale keyword appended to q
|
|
32
|
+
* type -> REJECTED (UnsupportedOptionError)
|
|
33
|
+
*
|
|
34
|
+
* Field mapping (Linkup wire → normalized, reader /fetch):
|
|
35
|
+
* url -> finalUrl (falls back to the requested url)
|
|
36
|
+
* markdown -> content (empty → ApiError; never cache an empty fetch)
|
|
37
|
+
* rawHtml / images -> (dropped)
|
|
38
|
+
* renderJs -> renderJs wire field (control honored; defaults true
|
|
39
|
+
* when absent for SPA compatibility)
|
|
40
|
+
*
|
|
41
|
+
* Research lifecycle (mirrors createExaDescriptor, Linkup SPEC §Research):
|
|
42
|
+
* POST /research — submit task once (no retry, double-charge
|
|
43
|
+
* prevention on a usage-based endpoint)
|
|
44
|
+
* GET /research/:id — poll every LINKUP_RESEARCH_POLL_INTERVAL_MS
|
|
45
|
+
* (default 5000) until completed/failed/not_found
|
|
46
|
+
* updatedAt / output.answer -> report (canonical object output)
|
|
47
|
+
* output (string) -> report (flat output fallback)
|
|
48
|
+
* markdown -> report (flat markdown fallback)
|
|
49
|
+
* output.sources[].name -> sources[].title (fallback url)
|
|
50
|
+
* output.sources[].url -> sources[].url
|
|
51
|
+
* `model` mapping: mini→"S", auto→"L", pro→"XL" (reasoningDepth)
|
|
52
|
+
* transient poll errors (429/5xx/network/timeout) are retried up to
|
|
53
|
+
* MAX_POLL_RETRIES=3 before propagating; failed poll -> ApiError 500
|
|
54
|
+
* with no raw body in the message.
|
|
55
|
+
* `not_found` triggers at most ONE recreation (Tavily/Parallel guard).
|
|
56
|
+
* State-file persistence via createAsyncJobStateFile + computeAsyncJobStateHash
|
|
57
|
+
* double-charge prevention; resume on Ctrl-C / crash.
|
|
58
|
+
*/
|
|
59
|
+
import { decodeReaderFetchResult } from "../../capabilities/reader.js";
|
|
60
|
+
import { decodeResearchResult } from "../../capabilities/research.js";
|
|
61
|
+
import { computeAsyncJobStateHash, createProductionAsyncJobStateFile, } from "../../lib/async-job-state.js";
|
|
62
|
+
import { withAsyncFileLock, DEFAULT_LOCK_TIMEOUT_MS, DEFAULT_LOCK_STALE_MS, } from "../../lib/async-file-lock.js";
|
|
63
|
+
import { asyncJobStateDir } from "../../lib/cache.js";
|
|
64
|
+
import { ApiError, NetworkError, TimeoutError, UnsupportedOptionError, ValidationError } from "../../lib/errors.js";
|
|
65
|
+
import { hashLinkupApiKey, isLinkupConfigured, requireLinkupApiKey } from "./credentials.js";
|
|
66
|
+
import { createLinkupResearch, fetchLinkupFetch, fetchLinkupSearch, pollLinkupResearch, } from "./client.js";
|
|
67
|
+
import { createLinkupQuotaCapability } from "./quota.js";
|
|
68
|
+
import { createLinkupDiagnosticsCapability } from "./diagnostics.js";
|
|
69
|
+
// ---------------------------------------------------------------------------
|
|
70
|
+
// Provider-owned credential fingerprint
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
function credentialFingerprint(apiKey) {
|
|
73
|
+
return hashLinkupApiKey(apiKey);
|
|
74
|
+
}
|
|
75
|
+
function resolveApiKey(env) {
|
|
76
|
+
return requireLinkupApiKey(env);
|
|
77
|
+
}
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
// Helpers
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
81
|
+
function isPlainObject(value) {
|
|
82
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
83
|
+
}
|
|
84
|
+
// ---------------------------------------------------------------------------
|
|
85
|
+
// Control mapping (SearchControls → Linkup-native API params)
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
/**
|
|
88
|
+
* Map a `SearchRecency` to a `fromDate`/`toDate` ISO 8601 date window
|
|
89
|
+
* (YYYY-MM-DD). `noLimit` (and unknown values) return `undefined` so
|
|
90
|
+
* no date fields are sent.
|
|
91
|
+
*/
|
|
92
|
+
function mapRecencyToDateWindow(recency, now) {
|
|
93
|
+
// All arithmetic uses UTC accessors so the month-end overflow guards
|
|
94
|
+
// reason on the same UTC timeline the window is serialized with
|
|
95
|
+
// (toISOString). Local-time accessors would skew the window by the
|
|
96
|
+
// zone offset whenever the local and UTC calendar days differ.
|
|
97
|
+
const current = new Date(now());
|
|
98
|
+
const from = new Date(now());
|
|
99
|
+
switch (recency) {
|
|
100
|
+
case "oneDay":
|
|
101
|
+
from.setUTCDate(from.getUTCDate() - 1);
|
|
102
|
+
break;
|
|
103
|
+
case "oneWeek":
|
|
104
|
+
from.setUTCDate(from.getUTCDate() - 7);
|
|
105
|
+
break;
|
|
106
|
+
case "oneMonth": {
|
|
107
|
+
const targetMonth = from.getUTCMonth() - 1;
|
|
108
|
+
from.setUTCMonth(targetMonth);
|
|
109
|
+
if (from.getUTCMonth() !== ((targetMonth % 12) + 12) % 12) {
|
|
110
|
+
from.setUTCDate(0);
|
|
111
|
+
}
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
case "oneYear": {
|
|
115
|
+
const originalMonth = current.getUTCMonth();
|
|
116
|
+
from.setUTCFullYear(current.getUTCFullYear() - 1);
|
|
117
|
+
if (from.getUTCMonth() !== originalMonth) {
|
|
118
|
+
from.setUTCDate(0);
|
|
119
|
+
}
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
default:
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
fromDate: from.toISOString().slice(0, 10),
|
|
127
|
+
toDate: current.toISOString().slice(0, 10),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Map a Provider-neutral `SearchRequest` into a Linkup-native wire
|
|
132
|
+
* request body.
|
|
133
|
+
*
|
|
134
|
+
* q -> q (with topic/location keywords appended)
|
|
135
|
+
* domain -> includeDomains: [domain]
|
|
136
|
+
* recency -> fromDate/toDate date window
|
|
137
|
+
* contentSize -> depth (high→"deep", else "standard")
|
|
138
|
+
*/
|
|
139
|
+
function mapSearchControls(request, now) {
|
|
140
|
+
const controls = request.controls;
|
|
141
|
+
const payload = {
|
|
142
|
+
q: request.query,
|
|
143
|
+
depth: controls?.contentSize === "high" ? "deep" : "standard",
|
|
144
|
+
};
|
|
145
|
+
if (controls?.domain) {
|
|
146
|
+
payload.includeDomains = [controls.domain];
|
|
147
|
+
}
|
|
148
|
+
if (controls?.recency) {
|
|
149
|
+
const window = mapRecencyToDateWindow(controls.recency, now);
|
|
150
|
+
if (window) {
|
|
151
|
+
payload.fromDate = window.fromDate;
|
|
152
|
+
payload.toDate = window.toDate;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// Topic and location have no native Linkup field; they are injected
|
|
156
|
+
// as keyword terms into the query (same pattern as Z.AI/MiniMax
|
|
157
|
+
// topic handling).
|
|
158
|
+
if (controls?.topic) {
|
|
159
|
+
payload.q = `${payload.q} ${controls.topic}`;
|
|
160
|
+
}
|
|
161
|
+
if (controls?.location) {
|
|
162
|
+
payload.q = `${payload.q} ${controls.location}`;
|
|
163
|
+
}
|
|
164
|
+
return payload;
|
|
165
|
+
}
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
// Response normalization
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
/**
|
|
170
|
+
* Normalize a raw Linkup search response into `SearchSource[]`.
|
|
171
|
+
*
|
|
172
|
+
* results[].name -> title (missing → "")
|
|
173
|
+
* results[].url -> url
|
|
174
|
+
* results[].content -> summary (falls back to snippet, then "")
|
|
175
|
+
* results[].favicon -> (dropped)
|
|
176
|
+
* results[].type -> (dropped)
|
|
177
|
+
*
|
|
178
|
+
* Any malformed shape is a retryable `ApiError` 500.
|
|
179
|
+
*/
|
|
180
|
+
function normalizeLinkupSearchResults(raw) {
|
|
181
|
+
if (!isPlainObject(raw)) {
|
|
182
|
+
throw new ApiError("Linkup search returned a malformed response", 500);
|
|
183
|
+
}
|
|
184
|
+
const results = raw.results;
|
|
185
|
+
if (!Array.isArray(results)) {
|
|
186
|
+
throw new ApiError("Linkup search returned a malformed response", 500);
|
|
187
|
+
}
|
|
188
|
+
const out = [];
|
|
189
|
+
for (const entry of results) {
|
|
190
|
+
if (!isPlainObject(entry)) {
|
|
191
|
+
throw new ApiError("Linkup search returned a malformed response", 500);
|
|
192
|
+
}
|
|
193
|
+
const url = entry.url;
|
|
194
|
+
if (typeof url !== "string") {
|
|
195
|
+
throw new ApiError("Linkup search returned a malformed response", 500);
|
|
196
|
+
}
|
|
197
|
+
const title = typeof entry.name === "string" ? entry.name : "";
|
|
198
|
+
const summary = typeof entry.content === "string"
|
|
199
|
+
? entry.content
|
|
200
|
+
: typeof entry.snippet === "string"
|
|
201
|
+
? entry.snippet
|
|
202
|
+
: "";
|
|
203
|
+
out.push({ title, url, summary });
|
|
204
|
+
}
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
function createLinkupSearchCapability(options) {
|
|
208
|
+
const { env, transport, now } = options;
|
|
209
|
+
const capability = {
|
|
210
|
+
validate(request) {
|
|
211
|
+
if (!request || typeof request.query !== "string" || request.query.trim() === "") {
|
|
212
|
+
throw new ValidationError("Search query must contain at least one non-whitespace character");
|
|
213
|
+
}
|
|
214
|
+
// Linkup supports domain, recency, contentSize, topic, and
|
|
215
|
+
// location (as query keywords). type (video content axis) is not
|
|
216
|
+
// supported by Linkup (Brave supplies video), so it is rejected
|
|
217
|
+
// before any credential resolution or transport call.
|
|
218
|
+
if (request.controls?.type !== undefined) {
|
|
219
|
+
throw new UnsupportedOptionError("linkup", "search", "type");
|
|
220
|
+
}
|
|
221
|
+
},
|
|
222
|
+
cacheIdentity(request) {
|
|
223
|
+
const apiKey = resolveApiKey(env);
|
|
224
|
+
const identityRequest = {
|
|
225
|
+
query: request.query,
|
|
226
|
+
};
|
|
227
|
+
if (request.controls) {
|
|
228
|
+
identityRequest.controls = request.controls;
|
|
229
|
+
}
|
|
230
|
+
return {
|
|
231
|
+
provider: "linkup",
|
|
232
|
+
capability: "search",
|
|
233
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
234
|
+
request: identityRequest,
|
|
235
|
+
// Linkup never probes legacy keys — no legacyCandidates.
|
|
236
|
+
};
|
|
237
|
+
},
|
|
238
|
+
async invoke(request) {
|
|
239
|
+
capability.validate(request);
|
|
240
|
+
const apiKey = resolveApiKey(env);
|
|
241
|
+
const wireRequest = mapSearchControls(request, now);
|
|
242
|
+
const raw = await fetchLinkupSearch(apiKey, wireRequest, transport);
|
|
243
|
+
return normalizeLinkupSearchResults(raw);
|
|
244
|
+
},
|
|
245
|
+
};
|
|
246
|
+
return capability;
|
|
247
|
+
}
|
|
248
|
+
// ---------------------------------------------------------------------------
|
|
249
|
+
// Reader Capability
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
251
|
+
/**
|
|
252
|
+
* Reader URL guard (Linkup SPEC: invalid URL → `ValidationError` before
|
|
253
|
+
* any credential resolution or transport call). Mirrors the Tavily
|
|
254
|
+
* reader guard.
|
|
255
|
+
*/
|
|
256
|
+
function assertHttpUrl(url) {
|
|
257
|
+
if (typeof url !== "string" || url.length === 0) {
|
|
258
|
+
throw new ValidationError("Linkup reader URL must be a non-empty string");
|
|
259
|
+
}
|
|
260
|
+
if (!/^https?:\/\//.test(url)) {
|
|
261
|
+
throw new ValidationError("URL must start with http:// or https://");
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Normalize a raw Linkup /fetch response into a `ReaderFetchResult`.
|
|
266
|
+
*
|
|
267
|
+
* url -> finalUrl (falls back to the requested url)
|
|
268
|
+
* markdown -> content (non-string or empty → ApiError; the Reader
|
|
269
|
+
* contract requires non-empty content — never cache an
|
|
270
|
+
* empty fetch)
|
|
271
|
+
* rawHtml -> (dropped)
|
|
272
|
+
* images -> (dropped)
|
|
273
|
+
*
|
|
274
|
+
* Linkup's wire shape carries no page title, so `title` is `null`.
|
|
275
|
+
*/
|
|
276
|
+
function normalizeLinkupFetchResult(url, raw) {
|
|
277
|
+
if (!isPlainObject(raw)) {
|
|
278
|
+
throw new ApiError("Linkup fetch returned a malformed response", 500);
|
|
279
|
+
}
|
|
280
|
+
const markdown = raw.markdown;
|
|
281
|
+
if (typeof markdown !== "string" || markdown.length === 0) {
|
|
282
|
+
throw new ApiError("Linkup fetch returned a malformed response", 500);
|
|
283
|
+
}
|
|
284
|
+
const finalUrl = typeof raw.url === "string" && raw.url.length > 0 ? raw.url : url;
|
|
285
|
+
return {
|
|
286
|
+
schemaVersion: 1,
|
|
287
|
+
url,
|
|
288
|
+
finalUrl,
|
|
289
|
+
title: null,
|
|
290
|
+
content: markdown,
|
|
291
|
+
contentFormat: "markdown",
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Reader controls with no Linkup `/fetch` wire equivalent. The wire body
|
|
296
|
+
* is `{ url, renderJs }` only; anything the user explicitly asks for
|
|
297
|
+
* that Linkup cannot honor is rejected at validation with the 3-arg
|
|
298
|
+
* `UnsupportedOptionError` — never silently accepted and dropped.
|
|
299
|
+
*
|
|
300
|
+
* Boolean semantics follow the shared reader convention: the read
|
|
301
|
+
* handler materializes `false` for absent flags, so `false` means "not
|
|
302
|
+
* requested" for the `true`-activated flags, while `retainImages` is
|
|
303
|
+
* the inverse (the handler sets `true` by default; only `false`
|
|
304
|
+
* carries user intent, and Linkup has no image-stripping knob).
|
|
305
|
+
* `format: "markdown"` is the adapter's native output and passes.
|
|
306
|
+
*/
|
|
307
|
+
const UNSUPPORTED_READER_CONDITIONS = [
|
|
308
|
+
{ key: "retainImages", active: false },
|
|
309
|
+
{ key: "withLinksSummary", active: true },
|
|
310
|
+
{ key: "noGfm", active: true },
|
|
311
|
+
{ key: "keepImgDataUrl", active: true },
|
|
312
|
+
{ key: "withImagesSummary", active: true },
|
|
313
|
+
];
|
|
314
|
+
function assertNoUnsupportedReaderControls(request) {
|
|
315
|
+
if (request.format === "text") {
|
|
316
|
+
throw new UnsupportedOptionError("linkup", "reader", "format");
|
|
317
|
+
}
|
|
318
|
+
for (const { key, active } of UNSUPPORTED_READER_CONDITIONS) {
|
|
319
|
+
if (request[key] === active) {
|
|
320
|
+
throw new UnsupportedOptionError("linkup", "reader", key);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
function createLinkupReaderCapability(options) {
|
|
325
|
+
const { env, transport } = options;
|
|
326
|
+
const fetch = {
|
|
327
|
+
kind: "reader-fetch",
|
|
328
|
+
validate(request) {
|
|
329
|
+
assertHttpUrl(request.url);
|
|
330
|
+
assertNoUnsupportedReaderControls(request);
|
|
331
|
+
},
|
|
332
|
+
cacheIdentity(request) {
|
|
333
|
+
const apiKey = resolveApiKey(env);
|
|
334
|
+
return {
|
|
335
|
+
provider: "linkup",
|
|
336
|
+
capability: "reader",
|
|
337
|
+
operation: "reader-fetch",
|
|
338
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
339
|
+
request,
|
|
340
|
+
// Linkup never probes legacy keys — no legacyCandidates.
|
|
341
|
+
legacyCandidates: [],
|
|
342
|
+
};
|
|
343
|
+
},
|
|
344
|
+
decodeCached(value) {
|
|
345
|
+
return decodeReaderFetchResult(value);
|
|
346
|
+
},
|
|
347
|
+
async invoke(request) {
|
|
348
|
+
fetch.validate(request);
|
|
349
|
+
const apiKey = resolveApiKey(env);
|
|
350
|
+
// renderJs is a Linkup /fetch wire field: honor the control, with
|
|
351
|
+
// headless-browser render (true) as the default when absent so
|
|
352
|
+
// dynamic SPAs extract like static pages.
|
|
353
|
+
const wireRequest = {
|
|
354
|
+
url: request.url,
|
|
355
|
+
renderJs: request.renderJs ?? true,
|
|
356
|
+
};
|
|
357
|
+
// --timeout wires to the client abort budget via a per-request
|
|
358
|
+
// LINKUP_TIMEOUT override (same seam Firecrawl uses; /fetch has no
|
|
359
|
+
// wire timeout field).
|
|
360
|
+
const requestDeps = request.timeout !== undefined
|
|
361
|
+
? {
|
|
362
|
+
...transport,
|
|
363
|
+
env: { LINKUP_TIMEOUT: String(Math.round(request.timeout * 1000)) },
|
|
364
|
+
}
|
|
365
|
+
: transport;
|
|
366
|
+
const raw = await fetchLinkupFetch(apiKey, wireRequest, requestDeps);
|
|
367
|
+
return normalizeLinkupFetchResult(request.url, raw);
|
|
368
|
+
},
|
|
369
|
+
};
|
|
370
|
+
return { fetch };
|
|
371
|
+
}
|
|
372
|
+
// ---------------------------------------------------------------------------
|
|
373
|
+
// Research Capability (async submit/poll lifecycle — Linkup SPEC §Research)
|
|
374
|
+
// ---------------------------------------------------------------------------
|
|
375
|
+
/**
|
|
376
|
+
* Default polling interval between `GET /research/:id` calls. Overridable
|
|
377
|
+
* via `LINKUP_RESEARCH_POLL_INTERVAL_MS` in the transport env so tests
|
|
378
|
+
* can poll instantly.
|
|
379
|
+
*/
|
|
380
|
+
const DEFAULT_RESEARCH_POLL_INTERVAL_MS = 5000;
|
|
381
|
+
function resolvePollIntervalMs(env) {
|
|
382
|
+
const raw = env?.LINKUP_RESEARCH_POLL_INTERVAL_MS;
|
|
383
|
+
const parsed = parseInt(raw ?? "", 10);
|
|
384
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_RESEARCH_POLL_INTERVAL_MS;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Build an abortable `sleep(ms)` from the injected timers. Cloned from
|
|
388
|
+
* the Tavily/Exa adapters — same mechanism, different transport type.
|
|
389
|
+
* A non-positive interval resolves via `setImmediate`; an aborted
|
|
390
|
+
* signal rejects with `TimeoutError` so the poll loop unwinds promptly.
|
|
391
|
+
*/
|
|
392
|
+
function makeSleep(deps, signal) {
|
|
393
|
+
const setT = deps?.setTimeout ?? setTimeout;
|
|
394
|
+
const clearT = deps?.clearTimeout ?? clearTimeout;
|
|
395
|
+
return (ms) => new Promise((resolve, reject) => {
|
|
396
|
+
if (signal?.aborted) {
|
|
397
|
+
reject(new TimeoutError(0, "Research polling aborted"));
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
if (ms <= 0) {
|
|
401
|
+
setImmediate(() => {
|
|
402
|
+
if (signal?.aborted) {
|
|
403
|
+
reject(new TimeoutError(0, "Research polling aborted"));
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
resolve();
|
|
407
|
+
});
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
const onAbort = () => {
|
|
411
|
+
clearT(id);
|
|
412
|
+
reject(new TimeoutError(0, "Research polling aborted"));
|
|
413
|
+
};
|
|
414
|
+
const id = setT(() => {
|
|
415
|
+
signal?.removeEventListener("abort", onAbort);
|
|
416
|
+
resolve();
|
|
417
|
+
}, ms);
|
|
418
|
+
signal?.addEventListener("abort", onAbort);
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
function isEexistError(err) {
|
|
422
|
+
return (typeof err === "object" &&
|
|
423
|
+
err !== null &&
|
|
424
|
+
"code" in err &&
|
|
425
|
+
err.code === "EEXIST");
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Map `ResearchRequest.model` -> Linkup `reasoningDepth` (SPEC §Research):
|
|
429
|
+
* mini -> "S"
|
|
430
|
+
* auto -> "L" (default)
|
|
431
|
+
* pro -> "XL"
|
|
432
|
+
*/
|
|
433
|
+
function mapModelToReasoningDepth(model) {
|
|
434
|
+
switch (model) {
|
|
435
|
+
case "mini":
|
|
436
|
+
return "S";
|
|
437
|
+
case "pro":
|
|
438
|
+
return "XL";
|
|
439
|
+
case "auto":
|
|
440
|
+
default:
|
|
441
|
+
return "L";
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Normalize a completed Linkup research poll into a `ResearchResult`.
|
|
446
|
+
*
|
|
447
|
+
* output.answer -> report (canonical object output)
|
|
448
|
+
* output (string) -> report (flat output fallback)
|
|
449
|
+
* markdown -> report (flat markdown fallback)
|
|
450
|
+
* output.sources[] -> sources[] ({name, url, snippet} -> {title, url})
|
|
451
|
+
* sources[] -> sources[] (top-level fallback)
|
|
452
|
+
* model -> echoed from the request (default "auto")
|
|
453
|
+
*/
|
|
454
|
+
function normalizeLinkupResearchResult(poll, request) {
|
|
455
|
+
let report;
|
|
456
|
+
if (isPlainObject(poll.output)) {
|
|
457
|
+
const answer = poll.output.answer;
|
|
458
|
+
if (typeof answer === "string")
|
|
459
|
+
report = answer;
|
|
460
|
+
}
|
|
461
|
+
else if (typeof poll.output === "string") {
|
|
462
|
+
report = poll.output;
|
|
463
|
+
}
|
|
464
|
+
if (report === undefined && typeof poll.markdown === "string") {
|
|
465
|
+
report = poll.markdown;
|
|
466
|
+
}
|
|
467
|
+
if (report === undefined) {
|
|
468
|
+
throw new ApiError("Linkup research returned a malformed response", 500);
|
|
469
|
+
}
|
|
470
|
+
let rawSources = undefined;
|
|
471
|
+
if (isPlainObject(poll.output) && poll.output.sources !== undefined) {
|
|
472
|
+
rawSources = poll.output.sources;
|
|
473
|
+
}
|
|
474
|
+
else if (poll.sources !== undefined) {
|
|
475
|
+
rawSources = poll.sources;
|
|
476
|
+
}
|
|
477
|
+
const sources = [];
|
|
478
|
+
if (Array.isArray(rawSources)) {
|
|
479
|
+
for (const entry of rawSources) {
|
|
480
|
+
if (!isPlainObject(entry))
|
|
481
|
+
continue;
|
|
482
|
+
const url = entry.url;
|
|
483
|
+
if (typeof url !== "string" || url.length === 0)
|
|
484
|
+
continue;
|
|
485
|
+
const title = typeof entry.name === "string" && entry.name.length > 0 ? entry.name : url;
|
|
486
|
+
sources.push({ title, url });
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
return {
|
|
490
|
+
schemaVersion: 1,
|
|
491
|
+
query: request.query,
|
|
492
|
+
model: request.model ?? "auto",
|
|
493
|
+
report,
|
|
494
|
+
sources,
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* True when a poll GET error is safe to retry. The poll is idempotent
|
|
499
|
+
* — retrying never creates a new run or charges the account. Only
|
|
500
|
+
* transient failures (429, 5xx, network, timeout) qualify; auth/quota/
|
|
501
|
+
* validation errors are terminal and propagate immediately.
|
|
502
|
+
*/
|
|
503
|
+
function isTransientPollError(err) {
|
|
504
|
+
if (err instanceof ApiError && typeof err.statusCode === "number") {
|
|
505
|
+
return err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode <= 599);
|
|
506
|
+
}
|
|
507
|
+
if (err instanceof NetworkError || err instanceof TimeoutError)
|
|
508
|
+
return true;
|
|
509
|
+
return false;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* POST /research to create a task, then persist its task id in the
|
|
513
|
+
* state file atomically. On EEXIST (a concurrent invocation already
|
|
514
|
+
* created a task for this request), read the existing state file and
|
|
515
|
+
* return its task id instead — the concurrent task is polled, not
|
|
516
|
+
* duplicated. Any other state-write failure never discards the created
|
|
517
|
+
* task id: the just-created (billable) task is polled in memory, since
|
|
518
|
+
* persistence is resumability best-effort, not a precondition.
|
|
519
|
+
*/
|
|
520
|
+
async function createResearchTask(apiKey, request, identityHash, stateFile, transport) {
|
|
521
|
+
const wireRequest = {
|
|
522
|
+
q: request.query,
|
|
523
|
+
mode: "research",
|
|
524
|
+
reasoningDepth: mapModelToReasoningDepth(request.model),
|
|
525
|
+
};
|
|
526
|
+
const created = await createLinkupResearch(apiKey, wireRequest, transport);
|
|
527
|
+
const taskId = created.id;
|
|
528
|
+
const state = {
|
|
529
|
+
requestId: taskId,
|
|
530
|
+
identityHash,
|
|
531
|
+
createdAt: new Date().toISOString(),
|
|
532
|
+
status: "pending",
|
|
533
|
+
};
|
|
534
|
+
try {
|
|
535
|
+
await stateFile.write(identityHash, state);
|
|
536
|
+
}
|
|
537
|
+
catch (err) {
|
|
538
|
+
if (isEexistError(err)) {
|
|
539
|
+
// Concurrent invocation won the race — poll its task instead.
|
|
540
|
+
const existing = await stateFile.read(identityHash);
|
|
541
|
+
if (existing !== null) {
|
|
542
|
+
return existing.requestId;
|
|
543
|
+
}
|
|
544
|
+
// Fall through: poll the task we just created (same as below).
|
|
545
|
+
}
|
|
546
|
+
else {
|
|
547
|
+
// The task already exists server-side and is billable; losing its
|
|
548
|
+
// id to a state-write failure would orphan the paid task and a
|
|
549
|
+
// retry would create (and pay for) a duplicate. Persistence is
|
|
550
|
+
// best-effort: poll the created task and complete the invocation.
|
|
551
|
+
// (The same shape Tavily documents for its corrupt-file path; the
|
|
552
|
+
// state write failure surfaces only if polling also fails.)
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return taskId;
|
|
556
|
+
}
|
|
557
|
+
function createLinkupResearchCapability(options) {
|
|
558
|
+
const { env, transport, researchStateFile, researchStateDir } = options;
|
|
559
|
+
const run = {
|
|
560
|
+
kind: "research-fetch",
|
|
561
|
+
validate(request) {
|
|
562
|
+
if (!request || typeof request.query !== "string" || request.query.trim() === "") {
|
|
563
|
+
throw new ValidationError("Research query must contain at least one non-whitespace character");
|
|
564
|
+
}
|
|
565
|
+
if (request.outputLength !== undefined) {
|
|
566
|
+
throw new UnsupportedOptionError("linkup", "research", "outputLength");
|
|
567
|
+
}
|
|
568
|
+
if (request.citationFormat !== undefined) {
|
|
569
|
+
throw new UnsupportedOptionError("linkup", "research", "citationFormat");
|
|
570
|
+
}
|
|
571
|
+
if (request.domain !== undefined) {
|
|
572
|
+
throw new UnsupportedOptionError("linkup", "research", "domain");
|
|
573
|
+
}
|
|
574
|
+
},
|
|
575
|
+
cacheIdentity(request) {
|
|
576
|
+
const apiKey = resolveApiKey(env);
|
|
577
|
+
return {
|
|
578
|
+
provider: "linkup",
|
|
579
|
+
capability: "research",
|
|
580
|
+
credentialFingerprint: credentialFingerprint(apiKey),
|
|
581
|
+
request,
|
|
582
|
+
};
|
|
583
|
+
},
|
|
584
|
+
decodeCached(value) {
|
|
585
|
+
return decodeResearchResult(value);
|
|
586
|
+
},
|
|
587
|
+
async invoke(request, signal) {
|
|
588
|
+
run.validate(request);
|
|
589
|
+
const apiKey = resolveApiKey(env);
|
|
590
|
+
const credFingerprint = credentialFingerprint(apiKey);
|
|
591
|
+
const identityHash = computeAsyncJobStateHash({
|
|
592
|
+
provider: "linkup",
|
|
593
|
+
capability: "research",
|
|
594
|
+
credentialFingerprint: credFingerprint,
|
|
595
|
+
request,
|
|
596
|
+
});
|
|
597
|
+
const pollIntervalMs = resolvePollIntervalMs(transport?.env);
|
|
598
|
+
const sleep = makeSleep(transport, signal);
|
|
599
|
+
const lockOpts = {
|
|
600
|
+
timeoutMs: DEFAULT_LOCK_TIMEOUT_MS,
|
|
601
|
+
staleMs: DEFAULT_LOCK_STALE_MS,
|
|
602
|
+
setTimeout: transport?.setTimeout,
|
|
603
|
+
signal,
|
|
604
|
+
timeoutLabel: "Linkup research",
|
|
605
|
+
};
|
|
606
|
+
// 1. Check for an in-flight task (resume after Ctrl-C / crash) or
|
|
607
|
+
// create a fresh task under the concurrent-create lock.
|
|
608
|
+
let taskId;
|
|
609
|
+
const existingState = await researchStateFile.read(identityHash);
|
|
610
|
+
if (existingState !== null) {
|
|
611
|
+
taskId = existingState.requestId;
|
|
612
|
+
}
|
|
613
|
+
else {
|
|
614
|
+
// Under the lock, re-check state before POSTing to ensure concurrent
|
|
615
|
+
// invocations don't duplicate paid POST requests.
|
|
616
|
+
taskId = await withAsyncFileLock(researchStateDir, identityHash, async () => {
|
|
617
|
+
const state = await researchStateFile.read(identityHash);
|
|
618
|
+
if (state !== null) {
|
|
619
|
+
return state.requestId;
|
|
620
|
+
}
|
|
621
|
+
return createResearchTask(apiKey, request, identityHash, researchStateFile, transport);
|
|
622
|
+
}, lockOpts);
|
|
623
|
+
}
|
|
624
|
+
// 2. Poll loop until terminal status. The GET (poll) is
|
|
625
|
+
// idempotent and safe to retry — transient 429/5xx/network
|
|
626
|
+
// errors on poll MUST NOT terminate a paid research run that
|
|
627
|
+
// is still active server-side. Transient poll failures are
|
|
628
|
+
// retried (bounded by MAX_POLL_RETRIES) before propagating.
|
|
629
|
+
const MAX_POLL_RETRIES = 3;
|
|
630
|
+
let consecutivePollFailures = 0;
|
|
631
|
+
let recreatedAfterNotFound = false;
|
|
632
|
+
for (;;) {
|
|
633
|
+
if (signal?.aborted) {
|
|
634
|
+
throw new TimeoutError(0, "Research polling aborted");
|
|
635
|
+
}
|
|
636
|
+
let poll;
|
|
637
|
+
try {
|
|
638
|
+
poll = await pollLinkupResearch(apiKey, taskId, transport, signal);
|
|
639
|
+
consecutivePollFailures = 0;
|
|
640
|
+
}
|
|
641
|
+
catch (pollErr) {
|
|
642
|
+
if (isTransientPollError(pollErr) && consecutivePollFailures < MAX_POLL_RETRIES) {
|
|
643
|
+
consecutivePollFailures++;
|
|
644
|
+
await sleep(pollIntervalMs);
|
|
645
|
+
continue;
|
|
646
|
+
}
|
|
647
|
+
throw pollErr;
|
|
648
|
+
}
|
|
649
|
+
if (poll.status === "completed") {
|
|
650
|
+
const result = normalizeLinkupResearchResult(poll, request);
|
|
651
|
+
await researchStateFile.remove(identityHash);
|
|
652
|
+
return result;
|
|
653
|
+
}
|
|
654
|
+
if (poll.status === "failed") {
|
|
655
|
+
await researchStateFile.remove(identityHash);
|
|
656
|
+
throw new ApiError("Linkup research task failed", 500);
|
|
657
|
+
}
|
|
658
|
+
if (poll.status === "not_found") {
|
|
659
|
+
// 404 — server-side task expired/disappeared. Allow at most ONE
|
|
660
|
+
// recreation; a second 404 terminates rather than risk unbounded
|
|
661
|
+
// paid creations (Tavily/Parallel guard).
|
|
662
|
+
if (recreatedAfterNotFound) {
|
|
663
|
+
await researchStateFile.remove(identityHash);
|
|
664
|
+
throw new ApiError("Linkup research task not found after recreation", 500);
|
|
665
|
+
}
|
|
666
|
+
recreatedAfterNotFound = true;
|
|
667
|
+
taskId = await withAsyncFileLock(researchStateDir, identityHash, async () => {
|
|
668
|
+
const state = await researchStateFile.read(identityHash);
|
|
669
|
+
if (state !== null && state.requestId !== taskId) {
|
|
670
|
+
return state.requestId;
|
|
671
|
+
}
|
|
672
|
+
await researchStateFile.remove(identityHash);
|
|
673
|
+
return createResearchTask(apiKey, request, identityHash, researchStateFile, transport);
|
|
674
|
+
}, lockOpts);
|
|
675
|
+
continue;
|
|
676
|
+
}
|
|
677
|
+
// pending or processing: sleep and poll again.
|
|
678
|
+
await sleep(pollIntervalMs);
|
|
679
|
+
}
|
|
680
|
+
},
|
|
681
|
+
};
|
|
682
|
+
return { run };
|
|
683
|
+
}
|
|
684
|
+
// ---------------------------------------------------------------------------
|
|
685
|
+
// Descriptor factory
|
|
686
|
+
// ---------------------------------------------------------------------------
|
|
687
|
+
/**
|
|
688
|
+
* Build the Linkup Provider Descriptor. The descriptor advertises the
|
|
689
|
+
* capabilities the constructed Adapter supplies and constructs an
|
|
690
|
+
* Adapter whose Search, Reader, Research, Quota, and Diagnostics
|
|
691
|
+
* Capabilities own credentials, transport,
|
|
692
|
+
* Provider field mapping, and failure normalization. Construction is
|
|
693
|
+
* side-effect-free; the transport is invoked per Capability call.
|
|
694
|
+
* Tests pass `transport` (typically a fake-fetch wrapper); production
|
|
695
|
+
* uses the no-argument factory which resolves to the global `fetch`
|
|
696
|
+
* and timers inside the transport Module.
|
|
697
|
+
*/
|
|
698
|
+
export function createLinkupDescriptor(dependencies) {
|
|
699
|
+
const transport = dependencies?.transport;
|
|
700
|
+
const now = dependencies?.now ?? (() => Date.now());
|
|
701
|
+
const researchStateDir = dependencies?.researchStateDir ?? asyncJobStateDir("research");
|
|
702
|
+
const researchStateFile = dependencies?.researchStateFile ??
|
|
703
|
+
createProductionAsyncJobStateFile(researchStateDir);
|
|
704
|
+
return {
|
|
705
|
+
id: "linkup",
|
|
706
|
+
isConfigured(env) {
|
|
707
|
+
return isLinkupConfigured(env);
|
|
708
|
+
},
|
|
709
|
+
capabilities() {
|
|
710
|
+
return new Set([
|
|
711
|
+
"search",
|
|
712
|
+
"reader",
|
|
713
|
+
"research",
|
|
714
|
+
"quota",
|
|
715
|
+
"diagnostics",
|
|
716
|
+
]);
|
|
717
|
+
},
|
|
718
|
+
create(context) {
|
|
719
|
+
const search = createLinkupSearchCapability({
|
|
720
|
+
env: context.env,
|
|
721
|
+
transport,
|
|
722
|
+
now,
|
|
723
|
+
});
|
|
724
|
+
const reader = createLinkupReaderCapability({
|
|
725
|
+
env: context.env,
|
|
726
|
+
transport,
|
|
727
|
+
});
|
|
728
|
+
const research = createLinkupResearchCapability({
|
|
729
|
+
env: context.env,
|
|
730
|
+
transport,
|
|
731
|
+
researchStateFile,
|
|
732
|
+
researchStateDir,
|
|
733
|
+
});
|
|
734
|
+
const quota = createLinkupQuotaCapability({
|
|
735
|
+
env: context.env,
|
|
736
|
+
transport,
|
|
737
|
+
});
|
|
738
|
+
const diagnostics = createLinkupDiagnosticsCapability({
|
|
739
|
+
env: context.env,
|
|
740
|
+
transport,
|
|
741
|
+
});
|
|
742
|
+
return { id: "linkup", search, reader, research, quota, diagnostics };
|
|
743
|
+
},
|
|
744
|
+
credentialEnvVars: ["LINKUP_API_KEY"],
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
//# sourceMappingURL=adapter.js.map
|