surf-skill 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +175 -0
- package/LICENSE +21 -0
- package/README.md +430 -0
- package/SKILL.md +278 -0
- package/bin/surf-skill.mjs +539 -0
- package/package.json +55 -0
- package/references/COSTS.md +72 -0
- package/references/parallel-api.md +155 -0
- package/references/tavily-api.md +90 -0
- package/src/env.mjs +125 -0
- package/src/index.mjs +22 -0
- package/src/install/postinstall.mjs +73 -0
- package/src/install/preuninstall.mjs +25 -0
- package/src/lib/api/crawl.mjs +55 -0
- package/src/lib/api/extract.mjs +46 -0
- package/src/lib/api/map.mjs +43 -0
- package/src/lib/api/research.mjs +96 -0
- package/src/lib/api/search.mjs +92 -0
- package/src/lib/audit.mjs +34 -0
- package/src/lib/cache.mjs +46 -0
- package/src/lib/cost.mjs +90 -0
- package/src/lib/dispatch.mjs +320 -0
- package/src/lib/flags.mjs +63 -0
- package/src/lib/format.mjs +110 -0
- package/src/lib/harness-install.mjs +149 -0
- package/src/lib/keys-cmd.mjs +138 -0
- package/src/lib/progress.mjs +81 -0
- package/src/lib/project-config.mjs +145 -0
- package/src/lib/providers/index.mjs +32 -0
- package/src/lib/providers/parallel.mjs +270 -0
- package/src/lib/providers/tavily.mjs +245 -0
- package/src/lib/setup.mjs +111 -0
- package/src/lib/state.mjs +197 -0
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// surf-skill — multi-provider web-skill CLI. Routes search/extract/crawl/map/research
|
|
3
|
+
// across Tavily and Parallel AI with automatic key + provider fallback.
|
|
4
|
+
|
|
5
|
+
import { readFile, writeFile, mkdir, unlink } from 'node:fs/promises';
|
|
6
|
+
import { parseFlags, sleep } from '../src/lib/flags.mjs';
|
|
7
|
+
import { dispatch, DispatchError } from '../src/lib/dispatch.mjs';
|
|
8
|
+
import { formatFor } from '../src/lib/format.mjs';
|
|
9
|
+
import { runKeysSubcommand } from '../src/lib/keys-cmd.mjs';
|
|
10
|
+
import { cacheClear } from '../src/lib/cache.mjs';
|
|
11
|
+
import { readUsage, USAGE_LOG } from '../src/lib/audit.mjs';
|
|
12
|
+
import { migrateLegacy } from '../src/lib/state.mjs';
|
|
13
|
+
import { runSetup } from '../src/lib/setup.mjs';
|
|
14
|
+
import { runProjectConfig, formatProjectConfigResult } from '../src/lib/project-config.mjs';
|
|
15
|
+
import { providerFromRequestId } from '../src/lib/providers/index.mjs';
|
|
16
|
+
import { progress, setSilent } from '../src/lib/progress.mjs';
|
|
17
|
+
|
|
18
|
+
const VERSION = '2.0.0';
|
|
19
|
+
|
|
20
|
+
// Catch SIGTERM/SIGINT so a harness-driven kill surfaces a useful message
|
|
21
|
+
// instead of dying silently. This is defense-in-depth: dispatch already
|
|
22
|
+
// tries to abort early via the self-budget check.
|
|
23
|
+
for (const sig of ['SIGTERM', 'SIGINT']) {
|
|
24
|
+
process.on(sig, () => {
|
|
25
|
+
process.stderr.write(
|
|
26
|
+
`❌ Error [KilledBySignal]: surf-skill received ${sig}. ` +
|
|
27
|
+
`If this came from the agent's bash timeout, run 'surf-skill project-config' ` +
|
|
28
|
+
`in this project to raise the limit, or use 'research-start' + 'research-poll' for long jobs.\n`
|
|
29
|
+
);
|
|
30
|
+
process.exit(143); // 128 + 15 (SIGTERM convention)
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const HELP = `surf-skill — multi-provider web skill (Tavily + Parallel AI)
|
|
35
|
+
|
|
36
|
+
Commands:
|
|
37
|
+
setup Interactive onboarding wizard (TTY required)
|
|
38
|
+
project-config [--harness <copilot|claude|pi|all>] [--yes]
|
|
39
|
+
Write per-project bash-timeout config so the
|
|
40
|
+
harness used in this project doesn't kill us.
|
|
41
|
+
Auto-detects via .github/, .claude/, .pi/.
|
|
42
|
+
REQUIRED for GH Copilot CLI projects.
|
|
43
|
+
search <q> [<q2> ...] Web search. Multiple positional args = batch
|
|
44
|
+
(sequential, partial failures reported inline).
|
|
45
|
+
extract <url> [url ...] Fetch & extract content from URLs
|
|
46
|
+
crawl <url> Crawl a site (Tavily only)
|
|
47
|
+
map <url> Discover URLs on a site (Tavily only)
|
|
48
|
+
research <topic> Sync deep research (~50s budget)
|
|
49
|
+
research-start <topic> Start async research; returns request_id
|
|
50
|
+
research-poll <request_id> Poll a research job
|
|
51
|
+
usage --provider <name> Account usage (per provider)
|
|
52
|
+
cache-clear Purge response cache
|
|
53
|
+
cost [--reset] Local credit ledger
|
|
54
|
+
keys <add|remove|list|reset|clear> [...]
|
|
55
|
+
|
|
56
|
+
Global flags:
|
|
57
|
+
--provider <tavily|parallel> Force provider, disables fallback
|
|
58
|
+
--no-fallback Stay on default provider, no cross-provider fallback
|
|
59
|
+
--no-cache Skip response cache
|
|
60
|
+
--json Print normalized envelope as JSON
|
|
61
|
+
--raw-json Print raw provider response (bypasses cache)
|
|
62
|
+
--confirm-expensive Allow operations estimated > 10 credits
|
|
63
|
+
--quiet Silence progress logs (stderr)
|
|
64
|
+
--help, -h Show this help
|
|
65
|
+
--version, -v Show version
|
|
66
|
+
|
|
67
|
+
Progress logs (stderr):
|
|
68
|
+
surf-skill emits one line per event to stderr, e.g.:
|
|
69
|
+
[surf 17:58:12] ▸ search → tavily (key #0)
|
|
70
|
+
[surf 17:58:14] ✓ search tavily 1234ms (2 credits)
|
|
71
|
+
Format is stable for agent parsing. Use --quiet or SURF_QUIET=1 to silence.
|
|
72
|
+
|
|
73
|
+
Examples:
|
|
74
|
+
surf-skill setup
|
|
75
|
+
surf-skill search "claude 4.7 release notes" --max 3
|
|
76
|
+
surf-skill search "topic A" "topic B" "topic C" # batch (3 queries)
|
|
77
|
+
surf-skill extract https://docs.anthropic.com/...
|
|
78
|
+
surf-skill research-start "compare X and Y" --model pro --confirm-expensive
|
|
79
|
+
surf-skill keys add --provider tavily tvly-...
|
|
80
|
+
surf-skill keys list
|
|
81
|
+
|
|
82
|
+
Key & state are stored in ~/.config/surf/keys.json (chmod 600).
|
|
83
|
+
Docs: ~/.agents/skills/surf-skill/SKILL.md`;
|
|
84
|
+
|
|
85
|
+
function die(msg, code = 1) {
|
|
86
|
+
process.stderr.write(`❌ Error: ${msg}\n`);
|
|
87
|
+
process.exit(code);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function out(msg) {
|
|
91
|
+
if (msg == null) return;
|
|
92
|
+
const s = typeof msg === 'string' ? msg : JSON.stringify(msg, null, 2);
|
|
93
|
+
process.stdout.write(s + (s.endsWith('\n') ? '' : '\n'));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function emitResult(envelope, flags) {
|
|
97
|
+
if (flags['raw-json']) {
|
|
98
|
+
out(JSON.stringify(envelope.raw, null, 2));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (flags.json) {
|
|
102
|
+
out(JSON.stringify({
|
|
103
|
+
provider: envelope.provider,
|
|
104
|
+
operation: envelope.operation,
|
|
105
|
+
latency_ms: envelope.latency_ms,
|
|
106
|
+
usage: envelope.usage,
|
|
107
|
+
data: envelope.data,
|
|
108
|
+
}, null, 2));
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
out(formatFor(envelope));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function buildSearchArgs(query, flags) {
|
|
115
|
+
return {
|
|
116
|
+
query,
|
|
117
|
+
depth: flags.depth || 'advanced',
|
|
118
|
+
max: flags.max,
|
|
119
|
+
topic: flags.topic,
|
|
120
|
+
time: flags.time,
|
|
121
|
+
startDate: flags['start-date'],
|
|
122
|
+
endDate: flags['end-date'],
|
|
123
|
+
domains: flags.domains,
|
|
124
|
+
excludeDomains: flags.exclude,
|
|
125
|
+
country: flags.country,
|
|
126
|
+
answer: flags.answer,
|
|
127
|
+
raw: flags.raw,
|
|
128
|
+
images: flags.images,
|
|
129
|
+
imageDesc: flags['image-desc'],
|
|
130
|
+
favicon: flags.favicon,
|
|
131
|
+
auto: flags.auto,
|
|
132
|
+
exactMatch: flags['exact-match'],
|
|
133
|
+
processor: flags.processor,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function cmdSearch(pos, flags) {
|
|
138
|
+
if (!pos.length) die('Usage: surf-skill search "query" [more queries ...]');
|
|
139
|
+
|
|
140
|
+
// Backward-compat: 1 positional arg = exactly one query (same as before).
|
|
141
|
+
if (pos.length === 1) {
|
|
142
|
+
const args = buildSearchArgs(pos[0], flags);
|
|
143
|
+
emitResult(await dispatch('search', args, flags), flags);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Batch mode: each positional arg is an independent query.
|
|
148
|
+
// Runs sequentially to avoid hammering one provider/key with N concurrent
|
|
149
|
+
// calls (which would trigger 429 rate limits).
|
|
150
|
+
await runSearchBatch(pos, flags);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async function runSearchBatch(queries, flags) {
|
|
154
|
+
progress.start(`batch: ${queries.length} queries`);
|
|
155
|
+
const batches = [];
|
|
156
|
+
let okCount = 0;
|
|
157
|
+
let failCount = 0;
|
|
158
|
+
let totalCredits = 0;
|
|
159
|
+
const t0 = Date.now();
|
|
160
|
+
|
|
161
|
+
for (let i = 0; i < queries.length; i++) {
|
|
162
|
+
const q = queries[i];
|
|
163
|
+
const label = `[${i + 1}/${queries.length}] "${q}"`;
|
|
164
|
+
progress.start(label);
|
|
165
|
+
const args = buildSearchArgs(q, flags);
|
|
166
|
+
try {
|
|
167
|
+
const env = await dispatch('search', args, flags);
|
|
168
|
+
okCount++;
|
|
169
|
+
const credits = env.usage && env.usage.credits;
|
|
170
|
+
if (credits != null) totalCredits += credits;
|
|
171
|
+
batches.push({
|
|
172
|
+
index: i,
|
|
173
|
+
query: q,
|
|
174
|
+
ok: true,
|
|
175
|
+
provider: env.provider,
|
|
176
|
+
latency_ms: env.latency_ms,
|
|
177
|
+
usage: env.usage,
|
|
178
|
+
data: env.data,
|
|
179
|
+
raw: env.raw,
|
|
180
|
+
});
|
|
181
|
+
} catch (e) {
|
|
182
|
+
failCount++;
|
|
183
|
+
const code = e.code || e.name || 'Error';
|
|
184
|
+
progress.fail(`${label} failed: [${code}] ${e.message || e}`);
|
|
185
|
+
batches.push({
|
|
186
|
+
index: i,
|
|
187
|
+
query: q,
|
|
188
|
+
ok: false,
|
|
189
|
+
error: { code, message: e.message || String(e), details: e.details },
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const elapsed = Date.now() - t0;
|
|
195
|
+
progress.done(`batch done: ${okCount}/${queries.length} ok, ${failCount} failed (${elapsed}ms, ${totalCredits} credits)`);
|
|
196
|
+
|
|
197
|
+
emitBatchResult({
|
|
198
|
+
operation: 'search-batch',
|
|
199
|
+
summary: { total: queries.length, succeeded: okCount, failed: failCount, total_credits: totalCredits, latency_ms: elapsed },
|
|
200
|
+
batches,
|
|
201
|
+
}, flags);
|
|
202
|
+
|
|
203
|
+
// Exit non-zero only when EVERY query failed.
|
|
204
|
+
if (okCount === 0 && failCount > 0) process.exitCode = 1;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function emitBatchResult(payload, flags) {
|
|
208
|
+
if (flags['raw-json']) {
|
|
209
|
+
out(JSON.stringify(payload.batches.map(b => b.raw ?? b.error), null, 2));
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (flags.json) {
|
|
213
|
+
// Strip `raw` from JSON output unless explicitly asked.
|
|
214
|
+
const safe = {
|
|
215
|
+
operation: payload.operation,
|
|
216
|
+
summary: payload.summary,
|
|
217
|
+
data: { batches: payload.batches.map(({ raw, ...rest }) => rest) },
|
|
218
|
+
};
|
|
219
|
+
out(JSON.stringify(safe, null, 2));
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
// Markdown
|
|
223
|
+
const { summary, batches } = payload;
|
|
224
|
+
let md = `# Search batch (${summary.total} queries · ${summary.succeeded} ok · ${summary.failed} failed)\n\n`;
|
|
225
|
+
md += `_total: ${summary.total_credits} credits · ${summary.latency_ms}ms_\n\n`;
|
|
226
|
+
for (const b of batches) {
|
|
227
|
+
md += `---\n\n## [${b.index + 1}/${summary.total}] ${b.query}\n\n`;
|
|
228
|
+
if (!b.ok) {
|
|
229
|
+
md += `**❌ Failed:** \`[${b.error.code}]\` ${b.error.message}\n\n`;
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
md += `_provider: ${b.provider} · ${b.latency_ms}ms`;
|
|
233
|
+
if (b.usage && b.usage.credits != null) md += ` · ${b.usage.credits} credits`;
|
|
234
|
+
md += `_\n\n`;
|
|
235
|
+
const r = b.data;
|
|
236
|
+
if (r.answer) md += `**Answer:** ${r.answer}\n\n`;
|
|
237
|
+
(r.results || []).forEach((it, i) => {
|
|
238
|
+
md += `### [${i + 1}] ${it.title || it.url}\n${it.url}\n`;
|
|
239
|
+
if (it.score != null) md += `*score: ${typeof it.score === 'number' ? it.score.toFixed(2) : it.score}*\n`;
|
|
240
|
+
if (it.published_date) md += `*published: ${it.published_date}*\n`;
|
|
241
|
+
const content = it.content || '';
|
|
242
|
+
md += `\n${content.length > 1500 ? content.slice(0, 1500) + '…' : content}\n\n`;
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
out(md);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function cmdExtract(pos, flags) {
|
|
249
|
+
if (!pos.length) die('Usage: surf-skill extract <url1> [url2 ...]');
|
|
250
|
+
if (pos.length > 20) die('extract supports at most 20 URLs per call.');
|
|
251
|
+
const args = {
|
|
252
|
+
urls: pos,
|
|
253
|
+
depth: flags.depth || 'basic',
|
|
254
|
+
format: flags.format || 'markdown',
|
|
255
|
+
images: flags.images,
|
|
256
|
+
favicon: flags.favicon,
|
|
257
|
+
query: flags.query,
|
|
258
|
+
chunks: flags.chunks,
|
|
259
|
+
extractTimeout: flags['extract-timeout'],
|
|
260
|
+
};
|
|
261
|
+
emitResult(await dispatch('extract', args, flags), flags);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function cmdCrawl(pos, flags) {
|
|
265
|
+
const url = pos[0];
|
|
266
|
+
if (!url) die('Usage: surf-skill crawl <url> [flags]');
|
|
267
|
+
const args = {
|
|
268
|
+
url,
|
|
269
|
+
maxDepth: flags['max-depth'],
|
|
270
|
+
maxBreadth: flags['max-breadth'],
|
|
271
|
+
limit: flags.limit,
|
|
272
|
+
instructions: flags.instructions,
|
|
273
|
+
selectPaths: flags['select-paths'],
|
|
274
|
+
selectDomains: flags['select-domains'],
|
|
275
|
+
excludePaths: flags['exclude-paths'],
|
|
276
|
+
excludeDomains: flags['exclude-domains'],
|
|
277
|
+
allowExternal: flags['allow-external'],
|
|
278
|
+
images: flags.images,
|
|
279
|
+
categories: flags.categories,
|
|
280
|
+
extractDepth: flags['extract-depth'] || 'basic',
|
|
281
|
+
format: flags.format || 'markdown',
|
|
282
|
+
query: flags.query,
|
|
283
|
+
chunks: flags.chunks,
|
|
284
|
+
timeout: flags.timeout,
|
|
285
|
+
};
|
|
286
|
+
emitResult(await dispatch('crawl', args, flags), flags);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
async function cmdMap(pos, flags) {
|
|
290
|
+
const url = pos[0];
|
|
291
|
+
if (!url) die('Usage: surf-skill map <url> [flags]');
|
|
292
|
+
const args = {
|
|
293
|
+
url,
|
|
294
|
+
maxDepth: flags['max-depth'],
|
|
295
|
+
maxBreadth: flags['max-breadth'],
|
|
296
|
+
limit: flags.limit,
|
|
297
|
+
instructions: flags.instructions,
|
|
298
|
+
selectPaths: flags['select-paths'],
|
|
299
|
+
selectDomains: flags['select-domains'],
|
|
300
|
+
excludePaths: flags['exclude-paths'],
|
|
301
|
+
excludeDomains: flags['exclude-domains'],
|
|
302
|
+
allowExternal: flags['allow-external'],
|
|
303
|
+
categories: flags.categories,
|
|
304
|
+
timeout: flags.timeout,
|
|
305
|
+
};
|
|
306
|
+
emitResult(await dispatch('map', args, flags), flags);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
async function cmdResearchStart(pos, flags) {
|
|
310
|
+
const input = pos.join(' ').trim();
|
|
311
|
+
if (!input) die('Usage: surf-skill research-start "topic" [--model mini|auto|pro]');
|
|
312
|
+
const args = {
|
|
313
|
+
input,
|
|
314
|
+
model: flags.model || 'auto',
|
|
315
|
+
citationFormat: flags.citations || 'numbered',
|
|
316
|
+
outputSchema: flags.schema ? JSON.parse(await readFile(flags.schema, 'utf8')) : undefined,
|
|
317
|
+
processor: flags.processor,
|
|
318
|
+
};
|
|
319
|
+
const envelope = await dispatch('research-start', args, flags);
|
|
320
|
+
await persistResearchHandle(envelope);
|
|
321
|
+
emitResult(envelope, flags);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
async function cmdResearchPoll(pos, flags) {
|
|
325
|
+
const id = pos[0];
|
|
326
|
+
if (!id) die('Usage: surf-skill research-poll <request_id>');
|
|
327
|
+
const decoded = providerFromRequestId(id);
|
|
328
|
+
if (!decoded) die(`unknown request_id prefix in '${id}' (expected tvly:... or pllx:...)`);
|
|
329
|
+
const envelope = await dispatch('research-poll', {}, { ...flags, __requestId: id });
|
|
330
|
+
if (envelope.data.status === 'completed' || envelope.data.status === 'failed') {
|
|
331
|
+
try { await unlink(`/tmp/surf-${id.replace(':', '_')}.json`); } catch {}
|
|
332
|
+
}
|
|
333
|
+
emitResult(envelope, flags);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
async function cmdResearch(pos, flags) {
|
|
337
|
+
const input = pos.join(' ').trim();
|
|
338
|
+
if (!input) die('Usage: surf-skill research "topic"');
|
|
339
|
+
const model = flags.model || 'mini';
|
|
340
|
+
if (model === 'pro' || model === 'ultra') {
|
|
341
|
+
die(`Refusing sync research with model=${model} (would exceed timeout). Use 'surf-skill research-start' + 'surf-skill research-poll'.`);
|
|
342
|
+
}
|
|
343
|
+
const startArgs = {
|
|
344
|
+
input,
|
|
345
|
+
model,
|
|
346
|
+
citationFormat: flags.citations || 'numbered',
|
|
347
|
+
processor: flags.processor,
|
|
348
|
+
};
|
|
349
|
+
const start = await dispatch('research-start', startArgs, flags);
|
|
350
|
+
await persistResearchHandle(start);
|
|
351
|
+
const id = start.data.request_id;
|
|
352
|
+
const deadline = Date.now() + 50_000;
|
|
353
|
+
while (Date.now() < deadline) {
|
|
354
|
+
await sleep(5000);
|
|
355
|
+
const poll = await dispatch('research-poll', {}, { ...flags, __requestId: id });
|
|
356
|
+
if (poll.data.status === 'completed' || poll.data.status === 'failed') {
|
|
357
|
+
try { await unlink(`/tmp/surf-${id.replace(':', '_')}.json`); } catch {}
|
|
358
|
+
emitResult(poll, flags);
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
out(`**Research did not finish in 50s.** Continue with: \`surf-skill research-poll ${id}\``);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
async function persistResearchHandle(envelope) {
|
|
366
|
+
try {
|
|
367
|
+
const id = envelope.data.request_id;
|
|
368
|
+
await mkdir('/tmp', { recursive: true }).catch(() => {});
|
|
369
|
+
await writeFile(`/tmp/surf-${id.replace(':', '_')}.json`, JSON.stringify({
|
|
370
|
+
started: Date.now(),
|
|
371
|
+
provider: envelope.provider,
|
|
372
|
+
request_id: id,
|
|
373
|
+
}));
|
|
374
|
+
} catch {}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
async function cmdUsage(_pos, flags) {
|
|
378
|
+
if (!flags.provider) die(`Usage: surf-skill usage --provider <tavily|parallel>`);
|
|
379
|
+
emitResult(await dispatch('usage', {}, flags), flags);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
async function cmdCacheClear() {
|
|
383
|
+
const n = await cacheClear();
|
|
384
|
+
out(`Cleared ${n} cache entr${n === 1 ? 'y' : 'ies'}.`);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
async function cmdCost(_pos, flags) {
|
|
388
|
+
if (flags.reset) {
|
|
389
|
+
try { await unlink(USAGE_LOG); } catch {}
|
|
390
|
+
out('Reset local usage ledger.');
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
const entries = await readUsage();
|
|
394
|
+
const total = entries.reduce((s, e) => s + (Number(e.credits) || 0), 0);
|
|
395
|
+
const live = entries.filter(e => !e.cached);
|
|
396
|
+
const hits = entries.filter(e => e.cached).length;
|
|
397
|
+
const byProvider = {};
|
|
398
|
+
for (const e of entries) {
|
|
399
|
+
const p = e.provider || 'unknown';
|
|
400
|
+
byProvider[p] = (byProvider[p] || 0) + (Number(e.credits) || 0);
|
|
401
|
+
}
|
|
402
|
+
if (flags.json) {
|
|
403
|
+
out(JSON.stringify({
|
|
404
|
+
totalCredits: total, byProvider,
|
|
405
|
+
entries: entries.length, liveCalls: live.length, cacheHits: hits,
|
|
406
|
+
recent: entries.slice(-20),
|
|
407
|
+
}, null, 2));
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
let md = `**Local recorded credits:** ${total}\n`;
|
|
411
|
+
for (const p of Object.keys(byProvider)) md += `- ${p}: ${byProvider[p]}\n`;
|
|
412
|
+
md += `\n- live API calls: ${live.length}\n- cache hits: ${hits}\n`;
|
|
413
|
+
if (!entries.length) {
|
|
414
|
+
md += '\n_No local usage recorded yet._\n';
|
|
415
|
+
} else {
|
|
416
|
+
md += '\n**Recent calls**\n';
|
|
417
|
+
for (const e of entries.slice(-20)) {
|
|
418
|
+
md += `- ${e.ts} [${e.provider || '?'}] ${e.op}: ${e.credits ?? '—'}${e.cached ? ' (cache hit)' : ''}\n`;
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
md += '\nUse `surf-skill cost --reset` to clear the local ledger.';
|
|
422
|
+
out(md);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
async function cmdKeys(pos, flags) {
|
|
426
|
+
const sub = pos[0];
|
|
427
|
+
if (!sub) die('Usage: surf-skill keys <add|remove|list|reset|clear> ...');
|
|
428
|
+
const subPos = pos.slice(1);
|
|
429
|
+
try {
|
|
430
|
+
const result = await runKeysSubcommand(sub, subPos, flags);
|
|
431
|
+
if (sub === 'list' || sub === 'ls' || sub === 'status') {
|
|
432
|
+
if (result.json) out(JSON.stringify(result.state, null, 2));
|
|
433
|
+
else out(result.text);
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
if (flags.json) {
|
|
437
|
+
out(JSON.stringify(result, null, 2));
|
|
438
|
+
} else if (sub === 'add') {
|
|
439
|
+
if (result.added) out(`✓ added [${result.index}] to ${result.provider}`);
|
|
440
|
+
else out(`already exists in ${result.provider} (no-op)`);
|
|
441
|
+
} else if (sub === 'remove' || sub === 'rm' || sub === 'delete') {
|
|
442
|
+
out(`✓ removed index ${result.index} from ${result.provider}`);
|
|
443
|
+
} else if (sub === 'reset') {
|
|
444
|
+
out(`✓ cleared burned for ${result.provider || 'all providers'}`);
|
|
445
|
+
} else if (sub === 'clear') {
|
|
446
|
+
out(`✓ cleared all keys${flags.all ? '' : ' for ' + (flags.provider || '?')}`);
|
|
447
|
+
}
|
|
448
|
+
} catch (e) {
|
|
449
|
+
if (e.code === 'NEEDS_YES') {
|
|
450
|
+
process.stderr.write(`❌ Error: ${e.message}\n`);
|
|
451
|
+
process.exit(2);
|
|
452
|
+
}
|
|
453
|
+
throw e;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// --- Main ---
|
|
458
|
+
|
|
459
|
+
await migrateLegacy();
|
|
460
|
+
|
|
461
|
+
const [, , cmd, ...rest] = process.argv;
|
|
462
|
+
|
|
463
|
+
if (!cmd || cmd === '--help' || cmd === '-h') {
|
|
464
|
+
out(HELP); process.exit(0);
|
|
465
|
+
}
|
|
466
|
+
if (cmd === '--version' || cmd === '-v') {
|
|
467
|
+
out(VERSION); process.exit(0);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const { pos, flags } = parseFlags(rest);
|
|
471
|
+
|
|
472
|
+
// Wire --quiet before any progress event fires.
|
|
473
|
+
if (flags.quiet) setSilent(true);
|
|
474
|
+
|
|
475
|
+
// Auto-launch setup wizard on first TTY use when no keys are configured.
|
|
476
|
+
// Commands that don't need keys (setup, keys, project-config, help, etc.)
|
|
477
|
+
// are excluded.
|
|
478
|
+
const NO_KEYS_NEEDED = new Set([
|
|
479
|
+
'setup', 'keys', 'project-config',
|
|
480
|
+
'cache-clear', 'cost',
|
|
481
|
+
'--help', '-h', '--version', '-v',
|
|
482
|
+
]);
|
|
483
|
+
if (!NO_KEYS_NEEDED.has(cmd) && process.stdin.isTTY) {
|
|
484
|
+
try {
|
|
485
|
+
const { loadState } = await import('../src/lib/state.mjs');
|
|
486
|
+
const state = await loadState();
|
|
487
|
+
const hasAny = (state.tavily.keys || []).length || (state.parallel.keys || []).length;
|
|
488
|
+
if (!hasAny) {
|
|
489
|
+
process.stderr.write('No keys configured. Launching setup wizard…\n\n');
|
|
490
|
+
await runSetup();
|
|
491
|
+
process.stderr.write('\n— Resuming your command —\n\n');
|
|
492
|
+
}
|
|
493
|
+
} catch {
|
|
494
|
+
// If anything goes wrong with the auto-wizard, fall through to the
|
|
495
|
+
// normal command which will produce its own actionable error.
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
try {
|
|
500
|
+
switch (cmd) {
|
|
501
|
+
case 'search': await cmdSearch(pos, flags); break;
|
|
502
|
+
case 'extract': await cmdExtract(pos, flags); break;
|
|
503
|
+
case 'crawl': await cmdCrawl(pos, flags); break;
|
|
504
|
+
case 'map': await cmdMap(pos, flags); break;
|
|
505
|
+
case 'research': await cmdResearch(pos, flags); break;
|
|
506
|
+
case 'research-start': await cmdResearchStart(pos, flags); break;
|
|
507
|
+
case 'research-poll': await cmdResearchPoll(pos, flags); break;
|
|
508
|
+
case 'usage': await cmdUsage(pos, flags); break;
|
|
509
|
+
case 'cache-clear': await cmdCacheClear(); break;
|
|
510
|
+
case 'cost': await cmdCost(pos, flags); break;
|
|
511
|
+
case 'keys': await cmdKeys(pos, flags); break;
|
|
512
|
+
case 'setup': await runSetup(); break;
|
|
513
|
+
case 'project-config': {
|
|
514
|
+
const result = await runProjectConfig(pos, flags);
|
|
515
|
+
out(formatProjectConfigResult(result, { json: !!flags.json }));
|
|
516
|
+
break;
|
|
517
|
+
}
|
|
518
|
+
default:
|
|
519
|
+
die(`Unknown command: ${cmd}. Try 'surf-skill --help'.`);
|
|
520
|
+
}
|
|
521
|
+
} catch (e) {
|
|
522
|
+
if (e instanceof DispatchError) {
|
|
523
|
+
process.stderr.write(`❌ Error [${e.code}]: ${e.message}\n`);
|
|
524
|
+
if (e.code === 'NoProviderAvailable' && process.stdin.isTTY) {
|
|
525
|
+
process.stderr.write(`→ Run 'surf-skill setup' to configure keys interactively.\n`);
|
|
526
|
+
}
|
|
527
|
+
process.exit(1);
|
|
528
|
+
}
|
|
529
|
+
if (e.code === 'PROJECT_CONFIG_NO_TTY' || e.code === 'PROJECT_CONFIG_BAD_HARNESS') {
|
|
530
|
+
process.stderr.write(`❌ Error: ${e.message}\n`);
|
|
531
|
+
process.exit(2);
|
|
532
|
+
}
|
|
533
|
+
if (e.code === 'NO_TTY') {
|
|
534
|
+
process.stderr.write(`❌ Error: ${e.message}\n`);
|
|
535
|
+
process.exit(2);
|
|
536
|
+
}
|
|
537
|
+
process.stderr.write(`❌ Error: ${e.message || String(e)}\n`);
|
|
538
|
+
process.exit(1);
|
|
539
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "surf-skill",
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Multi-provider web skill (Tavily + Parallel AI) for AI coding agents — CLI + Node library + Anthropic Agent Skill. Auto-fallback, multi-key rotation, per-project timeout config.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./src/index.mjs",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./src/index.mjs",
|
|
9
|
+
"./package.json": "./package.json"
|
|
10
|
+
},
|
|
11
|
+
"bin": {
|
|
12
|
+
"surf-skill": "./bin/surf-skill.mjs"
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"bin/",
|
|
16
|
+
"src/",
|
|
17
|
+
"references/",
|
|
18
|
+
"SKILL.md",
|
|
19
|
+
"README.md",
|
|
20
|
+
"CHANGELOG.md",
|
|
21
|
+
"LICENSE"
|
|
22
|
+
],
|
|
23
|
+
"scripts": {
|
|
24
|
+
"postinstall": "node ./src/install/postinstall.mjs || true",
|
|
25
|
+
"preuninstall": "node ./src/install/preuninstall.mjs || true",
|
|
26
|
+
"test:syntax": "node --check bin/surf-skill.mjs && for f in src/index.mjs src/env.mjs src/install/*.mjs src/lib/*.mjs src/lib/providers/*.mjs src/lib/api/*.mjs; do node --check \"$f\" || exit 1; done && echo syntax-ok"
|
|
27
|
+
},
|
|
28
|
+
"engines": {
|
|
29
|
+
"node": ">=18"
|
|
30
|
+
},
|
|
31
|
+
"keywords": [
|
|
32
|
+
"agent-skill",
|
|
33
|
+
"tavily",
|
|
34
|
+
"parallel-ai",
|
|
35
|
+
"web-search",
|
|
36
|
+
"connector",
|
|
37
|
+
"claude-code",
|
|
38
|
+
"copilot-cli",
|
|
39
|
+
"pi-coding-agent",
|
|
40
|
+
"anthropic-skill",
|
|
41
|
+
"fallback",
|
|
42
|
+
"multi-provider",
|
|
43
|
+
"research"
|
|
44
|
+
],
|
|
45
|
+
"repository": {
|
|
46
|
+
"type": "git",
|
|
47
|
+
"url": "git+https://github.com/frederico-kluser/surf-skill.git"
|
|
48
|
+
},
|
|
49
|
+
"homepage": "https://github.com/frederico-kluser/surf-skill#readme",
|
|
50
|
+
"bugs": {
|
|
51
|
+
"url": "https://github.com/frederico-kluser/surf-skill/issues"
|
|
52
|
+
},
|
|
53
|
+
"license": "MIT",
|
|
54
|
+
"author": "frederico-kluser"
|
|
55
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Credit cost reference
|
|
2
|
+
|
|
3
|
+
`surf` uses **Tavily credits** as its single unit. Parallel AI's per-call
|
|
4
|
+
pricing is not published, so the Parallel column below is a coarse estimate
|
|
5
|
+
that only powers the `--confirm-expensive` gate (>10 credits ⇒ blocked).
|
|
6
|
+
|
|
7
|
+
## Tavily — published
|
|
8
|
+
|
|
9
|
+
Pay-as-you-go: **US$ 0.008 / credit**. Free tier: **1,000 credits / month**.
|
|
10
|
+
|
|
11
|
+
| Plan | US$ / mo | Credits / mo |
|
|
12
|
+
|---|---|---|
|
|
13
|
+
| Researcher (Free) | 0 | 1,000 |
|
|
14
|
+
| Project | 30 | 4,000 |
|
|
15
|
+
| Bootstrap | 100 | 15,000 |
|
|
16
|
+
| Startup | 220 | 38,000 |
|
|
17
|
+
| Pro | 500 | 100,000 |
|
|
18
|
+
| Enterprise | custom | custom |
|
|
19
|
+
|
|
20
|
+
Per-call costs:
|
|
21
|
+
|
|
22
|
+
| Endpoint | Credits |
|
|
23
|
+
|---|---|
|
|
24
|
+
| `/search` basic / fast / ultra-fast | 1 |
|
|
25
|
+
| `/search` advanced or `auto_parameters=true` | 2 |
|
|
26
|
+
| `/extract` basic | 1 per 5 URLs |
|
|
27
|
+
| `/extract` advanced | 2 per 5 URLs |
|
|
28
|
+
| `/map` no instructions | 1 per 10 pages |
|
|
29
|
+
| `/map` with instructions | 2 per 10 pages |
|
|
30
|
+
| `/crawl` basic | mapping + 1/5 pages |
|
|
31
|
+
| `/crawl` advanced | mapping + 2/5 pages |
|
|
32
|
+
| `/research` mini | dynamic, ~5–15 |
|
|
33
|
+
| `/research` pro | dynamic, ~15–50 |
|
|
34
|
+
| `/research/{id}` (poll) | 0 |
|
|
35
|
+
| `/usage` | 0 |
|
|
36
|
+
|
|
37
|
+
## Parallel AI — estimated
|
|
38
|
+
|
|
39
|
+
Public pricing is opaque. The values below come from the relative tiering of
|
|
40
|
+
the processor model (`lite < base < core < pro < ultra < ultra8x`) and are
|
|
41
|
+
used only by `lib/cost.mjs` to gate expensive calls.
|
|
42
|
+
|
|
43
|
+
| Operation | Estimated "credits" |
|
|
44
|
+
|---|---|
|
|
45
|
+
| `search` (lite processor) | 1 |
|
|
46
|
+
| `search` (base processor — set by `--depth advanced`) | 2 |
|
|
47
|
+
| `extract` (any) | 1 per 5 URLs |
|
|
48
|
+
| `tasks/runs` lite (≈ `--model mini`) | 1 |
|
|
49
|
+
| `tasks/runs` base (≈ `--model auto`) | 2 |
|
|
50
|
+
| `tasks/runs` core | 5 |
|
|
51
|
+
| `tasks/runs` pro (≈ `--model pro`) | 8 |
|
|
52
|
+
| `tasks/runs` ultra (≈ `--model ultra`) | 25 |
|
|
53
|
+
| `tasks/runs` ultra8x | 200 |
|
|
54
|
+
| `crawl`, `map` | n/a (not supported) |
|
|
55
|
+
|
|
56
|
+
## Rules of thumb
|
|
57
|
+
|
|
58
|
+
- **Default to `--depth basic` and `--max 5`.** Escalating to `advanced`
|
|
59
|
+
doubles cost on both providers.
|
|
60
|
+
- **Always prefer `map` before `crawl`** when scoping a site — `map` returns
|
|
61
|
+
URLs cheaply, and you can re-run `crawl` on a filtered subset. Both are
|
|
62
|
+
Tavily-only operations.
|
|
63
|
+
- **`auto_parameters` always costs 2 Tavily credits**, even if it picks
|
|
64
|
+
`basic` internally.
|
|
65
|
+
- **`research --model pro` / `--processor ultra`** can be very expensive
|
|
66
|
+
— always confirm with the user.
|
|
67
|
+
- **Cache (TTL 6 h)** makes repeated identical queries free. Use
|
|
68
|
+
`--no-cache` only when freshness matters.
|
|
69
|
+
- **`research-poll` is free** — poll every 10–15 s without budget impact.
|
|
70
|
+
- **`--confirm-expensive` uses the WORST estimate across the eligible
|
|
71
|
+
providers**, so a search routed to either Tavily or Parallel will be
|
|
72
|
+
gated by the higher of the two estimates.
|