surf-skill 2.0.0

package/references/parallel-api.md

@@ -0,0 +1,155 @@
+# Parallel AI — endpoint reference
+
+Base URL: `https://api.parallel.ai`.
+
+**Auth**: header `x-api-key: <PARALLEL_API_KEY>` (**not** `Authorization: Bearer`).
+
+Get a key at <https://platform.parallel.ai>. Docs at <https://docs.parallel.ai>.
+
+## POST /v1/search
+
+Web search.
+
+| Field | Type | Notes |
+|---|---|---|
+| `objective` | string | required — natural language description of what you need |
+| `search_queries` | string[] | required — search queries to issue |
+| `processor` | `lite` \| `base` | optional; default `lite`. `base` ≈ Tavily's `advanced` depth |
+| `max_results` | int | optional |
+| `source_policy.include_domains` | string[] | optional |
+| `source_policy.exclude_domains` | string[] | optional |
+
+Response:
+
+```json
+{
+  "search_id": "...",
+  "results": [
+    {
+      "url": "...",
+      "title": "...",
+      "excerpts": ["...", "..."],
+      "publish_date": "2026-01-01"
+    }
+  ],
+  "warnings": [],
+  "usage": {}
+}
+```
+
+Doc: <https://docs.parallel.ai/search/search-quickstart>
+
+## POST /v1beta/extract  (beta)
+
+Pull content from known URLs.
+
+| Field | Type | Notes |
+|---|---|---|
+| `urls` | string[] | required |
+| `objective` | string | optional — focuses extraction on specific information |
+| `excerpts` | bool | optional — return objective-focused excerpts |
+| `full_content` | bool | optional — return full Markdown of each page |
+
+Response:
+
+```json
+{
+  "extract_id": "...",
+  "results": [
+    {
+      "url": "...",
+      "title": "...",
+      "publish_date": "...",
+      "excerpts": ["..."],
+      "full_content": "..."
+    }
+  ],
+  "errors": []
+}
+```
+
+Doc: <https://docs.parallel.ai/extract/extract-quickstart>
+
+## POST /v1/tasks/runs   →  GET /v1/tasks/runs/{run_id}/result
+
+Async Task API — Parallel's deep-research equivalent.
+
+`POST /v1/tasks/runs` body:
+
+| Field | Type | Notes |
+|---|---|---|
+| `input` | string \| object | required |
+| `processor` | `lite` \| `base` \| `core` \| `pro` \| `ultra` \| `ultra8x` | required |
+| `task_spec.output_schema` | JSON Schema / text / auto | optional — structured output |
+| `metadata` | object | optional |
+
+Returns **HTTP 202** with `{ run_id, status: "queued" \| "running", is_active, processor }`.
+
+`GET /v1/tasks/runs/{run_id}` returns status only.
+
+`GET /v1/tasks/runs/{run_id}/result` returns the result **only after `status: "completed"`**:
+
+```json
+{
+  "run_id": "...",
+  "status": "completed",
+  "output": {
+    "content": "...",
+    "basis": [
+      { "url": "...", "title": "..." }
+    ]
+  }
+}
+```
+
+Processor mapping used by `surf-skill research`:
+
+| `--model` | Parallel processor |
+|---|---|
+| `mini` | `lite` |
+| `auto` | `base` |
+| `pro` | `pro` |
+| `ultra` | `ultra` |
+
+Doc: <https://docs.parallel.ai/task-api/task-quickstart>,
+<https://docs.parallel.ai/task-api/guides/choose-a-processor>
+
+## Capabilities not provided by Parallel
+
+- **No crawl endpoint** (no recursive site walk).
+- **No URL-map endpoint** (no sitemap discovery without fetching content).
+- **No public `/usage` endpoint** at the time of writing.
+
+`surf` routes `crawl` and `map` to Tavily only.
+
+## Error format
+
+Non-2xx responses use:
+
+```json
+{
+  "type": "error",
+  "error": {
+    "ref_id": "...",
+    "message": "human-readable message",
+    "detail": {}
+  }
+}
+```
+
+Surf classifies them as:
+
+- `401` → `auth` (burn key, try next)
+- `402` insufficient credits → `auth` (burn key, try next)
+- `403` → `auth` **unless** body says "processor" → `caller_4xx`
+- `429` → `rate_limit_429` (retry with backoff)
+- `5xx` → `server_5xx` (retry; after 3 attempts, burn key)
+- other 4xx → `caller_4xx` (no fallback, throw)
+
+## Notes
+
+- Free tier / RPS / `Retry-After` header are **not publicly documented**.
+- The `parallel-web` Node SDK exists (v0.4.1) but requires Node 20+; surf
+  uses `fetch` directly to stay zero-deps and Node 18+ compatible.
+- Extract is in `v1beta`; the path may change. The adapter is in
+  `lib/providers/parallel.mjs` if you need to update it.

package/references/tavily-api.md

@@ -0,0 +1,90 @@
+# Tavily API — endpoint reference
+
+Base: `https://api.tavily.com`. Auth: `Authorization: Bearer tvly-…`.
+
+## POST /search
+Search the web.
+
+| Field | Type | Notes |
+|---|---|---|
+| `query` | string | required |
+| `search_depth` | `basic` \| `advanced` \| `fast` \| `ultra-fast` | default `basic` |
+| `max_results` | int | ≤20 |
+| `topic` | `general` \| `news` \| `finance` | |
+| `time_range` | `day` \| `week` \| `month` \| `year` | |
+| `start_date`, `end_date` | ISO date | |
+| `include_domains` | string[] | ≤300 |
+| `exclude_domains` | string[] | ≤150 |
+| `include_answer` | `false` \| `basic` \| `advanced` | |
+| `include_raw_content` | `false` \| `markdown` \| `text` | |
+| `include_images`, `include_image_descriptions`, `include_favicon` | bool | |
+| `country` | string | only with `topic=general` |
+| `auto_parameters` | bool | always 2 credits |
+| `exact_match` | bool | |
+| `include_usage` | bool | include credit usage in response |
+
+## POST /extract
+Pull content from known URLs.
+
+| Field | Type | Notes |
+|---|---|---|
+| `urls` | string[] | up to 20 |
+| `extract_depth` | `basic` \| `advanced` | |
+| `format` | `markdown` \| `text` | |
+| `include_images`, `include_favicon` | bool | |
+| `query` | string | filter relevance |
+| `chunks_per_source` | 1–5 | |
+| `timeout` | float | 1.0–60.0 s |
+
+## POST /crawl
+Mapping + extraction starting from a URL.
+
+| Field | Type | Notes |
+|---|---|---|
+| `url` | string | required |
+| `max_depth`, `max_breadth`, `limit` | int | |
+| `instructions` | string | natural language guidance |
+| `select_paths`, `select_domains` | string[] | regex |
+| `exclude_paths`, `exclude_domains` | string[] | |
+| `allow_external` | bool | |
+| `include_images` | bool | |
+| `categories` | string[] | |
+| `extract_depth` | `basic` \| `advanced` | |
+| `format` | `markdown` \| `text` | |
+| `query` | string | optional focus filter |
+| `chunks_per_source` | 1–5 | |
+| `timeout` | float | custom server-side timeout |
+| `include_usage` | bool | include credit usage in response |
+
+## POST /map
+Discover URLs without extracting (cheaper).
+
+Same selection fields as crawl, no `extract_depth`. API also supports `timeout` and `include_usage`.
+
+## POST /research
+Start an async research job.
+
+| Field | Type | Notes |
+|---|---|---|
+| `input` | string | required |
+| `model` | `mini` \| `pro` \| `auto` | |
+| `stream` | bool | SSE |
+| `output_schema` | JSON Schema | |
+| `citation_format` | `numbered` \| `mla` \| `apa` \| `chicago` | |
+
+Returns `{ request_id, status: "pending", model }` quickly.
+
+## GET /research/{request_id}
+Poll a research job. Free.
+
+Returns `{ status: "pending" | "completed" | "failed", content, sources }`.
+
+## GET /usage
+Account usage. Free.
+
+## Status codes
+
+- `200` ok
+- `401` bad/missing key
+- `429` rate limit (retry with backoff)
+- `432`, `433` plan/quota limits — escalate to user

package/src/env.mjs

@@ -0,0 +1,125 @@
+// Key discovery for library mode.
+// Priority (each level can contribute; results merged + deduped):
+//   1. Explicit opts (opts.tavilyKey / opts.tavilyKeys / parallel*)
+//   2. process.env (TAVILY_API_KEYS comma-separated + TAVILY_API_KEY)
+//   3. .env file at process.cwd() (lightweight regex parser, no dotenv dep)
+//   4. ~/.config/surf/keys.json (CLI persistent store, fallback only)
+
+import { existsSync, promises as fs } from 'node:fs';
+import path from 'node:path';
+import { loadState } from './lib/state.mjs';
+
+const ENV_FILE_CACHE = new Map();
+
+async function loadDotenv(dir) {
+  if (ENV_FILE_CACHE.has(dir)) return ENV_FILE_CACHE.get(dir);
+  const p = path.join(dir, '.env');
+  const out = {};
+  if (existsSync(p)) {
+    try {
+      const txt = await fs.readFile(p, 'utf8');
+      for (const line of txt.split('\n')) {
+        const m = line.match(/^\s*([A-Z][A-Z0-9_]*)\s*=\s*"?([^"#]*?)"?\s*(?:#.*)?$/);
+        if (m) out[m[1]] = m[2].trim();
+      }
+    } catch {}
+  }
+  ENV_FILE_CACHE.set(dir, out);
+  return out;
+}
+
+function splitCsv(s) {
+  return typeof s === 'string'
+    ? s.split(',').map(x => x.trim()).filter(Boolean)
+    : [];
+}
+
+function arrayify(v) {
+  if (!v) return [];
+  return Array.isArray(v) ? v.filter(Boolean) : [v].filter(Boolean);
+}
+
+/**
+ * Resolve API keys for both providers using the discovery hierarchy.
+ *
+ * @param {object} opts
+ * @param {string|string[]} [opts.tavilyKey] - single key or array
+ * @param {string[]} [opts.tavilyKeys] - array (alias)
+ * @param {string|string[]} [opts.parallelKey] - single or array
+ * @param {string[]} [opts.parallelKeys] - array (alias)
+ * @param {boolean} [opts.skipDotenv=false] - skip .env scanning
+ * @param {boolean} [opts.skipConfigFile=false] - skip ~/.config/surf/keys.json
+ * @param {string} [opts.cwd=process.cwd()]
+ * @returns {Promise<{tavily: string[], parallel: string[]}>}
+ */
+export async function discoverKeys(opts = {}) {
+  const cwd = opts.cwd || process.cwd();
+
+  // Level 1: explicit
+  const explicitTavily = [
+    ...arrayify(opts.tavilyKey),
+    ...arrayify(opts.tavilyKeys),
+  ];
+  const explicitParallel = [
+    ...arrayify(opts.parallelKey),
+    ...arrayify(opts.parallelKeys),
+  ];
+
+  // Level 2: process.env
+  const envTavily = [
+    ...splitCsv(process.env.TAVILY_API_KEYS),
+    process.env.TAVILY_API_KEY,
+  ].filter(Boolean);
+  const envParallel = [
+    ...splitCsv(process.env.PARALLEL_API_KEYS),
+    process.env.PARALLEL_API_KEY,
+  ].filter(Boolean);
+
+  // Level 3: .env file
+  let dotenvTavily = [];
+  let dotenvParallel = [];
+  if (!opts.skipDotenv) {
+    const env = await loadDotenv(cwd);
+    dotenvTavily = [
+      ...splitCsv(env.TAVILY_API_KEYS),
+      env.TAVILY_API_KEY,
+    ].filter(Boolean);
+    dotenvParallel = [
+      ...splitCsv(env.PARALLEL_API_KEYS),
+      env.PARALLEL_API_KEY,
+    ].filter(Boolean);
+  }
+
+  // Level 4: ~/.config/surf/keys.json (only if nothing yet from 1-3)
+  let cfgTavily = [];
+  let cfgParallel = [];
+  const noneSoFarTavily = !explicitTavily.length && !envTavily.length && !dotenvTavily.length;
+  const noneSoFarParallel = !explicitParallel.length && !envParallel.length && !dotenvParallel.length;
+  if (!opts.skipConfigFile && (noneSoFarTavily || noneSoFarParallel)) {
+    try {
+      const state = await loadState();
+      if (noneSoFarTavily) cfgTavily = state.tavily.keys || [];
+      if (noneSoFarParallel) cfgParallel = state.parallel.keys || [];
+    } catch {}
+  }
+
+  return {
+    tavily: [...new Set([...explicitTavily, ...envTavily, ...dotenvTavily, ...cfgTavily])],
+    parallel: [...new Set([...explicitParallel, ...envParallel, ...dotenvParallel, ...cfgParallel])],
+  };
+}
+
+/**
+ * Build an in-memory state object that the dispatch layer can use directly
+ * without touching ~/.config/surf/keys.json.
+ */
+export async function buildInMemoryState(opts = {}) {
+  const { tavily, parallel } = await discoverKeys(opts);
+  return {
+    schema_version: 1,
+    tavily: { keys: tavily, current: 0, burned: [] },
+    parallel: { keys: parallel, current: 0, burned: [] },
+    last_ok_provider: null,
+    _inMemory: true,
+  };
+}

package/src/index.mjs

@@ -0,0 +1,22 @@
+// surf-skill — library entry point.
+// Named exports for each operation. CLI is at bin/surf-skill.mjs.
+//
+// Usage:
+//   import { search, extract, research } from 'surf-skill';
+//   const r = await search('claude api', { max: 3 });
+//
+// Keys are auto-discovered (opts > process.env > .env > ~/.config/surf/keys.json).
+// Pass `tavilyKeys: [...]` / `parallelKeys: [...]` to override.
+
+export { search } from './lib/api/search.mjs';
+export { extract } from './lib/api/extract.mjs';
+export { crawl } from './lib/api/crawl.mjs';
+export { map } from './lib/api/map.mjs';
+export {
+  research,
+  researchStart,
+  researchPoll,
+} from './lib/api/research.mjs';
+
+export { discoverKeys, buildInMemoryState } from './env.mjs';
+export { setSilent } from './lib/progress.mjs';

package/src/install/postinstall.mjs

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+// Runs after `npm install`. Idempotent. Must never fail npm install.
+// - Detects global vs local install.
+// - Global: symlinks/copies package into the 4 supported harness skill dirs,
+//   creates ~/.config/surf/keys.json skeleton, cleans legacy 'tavily'/'surf'/'tvly'
+//   symlinks from prior versions.
+// - Local: just prints "installed as library" and exits.
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  installSkill,
+  cleanupLegacy,
+  ensureKeysSkeleton,
+} from '../lib/harness-install.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const pkgRoot = path.resolve(__dirname, '..', '..');
+
+const isGlobal = process.env.npm_config_global === 'true';
+// Some CI environments don't set npm_config_global; also treat the case where
+// __dirname is under a global node_modules path as "global enough".
+const looksGlobal = /node_modules\/surf-skill\/src\/install$/.test(__dirname) ||
+                    /node_modules\\surf-skill\\src\\install$/.test(__dirname);
+
+async function main() {
+  if (!isGlobal && !looksGlobal) {
+    // Local install: don't touch user system. Library mode.
+    process.stdout.write('surf-skill installed as a library (npm i surf-skill).\n');
+    process.stdout.write('  → For the global CLI: npm i -g surf-skill\n');
+    process.stdout.write('  → To import: import { search } from "surf-skill"\n');
+    return;
+  }
+
+  // Cleanup legacy symlinks from earlier versions BEFORE creating new ones.
+  const legacy = await cleanupLegacy();
+  for (const r of legacy) {
+    process.stdout.write(`✓ removed legacy ${r.removed}\n`);
+  }
+
+  // Install symlinks into each harness skill dir.
+  const installed = await installSkill(pkgRoot);
+  for (const r of installed) {
+    if (r.action === 'error') {
+      process.stdout.write(`⚠ ${r.dir}: ${r.error}\n`);
+    } else {
+      const verb = {
+        symlinked: '✓ symlinked',
+        copied: '✓ copied (no symlink permission)',
+        'kept-symlink': '✓ already linked',
+        'preserved-existing': 'ℹ preserved your existing copy at',
+      }[r.action] || r.action;
+      process.stdout.write(`${verb} ${r.dir}\n`);
+    }
+  }
+
+  // Create state dir + skeleton keys.json.
+  const skel = await ensureKeysSkeleton();
+  if (skel.created) process.stdout.write(`✓ created ${skel.created} (chmod 600)\n`);
+
+  process.stdout.write('\n');
+  process.stdout.write('✓ surf-skill 2.0.0 installed globally\n');
+  process.stdout.write('  → Run `surf-skill setup` to add Tavily/Parallel keys\n');
+  process.stdout.write('    (or just run any command — wizard auto-launches in TTY)\n');
+  process.stdout.write('  → `surf-skill --help` for the full command list\n');
+}
+
+main().catch(e => {
+  // NEVER fail npm install. Print warning + exit 0.
+  process.stderr.write(`surf-skill postinstall warning: ${e.message}\n`);
+  process.stderr.write('  (skill is installed; harness symlinks may need manual setup)\n');
+  process.exit(0);
+});

package/src/install/preuninstall.mjs

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+// Runs before `npm rm -g surf-skill`. Removes our symlinks; leaves user state.
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { uninstallSkill } from '../lib/harness-install.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const pkgRoot = path.resolve(__dirname, '..', '..');
+
+async function main() {
+  const results = await uninstallSkill(pkgRoot);
+  for (const r of results) {
+    if (r.removed) process.stdout.write(`✓ removed ${r.dir}\n`);
+    else if (r.error) process.stdout.write(`⚠ ${r.dir}: ${r.error}\n`);
+  }
+  process.stdout.write('\nsurf-skill uninstalled.\n');
+  process.stdout.write('  → Your keys at ~/.config/surf/keys.json are preserved.\n');
+  process.stdout.write('  → To wipe: rm -rf ~/.config/surf ~/.cache/surf\n');
+}
+
+main().catch(e => {
+  process.stderr.write(`surf-skill preuninstall warning: ${e.message}\n`);
+  process.exit(0);
+});

package/src/lib/api/crawl.mjs

@@ -0,0 +1,55 @@
+// Library wrapper for `crawl` (Tavily only).
+
+import { dispatch } from '../dispatch.mjs';
+import { buildInMemoryState } from '../../env.mjs';
+import { setSilent } from '../progress.mjs';
+
+/**
+ * Recursive site crawl. Tavily-only.
+ *
+ * @param {string} url - root URL
+ * @param {object} [opts]
+ * @param {number} [opts.maxDepth=1]
+ * @param {number} [opts.maxBreadth=20]
+ * @param {number} [opts.limit=50]
+ * @param {string} [opts.instructions]
+ * @param {string|string[]} [opts.selectPaths]
+ * @param {string|string[]} [opts.excludePaths]
+ * @param {string} [opts.tavilyKey]
+ * @returns {Promise<object>} normalized envelope
+ */
+export async function crawl(url, opts = {}) {
+  if (opts.quiet !== false) setSilent(true);
+  if (!url || typeof url !== 'string') throw new Error('crawl: url required');
+
+  const state = await buildInMemoryState(opts);
+  return dispatch(
+    'crawl',
+    {
+      url,
+      maxDepth: opts.maxDepth,
+      maxBreadth: opts.maxBreadth,
+      limit: opts.limit,
+      instructions: opts.instructions,
+      selectPaths: opts.selectPaths,
+      selectDomains: opts.selectDomains,
+      excludePaths: opts.excludePaths,
+      excludeDomains: opts.excludeDomains,
+      allowExternal: opts.allowExternal,
+      images: opts.images,
+      categories: opts.categories,
+      extractDepth: opts.extractDepth || 'basic',
+      format: opts.format || 'markdown',
+      query: opts.query,
+      chunks: opts.chunks,
+      timeout: opts.timeout,
+    },
+    {
+      provider: opts.provider,
+      'no-fallback': opts.noFallback,
+      'no-cache': opts.noCache,
+      'confirm-expensive': true,
+    },
+    { state }
+  );
+}

package/src/lib/api/extract.mjs

@@ -0,0 +1,46 @@
+// Library wrapper for `extract`.
+
+import { dispatch } from '../dispatch.mjs';
+import { buildInMemoryState } from '../../env.mjs';
+import { setSilent } from '../progress.mjs';
+
+/**
+ * Extract clean content from URLs.
+ *
+ * @param {string|string[]} urls - one or more URLs (max 20)
+ * @param {object} [opts]
+ * @param {string|string[]} [opts.tavilyKey|opts.tavilyKeys]
+ * @param {string|string[]} [opts.parallelKey|opts.parallelKeys]
+ * @param {'tavily'|'parallel'} [opts.provider]
+ * @param {'basic'|'advanced'} [opts.depth='basic']
+ * @param {string} [opts.query] - focus extraction on this topic
+ * @param {boolean} [opts.quiet=true]
+ * @returns {Promise<object>} normalized envelope
+ */
+export async function extract(urls, opts = {}) {
+  if (opts.quiet !== false) setSilent(true);
+  const urlList = Array.isArray(urls) ? urls : [urls];
+  if (!urlList.length) throw new Error('extract: at least 1 URL required');
+  if (urlList.length > 20) throw new Error('extract: max 20 URLs per call');
+
+  const state = await buildInMemoryState(opts);
+  return dispatch(
+    'extract',
+    {
+      urls: urlList,
+      depth: opts.depth || 'basic',
+      format: opts.format || 'markdown',
+      query: opts.query,
+      chunks: opts.chunks,
+      images: opts.images,
+      extractTimeout: opts.extractTimeout,
+    },
+    {
+      provider: opts.provider,
+      'no-fallback': opts.noFallback,
+      'no-cache': opts.noCache,
+      'confirm-expensive': true,
+    },
+    { state }
+  );
+}

package/src/lib/api/map.mjs

@@ -0,0 +1,43 @@
+// Library wrapper for `map` (URL discovery; Tavily only).
+
+import { dispatch } from '../dispatch.mjs';
+import { buildInMemoryState } from '../../env.mjs';
+import { setSilent } from '../progress.mjs';
+
+/**
+ * Discover URLs on a site without fetching content. Tavily-only.
+ *
+ * @param {string} url - root URL
+ * @param {object} [opts]
+ * @returns {Promise<object>} normalized envelope with { base_url, urls[] }
+ */
+export async function map(url, opts = {}) {
+  if (opts.quiet !== false) setSilent(true);
+  if (!url || typeof url !== 'string') throw new Error('map: url required');
+
+  const state = await buildInMemoryState(opts);
+  return dispatch(
+    'map',
+    {
+      url,
+      maxDepth: opts.maxDepth,
+      maxBreadth: opts.maxBreadth,
+      limit: opts.limit,
+      instructions: opts.instructions,
+      selectPaths: opts.selectPaths,
+      selectDomains: opts.selectDomains,
+      excludePaths: opts.excludePaths,
+      excludeDomains: opts.excludeDomains,
+      allowExternal: opts.allowExternal,
+      categories: opts.categories,
+      timeout: opts.timeout,
+    },
+    {
+      provider: opts.provider,
+      'no-fallback': opts.noFallback,
+      'no-cache': opts.noCache,
+      'confirm-expensive': true,
+    },
+    { state }
+  );
+}