webscraping-ai-cli 1.0.2 → 1.2.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 // src/cli.ts
-import { Command as Command10 } from "commander";
+import { Command as Command12 } from "commander";
 import { readFileSync } from "fs";
 import { dirname as dirname3, resolve as resolve2 } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
@@ -67,24 +67,100 @@ function maskApiKey(key) {
 
 // src/lib/output.ts
 import { promises as fs2 } from "fs";
-async function emit(value, opts = {}) {
+
+// src/lib/stdin.ts
+import readline from "readline";
+function urlIsStdin(url) {
+  return url === "-";
+}
+async function* readUrlsFromStdin() {
+  const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
+  for await (const raw of rl) {
+    const line = raw.trim();
+    if (line === "" || line.startsWith("#")) continue;
+    yield line;
+  }
+}
+async function* readQueriesFromStdin(input = process.stdin) {
+  const rl = readline.createInterface({ input, crlfDelay: Infinity });
+  for await (const line of rl) {
+    if (line.trim() === "") continue;
+    yield line;
+  }
+}
+async function* urlIterator(arg) {
+  if (urlIsStdin(arg)) {
+    yield* readUrlsFromStdin();
+    return;
+  }
+  yield arg;
+}
+async function* queryIterator(arg) {
+  if (urlIsStdin(arg)) {
+    yield* readQueriesFromStdin();
+    return;
+  }
+  yield arg;
+}
+
+// src/lib/output.ts
+async function emit(value, opts = {}, write = {}) {
   const text = stringify(value, opts);
   if (opts.output && opts.output.trim() !== "") {
-    await fs2.writeFile(opts.output, text + (text.endsWith("\n") ? "" : "\n"));
+    const data = text + (text.endsWith("\n") ? "" : "\n");
+    if (write.append) await fs2.appendFile(opts.output, data);
+    else await fs2.writeFile(opts.output, data);
     return;
   }
   process.stdout.write(text);
   if (!text.endsWith("\n")) process.stdout.write("\n");
 }
+function createEmitter(opts = {}) {
+  let first = true;
+  return async (value) => {
+    const append = !first;
+    first = false;
+    await emit(value, opts, { append });
+  };
+}
 function stringify(value, opts = {}) {
   if (typeof value === "string") return value;
   const pretty = opts.pretty ?? process.stdout.isTTY ?? false;
   return JSON.stringify(value, null, pretty ? 2 : 0);
 }
+async function openEmitter(opts, arg) {
+  if (urlIsStdin(arg) && opts.output && opts.output.trim() !== "") {
+    await fs2.writeFile(opts.output, "");
+    return (value) => emit(value, opts, { append: true });
+  }
+  return createEmitter(opts);
+}
 
 // src/lib/sdk.ts
 import { WebScrapingAI } from "webscraping-ai";
+
+// src/lib/redact.ts
+var secrets = /* @__PURE__ */ new Set();
+function registerSecret(value) {
+  if (value && value.trim() !== "") secrets.add(value);
+}
+var API_KEY_PARAM = /(api_key=)[^&\s"'<>]*/gi;
+function redactSecrets(text) {
+  let out = text.replace(API_KEY_PARAM, "$1[REDACTED]");
+  const candidates = new Set(secrets);
+  const env = process.env.WEBSCRAPING_AI_API_KEY?.trim();
+  if (env) candidates.add(env);
+  for (const secret of candidates) {
+    for (const form of /* @__PURE__ */ new Set([secret, encodeURIComponent(secret)])) {
+      out = out.split(form).join("[REDACTED]");
+    }
+  }
+  return out;
+}
+
+// src/lib/sdk.ts
 function createClient(options) {
+  registerSecret(options.apiKey);
   const inner = new WebScrapingAI({
     apiKey: options.apiKey,
     timeoutMs: options.requestTimeoutMs
@@ -107,6 +183,10 @@ function createClient(options) {
           return (o) => target.question({ ...o, ...tag });
         case "fields":
           return (o) => target.fields({ ...o, ...tag });
+        case "serp":
+          return (o) => target.serp({ ...o, params: { ...o.params, from_cli: true } });
+        case "data":
+          return (o) => target.data({ ...o, params: { ...o.params, from_cli: true } });
         case "account":
           return value.bind(target);
         default:
@@ -177,19 +257,66 @@ function buildQuestionOptions(url, question, flags, resolved) {
 function buildFieldsOptions(url, fields, flags, resolved) {
   return { url, fields, ...buildCommonOptions(flags, resolved) };
 }
+function buildSerpOptions(query, flags = {}) {
+  if (typeof query !== "string" || query.trim() === "") {
+    throw new Error("serp requires a non-empty query");
+  }
+  const out = { q: query };
+  if (flags.engine !== void 0) out.engine = flags.engine;
+  if (flags.gl) out.gl = flags.gl;
+  if (flags.hl) out.hl = flags.hl;
+  if (flags.page !== void 0) {
+    if (!Number.isSafeInteger(flags.page) || flags.page < 1) {
+      throw new Error(`--page must be an integer >= 1, got: ${flags.page}`);
+    }
+    out.page = flags.page;
+  }
+  return out;
+}
+var RESERVED_DATA_PARAMS = /* @__PURE__ */ new Set(["api_key", "url", "from_cli", "__proto__"]);
+var NAMED_DATA_PARAMS = {
+  country: "--country",
+  transcript: "--transcript",
+  transcript_language: "--transcript-language"
+};
+function parseDataParams(raw = []) {
+  const out = /* @__PURE__ */ Object.create(null);
+  for (const entry of raw) {
+    const eq = entry.indexOf("=");
+    if (eq <= 0) throw new Error(`--param must be key=value, got: ${entry}`);
+    const key = entry.slice(0, eq);
+    if (RESERVED_DATA_PARAMS.has(key)) {
+      throw new Error(`--param must not set "${key}"`);
+    }
+    if (Object.hasOwn(NAMED_DATA_PARAMS, key)) {
+      throw new Error(`--param must not set "${key}"; use ${NAMED_DATA_PARAMS[key]}`);
+    }
+    if (Object.hasOwn(out, key)) {
+      throw new Error(`--param "${key}" given more than once`);
+    }
+    out[key] = entry.slice(eq + 1);
+  }
+  return { ...out };
+}
+function buildDataOptions(url, flags = {}) {
+  if (typeof url !== "string" || url.trim() === "") {
+    throw new Error("data requires a non-empty url");
+  }
+  const out = { url };
+  if (flags.country) out.country = flags.country;
+  if (flags.transcript !== void 0) out.transcript = flags.transcript;
+  if (flags.transcriptLanguage) out.transcript_language = flags.transcriptLanguage;
+  const params = parseDataParams(flags.param);
+  if (Object.keys(params).length > 0) out.params = params;
+  return out;
+}
 
 // src/lib/options.ts
 import { promises as fs3 } from "fs";
 import { Option } from "commander";
 function addCommonScrapeOptions(cmd) {
-  return cmd.option("-k, --api-key <key>", "API key (overrides $WEBSCRAPING_AI_API_KEY and config file)").option("-o, --output <file>", "write result to FILE instead of stdout").option("--pretty", "pretty-print JSON output (default on TTY)").addOption(
-    new Option("--no-pretty", "force single-line JSON output").conflicts("pretty")
-  ).option("--js", "execute JS via a headless browser (default: enabled)").option("--no-js", "disable JS execution (faster, cheaper)").addOption(
-    new Option("--proxy <type>", "proxy pool").choices([
-      "datacenter",
-      "residential",
-      "stealth"
-    ])
+  return cmd.option("-k, --api-key <key>", "API key (overrides $WEBSCRAPING_AI_API_KEY and config file)").option("-o, --output <file>", "write result to FILE instead of stdout").option("--pretty", "pretty-print JSON output (default on TTY)").option("--no-pretty", "force single-line JSON output").option("--js", "execute JS via a headless browser (default: enabled)").option("--no-js", "disable JS execution (faster, cheaper)").addOption(
+    new Option("--proxy <type>", "proxy pool").choices(["datacenter", "residential", "stealth"])
   ).option("--country <code>", "two-letter proxy country code, e.g. us, gb, de").addOption(
     new Option("--device <type>", "device emulation").choices(["desktop", "mobile", "tablet"])
   ).option("--timeout <ms>", "page-retrieval timeout in ms (max 30000)", integerParser).option("--js-timeout <ms>", "JS rendering time in ms after load (max 20000)", integerParser).option("--wait-for <selector>", "CSS selector to wait for before returning content").option(
@@ -197,6 +324,9 @@ function addCommonScrapeOptions(cmd) {
     "HTTP headers to forward to the target page; JSON object or @file.json"
   ).option("--js-script <code>", "custom JS to evaluate on the target page").option("--custom-proxy <url>", "custom proxy URL: http://user:pass@host:port").option("--error-on-404", "return error on 404").option("--error-on-redirect", "return error on redirect");
 }
+function isPositiveIntegerString(value) {
+  return /^[1-9]\d*$/.test(value) && Number.isSafeInteger(Number(value));
+}
 function integerParser(value) {
   const n = Number.parseInt(value, 10);
   if (!Number.isFinite(n)) throw new Error(`expected integer, got: ${value}`);
@@ -233,27 +363,6 @@ function parseJsonObject(text, label) {
   return out;
 }
 
-// src/lib/stdin.ts
-import readline from "readline";
-function urlIsStdin(url) {
-  return url === "-";
-}
-async function* readUrlsFromStdin() {
-  const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
-  for await (const raw of rl) {
-    const line = raw.trim();
-    if (line === "" || line.startsWith("#")) continue;
-    yield line;
-  }
-}
-async function* urlIterator(arg) {
-  if (urlIsStdin(arg)) {
-    yield* readUrlsFromStdin();
-    return;
-  }
-  yield arg;
-}
-
 // src/commands/ask.ts
 function askCommand() {
   const cmd = new Command2("ask").description("Ask an AI question about a page (wraps /ai/question)").argument("<url>", "page URL (or `-` to read URLs from stdin)").requiredOption("-q, --question <text>", "the question / instructions for the model").addOption(new Option2("--format <fmt>", "response wrapping").choices(["text", "json"]));
@@ -263,10 +372,11 @@ function askCommand() {
     const apiKey = await resolveApiKey({ flag: opts.apiKey });
     const headers = await parseHeaders(opts.headers);
     const client = createClient({ apiKey, requestTimeoutMs: opts.timeout });
+    const write = await openEmitter(opts, url);
     for await (const target of urlIterator(url)) {
       const sdkOptions = buildQuestionOptions(target, opts.question, opts, { headers });
       const result = await client.question(sdkOptions);
-      await emit(result, opts);
+      await write(result);
     }
   });
   return cmd;
@@ -286,8 +396,10 @@ function authCommand() {
     const persisted = await readPersistedConfig();
     const env = process.env.WEBSCRAPING_AI_API_KEY;
     if (env && env.trim() !== "") {
-      process.stdout.write(`Active key (env WEBSCRAPING_AI_API_KEY): ${maskApiKey(env.trim())}
-`);
+      process.stdout.write(
+        `Active key (env WEBSCRAPING_AI_API_KEY): ${maskApiKey(env.trim())}
+`
+      );
     }
     if (persisted.apiKey && persisted.apiKey.trim() !== "") {
       process.stdout.write(
@@ -301,16 +413,125 @@ function authCommand() {
   });
   cmd.command("clear").description("Remove the stored API key file").action(async () => {
     const removed = await clearPersistedConfig();
-    process.stdout.write(removed ? `Removed ${configPath()}
-` : "No stored API key to remove.\n");
+    process.stdout.write(
+      removed ? `Removed ${configPath()}
+` : "No stored API key to remove.\n"
+    );
   });
   return cmd;
 }
 
-// src/commands/extract.ts
+// src/commands/data.ts
 import { Command as Command4 } from "commander";
+
+// src/lib/errors.ts
+import {
+  APIConnectionError,
+  APIError,
+  APITimeoutError,
+  AuthenticationError,
+  BadRequestError,
+  GatewayTimeoutError,
+  PaymentRequiredError,
+  RateLimitError,
+  ServerError,
+  WebScrapingAIError
+} from "webscraping-ai";
+var EXIT_CODES = {
+  ok: 0,
+  generic: 1,
+  usage: 2,
+  badRequest: 3,
+  auth: 4,
+  paymentRequired: 5,
+  rateLimit: 6,
+  serverError: 7,
+  timeout: 8,
+  connection: 9
+};
+function exitCodeFor(err) {
+  if (err instanceof BadRequestError) return EXIT_CODES.badRequest;
+  if (err instanceof AuthenticationError) return EXIT_CODES.auth;
+  if (err instanceof PaymentRequiredError) return EXIT_CODES.paymentRequired;
+  if (err instanceof RateLimitError) return EXIT_CODES.rateLimit;
+  if (err instanceof ServerError) return EXIT_CODES.serverError;
+  if (err instanceof GatewayTimeoutError) return EXIT_CODES.timeout;
+  if (err instanceof APITimeoutError) return EXIT_CODES.timeout;
+  if (err instanceof APIConnectionError) return EXIT_CODES.connection;
+  if (err instanceof APIError) return EXIT_CODES.serverError;
+  if (err instanceof WebScrapingAIError) return EXIT_CODES.generic;
+  return EXIT_CODES.generic;
+}
+function formatError(err) {
+  return redactSecrets(rawMessage(err));
+}
+function rawMessage(err) {
+  if (err instanceof APIError) {
+    const parts = [err.message];
+    if (err.status) parts.push(`(HTTP ${err.status})`);
+    return parts.join(" ");
+  }
+  if (err instanceof Error) return err.message;
+  return String(err);
+}
+
+// src/commands/data.ts
+function collect(value, previous = []) {
+  return [...previous, value];
+}
+function dataCommand() {
+  const cmd = new Command4("data").description(
+    "Structured JSON for a page on a supported site, e.g. YouTube, TikTok, X, LinkedIn, Instagram, Reddit (wraps /data, 15 credits/request; more sites are added server-side, an unsupported URL or page type gets a 400, not charged, listing what is supported)"
+  ).argument("<url>", "page URL (or `-` to read URLs from stdin, one per line)").option(
+    "--country <code>",
+    "two-letter country code of the proxy used to fetch the page (default: us)"
+  ).option(
+    "--transcript",
+    "YouTube videos only: also fetch the transcript into data.transcript (null without matching captions; a failed transcript fetch fails the request with a 500, not charged)"
+  ).option(
+    "--transcript-language <code>",
+    "caption language to pick, e.g. en or de (default: English, then the first track)"
+  ).option(
+    "--param <key=value>",
+    "extra site-specific query parameter sent as-is (repeatable)",
+    collect
+  ).option("-k, --api-key <key>", "API key (overrides $WEBSCRAPING_AI_API_KEY and config file)").option("-o, --output <file>", "write result to FILE instead of stdout").option("--pretty", "pretty-print JSON output (default on TTY)").option("--no-pretty", "force single-line JSON output");
+  cmd.action(async (url, opts) => {
+    if (!urlIsStdin(url) && url.trim() === "") {
+      cmd.error("error: data requires a non-empty url", {
+        exitCode: EXIT_CODES.usage,
+        code: "wsai.emptyUrl"
+      });
+    }
+    try {
+      parseDataParams(opts.param);
+    } catch (err) {
+      cmd.error(`error: ${err.message}`, {
+        exitCode: EXIT_CODES.usage,
+        code: "wsai.invalidParam"
+      });
+    }
+    const flags = {
+      country: opts.country,
+      transcript: opts.transcript,
+      transcriptLanguage: opts.transcriptLanguage,
+      param: opts.param
+    };
+    const apiKey = await resolveApiKey({ flag: opts.apiKey });
+    const client = createClient({ apiKey });
+    const write = await openEmitter(opts, url);
+    for await (const target of urlIterator(url)) {
+      const result = await client.data(buildDataOptions(target, flags));
+      await write(result);
+    }
+  });
+  return cmd;
+}
+
+// src/commands/extract.ts
+import { Command as Command5 } from "commander";
 function extractCommand() {
-  const cmd = new Command4("extract").description("Extract structured fields from a page using AI (wraps /ai/fields)").argument("<url>", "page URL (or `-` to read URLs from stdin)").requiredOption(
+  const cmd = new Command5("extract").description("Extract structured fields from a page using AI (wraps /ai/fields)").argument("<url>", "page URL (or `-` to read URLs from stdin)").requiredOption(
     "-f, --fields <json>",
     "fields to extract \u2014 JSON object {name: description} or @file.json"
   );
@@ -321,39 +542,48 @@ function extractCommand() {
     const headers = await parseHeaders(opts.headers);
     const fields = await parseFields(opts.fields);
     const client = createClient({ apiKey, requestTimeoutMs: opts.timeout });
+    const write = await openEmitter(opts, url);
     for await (const target of urlIterator(url)) {
       const sdkOptions = buildFieldsOptions(target, fields, opts, { headers });
       const result = await client.fields(sdkOptions);
-      await emit(result, opts);
+      await write(result);
     }
   });
   return cmd;
 }
 
 // src/commands/html.ts
-import { Command as Command5, Option as Option3 } from "commander";
+import { Command as Command6, Option as Option3 } from "commander";
 function htmlCommand() {
-  const cmd = new Command5("html").description("Fetch the full HTML of a page").argument("<url>", "page URL (or `-` to read URLs from stdin)").addOption(new Option3("--return-script-result", "return the result of --js-script instead of HTML")).addOption(new Option3("--format <fmt>", "response wrapping").choices(["text", "json"]));
+  const cmd = new Command6("html").description("Fetch the full HTML of a page").argument("<url>", "page URL (or `-` to read URLs from stdin)").addOption(
+    new Option3("--return-script-result", "return the result of --js-script instead of HTML")
+  ).addOption(new Option3("--format <fmt>", "response wrapping").choices(["text", "json"]));
   addCommonScrapeOptions(cmd);
   cmd.action(async (url, opts) => {
     const apiKey = await resolveApiKey({ flag: opts.apiKey });
     const headers = await parseHeaders(opts.headers);
     const client = createClient({ apiKey, requestTimeoutMs: opts.timeout });
+    const write = await openEmitter(opts, url);
     for await (const target of urlIterator(url)) {
-      const sdkOptions = buildHtmlOptions(target, opts, { headers }, {
-        returnScriptResult: opts.returnScriptResult
-      });
+      const sdkOptions = buildHtmlOptions(
+        target,
+        opts,
+        { headers },
+        {
+          returnScriptResult: opts.returnScriptResult
+        }
+      );
       const result = await client.html(sdkOptions);
-      await emit(result, opts);
+      await write(result);
     }
   });
   return cmd;
 }
 
 // src/commands/selected.ts
-import { Command as Command6, Option as Option4 } from "commander";
+import { Command as Command7, Option as Option4 } from "commander";
 function selectedCommand() {
-  const cmd = new Command6("selected").description("Fetch HTML of one CSS-selected page area").argument("<url>", "page URL (or `-` to read URLs from stdin)").option(
+  const cmd = new Command7("selected").description("Fetch HTML of one CSS-selected page area").argument("<url>", "page URL (or `-` to read URLs from stdin)").option(
     "-s, --selector <css>",
     "CSS selector for the target element(s); omit for whole-page HTML"
   ).addOption(new Option4("--format <fmt>", "response wrapping").choices(["text", "json"]));
@@ -362,19 +592,20 @@ function selectedCommand() {
     const apiKey = await resolveApiKey({ flag: opts.apiKey });
     const headers = await parseHeaders(opts.headers);
     const client = createClient({ apiKey, requestTimeoutMs: opts.timeout });
+    const write = await openEmitter(opts, url);
     for await (const target of urlIterator(url)) {
       const sdkOptions = buildSelectedOptions(target, opts.selector, opts, { headers });
       const result = await client.selected(sdkOptions);
-      await emit(result, opts);
+      await write(result);
     }
   });
   return cmd;
 }
 
 // src/commands/selectedMultiple.ts
-import { Command as Command7 } from "commander";
+import { Command as Command8 } from "commander";
 function selectedMultipleCommand() {
-  const cmd = new Command7("selected-multiple").description("Fetch HTML of multiple CSS-selected page areas").argument("<url>", "page URL (or `-` to read URLs from stdin)").option(
+  const cmd = new Command8("selected-multiple").description("Fetch HTML of multiple CSS-selected page areas").argument("<url>", "page URL (or `-` to read URLs from stdin)").option(
     "-s, --selector <css>",
     "CSS selector \u2014 pass multiple times for multiple selectors",
     collectMulti,
@@ -386,10 +617,11 @@ function selectedMultipleCommand() {
     const apiKey = await resolveApiKey({ flag: opts.apiKey });
     const headers = await parseHeaders(opts.headers);
     const client = createClient({ apiKey, requestTimeoutMs: opts.timeout });
+    const write = await openEmitter(opts, url);
     for await (const target of urlIterator(url)) {
       const sdkOptions = buildSelectedMultipleOptions(target, selectors, opts, { headers });
       const result = await client.selectedMultiple(sdkOptions);
-      await emit(result, opts);
+      await write(result);
     }
   });
   return cmd;
@@ -398,8 +630,45 @@ function collectMulti(value, previous) {
   return [...previous, value];
 }
 
+// src/commands/serp.ts
+import { Command as Command9, Option as Option5 } from "commander";
+function serpCommand() {
+  const cmd = new Command9("serp").alias("search").description("Search Google and print parsed results as JSON (wraps /serp, 15 credits/search)").argument("<query...>", "search query (or `-` to read queries from stdin, one per line)").addOption(new Option5("--engine <name>", "search engine (default: google)").choices(["google"])).option("--gl <code>", "two-letter country code for search geolocation (default: us)").option("--hl <code>", "two-letter language code for the results (default: en)").option(
+    "--page <n>",
+    "results page number, integer >= 1, 10 results per page (default: 1; server rejects > 100 with a 400)"
+  ).option("-k, --api-key <key>", "API key (overrides $WEBSCRAPING_AI_API_KEY and config file)").option("-o, --output <file>", "write result to FILE instead of stdout").option("--pretty", "pretty-print JSON output (default on TTY)").option("--no-pretty", "force single-line JSON output");
+  cmd.action(async (words, opts) => {
+    const query = words.join(" ");
+    let page;
+    if (opts.page !== void 0) {
+      if (!isPositiveIntegerString(opts.page)) {
+        cmd.error(`error: --page must be an integer >= 1, got: ${opts.page}`, {
+          exitCode: EXIT_CODES.usage,
+          code: "wsai.invalidPage"
+        });
+      }
+      page = Number(opts.page);
+    }
+    if (!urlIsStdin(query) && query.trim() === "") {
+      cmd.error("error: serp requires a non-empty query", {
+        exitCode: EXIT_CODES.usage,
+        code: "wsai.emptyQuery"
+      });
+    }
+    const flags = { engine: opts.engine, gl: opts.gl, hl: opts.hl, page };
+    const apiKey = await resolveApiKey({ flag: opts.apiKey });
+    const client = createClient({ apiKey });
+    const write = await openEmitter(opts, query);
+    for await (const q of queryIterator(query)) {
+      const result = await client.serp(buildSerpOptions(q, flags));
+      await write(result);
+    }
+  });
+  return cmd;
+}
+
 // src/commands/setupSkill.ts
-import { Command as Command8 } from "commander";
+import { Command as Command10 } from "commander";
 
 // src/lib/skill.ts
 import { promises as fs4 } from "fs";
@@ -408,24 +677,72 @@ import { dirname as dirname2, join as join2, resolve } from "path";
 import { fileURLToPath } from "url";
 var EDITORS = [
   // Claude Code: user-scope skills live under ~/.claude/skills/<name>/
-  { id: "claude-code", label: "Claude Code", mode: "dir", kind: "user", path: ".claude/skills/webscraping-ai" },
+  {
+    id: "claude-code",
+    label: "Claude Code",
+    mode: "dir",
+    kind: "user",
+    path: ".claude/skills/webscraping-ai"
+  },
   // Cursor: project-scope rule file. Cursor doesn't have a user-wide skill
   // dir, so this is the editor's "install" surface.
-  { id: "cursor", label: "Cursor", mode: "file", kind: "project", path: ".cursor/rules/webscraping-ai.mdc" },
+  {
+    id: "cursor",
+    label: "Cursor",
+    mode: "file",
+    kind: "project",
+    path: ".cursor/rules/webscraping-ai.mdc"
+  },
   // Windsurf: project-scope rules. Same shape as Cursor.
-  { id: "windsurf", label: "Windsurf", mode: "file", kind: "project", path: ".windsurf/rules/webscraping-ai.md" },
+  {
+    id: "windsurf",
+    label: "Windsurf",
+    mode: "file",
+    kind: "project",
+    path: ".windsurf/rules/webscraping-ai.md"
+  },
   // Kiro: user-scope steering directory.
   { id: "kiro", label: "Kiro", mode: "dir", kind: "user", path: ".kiro/steering/webscraping-ai" },
   // OpenCode: project-scope skills (mirrors Claude shape).
-  { id: "opencode", label: "OpenCode", mode: "dir", kind: "project", path: ".opencode/skills/webscraping-ai" },
+  {
+    id: "opencode",
+    label: "OpenCode",
+    mode: "dir",
+    kind: "project",
+    path: ".opencode/skills/webscraping-ai"
+  },
   // Gemini CLI: user-scope extension dir with a GEMINI.md.
-  { id: "gemini", label: "Gemini CLI", mode: "file", kind: "user", path: ".gemini/extensions/webscraping-ai/GEMINI.md" },
+  {
+    id: "gemini",
+    label: "Gemini CLI",
+    mode: "file",
+    kind: "user",
+    path: ".gemini/extensions/webscraping-ai/GEMINI.md"
+  },
   // GitHub Copilot: project-scope custom instructions file. Single file.
-  { id: "copilot", label: "GitHub Copilot", mode: "file", kind: "project", path: ".github/copilot-instructions.md" },
+  {
+    id: "copilot",
+    label: "GitHub Copilot",
+    mode: "file",
+    kind: "project",
+    path: ".github/copilot-instructions.md"
+  },
   // Augment: project-scope rules file.
-  { id: "augment", label: "Augment", mode: "file", kind: "project", path: ".augment/rules/webscraping-ai.md" },
+  {
+    id: "augment",
+    label: "Augment",
+    mode: "file",
+    kind: "project",
+    path: ".augment/rules/webscraping-ai.md"
+  },
   // Factory: project-scope rules file.
-  { id: "factory", label: "Factory", mode: "file", kind: "project", path: ".factory/rules/webscraping-ai.md" }
+  {
+    id: "factory",
+    label: "Factory",
+    mode: "file",
+    kind: "project",
+    path: ".factory/rules/webscraping-ai.md"
+  }
 ];
 function editorById(id) {
   return EDITORS.find((e) => e.id === id);
@@ -468,11 +785,11 @@ async function copyDirectory(source, destination) {
 
 // src/commands/setupSkill.ts
 function setupCommand() {
-  const cmd = new Command8("setup").description("Install the AI agent skill into your editor(s)");
+  const cmd = new Command10("setup").description("Install the AI agent skill into your editor(s)");
   cmd.command("skill").description("Install the WebScraping.AI skill into one or more AI coding editors").option(
     "-e, --editor <id>",
     `editor id (repeatable). Supported: ${EDITORS.map((e) => e.id).join(", ")}`,
-    collect,
+    collect2,
     []
   ).option("-a, --all", "install into every supported editor").option("-f, --force", "overwrite existing skill files").action(async (opts) => {
     const selected = resolveSelection(opts);
@@ -497,7 +814,7 @@ function setupCommand() {
   });
   return cmd;
 }
-function collect(value, previous) {
+function collect2(value, previous) {
   return [...previous, value];
 }
 function resolveSelection(opts) {
@@ -511,76 +828,34 @@ function resolveSelection(opts) {
 }
 
 // src/commands/text.ts
-import { Command as Command9, Option as Option5 } from "commander";
+import { Command as Command11, Option as Option6 } from "commander";
 function textCommand() {
-  const cmd = new Command9("text").description("Fetch the visible text of a page").argument("<url>", "page URL (or `-` to read URLs from stdin)").addOption(
-    new Option5("--text-format <fmt>", "text response format").choices(["plain", "json", "xml"])
+  const cmd = new Command11("text").description("Fetch the visible text of a page").argument("<url>", "page URL (or `-` to read URLs from stdin)").addOption(
+    new Option6("--text-format <fmt>", "text response format").choices(["plain", "json", "xml"])
   ).option("--return-links", "include page links (text-format=json only)");
   addCommonScrapeOptions(cmd);
   cmd.action(async (url, opts) => {
     const apiKey = await resolveApiKey({ flag: opts.apiKey });
     const headers = await parseHeaders(opts.headers);
     const client = createClient({ apiKey, requestTimeoutMs: opts.timeout });
+    const write = await openEmitter(opts, url);
     for await (const target of urlIterator(url)) {
-      const sdkOptions = buildTextOptions(target, opts, { headers }, {
-        textFormat: opts.textFormat,
-        returnLinks: opts.returnLinks
-      });
+      const sdkOptions = buildTextOptions(
+        target,
+        opts,
+        { headers },
+        {
+          textFormat: opts.textFormat,
+          returnLinks: opts.returnLinks
+        }
+      );
       const result = await client.text(sdkOptions);
-      await emit(result, opts);
+      await write(result);
     }
   });
   return cmd;
 }
 
-// src/lib/errors.ts
-import {
-  APIConnectionError,
-  APIError,
-  APITimeoutError,
-  AuthenticationError,
-  BadRequestError,
-  GatewayTimeoutError,
-  PaymentRequiredError,
-  RateLimitError,
-  ServerError,
-  WebScrapingAIError
-} from "webscraping-ai";
-var EXIT_CODES = {
-  ok: 0,
-  generic: 1,
-  usage: 2,
-  badRequest: 3,
-  auth: 4,
-  paymentRequired: 5,
-  rateLimit: 6,
-  serverError: 7,
-  timeout: 8,
-  connection: 9
-};
-function exitCodeFor(err) {
-  if (err instanceof BadRequestError) return EXIT_CODES.badRequest;
-  if (err instanceof AuthenticationError) return EXIT_CODES.auth;
-  if (err instanceof PaymentRequiredError) return EXIT_CODES.paymentRequired;
-  if (err instanceof RateLimitError) return EXIT_CODES.rateLimit;
-  if (err instanceof ServerError) return EXIT_CODES.serverError;
-  if (err instanceof GatewayTimeoutError) return EXIT_CODES.timeout;
-  if (err instanceof APITimeoutError) return EXIT_CODES.timeout;
-  if (err instanceof APIConnectionError) return EXIT_CODES.connection;
-  if (err instanceof APIError) return EXIT_CODES.serverError;
-  if (err instanceof WebScrapingAIError) return EXIT_CODES.generic;
-  return EXIT_CODES.generic;
-}
-function formatError(err) {
-  if (err instanceof APIError) {
-    const parts = [err.message];
-    if (err.status) parts.push(`(HTTP ${err.status})`);
-    return parts.join(" ");
-  }
-  if (err instanceof Error) return err.message;
-  return String(err);
-}
-
 // src/cli.ts
 function packageVersion() {
   const here = dirname3(fileURLToPath2(import.meta.url));
@@ -589,9 +864,9 @@ function packageVersion() {
   return raw.version;
 }
 function buildProgram() {
-  const program = new Command10();
+  const program = new Command12();
   program.name("webscraping-ai").description(
-    "Official CLI for the WebScraping.AI API. Fetch pages, ask AI, extract structured data.\nSet $WEBSCRAPING_AI_API_KEY or run `webscraping-ai auth set <key>` first."
+    "Official CLI for the WebScraping.AI API. Fetch pages, search Google, get structured site data, ask AI, extract fields.\nSet $WEBSCRAPING_AI_API_KEY or run `webscraping-ai auth set <key>` first."
   ).version(packageVersion(), "-v, --version", "output the version number").showHelpAfterError();
   program.addCommand(htmlCommand());
   program.addCommand(textCommand());
@@ -599,6 +874,8 @@ function buildProgram() {
   program.addCommand(selectedMultipleCommand());
   program.addCommand(askCommand());
   program.addCommand(extractCommand());
+  program.addCommand(serpCommand());
+  program.addCommand(dataCommand());
   program.addCommand(accountCommand());
   program.addCommand(authCommand());
   program.addCommand(setupCommand());