webforai 3.0.0 → 4.0.0

package/dist/bin.js

@@ -6,24 +6,35 @@ import { CommanderError, program } from "commander";
 // package.json
 var package_default = {
   name: "webforai",
-  version: "3.0.0",
-  description: "A library that provides a web interface for AI",
+  version: "4.0.0",
+  description: "Convert web pages and local HTML to clean, LLM-ready Markdown: library, CLI (npx webforai) and hosted-API client.",
   author: "inaridiy",
   license: "Apache-2.0",
   keywords: [
-    "web",
+    "html-to-markdown",
+    "markdown",
+    "llm",
     "ai",
+    "scraper",
+    "crawler",
+    "readability",
+    "web-scraping",
+    "cli",
+    "rag",
     "html",
     "html2md",
-    "markdown",
     "mdast",
     "hast"
   ],
   repository: {
     type: "git",
-    url: "https://github.com/inaridiy/webforai.git"
+    url: "https://github.com/inaridiy/webforai.git",
+    directory: "packages/webforai"
   },
   homepage: "https://webforai.dev/",
+  bugs: {
+    url: "https://github.com/inaridiy/webforai/issues"
+  },
   scripts: {
     "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
     clean: "rimraf dist",
@@ -154,163 +165,6 @@ var package_default = {
   }
 };
 
-// src/cli/commands/skill/index.ts
-import { spawn } from "node:child_process";
-import fs from "node:fs/promises";
-import os from "node:os";
-import path from "node:path";
-
-// src/cli/commands/skill/content.ts
-var renderSkillMarkdown = (version) => `---
-name: webforai
-description: Convert web pages or local HTML files to clean, LLM-ready Markdown with the webforai CLI. Use when a task needs page content as Markdown - reading documentation or articles, scraping, summarising URLs, building text corpora - including JavaScript-heavy or bot-protected pages via the hosted platform.
----
-
-# webforai CLI${version ? ` (v${version})` : ""}
-
-Convert a URL or local HTML file to Markdown. Non-interactive by default: the Markdown goes
-to stdout, every log goes to stderr, so output can be piped or captured directly.
-
-## Basic usage
-
-\`\`\`bash
-npx webforai https://example.com/article            # Markdown to stdout
-npx webforai https://example.com/article -o out.md  # write to a file instead
-npx webforai ./page.html                            # convert a local HTML file
-npx webforai https://example.com --json             # machine-readable envelope (see below)
-\`\`\`
-
-## Flags
-
-- \`-o, --output <path>\` \u2014 write Markdown to a file instead of stdout.
-- \`-l, --loader <fetch|playwright|platform>\` \u2014 how HTML is acquired. Default: \`fetch\` for
-  URLs, automatic \`local\` for file paths. \`playwright\` renders JavaScript locally (needs
-  \`npx playwright install chromium\`). \`platform\` uses the hosted API (below).
-- \`-m, --mode <default|ai>\` \u2014 \`ai\` strips links/tables/images down to plain text
-  (local conversion only).
-- \`--extractor <auto|takumi|minimal|none>\` \u2014 main-content extraction preset
-  (\`none\` converts the whole page).
-- \`--frontmatter\` \u2014 prepend YAML front matter (title, author, canonical URL, ...).
-- \`--json\` \u2014 print a JSON envelope to stdout instead of raw Markdown.
-- \`--engine <fetch|browser|proxy-fetch|proxy-browser>\` \u2014 platform acquisition engine
-  (implies \`-l platform\`). \`browser\`/\`proxy-browser\` render JavaScript; \`proxy-*\` engines
-  egress through rotating proxies for bot-protected sites.
-- \`--region <auto|us|eu|uk|jp|asia>\` \u2014 platform proxy egress region (implies \`-l platform\`).
-- \`--screenshot\` \u2014 platform browser engines only; the envelope carries an expiring
-  \`screenshotUrl\`.
-- \`--api-key <key>\` / \`--platform-url <url>\` \u2014 platform credentials and host; usually set
-  via env vars instead.
-- \`-i, --interactive\` \u2014 guided prompt flow (for humans; do not use from an agent).
-- \`-d, --debug\` \u2014 verbose logs on stderr.
-
-## JSON envelope (\`--json\`)
-
-\`\`\`json
-{
-  "source": "https://example.com",
-  "loader": "fetch",
-  "url": "https://example.com",
-  "markdown": "# ...",
-  "metadata": { "title": "..." },
-  "engine": "browser",
-  "region": "auto",
-  "credits": 5,
-  "screenshotUrl": "https://...",
-  "output": "out.md"
-}
-\`\`\`
-
-\`engine\`/\`region\`/\`credits\`/\`screenshotUrl\` appear only for the platform loader; \`output\`
-only when \`-o\` also wrote a file.
-
-## Hosted platform
-
-For JavaScript-heavy or bot-protected pages, the hosted platform fetches server-side:
-
-\`\`\`bash
-export WEBFORAI_API_KEY=wfa_...       # from https://platform.webforai.dev/dashboard
-npx webforai https://example.com --engine browser --json
-\`\`\`
-
-\`WEBFORAI_PLATFORM_URL\` overrides the host for self-hosted deployments. Requests are metered
-in credits (fetch 1, proxy-fetch 2, browser/proxy-browser 5); failures are never billed.
-
-## Exit codes
-
-- \`0\` \u2014 success.
-- \`1\` \u2014 runtime failure (network, conversion, platform error; message on stderr).
-- \`2\` \u2014 usage error (unknown flag or invalid value; message on stderr).
-
-## Errors worth handling
-
-- \`payment_required\` on stderr \u2192 platform credits exhausted.
-- \`rate_limited\` \u2192 back off (a \`retryAfter\` seconds hint is included when known).
-- Playwright loader errors explain the exact \`npx playwright install\` command to run.
-`;
-
-// src/cli/commands/skill/index.ts
-var buildSkillsAddArgs = (source, flags) => {
-  const args = ["-y", "skills", "add", source];
-  if (flags.global) {
-    args.push("--global");
-  }
-  if (flags.agent) {
-    args.push("--agent", flags.agent);
-  }
-  if (flags.yes) {
-    args.push("-y");
-  }
-  if (flags.copy) {
-    args.push("--copy");
-  }
-  if (flags.all) {
-    args.push("--all");
-  }
-  return args;
-};
-var runSkillsAdd = (args) => new Promise((resolve, reject) => {
-  const child = spawn("npx", args, { stdio: ["inherit", "inherit", "inherit"] });
-  child.on("error", reject);
-  child.on("close", (code) => resolve(code ?? 1));
-});
-var skillCommand = async (flags) => {
-  const markdown = renderSkillMarkdown(package_default.version);
-  if (!(flags.install || flags.dir)) {
-    process.stdout.write(markdown);
-    return;
-  }
-  if (flags.dir) {
-    const target = path.join(flags.dir, "webforai", "SKILL.md");
-    await fs.mkdir(path.dirname(target), { recursive: true });
-    await fs.writeFile(target, markdown);
-    console.error(`[webforai] Agent Skill written to ${target}`);
-    return;
-  }
-  const stage = await fs.mkdtemp(path.join(os.tmpdir(), "webforai-skill-"));
-  try {
-    await fs.mkdir(path.join(stage, "webforai"), { recursive: true });
-    await fs.writeFile(path.join(stage, "webforai", "SKILL.md"), markdown);
-    const args = buildSkillsAddArgs(stage, flags);
-    console.error(`[webforai] running: npx ${args.join(" ")}`);
-    const code = await runSkillsAdd(args).catch((error) => {
-      console.error(
-        `[webforai] could not launch the skills CLI (${error instanceof Error ? error.message : String(error)}).`
-      );
-      console.error("[webforai] fallback: `webforai skill --dir .claude/skills` writes the file directly.");
-      return 1;
-    });
-    if (code !== 0) {
-      process.exitCode = 1;
-    }
-  } finally {
-    await fs.rm(stage, { recursive: true, force: true });
-  }
-};
-
-// src/cli/commands/webforai/interactive.ts
-import { intro, log, outro, select as select7, spinner, text as text5 } from "@clack/prompts";
-import pc2 from "picocolors";
-
 // src/platform/error.ts
 var PlatformApiError = class extends Error {
   constructor(code, message, status, retryAfter) {
@@ -336,7 +190,9 @@ var resolveFetch = (custom) => {
 
 // src/platform/types.ts
 var ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
-var REGIONS = ["auto", "us", "eu", "uk", "jp", "asia"];
+var REQUESTED_ENGINES = [...ENGINES, "auto"];
+var REGIONS = ["auto", "jp"];
+var CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
 var isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
 
 // src/platform/client.ts
@@ -365,16 +221,30 @@ var parseRetryAfter = (body, response) => {
   const header = Number(response.headers.get("retry-after"));
   return Number.isFinite(header) && header > 0 ? header : void 0;
 };
-var toApiError = (response, body) => {
+var dashboardHint = (status, dashboardUrl) => {
+  if (status === 401) {
+    return `create or check API keys at ${dashboardUrl}`;
+  }
+  if (status === 402) {
+    return `manage billing, credits and the spend cap at ${dashboardUrl}`;
+  }
+  return void 0;
+};
+var toApiError = (response, body, dashboardUrl) => {
   const envelope = body;
   const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
-  const message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
+  if (hint && !message.includes("/dashboard")) {
+    message = `${message} (${hint})`;
+  }
   return new PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
 };
 var createPlatformClient = (options = {}) => {
   const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const dashboardUrl = `${baseUrl}/dashboard`;
   const fetchImpl = resolveFetch(options.fetch);
-  const request = async (path4, init) => {
+  const request = async (path6, init) => {
     const headers = { accept: "application/json" };
     if (init.body !== void 0) {
       headers["content-type"] = "application/json";
@@ -383,27 +253,28 @@ var createPlatformClient = (options = {}) => {
       if (!options.apiKey) {
         throw new PlatformApiError(
           "missing_api_key",
-          "This endpoint needs an API key: pass `apiKey` to createPlatformClient (create one on the platform dashboard).",
+          `This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
           0
         );
       }
       headers.authorization = `Bearer ${options.apiKey}`;
     }
-    const response = await fetchImpl(`${baseUrl}${path4}`, {
+    const response = await fetchImpl(`${baseUrl}${path6}`, {
       method: init.method,
       headers,
-      body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
+      body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
+      ...init.signal ? { signal: init.signal } : {}
     });
     const body = await response.json().catch(() => void 0);
     if (!response.ok) {
-      throw toApiError(response, body);
+      throw toApiError(response, body, dashboardUrl);
     }
     if (body === void 0) {
       throw new PlatformApiError("invalid_response", "response body is not JSON", response.status);
     }
     return body;
   };
-  const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
+  const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
   const getJobResults = (jobId, opts) => {
     const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
     return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
@@ -419,6 +290,17 @@ var createPlatformClient = (options = {}) => {
     }
     return await response.json();
   };
+  const getJobOrBackOff = async (jobId, signal, interval) => {
+    try {
+      return await getJob(jobId, signal);
+    } catch (error) {
+      if (!(error instanceof PlatformApiError && error.status === 429)) {
+        throw error;
+      }
+      await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
+      return void 0;
+    }
+  };
   return {
     scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
     scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
@@ -429,17 +311,48 @@ var createPlatformClient = (options = {}) => {
     waitForJob: async (jobId, opts = {}) => {
       const interval = opts.pollIntervalMs ?? 2e3;
       const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
-      const startedAt = Date.now();
-      for (; ; ) {
-        const status = await getJob(jobId);
-        opts.onStatus?.(status);
-        if (TERMINAL_STATES.has(status.status)) {
-          return status;
-        }
-        if (Date.now() - startedAt + interval > timeout) {
-          throw new PlatformApiError("poll_timeout", `job ${jobId} still ${status.status} after ${timeout}ms`, 0);
+      if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
+        throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
+      }
+      if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
+        throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
+      }
+      opts.signal?.throwIfAborted();
+      const controller = new AbortController();
+      const { signal } = controller;
+      const onAbort = () => controller.abort(opts.signal?.reason);
+      opts.signal?.addEventListener("abort", onAbort, { once: true });
+      const timer = setTimeout(
+        () => controller.abort(new PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
+        timeout
+      );
+      let rejectOnAbort = () => {
+      };
+      const aborted = new Promise((_resolve, reject) => {
+        rejectOnAbort = () => reject(signal.reason);
+        signal.addEventListener("abort", rejectOnAbort, { once: true });
+      });
+      const poll = async () => {
+        for (; ; ) {
+          signal.throwIfAborted();
+          const status = await getJobOrBackOff(jobId, signal, interval);
+          signal.throwIfAborted();
+          if (status === void 0) {
+            continue;
+          }
+          opts.onStatus?.(status);
+          if (TERMINAL_STATES.has(status.status)) {
+            return status;
+          }
+          await sleep(interval, signal);
         }
-        await sleep(interval, opts.signal);
+      };
+      try {
+        return await Promise.race([poll(), aborted]);
+      } finally {
+        clearTimeout(timer);
+        opts.signal?.removeEventListener("abort", onAbort);
+        signal.removeEventListener("abort", rejectOnAbort);
       }
     },
     jobResults: async function* (jobId) {
@@ -456,6 +369,9 @@ var createPlatformClient = (options = {}) => {
   };
 };
 
+// src/cli/commands/jobs/options.ts
+import fs3 from "node:fs";
+
 // src/cli/constants.ts
 var DEFAULT_PATH = "https://example.com";
 var LOADERS = ["fetch", "playwright", "platform"];
@@ -464,6 +380,850 @@ var EXTRACTORS = ["auto", "takumi", "minimal", "none"];
 var API_KEY_ENV = "WEBFORAI_API_KEY";
 var PLATFORM_URL_ENV = "WEBFORAI_PLATFORM_URL";
 
+// src/cli/commands/webforai/options.ts
+import fs2 from "node:fs";
+
+// src/cli/utils.ts
+import fs from "node:fs";
+import path from "node:path";
+var isUrl = (maybeUrl) => {
+  try {
+    new URL(maybeUrl);
+    return true;
+  } catch {
+    return false;
+  }
+};
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
+      }
+      if (acc.length >= 2) {
+        return acc;
+      }
+      if (part === "www") {
+        return acc;
+      }
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+var sourcePathToOutputPath = (sourcePath) => {
+  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+};
+function getNextAvailableFilePath(filePath, exists = fs.existsSync) {
+  const parsedPath = path.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  const match = firstPart.match(/_(\d+)$/);
+  let counter = match ? Number.parseInt(match[1], 10) + 1 : 1;
+  let nextFilePath = filePath;
+  while (exists(nextFilePath)) {
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = path.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+var platformDashboardUrl = (platformUrl) => `${(platformUrl || DEFAULT_BASE_URL).replace(/\/+$/u, "")}/dashboard`;
+
+// src/cli/commands/webforai/options.ts
+var UsageError = class extends Error {
+};
+var oneOf = (name, value, allowed) => {
+  if (!allowed.includes(value)) {
+    throw new UsageError(`invalid --${name} "${value}" (expected: ${allowed.join(" | ")})`);
+  }
+  return value;
+};
+var impliesPlatform = (flags) => flags.engine !== void 0 || flags.region !== void 0;
+var resolveLoader = (sourceIsUrl, flags) => {
+  if (flags.loader) {
+    const loader = oneOf("loader", flags.loader, LOADERS);
+    if (impliesPlatform(flags) && loader !== "platform") {
+      throw new UsageError(`--engine/--region require --loader platform (received --loader ${loader})`);
+    }
+    return loader;
+  }
+  if (impliesPlatform(flags)) {
+    return "platform";
+  }
+  return sourceIsUrl ? "fetch" : "local";
+};
+var assertUsableLocalSource = (source, loader, flags) => {
+  if (loader !== "local" && flags.loader) {
+    throw new UsageError(`--loader ${loader} needs a URL source, but "${source}" is a local path`);
+  }
+  if (impliesPlatform(flags)) {
+    throw new UsageError("--engine/--region need a URL source (the platform cannot read local files)");
+  }
+  if (!fs2.existsSync(source)) {
+    throw new UsageError(`source not found: "${source}" (URLs must start with http:// or https://)`);
+  }
+};
+var applyPlatformFields = (resolved, flags, env) => {
+  resolved.engine = flags.engine ? oneOf("engine", flags.engine, REQUESTED_ENGINES) : void 0;
+  resolved.region = flags.region ? oneOf("region", flags.region, REGIONS) : void 0;
+  resolved.apiKey = flags.apiKey ?? env[API_KEY_ENV];
+  resolved.platformUrl = flags.platformUrl ?? env[PLATFORM_URL_ENV];
+  if (!resolved.apiKey) {
+    throw missingApiKeyError(resolved.platformUrl);
+  }
+};
+var missingApiKeyError = (platformUrl) => new UsageError(
+  `the platform loader needs an API key: pass --api-key or set ${API_KEY_ENV} (create one at ${platformDashboardUrl(
+    platformUrl
+  )}; 1,000 free credits/month, no card)`
+);
+var resolveRunOptions = (source, flags, env = process.env) => {
+  const sourceIsUrl = isUrl(source);
+  const loader = resolveLoader(sourceIsUrl, flags);
+  if (!sourceIsUrl) {
+    assertUsableLocalSource(source, loader, flags);
+  }
+  const resolved = {
+    source,
+    loader: sourceIsUrl ? loader : "local",
+    mode: flags.mode ? oneOf("mode", flags.mode, MODES) : "default",
+    extractor: flags.extractor ? oneOf("extractor", flags.extractor, EXTRACTORS) : "auto",
+    frontmatter: flags.frontmatter ?? false,
+    output: flags.output,
+    json: flags.json ?? false,
+    debug: flags.debug ?? false,
+    screenshot: flags.screenshot ?? false,
+    respectRobotsTxt: flags.respectRobots ?? false
+  };
+  if (resolved.loader === "platform") {
+    applyPlatformFields(resolved, flags, env);
+  } else if (flags.screenshot) {
+    throw new UsageError("--screenshot is only available with the platform loader (browser engines)");
+  } else if (flags.respectRobots) {
+    throw new UsageError("--respect-robots is only available with the platform loader");
+  }
+  return resolved;
+};
+
+// src/cli/commands/jobs/options.ts
+var MAX_BATCH_URLS = 100;
+var MAX_CRAWL_DEPTH = 5;
+var MAX_CRAWL_LIMIT = 500;
+var DEFAULT_TIMEOUT_SECONDS = 30 * 60;
+var isHttpUrl = (value) => {
+  try {
+    const { protocol } = new URL(value);
+    return protocol === "http:" || protocol === "https:";
+  } catch {
+    return false;
+  }
+};
+var oneOf2 = (name, value, allowed) => {
+  if (!allowed.includes(value)) {
+    throw new UsageError(`invalid --${name} "${value}" (expected: ${allowed.join(" | ")})`);
+  }
+  return value;
+};
+var integerIn = (name, value, min, max) => {
+  if (value === void 0) {
+    return void 0;
+  }
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
+    throw new UsageError(`invalid --${name} "${value}" (expected an integer from ${min} to ${max})`);
+  }
+  return parsed;
+};
+var patterns = (name, values) => {
+  if (!values || values.length === 0) {
+    return void 0;
+  }
+  for (const value of values) {
+    try {
+      new RegExp(value);
+    } catch {
+      throw new UsageError(`invalid --${name} pattern "${value}" (expected a regular expression)`);
+    }
+  }
+  return values;
+};
+var parseUrlList = (text6) => text6.split(/\r?\n/u).map((line) => line.trim()).filter((line) => line !== "" && !line.startsWith("#"));
+var readUrlFile = (file, readFile) => {
+  try {
+    return parseUrlList(readFile(file));
+  } catch (error) {
+    throw new UsageError(`cannot read --file "${file}": ${error instanceof Error ? error.message : String(error)}`);
+  }
+};
+var defaultReadFile = (file) => fs3.readFileSync(file === "-" ? 0 : file, "utf-8");
+var resolveJobOptions = (kind, sources, flags, env = process.env, readFile = defaultReadFile) => {
+  const fromFile = kind === "batch" && flags.file ? readUrlFile(flags.file, readFile) : [];
+  const urls = [.../* @__PURE__ */ new Set([...sources, ...fromFile])];
+  if (kind === "crawl" && urls.length !== 1) {
+    throw new UsageError("crawl takes exactly one seed URL");
+  }
+  if (urls.length === 0) {
+    throw new UsageError("batch needs at least one URL (as arguments or via --file <path|->)");
+  }
+  if (urls.length > MAX_BATCH_URLS) {
+    throw new UsageError(
+      `batch takes at most ${MAX_BATCH_URLS} URLs per job (received ${urls.length}); split the list`
+    );
+  }
+  const invalid = urls.find((url) => !isHttpUrl(url));
+  if (invalid !== void 0) {
+    throw new UsageError(`not an http(s) URL: "${invalid}"`);
+  }
+  if (!flags.output) {
+    throw new UsageError(`${kind} writes one Markdown file per page: pass -o, --output <dir>`);
+  }
+  const apiKey = flags.apiKey ?? env[API_KEY_ENV];
+  const platformUrl = flags.platformUrl ?? env[PLATFORM_URL_ENV];
+  if (!apiKey) {
+    throw missingApiKeyError(platformUrl);
+  }
+  const timeoutSeconds = integerIn("timeout", flags.timeout, 1, 24 * 60 * 60) ?? DEFAULT_TIMEOUT_SECONDS;
+  return {
+    kind,
+    urls,
+    outputDir: flags.output,
+    engine: flags.engine ? oneOf2("engine", flags.engine, REQUESTED_ENGINES) : void 0,
+    region: flags.region ? oneOf2("region", flags.region, REGIONS) : void 0,
+    extractor: flags.extractor ? oneOf2("extractor", flags.extractor, EXTRACTORS) : "auto",
+    frontmatter: flags.frontmatter ?? false,
+    respectRobotsTxt: flags.respectRobots,
+    apiKey,
+    platformUrl,
+    json: flags.json ?? false,
+    debug: flags.debug ?? false,
+    timeoutMs: timeoutSeconds * 1e3,
+    maxDepth: integerIn("max-depth", flags.maxDepth, 0, MAX_CRAWL_DEPTH),
+    limit: integerIn("limit", flags.limit, 1, MAX_CRAWL_LIMIT),
+    includePaths: patterns("include", flags.include),
+    excludePaths: patterns("exclude", flags.exclude),
+    sitemap: flags.sitemap ? oneOf2("sitemap", flags.sitemap, CRAWL_SITEMAP_MODES) : void 0,
+    llmsTxt: flags.llmsTxt ?? false
+  };
+};
+
+// src/cli/commands/jobs/run.ts
+import fs4 from "node:fs/promises";
+import path3 from "node:path";
+
+// src/cli/commands/jobs/files.ts
+import { createHash } from "node:crypto";
+import path2 from "node:path";
+var MAX_SEGMENT_LENGTH = 80;
+var MAX_DEPTH = 12;
+var PAGE_EXTENSION = /\.(?:html?|php|aspx?|jsp|md)$/iu;
+var shortHash = (value) => createHash("sha256").update(value).digest("hex").slice(0, 8);
+var decodeSegment = (segment) => {
+  try {
+    return decodeURIComponent(segment);
+  } catch {
+    return segment;
+  }
+};
+var sanitizeSegment = (segment) => {
+  const cleaned = decodeSegment(segment).normalize("NFC").replace(/[<>:"/\\|?*\u0000-\u001f\u007f]+/gu, "-").replace(/\s+/gu, "-").replace(/-{2,}/gu, "-").replace(/^[-.]+|[-.]+$/gu, "");
+  if (cleaned.length <= MAX_SEGMENT_LENGTH) {
+    return cleaned;
+  }
+  return `${cleaned.slice(0, MAX_SEGMENT_LENGTH - 9)}-${shortHash(cleaned)}`;
+};
+var pageRelativePath = (pageUrl, options = {}) => {
+  const url = new URL(pageUrl);
+  const rawSegments = url.pathname.split("/");
+  const trailingSlash = url.pathname.endsWith("/");
+  const segments = rawSegments.map(sanitizeSegment).filter(Boolean).slice(0, MAX_DEPTH);
+  let leaf = trailingSlash || segments.length === 0 ? "index" : segments.pop() ?? "index";
+  leaf = leaf.replace(PAGE_EXTENSION, "") || "index";
+  if (url.search.length > 1) {
+    leaf = `${leaf}_q-${shortHash(url.search)}`;
+  }
+  const directories = options.includeHost ? [sanitizeSegment(url.host) || "host", ...segments] : segments;
+  return path2.posix.join(...directories, `${leaf}.md`);
+};
+var planOutputFiles = (urls, options = {}) => {
+  const taken = /* @__PURE__ */ new Set();
+  const sorted = [...new Set(urls)].sort();
+  return sorted.map((url) => {
+    const candidate = pageRelativePath(url, options);
+    const relativePath = getNextAvailableFilePath(candidate, (name) => taken.has(name.toLowerCase()));
+    taken.add(relativePath.toLowerCase());
+    return { url, relativePath };
+  });
+};
+var resolveInside = (outputDir, relativePath) => {
+  const root = path2.resolve(outputDir);
+  const target = path2.resolve(root, ...relativePath.split("/"));
+  if (target !== root && !target.startsWith(`${root}${path2.sep}`)) {
+    throw new Error(`refusing to write outside the output directory: ${relativePath}`);
+  }
+  return target;
+};
+
+// src/cli/commands/jobs/llms-txt.ts
+var str = (value) => typeof value === "string" && value.trim() !== "" ? value.trim().replace(/\s+/gu, " ") : void 0;
+var firstHeading = (markdown) => {
+  const match = /^#{1,2}[ \t]+(.+?)[ \t#]*$/mu.exec(markdown);
+  return match ? str(match[1]) : void 0;
+};
+var escapeLinkText = (text6) => text6.replace(/([\\[\]])/gu, "\\$1");
+var pageTitle = (page) => str(page.metadata.title) ?? firstHeading(page.markdown) ?? (new URL(page.url).pathname || page.url);
+var normalizeUrl = (value) => {
+  const url = new URL(value);
+  url.hash = "";
+  return url.href;
+};
+var findSeed = (site) => {
+  const seed = normalizeUrl(site.seedUrl);
+  return site.pages.find((page) => normalizeUrl(page.url) === seed);
+};
+var siteTitle = (site, seed) => str(seed?.metadata.siteName) ?? str(site.pages.find((page) => str(page.metadata.siteName))?.metadata.siteName) ?? str(seed?.metadata.title) ?? new URL(site.seedUrl).host;
+var sectionOf = (pageUrl) => {
+  const { pathname } = new URL(pageUrl);
+  const segments = pathname.split("/").filter(Boolean);
+  const first = segments[0];
+  if (!first || segments.length === 1 && !pathname.endsWith("/")) {
+    return "";
+  }
+  let decoded = first;
+  try {
+    decoded = decodeURIComponent(first);
+  } catch {
+  }
+  const words = decoded.replace(/[-_]+/gu, " ").trim();
+  return words.charAt(0).toUpperCase() + words.slice(1);
+};
+var orderPages = (pages) => {
+  const groups = /* @__PURE__ */ new Map();
+  for (const page of [...pages].sort((a, b) => a.url < b.url ? -1 : a.url > b.url ? 1 : 0)) {
+    const section = sectionOf(page.url);
+    groups.set(section, [...groups.get(section) ?? [], page]);
+  }
+  return [...groups.entries()].sort(([a], [b]) => a === "" ? -1 : b === "" ? 1 : a.localeCompare(b)).map(([section, grouped]) => ({ section, pages: grouped }));
+};
+var renderLlmsTxt = (site) => {
+  const seed = findSeed(site);
+  const lines = [`# ${siteTitle(site, seed)}`, ""];
+  const summary = str(seed?.metadata.description);
+  if (summary) {
+    lines.push(`> ${summary}`, "");
+  }
+  for (const { section, pages } of orderPages(site.pages)) {
+    lines.push(`## ${section || "Pages"}`, "");
+    for (const page of pages) {
+      const description = str(page.metadata.description);
+      const link = `- [${escapeLinkText(pageTitle(page))}](${page.url})`;
+      lines.push(description ? `${link}: ${description}` : link);
+    }
+    lines.push("");
+  }
+  return `${lines.join("\n").trimEnd()}
+`;
+};
+var renderLlmsFullTxt = (site) => {
+  const seed = findSeed(site);
+  const parts = [`# ${siteTitle(site, seed)}`];
+  for (const { pages } of orderPages(site.pages)) {
+    for (const page of pages) {
+      const body = page.markdown.trim();
+      const heading2 = /^#[ \t]/u.test(body) ? "" : `# ${pageTitle(page)}
+
+`;
+      parts.push(`${heading2}Source: ${page.url}
+
+${body}`);
+    }
+  }
+  return `${parts.join("\n\n---\n\n")}
+`;
+};
+
+// src/cli/commands/jobs/run.ts
+var POLL_INTERVAL_MS = 2e3;
+var MAX_RATE_LIMIT_RETRIES = 5;
+var MAX_LISTED_FAILURES = 20;
+var defaultIo = () => ({
+  stdout: (text6) => process.stdout.write(text6),
+  stderr: (text6) => process.stderr.write(text6),
+  isTTY: Boolean(process.stderr.isTTY),
+  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  fetch: globalThis.fetch.bind(globalThis)
+});
+var JobIncompleteError = class extends Error {
+};
+var progressLine = (kind, status) => {
+  const total = status.total > 0 ? `/${status.total}` : "";
+  const failed = status.failed > 0 ? ` \xB7 ${status.failed} failed` : "";
+  return `${kind} ${status.jobId} \xB7 ${status.status} \xB7 ${status.completed}${total} pages${failed} \xB7 ${status.credits} credits`;
+};
+var withRateLimitRetry = async (io, log2, call) => {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await call();
+    } catch (error) {
+      if (!(error instanceof PlatformApiError) || error.code !== "rate_limited" || attempt > MAX_RATE_LIMIT_RETRIES) {
+        throw error;
+      }
+      const seconds = Math.min(error.retryAfter ?? 2 ** attempt, 120);
+      log2(`rate limited; retrying in ${seconds}s`);
+      await io.sleep(seconds * 1e3);
+    }
+  }
+};
+var submit = (client, job) => {
+  const common = {
+    engine: job.engine,
+    region: job.region,
+    respectRobotsTxt: job.respectRobotsTxt,
+    convert: { extractor: job.extractor, frontmatter: job.frontmatter || void 0 }
+  };
+  if (job.kind === "batch") {
+    return client.batch({ urls: job.urls, ...common });
+  }
+  return client.crawl({
+    url: job.urls[0],
+    maxDepth: job.maxDepth,
+    limit: job.limit,
+    includePaths: job.includePaths,
+    excludePaths: job.excludePaths,
+    sitemap: job.sitemap,
+    ...common
+  });
+};
+var resolveItem = async (io, item) => {
+  if (!isStoredPageStub(item)) {
+    return item;
+  }
+  const response = await io.fetch(item.resultUrl);
+  if (!response.ok) {
+    throw new PlatformApiError(
+      "artifact_fetch_failed",
+      `stored result for ${item.url} could not be downloaded (HTTP ${response.status})`,
+      response.status
+    );
+  }
+  return await response.json();
+};
+var createReporter = (io, job) => {
+  let liveLine = false;
+  const clearLive = () => {
+    if (liveLine) {
+      io.stderr("\r\x1B[2K");
+      liveLine = false;
+    }
+  };
+  const log2 = (line) => {
+    clearLive();
+    io.stderr(`[webforai] ${line}
+`);
+  };
+  const status = (current) => {
+    if (io.isTTY) {
+      io.stderr(`\r\x1B[2K[webforai] ${progressLine(job.kind, current)}`);
+      liveLine = true;
+    } else if (job.debug) {
+      log2(progressLine(job.kind, current));
+    }
+  };
+  return { log: log2, status, clearLive };
+};
+var waitWithProgress = async (client, job, jobId, io, reporter) => {
+  const deadline = Date.now() + job.timeoutMs;
+  try {
+    return await withRateLimitRetry(
+      io,
+      reporter.log,
+      () => client.waitForJob(jobId, {
+        pollIntervalMs: POLL_INTERVAL_MS,
+        timeoutMs: Math.max(1, deadline - Date.now()),
+        onStatus: reporter.status
+      })
+    );
+  } catch (error) {
+    if (error instanceof PlatformApiError && error.code === "poll_timeout") {
+      throw new PlatformApiError(
+        "poll_timeout",
+        `${error.message}; the job keeps running on the platform and its results stay available for 7 days (GET /v1/jobs/${jobId}/results)`,
+        0
+      );
+    }
+    throw error;
+  } finally {
+    reporter.clearLive();
+  }
+};
+var collectResults = async (client, jobId, io, reporter) => {
+  const successes = [];
+  const failures = [];
+  let cursor;
+  do {
+    const page = await withRateLimitRetry(io, reporter.log, () => client.getJobResults(jobId, { cursor }));
+    for (const item of page.results) {
+      const result = await resolveItem(io, item);
+      if (result.status === "ok") {
+        successes.push(result);
+      } else {
+        failures.push(result);
+      }
+    }
+    cursor = page.cursor;
+  } while (cursor);
+  return { successes, failures };
+};
+var writePages = async (job, successes) => {
+  const byUrl = new Map(successes.map((page) => [page.url, page]));
+  const written = [];
+  await fs4.mkdir(job.outputDir, { recursive: true });
+  for (const { url, relativePath } of planOutputFiles([...byUrl.keys()], { includeHost: job.kind === "batch" })) {
+    const page = byUrl.get(url);
+    const target = resolveInside(job.outputDir, relativePath);
+    await fs4.mkdir(path3.dirname(target), { recursive: true });
+    await fs4.writeFile(target, page.markdown.endsWith("\n") ? page.markdown : `${page.markdown}
+`);
+    written.push({
+      url,
+      file: path3.join(job.outputDir, relativePath),
+      title: typeof page.metadata.title === "string" ? page.metadata.title : void 0,
+      engine: page.engine,
+      credits: page.credits,
+      warning: page.warning
+    });
+  }
+  return written;
+};
+var writeLlmsFiles = async (job, successes, envelope) => {
+  const pages = [...new Map(successes.map((page) => [page.url, page])).values()];
+  const site = { seedUrl: job.urls[0], pages };
+  envelope.llmsTxt = path3.join(job.outputDir, "llms.txt");
+  envelope.llmsFullTxt = path3.join(job.outputDir, "llms-full.txt");
+  await fs4.writeFile(envelope.llmsTxt, renderLlmsTxt(site));
+  await fs4.writeFile(envelope.llmsFullTxt, renderLlmsFullTxt(site));
+};
+var report = (job, envelope, io, log2) => {
+  if (job.json) {
+    io.stdout(`${JSON.stringify(envelope, null, 2)}
+`);
+  } else {
+    for (const page of envelope.pages) {
+      io.stdout(`${page.file}
+`);
+    }
+  }
+  const failed = envelope.failures.length;
+  log2(
+    `${job.kind} ${envelope.jobId} ${envelope.status}: ${envelope.pages.length} page(s) written to ${job.outputDir}${failed > 0 ? `, ${failed} failed` : ""} \xB7 ${envelope.credits} credits`
+  );
+  if (envelope.llmsTxt) {
+    log2(`llms.txt and llms-full.txt written to ${job.outputDir}`);
+  }
+  for (const failure of envelope.failures.slice(0, MAX_LISTED_FAILURES)) {
+    log2(`  failed ${failure.url} \u2014 ${failure.code}: ${failure.message}`);
+  }
+  if (failed > MAX_LISTED_FAILURES) {
+    log2(`  \u2026 and ${failed - MAX_LISTED_FAILURES} more (see --json)`);
+  }
+};
+var runJobCommand = async (job, io = defaultIo()) => {
+  const client = createPlatformClient({ apiKey: job.apiKey, baseUrl: job.platformUrl, fetch: io.fetch });
+  const reporter = createReporter(io, job);
+  const { jobId } = await withRateLimitRetry(io, reporter.log, () => submit(client, job));
+  reporter.log(
+    `${job.kind} job ${jobId} submitted (${job.kind === "crawl" ? job.urls[0] : `${job.urls.length} URLs`})`
+  );
+  const final = await waitWithProgress(client, job, jobId, io, reporter);
+  const { successes, failures } = await collectResults(client, jobId, io, reporter);
+  const envelope = {
+    jobId,
+    type: job.kind,
+    status: final.status,
+    outputDir: job.outputDir,
+    credits: final.credits,
+    pages: await writePages(job, successes),
+    failures: failures.map(({ url, error }) => ({ url, code: error.code, message: error.message })),
+    error: final.error
+  };
+  if (job.llmsTxt && successes.length > 0) {
+    await writeLlmsFiles(job, successes, envelope);
+  }
+  report(job, envelope, io, reporter.log);
+  if (final.status === "failed") {
+    throw new JobIncompleteError(`${job.kind} job ${jobId} failed${final.error ? `: ${final.error}` : ""}`);
+  }
+  if (envelope.pages.length === 0) {
+    throw new JobIncompleteError(`${job.kind} job ${jobId} converted no pages`);
+  }
+  return envelope;
+};
+
+// src/cli/commands/skill/index.ts
+import { spawn } from "node:child_process";
+import fs5 from "node:fs/promises";
+import os from "node:os";
+import path4 from "node:path";
+
+// src/cli/commands/skill/content.ts
+var renderSkillMarkdown = (version) => `---
+name: webforai
+description: Convert web pages or local HTML files to clean, LLM-ready Markdown with the webforai CLI. Use when a task needs page content as Markdown - reading documentation or articles, scraping, summarising URLs, crawling a docs site into files or llms.txt, building text corpora - including JavaScript-heavy or bot-protected pages via the hosted platform.
+---
+
+# webforai CLI${version ? ` (v${version})` : ""}
+
+Convert a URL or local HTML file to Markdown. Non-interactive by default: the Markdown goes
+to stdout, every log goes to stderr, so output can be piped or captured directly.
+
+## Basic usage
+
+\`\`\`bash
+npx webforai https://example.com/article            # Markdown to stdout
+npx webforai https://example.com/article -o out.md  # write to a file instead
+npx webforai ./page.html                            # convert a local HTML file
+npx webforai https://example.com --json             # machine-readable envelope (see below)
+\`\`\`
+
+## Which command
+
+- One page, or a handful you will read right away \u2192 \`npx webforai <url>\` (free, local;
+  add \`--engine auto\` when the page needs JavaScript or blocks plain fetches).
+- A known list of URLs (2\u2013100) to keep as files \u2192 \`npx webforai batch\` (hosted platform).
+- A whole site or docs section \u2192 \`npx webforai crawl\` (hosted platform); add \`--llms-txt\`
+  for a single \`llms-full.txt\` you can read in one go.
+
+## Flags
+
+- \`-o, --output <path>\` \u2014 write Markdown to a file instead of stdout.
+- \`-l, --loader <fetch|playwright|platform>\` \u2014 how HTML is acquired. Default: \`fetch\` for
+  URLs, automatic \`local\` for file paths. \`playwright\` renders JavaScript locally (needs
+  \`npx playwright install chromium\`). \`platform\` uses the hosted API (below).
+- \`-m, --mode <default|ai>\` \u2014 \`ai\` strips links/tables/images down to plain text
+  (local conversion only).
+- \`--extractor <auto|takumi|minimal|none>\` \u2014 main-content extraction preset
+  (\`none\` converts the whole page).
+- \`--frontmatter\` \u2014 prepend YAML front matter (title, author, canonical URL, ...).
+- \`--json\` \u2014 print a JSON envelope to stdout instead of raw Markdown.
+- \`--engine <auto|fetch|browser|proxy-fetch|proxy-browser>\` \u2014 platform acquisition engine
+  (implies \`-l platform\`). \`auto\` (the platform default) starts with a plain fetch and
+  escalates to browser rendering when the page is a client-rendered shell;
+  \`browser\`/\`proxy-browser\` always render JavaScript; \`proxy-*\` engines egress through
+  rotating proxies for bot-protected sites (paid plan only).
+- \`--region <auto|jp>\` \u2014 platform proxy egress region (implies \`-l platform\`); \`jp\` runs
+  on the proxy tier, so it needs a paid plan too.
+- \`--screenshot\` \u2014 platform browser engines only; the envelope carries an expiring
+  \`screenshotUrl\`.
+- \`--respect-robots\` \u2014 platform loader only; honor the target site's robots.txt rules
+  (off by default). A disallowed URL fails with \`robots_disallowed\` and is not billed.
+- \`--api-key <key>\` / \`--platform-url <url>\` \u2014 platform credentials and host; usually set
+  via env vars instead.
+- \`-i, --interactive\` \u2014 guided prompt flow (for humans; do not use from an agent).
+- \`-d, --debug\` \u2014 verbose logs on stderr.
+
+## JSON envelope (\`--json\`)
+
+\`\`\`json
+{
+  "source": "https://example.com",
+  "loader": "fetch",
+  "url": "https://example.com",
+  "markdown": "# ...",
+  "metadata": { "title": "..." },
+  "engine": "browser",
+  "region": "auto",
+  "credits": 5,
+  "screenshotUrl": "https://...",
+  "output": "out.md"
+}
+\`\`\`
+
+\`engine\`/\`region\`/\`credits\`/\`screenshotUrl\` appear only for the platform loader; \`output\`
+only when \`-o\` also wrote a file. A \`warning\` field appears when the result is probably
+degraded \u2014 most commonly a client-rendered shell fetched without JavaScript; rerun with
+\`--engine auto\` (or \`-l playwright\`) when you see one.
+
+## Hosted platform
+
+For JavaScript-heavy or bot-protected pages, the hosted platform fetches server-side:
+
+\`\`\`bash
+export WEBFORAI_API_KEY=wfa_...       # from https://platform.webforai.dev/dashboard
+npx webforai https://example.com --engine auto --json
+\`\`\`
+
+\`WEBFORAI_PLATFORM_URL\` overrides the host for self-hosted deployments. Accounts get 1,000
+free credits per month; failures are never billed.
+
+| operation | credits |
+| --- | --- |
+| \`fetch\` | 1 |
+| \`browser\` | 2 |
+| \`proxy-fetch\` (paid plan only) | 2 |
+| \`proxy-browser\` (paid plan only) | 3 |
+| \`--screenshot\` | +1 |
+| image rehosting (API \`rehostImages\`) | +1 per started 5 images |
+
+\`auto\` bills the engine that actually ran (1, or 2 when it needed a browser). The proxy
+engines \u2014 and \`--region jp\`, which runs on them \u2014 need an active subscription; on the free
+allowance they fail with \`payment_required\`. Prefer \`--engine auto\` unless a site blocks
+it. Batch and crawl jobs bill per converted page.
+
+## Crawl and batch (hosted platform)
+
+\`\`\`bash
+# a docs site \u2192 one .md per page, mirroring URL paths, plus llms.txt / llms-full.txt
+npx webforai crawl https://docs.example.com -o docs --limit 100 --include '^/docs' --llms-txt
+
+# a list of URLs (arguments, or one per line via --file; '-' reads stdin)
+npx webforai batch https://a.example/post https://b.example/news -o pages
+npx webforai batch --file urls.txt -o pages --json
+\`\`\`
+
+Both submit an async job, wait for it, then write every converted page under \`-o <dir>\`
+(\`/docs/intro\` \u2192 \`docs/intro.md\`; batch prefixes the host: \`a.example/post.md\`). stdout
+lists the written files, one per line; \`--json\` prints \`{ jobId, status, credits, pages:
+[{ url, file, title }], failures: [{ url, code, message }], llmsTxt?, llmsFullTxt? }\`
+instead. Progress and the summary go to stderr. Exit code 1 when the job failed or no page
+converted; individual failed pages are listed but do not fail the command.
+
+- \`crawl <url>\` \u2014 same-origin only. \`--max-depth <0-5>\` (default 2), \`--limit <1-500>\`
+  (default 50), \`--include <regex>\` / \`--exclude <regex>\` on the pathname (repeatable).
+  \`--sitemap <skip|include|only>\`: \`include\` adds the site's sitemap URLs to the links it
+  follows, \`only\` crawls just the seed plus sitemap URLs (best for blogs and large docs with
+  a good sitemap). Honors robots.txt by default; \`--no-respect-robots\` turns that off.
+- \`--llms-txt\` (crawl) \u2014 also write \`llms.txt\` (site title, summary, \`- [title](url):
+  description\` links grouped by top-level path) and \`llms-full.txt\` (every page's Markdown
+  concatenated) per llmstxt.org. Read \`llms-full.txt\` when you need the whole site in context.
+- \`batch [urls...]\` \u2014 up to 100 URLs per job; \`-f, --file <path|->\` adds URLs from a file.
+  \`--respect-robots\` opts in to robots.txt checks.
+- Shared: \`--engine\`, \`--region\`, \`--extractor\`, \`--frontmatter\`, \`--json\`,
+  \`--timeout <seconds>\` (default 1800; the job keeps running after a timeout and results stay
+  available for 7 days), \`--api-key\`, \`--platform-url\`, \`-d\`.
+
+## Exit codes
+
+- \`0\` \u2014 success.
+- \`1\` \u2014 runtime failure (network, conversion, platform error; message on stderr).
+- \`2\` \u2014 usage error (unknown flag or invalid value; message on stderr).
+
+## Errors worth handling
+
+Platform errors print as \`error: <code>: <message>\` on stderr (exit 1). What to do:
+
+- \`invalid_api_key\` (401) \u2192 the key is missing, revoked or mistyped. Do not retry; ask the
+  user for a key from the dashboard named in the message.
+- \`payment_required\` (402) \u2192 the free monthly allowance is used up, or a proxy engine /
+  \`--region jp\` was requested without a subscription. Do not retry; fall back to
+  \`--engine auto\` (or the local loaders) for proxy requests, otherwise tell the user.
+- \`spend_cap_reached\` (402) \u2192 the account's monthly spend cap (default $50) is reached; the
+  user can raise it on the dashboard. Do not retry.
+- \`rate_limited\` (429) \u2192 per-account limit: 60 requests/minute (free), 600 (paid). Wait the
+  \`retryAfter\` seconds shown, then retry; for many URLs use one \`batch\` job instead of a
+  loop of single requests. \`crawl\`/\`batch\` retry this automatically.
+- \`too_many_jobs\` (429) \u2192 3 (free) / 20 (paid) batch/crawl jobs already queued or running.
+  Wait for one to finish, then resubmit.
+- \`robots_disallowed\` (403) \u2192 robots.txt disallows the URL (\`--respect-robots\`, or a crawl
+  by default). Not billed; respect it unless the user explicitly decides otherwise.
+- \`invalid_request\` (400) \u2192 a bad flag value or URL; fix the input.
+- \`engine_unavailable\` / \`scheduling_unknown\` / \`result_unavailable\` (503) \u2192 transient;
+  retry later.
+- Playwright loader errors explain the exact \`npx playwright install\` command to run.
+`;
+
+// src/cli/commands/skill/index.ts
+var buildSkillsAddArgs = (source, flags) => {
+  const args = ["-y", "skills", "add", source];
+  if (flags.global) {
+    args.push("--global");
+  }
+  if (flags.agent) {
+    args.push("--agent", flags.agent);
+  }
+  if (flags.yes) {
+    args.push("-y");
+  }
+  if (flags.copy) {
+    args.push("--copy");
+  }
+  if (flags.all) {
+    args.push("--all");
+  }
+  return args;
+};
+var runSkillsAdd = (args) => new Promise((resolve, reject) => {
+  const child = spawn("npx", args, { stdio: ["inherit", "inherit", "inherit"] });
+  child.on("error", reject);
+  child.on("close", (code) => resolve(code ?? 1));
+});
+var skillCommand = async (flags) => {
+  const markdown = renderSkillMarkdown(package_default.version);
+  if (!(flags.install || flags.dir)) {
+    process.stdout.write(markdown);
+    return;
+  }
+  if (flags.dir) {
+    const target = path4.join(flags.dir, "webforai", "SKILL.md");
+    await fs5.mkdir(path4.dirname(target), { recursive: true });
+    await fs5.writeFile(target, markdown);
+    console.error(`[webforai] Agent Skill written to ${target}`);
+    return;
+  }
+  const stage = await fs5.mkdtemp(path4.join(os.tmpdir(), "webforai-skill-"));
+  try {
+    await fs5.mkdir(path4.join(stage, "webforai"), { recursive: true });
+    await fs5.writeFile(path4.join(stage, "webforai", "SKILL.md"), markdown);
+    const args = buildSkillsAddArgs(stage, flags);
+    console.error(`[webforai] running: npx ${args.join(" ")}`);
+    const code = await runSkillsAdd(args).catch((error) => {
+      console.error(
+        `[webforai] could not launch the skills CLI (${error instanceof Error ? error.message : String(error)}).`
+      );
+      console.error("[webforai] fallback: `webforai skill --dir .claude/skills` writes the file directly.");
+      return 1;
+    });
+    if (code !== 0) {
+      process.exitCode = 1;
+    }
+  } finally {
+    await fs5.rm(stage, { recursive: true, force: true });
+  }
+};
+
+// src/cli/commands/webforai/interactive.ts
+import { intro, log, outro, select as select7, spinner, text as text5 } from "@clack/prompts";
+import pc2 from "picocolors";
+
 // src/cli/helpers/assertContinue.ts
 import { cancel, isCancel } from "@clack/prompts";
 function assertContinue(message, cancelMessage = "Canceled.") {
@@ -474,89 +1234,8 @@ function assertContinue(message, cancelMessage = "Canceled.") {
 }
 
 // src/cli/helpers/inputOutputPath.ts
-import fs3 from "node:fs";
+import fs6 from "node:fs";
 import { confirm, text } from "@clack/prompts";
-
-// src/cli/utils.ts
-import fs2 from "node:fs";
-import path2 from "node:path";
-var isUrl = (maybeUrl) => {
-  try {
-    new URL(maybeUrl);
-    return true;
-  } catch {
-    return false;
-  }
-};
-function changeFileExtension(filePath, newExtension) {
-  const parsedPath = filePath.split("/");
-  const fileName = parsedPath[parsedPath.length - 1];
-  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
-  if (fileName.startsWith(".")) {
-    const parts = fileName.split(".");
-    if (parts.length === 2) {
-      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
-    }
-    parts[parts.length - 1] = newExtension.replace(/^\./, "");
-    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
-  }
-  const lastDotIndex = fileName.lastIndexOf(".");
-  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
-  const newFileName = `${baseName}${formattedNewExtension}`;
-  parsedPath[parsedPath.length - 1] = newFileName;
-  return parsedPath.join("/");
-}
-function urlToFilename(url) {
-  try {
-    const urlObj = new URL(url);
-    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
-      if (index === 0) {
-        return acc;
-      }
-      if (acc.length >= 2) {
-        return acc;
-      }
-      if (part === "www") {
-        return acc;
-      }
-      return [part, ...acc];
-    }, []);
-    const domainString = domainParts.reverse().join("-");
-    const pathParts = urlObj.pathname.split("/").filter(Boolean);
-    const relevantPathParts = pathParts.slice(-2);
-    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
-    let filename = [domainString, pathString].filter(Boolean).join("-");
-    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
-    return filename || "output";
-  } catch {
-    return "output";
-  }
-}
-var sourcePathToOutputPath = (sourcePath) => {
-  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
-};
-function getNextAvailableFilePath(filePath) {
-  const parsedPath = path2.parse(filePath);
-  const directory = parsedPath.dir;
-  const fullName = parsedPath.base;
-  const [firstPart, ...restParts] = fullName.split(".");
-  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
-  const baseName = firstPart.replace(/_\d+$/, "");
-  let counter = 1;
-  let nextFilePath = filePath;
-  while (fs2.existsSync(nextFilePath)) {
-    const match = firstPart.match(/_(\d+)$/);
-    if (match) {
-      counter = Number.parseInt(match[1], 10) + 1;
-    }
-    const newName = `${baseName}_${counter}${restName}`;
-    nextFilePath = path2.join(directory, newName);
-    counter++;
-  }
-  return nextFilePath;
-}
-
-// src/cli/helpers/inputOutputPath.ts
 var inputOutputPath = async (sourcePath) => {
   const outputPath = await text({
     message: "Enter the output file path:",
@@ -566,16 +1245,16 @@ var inputOutputPath = async (sourcePath) => {
       if (value.trim() === "") {
         return "Output path is required";
       }
-      if (fs3.existsSync(value) && fs3.statSync(value).isDirectory()) {
+      if (fs6.existsSync(value) && fs6.statSync(value).isDirectory()) {
         return "No directory can be specified.";
       }
-      if (!fs3.existsSync(value) && value.endsWith("/")) {
+      if (!fs6.existsSync(value) && value.endsWith("/")) {
         return "No directory can be specified.";
       }
     }
   });
   assertContinue(outputPath);
-  if (!fs3.existsSync(outputPath)) {
+  if (!fs6.existsSync(outputPath)) {
     return outputPath;
   }
   const isOutputFileOverwrite = await confirm({
@@ -594,10 +1273,10 @@ var inputOutputPath = async (sourcePath) => {
       if (value.trim() === "") {
         return "Output path is required";
       }
-      if (fs3.existsSync(value)) {
+      if (fs6.existsSync(value)) {
         return "The file already exists";
       }
-      if (!fs3.existsSync(value) && value.endsWith("/")) {
+      if (!fs6.existsSync(value) && value.endsWith("/")) {
         return "No directory can be specified.";
       }
     }
@@ -607,7 +1286,7 @@ var inputOutputPath = async (sourcePath) => {
 };
 
 // src/cli/helpers/inputSourcePath.ts
-import fs4 from "node:fs";
+import fs7 from "node:fs";
 import { text as text2 } from "@clack/prompts";
 var inputSourcePath = async () => {
   const result = await text2({
@@ -619,10 +1298,10 @@ var inputSourcePath = async () => {
         return "Source is required";
       }
       if (!isUrl(value)) {
-        if (!fs4.existsSync(value)) {
+        if (!fs7.existsSync(value)) {
           return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
         }
-        if (fs4.statSync(value).isDirectory()) {
+        if (fs7.statSync(value).isDirectory()) {
           return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
         }
       }
@@ -662,8 +1341,8 @@ var selectLoader = async () => {
 };
 
 // src/cli/commands/webforai/run.ts
-import fs6 from "node:fs/promises";
-import path3 from "node:path";
+import fs9 from "node:fs/promises";
+import path5 from "node:path";
 
 // src/html-to-markdown.ts
 import { fromHtml as fromHtml2 } from "hast-util-from-html";
@@ -1494,7 +2173,11 @@ var countAnchors = (element3) => {
   }
   return count;
 };
+var LITERAL_TEXT_TAGS = /* @__PURE__ */ new Set(["code", "kbd", "samp", "var", "pre"]);
 var isMidArticleWidget = (element3) => {
+  if (LITERAL_TEXT_TAGS.has(element3.tagName)) {
+    return false;
+  }
   if (element3.tagName === "form" || element3.tagName === "fieldset") {
     return true;
   }
@@ -2684,14 +3367,22 @@ var detectLanguage = (code) => {
 };
 
 // src/mdast-handlers/custom-code-handler.ts
-var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
+var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
+var attributeLanguage = (element3) => {
+  for (const name of LANGUAGE_ATTRIBUTES) {
+    const value = element3.properties?.[name];
+    if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
+      return value.trim();
+    }
+  }
+  return void 0;
+};
+var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
 var customCodeHandler = (state, node) => {
-  const classNames = node.properties?.className || [];
-  const codeValue = trimTrailingLines(toText(node)).trim();
-  const classLang = classNames.map((className) => {
-    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
-    return match?.[1];
-  }).find((className) => className);
+  const code = node.children.find((child) => isElement(child) && child.tagName === "code");
+  const codeValue = trimTrailingLines(toText(node));
+  const classLang = codeLanguage(...isElement(code) ? [code, node] : [node]);
   const lang = classLang || detectLanguage(codeValue) || null;
   const result = { type: "code", lang, meta: null, value: codeValue };
   state.patch(node, result);
@@ -2706,42 +3397,19 @@ import { toText as toText2 } from "hast-util-to-text";
 import { trimTrailingLines as trimTrailingLines2 } from "trim-trailing-lines";
 var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
 var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
-var LANGUAGE_MATCH_REGEX2 = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
-var findRecursive = (array, condition, maxDepth = 3) => {
-  if (maxDepth <= 0) {
-    return null;
-  }
-  for (const value of array) {
-    const result = condition(value);
-    if (Array.isArray(result)) {
-      return findRecursive(result, condition, maxDepth - 1);
-    }
-    if (result) {
-      return value;
-    }
-  }
-  return null;
-};
 var customDivHandler = (state, node) => {
-  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
-  const codeBlock = findRecursive(node.children, (child) => {
-    if (child.type !== "element") {
-      return false;
-    }
-    if (child.tagName === "pre") {
-      return true;
-    }
-    return child.children.filter((child2) => child2.type === "element");
-  });
-  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
-    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
-    const codeValue = trimTrailingLines2(toText2(codeBlock)).trim();
+  const classNames = classList(node);
+  if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    return defaultHandlers2.div(state, node);
+  }
+  const codeBlocks = collectElements(node, (element3) => element3.tagName === "pre");
+  const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
+  if (codeBlock) {
+    const codeValue = trimTrailingLines2(toText2(codeBlock));
     const filenameElement = select5(CODE_FILENAME_SELECTORS, node);
     const fileLang = filenameElement ? hastToString2(filenameElement).match(/\.(\w+)$/)?.[1] : null;
-    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
-      const match = LANGUAGE_MATCH_REGEX2.map((regex) => className.match(regex)).find((match2) => match2);
-      return match?.[1];
-    }).find((className) => className);
+    const code = codeBlock.children.find((child) => isElement(child) && child.tagName === "code");
+    const classLang = codeLanguage(...isElement(code) ? [code, codeBlock, node] : [codeBlock, node]);
     const lang = fileLang || classLang || detectLanguage(codeValue) || null;
     const result = { type: "code", lang, meta: null, value: codeValue };
     state.patch(node, result);
@@ -2837,6 +3505,20 @@ var hoistBoundaryWhitespace = (nodes) => {
   }
   return result;
 };
+var termsAndDefinitions = (parent) => {
+  const found = [];
+  for (const child of parent.children) {
+    if (child.type !== "element") {
+      continue;
+    }
+    if (child.tagName === "dt" || child.tagName === "dd") {
+      found.push(child);
+    } else if (child.tagName === "div") {
+      found.push(...termsAndDefinitions(child));
+    }
+  }
+  return found;
+};
 var definitionListHandler = (state, node) => {
   const items = [];
   let current;
@@ -2851,11 +3533,7 @@ var definitionListHandler = (state, node) => {
     items.push({ type: "listItem", spread: false, children: [heading2, ...current.definitions] });
     current = void 0;
   };
-  for (const child of node.children) {
-    if (child.type !== "element") {
-      continue;
-    }
-    const element3 = child;
+  for (const element3 of termsAndDefinitions(node)) {
     if (element3.tagName === "dt") {
       if (current && current.definitions.length > 0) {
         flush();
@@ -2874,7 +3552,8 @@ var definitionListHandler = (state, node) => {
   }
   flush();
   if (items.length === 0) {
-    return void 0;
+    const fallback = state.toFlow(state.all(node));
+    return fallback.length > 0 ? fallback : void 0;
   }
   const result = { type: "list", ordered: false, spread: false, children: items };
   state.patch(node, result);
@@ -2987,14 +3666,16 @@ var katexDuplicates = (node) => {
   if (!(classes.includes("katex") || classes.includes("katex-display"))) {
     return [];
   }
-  const hasMathml = node.children.some((child) => isElement(child) && classList(child).includes("katex-mathml"));
+  const hasMathml = node.children.some(
+    (child) => isElement(child) && classList(child).includes("katex-mathml") && hasMathSource(child)
+  );
   if (!hasMathml) {
     return [];
   }
   return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
 };
 var mediawikiDuplicates = (node) => {
-  if (!classList(node).includes("mwe-math-element")) {
+  if (!classList(node).includes("mwe-math-element") || !hasMathSource(node)) {
     return [];
   }
   return node.children.filter(
@@ -3005,8 +3686,15 @@ var mathjaxDuplicates = (node) => {
   if (node.tagName !== "mjx-container") {
     return [];
   }
+  const hasMathml = node.children.some(
+    (child) => isElement(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
+  );
+  if (!hasMathml) {
+    return [];
+  }
   return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
 };
+var hasMathSource = (node) => Boolean(findElement(node, (element3) => element3.tagName === "math" && element3.children.length > 0));
 var dedupeRenderedMath = (tree) => {
   const doomed = /* @__PURE__ */ new Set();
   walk(tree, (node) => {
@@ -3068,8 +3756,8 @@ var getLangFromHast = (node) => {
   }
   return void 0;
 };
-var getLangFromStr = (str) => {
-  const match = str.match(/lang=["']([^"']+)["']/);
+var getLangFromStr = (str2) => {
+  const match = str2.match(/lang=["']([^"']+)["']/);
   if (match) {
     return match[1];
   }
@@ -3096,7 +3784,7 @@ var htmlToMdast = (htmlOrHast, options) => {
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
   const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || getUrlFromHast(hast);
-  const extractedHast = pipeExtractors({ hast, lang, url, owned: isOwned }, extractors);
+  const extractedHast = pipeExtractors({ hast: isOwned ? hast : cloneHast(hast), lang, url, owned: true }, extractors);
   normalizeHast(extractedHast, options?.normalize);
   const mdast = toMdast(extractedHast, {
     handlers: {
@@ -3121,19 +3809,25 @@ import { mathToMarkdown } from "mdast-util-math";
 import { toMarkdown } from "mdast-util-to-markdown";
 
 // src/link-replacer.ts
-var linkReplacer = (markdown, base) => {
-  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
-  return markdown.replace(regex, (match, pre, url, post) => {
-    if (/^(https?:|#)/.test(url)) {
-      return match;
-    }
-    try {
-      const absoluteUrl = new URL(url, base).href;
-      return `${pre}${absoluteUrl}${post}`;
-    } catch {
-      return match;
-    }
-  });
+var absoluteUrl = (url, base) => {
+  if (/^(?:[a-z][a-z\d+.-]*:|#)/i.test(url)) {
+    return url;
+  }
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+};
+var linkReplacer = (node, base) => {
+  let result = node;
+  if (node.type === "link" || node.type === "image" || node.type === "definition") {
+    result = { ...node, url: absoluteUrl(node.url, base) };
+  }
+  if ("children" in result) {
+    return { ...result, children: result.children.map((child) => linkReplacer(child, base)) };
+  }
+  return result;
 };
 
 // src/utils/mdast-utils.ts
@@ -3151,11 +3845,8 @@ var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 var mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
-  if (baseUrl) {
-    markdown = linkReplacer(markdown, baseUrl);
-  }
-  return markdown;
+  const tree = warpRoot(mdast);
+  return toMarkdown(baseUrl ? linkReplacer(tree, baseUrl) : tree, toMarkdownOptions);
 };
 
 // src/metadata/index.ts
@@ -3269,7 +3960,7 @@ var documentTitle = (tree) => {
   }
   return title.children.map((child) => child.type === "text" ? child.value : "").join("");
 };
-var firstHeading = (tree) => {
+var firstHeading2 = (tree) => {
   const heading2 = findElement(tree, (element3) => element3.tagName === "h1");
   if (!heading2) {
     return void 0;
@@ -3319,7 +4010,7 @@ var extractMetadata = (tree) => {
     }
     return void 0;
   };
-  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading2(tree));
   return {
     title,
     description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),
@@ -3390,8 +4081,101 @@ ${body}`;
 };
 var normalizeForCompare = (value) => value.replace(/\s+/g, " ").trim().toLowerCase();
 
+// src/detect-client-shell.ts
+var SHELL_SUSPICION_THRESHOLD = 300;
+var SHELL_EMPTY_THRESHOLD = 50;
+var BODY_PATTERN = /<body[^>]*>([\s\S]*?)<\/body>/i;
+var HEAD_PATTERN = /<head[^>]*>[\s\S]*?<\/head>/gi;
+var NOSCRIPT_PATTERN = /<noscript[^>]*>([\s\S]*?)<\/noscript>/gi;
+var INVISIBLE_ELEMENT_PATTERN = /<(script|style|template|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi;
+var COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
+var TAG_PATTERN = /<[^>]+>/g;
+var MOUNT_POINT_PATTERN = /<[a-z][^>]*\sid=["'](?:root|app|__next|___gatsby|__nuxt|q-app|svelte|react-root)["']|<app-root[\s>]|\sdata-reactroot[\s>=]/i;
+var CHALLENGE_PATTERN = /cf-chl|challenge-platform|cf_chl_opt|just a moment|attention required|checking your browser|_incapsula_|px-captcha|datadome/i;
+var ENTITIES = {
+  "&nbsp;": " ",
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+var visibleTextOf = (markup) => markup.replace(COMMENT_PATTERN, " ").replace(INVISIBLE_ELEMENT_PATTERN, " ").replace(TAG_PATTERN, " ").replace(/&(?:nbsp|amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? " ").replace(/\s+/g, " ").trim();
+var noscriptMentionsJavascript = (html) => {
+  for (const match of html.matchAll(NOSCRIPT_PATTERN)) {
+    if (/javascript/i.test(match[1] ?? "")) {
+      return true;
+    }
+  }
+  return false;
+};
+var detectClientShell = (html) => {
+  const body = BODY_PATTERN.exec(html)?.[1] ?? html.replace(HEAD_PATTERN, " ");
+  const visibleTextLength = visibleTextOf(body).length;
+  if (visibleTextLength >= SHELL_SUSPICION_THRESHOLD) {
+    return { isShell: false, visibleTextLength };
+  }
+  if (CHALLENGE_PATTERN.test(html)) {
+    return { isShell: true, reason: "anti-bot-challenge", visibleTextLength };
+  }
+  if (MOUNT_POINT_PATTERN.test(body)) {
+    return { isShell: true, reason: "spa-shell", visibleTextLength };
+  }
+  if (noscriptMentionsJavascript(html)) {
+    return { isShell: true, reason: "noscript-only", visibleTextLength };
+  }
+  if (visibleTextLength < SHELL_EMPTY_THRESHOLD) {
+    return { isShell: true, reason: "empty-body", visibleTextLength };
+  }
+  return { isShell: false, visibleTextLength };
+};
+
+// src/extract-meta-refresh.ts
+var MAX_META_REFRESH_DELAY_SECONDS = 10;
+var META_TAG_PATTERN = /<meta\b[^>]*>/gi;
+var HTTP_EQUIV_REFRESH_PATTERN = /\bhttp-equiv\s*=\s*("\s*refresh\s*"|'\s*refresh\s*'|refresh\b)/i;
+var CONTENT_ATTR_PATTERN = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i;
+var CONTENT_VALUE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*[;,]\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+?))\s*$/i;
+var resolveTarget = (raw, baseUrl) => {
+  try {
+    return baseUrl === void 0 ? new URL(raw) : new URL(raw, baseUrl);
+  } catch {
+    return void 0;
+  }
+};
+var extractMetaRefresh = (html, baseUrl) => {
+  for (const [tag] of html.matchAll(META_TAG_PATTERN)) {
+    if (!HTTP_EQUIV_REFRESH_PATTERN.test(tag)) {
+      continue;
+    }
+    const contentMatch = CONTENT_ATTR_PATTERN.exec(tag);
+    const content = contentMatch?.[1] ?? contentMatch?.[2] ?? contentMatch?.[3];
+    if (!content) {
+      continue;
+    }
+    const value = CONTENT_VALUE_PATTERN.exec(content);
+    if (!value) {
+      continue;
+    }
+    const delaySeconds = Number.parseFloat(value[1] ?? "");
+    const raw = (value[2] ?? value[3] ?? value[4] ?? "").trim();
+    if (!raw || delaySeconds > MAX_META_REFRESH_DELAY_SECONDS) {
+      continue;
+    }
+    const target = resolveTarget(raw, baseUrl);
+    if (!target || target.protocol !== "http:" && target.protocol !== "https:") {
+      continue;
+    }
+    if (baseUrl !== void 0 && resolveTarget(baseUrl, void 0)?.href === target.href) {
+      continue;
+    }
+    return { url: target.href, delaySeconds };
+  }
+  return void 0;
+};
+
 // src/cli/commands/webforai/loadHtml.ts
-import fs5 from "node:fs/promises";
+import fs8 from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import boxen from "boxen";
 import pc from "picocolors";
@@ -3399,9 +4183,20 @@ import { chromium as chromium2 } from "playwright-core";
 
 // src/loaders/fetch.ts
 var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+var MAX_META_REFRESH_HOPS = 3;
 var loadHtml = async (url, userAgent = USER_AGENT) => {
-  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
-  return response.text();
+  let current = url;
+  let html = "";
+  for (let hop = 0; hop <= MAX_META_REFRESH_HOPS; hop++) {
+    const response = await fetch(current, { headers: { "User-Agent": userAgent } });
+    html = await response.text();
+    const refresh = extractMetaRefresh(html, response.url || current);
+    if (!refresh) {
+      break;
+    }
+    current = refresh.url;
+  }
+  return html;
 };
 
 // src/loaders/playwright.ts
@@ -3419,40 +4214,42 @@ var SUPER_BYPASS_DEVICE = {
 var loadHtml2 = async (url, options) => {
   const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
   const _browser = browser ?? await chromium.launch({ headless: true });
-  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
-  if (superBypassMode) {
-    await context.addInitScript(() => {
-      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
-      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
-      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
-    });
-  }
-  const page = await context.newPage();
-  if (superBypassMode) {
-    await page.route("**/*.js", (route) => {
-      if (route.request().url().includes("captcha-delivery")) {
-        return route.abort();
+  let context;
+  try {
+    context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
+    if (superBypassMode) {
+      await context.addInitScript(() => {
+        Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+        Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+        Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+      });
+    }
+    const page = await context.newPage();
+    if (superBypassMode) {
+      await page.route("**/*.js", (route) => {
+        if (route.request().url().includes("captcha-delivery")) {
+          return route.abort();
+        }
+        return route.continue();
+      });
+    }
+    await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
+    await page.evaluate(() => {
+      const elements = document.querySelectorAll("*");
+      for (const element3 of elements) {
+        const rect = element3.getBoundingClientRect();
+        element3.setAttribute("data-rwidth", rect.width.toString());
+        element3.setAttribute("data-rheight", rect.height.toString());
       }
-      return route.continue();
     });
-  }
-  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
-  await page.evaluate(() => {
-    const elements = document.querySelectorAll("*");
-    for (const element3 of elements) {
-      const rect = element3.getBoundingClientRect();
-      element3.setAttribute("data-rwidth", rect.width.toString());
-      element3.setAttribute("data-rheight", rect.height.toString());
+    return await page.content();
+  } finally {
+    if (browser) {
+      await context?.close();
+    } else {
+      await _browser.close();
     }
-  });
-  const html = await page.content();
-  await page.close();
-  if (browser) {
-    await context.close();
-  } else {
-    await _browser.close();
   }
-  return html;
 };
 
 // src/cli/commands/webforai/loadHtml.ts
@@ -3462,23 +4259,23 @@ var debugLog = (enabled, message) => {
   }
 };
 var checkPlaywrightAvailable = async () => {
-  const path4 = chromium2.executablePath();
+  const path6 = chromium2.executablePath();
   try {
-    await fs5.access(path4);
+    await fs8.access(path6);
     return true;
   } catch {
     return false;
   }
 };
 var getPlaywrightVersion = async () => {
-  const path4 = await import.meta.resolve("playwright-core/package.json");
-  const pwPackageJson = await fs5.readFile(fileURLToPath(path4), "utf-8").then((res) => JSON.parse(res.toString())).catch(() => null);
+  const path6 = await import.meta.resolve("playwright-core/package.json");
+  const pwPackageJson = await fs8.readFile(fileURLToPath(path6), "utf-8").then((res) => JSON.parse(res.toString())).catch(() => null);
   return pwPackageJson?.version;
 };
 var loadHtml3 = async (sourcePath, loader, options) => {
   if (loader === "local") {
     debugLog(options.debug, `Loading HTML from local file: ${sourcePath}`);
-    const content = await fs5.readFile(sourcePath, "utf-8");
+    const content = await fs8.readFile(sourcePath, "utf-8");
     debugLog(options.debug, `HTML loaded: ${content.slice(0, 100)}`);
     return content;
   }
@@ -3545,12 +4342,13 @@ var convertViaPlatform = async (run) => {
   if (run.mode !== "default") {
     console.error("[webforai] --mode is applied by local conversion only and is ignored by the platform loader");
   }
-  debugLog2(run.debug, `platform scrape: ${run.source} (engine=${run.engine ?? "fetch"})`);
+  debugLog2(run.debug, `platform scrape: ${run.source} (engine=${run.engine ?? "auto"})`);
   const result = await platform.scrape({
     url: run.source,
     engine: run.engine,
     region: run.region,
     screenshot: run.screenshot ? true : void 0,
+    respectRobotsTxt: run.respectRobotsTxt ? true : void 0,
     convert: { extractor: run.extractor, frontmatter: run.frontmatter }
   });
   return {
@@ -3562,7 +4360,8 @@ var convertViaPlatform = async (run) => {
     markdown: result.markdown,
     metadata: result.metadata,
     credits: result.credits,
-    screenshotUrl: result.screenshotUrl
+    screenshotUrl: result.screenshotUrl,
+    warning: result.warning
   };
 };
 var convertLocally = async (run) => {
@@ -3575,18 +4374,21 @@ var convertLocally = async (run) => {
     extractors: extractorOptions(run.extractor),
     ...run.mode === "ai" ? AI_MODE_OPTIONS : {}
   });
+  const shell = run.loader === "fetch" ? detectClientShell(html) : void 0;
+  const warning = shell?.isShell ? "This page looks like it renders client-side; the fetched HTML has no readable content. Try --engine auto (hosted platform, renders JavaScript) or --loader playwright." : void 0;
   return {
     source: run.source,
     loader: run.loader,
     url: sourceUrl,
     markdown,
-    metadata
+    metadata,
+    warning
   };
 };
 var executeRun = (run) => run.loader === "platform" ? convertViaPlatform(run) : convertLocally(run);
 var writeOutputFile = async (outputPath, markdown) => {
-  await fs6.mkdir(path3.dirname(path3.resolve(outputPath)), { recursive: true });
-  await fs6.writeFile(outputPath, markdown);
+  await fs9.mkdir(path5.dirname(path5.resolve(outputPath)), { recursive: true });
+  await fs9.writeFile(outputPath, markdown);
 };
 var runCommand = async (run) => {
   const envelope = await executeRun(run);
@@ -3604,6 +4406,9 @@ var runCommand = async (run) => {
     process.stdout.write(envelope.markdown.endsWith("\n") ? envelope.markdown : `${envelope.markdown}
 `);
   }
+  if (envelope.warning) {
+    console.error(`[webforai] warning: ${envelope.warning}`);
+  }
   if (envelope.screenshotUrl) {
     console.error(`[webforai] screenshot (expires ~24h): ${envelope.screenshotUrl}`);
   }
@@ -3614,20 +4419,21 @@ var runCommand = async (run) => {
 
 // src/cli/commands/webforai/interactive.ts
 var ENGINE_HINTS = {
+  auto: "Cheapest first, renders when the page needs JavaScript \xB7 1\u20132 credits",
   fetch: "Plain fetch \xB7 1 credit",
-  browser: "Browser rendering, screenshots \xB7 5 credits",
-  "proxy-fetch": "Rotating-proxy egress \xB7 2 credits",
-  "proxy-browser": "Browser rendering behind the proxy \xB7 5 credits"
+  browser: "Browser rendering, screenshots \xB7 2 credits",
+  "proxy-fetch": "Rotating-proxy egress \xB7 2 credits \xB7 paid plan",
+  "proxy-browser": "Browser rendering behind the proxy \xB7 3 credits \xB7 paid plan"
 };
 var promptPlatformOptions = async (flags, env) => {
   const engine = flags.engine ?? await select7({
     message: "Select platform engine:",
-    initialValue: "fetch",
-    options: ENGINES.map((value) => ({ value, label: value, hint: ENGINE_HINTS[value] }))
+    initialValue: "auto",
+    options: REQUESTED_ENGINES.map((value) => ({ value, label: value, hint: ENGINE_HINTS[value] }))
   });
   assertContinue(engine);
   const region = flags.region ?? await select7({
-    message: "Select egress region (proxy engines only):",
+    message: "Select egress region (proxy engines only; jp needs a paid plan):",
     initialValue: "auto",
     options: REGIONS.map((value) => ({ value, label: value }))
   });
@@ -3647,26 +4453,18 @@ var promptPlatformOptions = async (flags, env) => {
 var interactiveCommand = async (initialSource, flags, env = process.env) => {
   intro(pc2.bold(pc2.green(`webforai CLI version ${package_default.version}`)));
   const source = initialSource ?? await inputSourcePath();
-  const loader = isUrl(source) ? flags.loader ?? await selectLoader() : "local";
-  const run = {
-    source,
-    loader,
-    mode: "default",
-    extractor: flags.extractor ?? "auto",
-    frontmatter: flags.frontmatter ?? false,
-    json: false,
-    debug: flags.debug ?? false,
-    screenshot: flags.screenshot ?? false
+  const loader = isUrl(source) ? flags.loader ?? (flags.engine !== void 0 || flags.region !== void 0 ? "platform" : await selectLoader()) : "local";
+  const selectedFlags = {
+    ...flags,
+    loader: loader === "local" ? flags.loader : loader
   };
   if (loader === "platform") {
     const platform = await promptPlatformOptions(flags, env);
-    run.engine = platform.engine;
-    run.region = platform.region;
-    run.apiKey = platform.apiKey;
-    run.platformUrl = flags.platformUrl;
+    Object.assign(selectedFlags, platform);
   } else {
-    run.mode = flags.mode ?? await selectExtractMode();
+    selectedFlags.mode = flags.mode ?? await selectExtractMode();
   }
+  const run = resolveRunOptions(source, selectedFlags, env);
   const outputPath = flags.output ?? await inputOutputPath(source);
   const s = spinner();
   s.start(loader === "platform" ? "Converting via the platform..." : "Loading and converting...");
@@ -3689,73 +4487,6 @@ var interactiveCommand = async (initialSource, flags, env = process.env) => {
   outro(pc2.green(`${pc2.bold("Done!")} Markdown saved to ${outputPath}`));
 };
 
-// src/cli/commands/webforai/options.ts
-import fs7 from "node:fs";
-var UsageError = class extends Error {
-};
-var oneOf = (name, value, allowed) => {
-  if (!allowed.includes(value)) {
-    throw new UsageError(`invalid --${name} "${value}" (expected: ${allowed.join(" | ")})`);
-  }
-  return value;
-};
-var impliesPlatform = (flags) => flags.engine !== void 0 || flags.region !== void 0;
-var resolveLoader = (sourceIsUrl, flags) => {
-  if (flags.loader) {
-    return oneOf("loader", flags.loader, LOADERS);
-  }
-  if (impliesPlatform(flags)) {
-    return "platform";
-  }
-  return sourceIsUrl ? "fetch" : "local";
-};
-var assertUsableLocalSource = (source, loader, flags) => {
-  if (loader !== "local" && flags.loader) {
-    throw new UsageError(`--loader ${loader} needs a URL source, but "${source}" is a local path`);
-  }
-  if (impliesPlatform(flags)) {
-    throw new UsageError("--engine/--region need a URL source (the platform cannot read local files)");
-  }
-  if (!fs7.existsSync(source)) {
-    throw new UsageError(`source not found: "${source}" (URLs must start with http:// or https://)`);
-  }
-};
-var applyPlatformFields = (resolved, flags, env) => {
-  resolved.engine = flags.engine ? oneOf("engine", flags.engine, ENGINES) : void 0;
-  resolved.region = flags.region ? oneOf("region", flags.region, REGIONS) : void 0;
-  resolved.apiKey = flags.apiKey ?? env[API_KEY_ENV];
-  resolved.platformUrl = flags.platformUrl ?? env[PLATFORM_URL_ENV];
-  if (!resolved.apiKey) {
-    throw new UsageError(
-      `the platform loader needs an API key: pass --api-key or set ${API_KEY_ENV} (create one at https://platform.webforai.dev/dashboard)`
-    );
-  }
-};
-var resolveRunOptions = (source, flags, env = process.env) => {
-  const sourceIsUrl = isUrl(source);
-  const loader = resolveLoader(sourceIsUrl, flags);
-  if (!sourceIsUrl) {
-    assertUsableLocalSource(source, loader, flags);
-  }
-  const resolved = {
-    source,
-    loader: sourceIsUrl ? loader : "local",
-    mode: flags.mode ? oneOf("mode", flags.mode, MODES) : "default",
-    extractor: flags.extractor ? oneOf("extractor", flags.extractor, EXTRACTORS) : "auto",
-    frontmatter: flags.frontmatter ?? false,
-    output: flags.output,
-    json: flags.json ?? false,
-    debug: flags.debug ?? false,
-    screenshot: flags.screenshot ?? false
-  };
-  if (resolved.loader === "platform") {
-    applyPlatformFields(resolved, flags, env);
-  } else if (flags.screenshot) {
-    throw new UsageError("--screenshot is only available with the platform loader (browser engines)");
-  }
-  return resolved;
-};
-
 // src/cli/bin.ts
 var HELP_EPILOGUE = `
 Examples:
@@ -3764,7 +4495,9 @@ Examples:
   $ webforai ./page.html --extractor none             convert a local file, no extraction
   $ webforai https://example.com --json | jq .markdown
   $ webforai https://spa.example.com -l playwright    render JS in local Chromium
-  $ webforai https://example.com --engine browser     render via the hosted platform
+  $ webforai https://example.com --engine auto        hosted platform, renders JS when needed
+  $ webforai crawl https://docs.example.com -o docs --llms-txt   crawl a site (platform) + llms.txt
+  $ webforai batch --file urls.txt -o pages           many URLs, one .md per page (platform)
   $ webforai -i                                       guided interactive mode
   $ webforai skill --install                          install the Agent Skill for this CLI
 
@@ -3777,9 +4510,23 @@ Exit codes:
 
 Output contract:
   stdout carries only the Markdown (or the --json envelope); all logs go to stderr.
+
+Docs:
+  https://webforai.dev/cli                    this CLI
+  https://webforai.dev/platform               the hosted platform (--engine / --region)
+  ${DEFAULT_BASE_URL}/dashboard     API keys, usage and billing
 `;
-program.name("webforai").description("Convert web pages and local HTML to clean, LLM-ready Markdown.").version(package_default.version, "-v, --version", "output the current version").addHelpText("after", HELP_EPILOGUE).showSuggestionAfterError();
-program.argument("[source]", "URL or local HTML file to convert").option("-o, --output <path>", "write Markdown to a file instead of stdout").option("-l, --loader <loader>", `how HTML is acquired (${LOADERS.join(" | ")}; local files are automatic)`).option("-m, --mode <mode>", `conversion mode (${MODES.join(" | ")}); ai strips links/tables/images`).option("--extractor <extractor>", `main-content extraction preset (${EXTRACTORS.join(" | ")})`, void 0).option("--frontmatter", "prepend YAML front matter built from the page metadata").option("--json", "print a JSON envelope {source, loader, url, markdown, metadata, ...} to stdout").option("--engine <engine>", "platform engine (fetch | browser | proxy-fetch | proxy-browser); implies -l platform").option("--region <region>", "platform proxy egress region (auto | us | eu | uk | jp | asia); implies -l platform").option("--screenshot", "platform browser engines: also capture a screenshot (expiring URL)").option("--api-key <key>", `platform API key (defaults to $${API_KEY_ENV})`).option("--platform-url <url>", `platform base URL (defaults to $${PLATFORM_URL_ENV} or the hosted instance)`).option("-i, --interactive", "guided prompt flow (the pre-v3 default)").option("-d, --debug", "verbose logs on stderr").action(async (source, flags) => {
+var DASHBOARD_ERRORS = /* @__PURE__ */ new Set(["invalid_api_key", "payment_required", "spend_cap_reached"]);
+var activePlatformUrl;
+var rememberPlatformUrl = (flags) => {
+  activePlatformUrl = flags.platformUrl ?? process.env[PLATFORM_URL_ENV];
+};
+program.name("webforai").enablePositionalOptions().exitOverride().description("Convert web pages and local HTML to clean, LLM-ready Markdown.").version(package_default.version, "-v, --version", "output the current version").addHelpText("after", HELP_EPILOGUE).showSuggestionAfterError();
+program.argument("[source]", "URL or local HTML file to convert").option("-o, --output <path>", "write Markdown to a file instead of stdout").option("-l, --loader <loader>", `how HTML is acquired (${LOADERS.join(" | ")}; local files are automatic)`).option("-m, --mode <mode>", `conversion mode (${MODES.join(" | ")}); ai strips links/tables/images`).option("--extractor <extractor>", `main-content extraction preset (${EXTRACTORS.join(" | ")})`, void 0).option("--frontmatter", "prepend YAML front matter built from the page metadata").option("--json", "print a JSON envelope {source, loader, url, markdown, metadata, ...} to stdout").option(
+  "--engine <engine>",
+  "platform engine (auto | fetch | browser | proxy-fetch | proxy-browser; proxy-* need a paid plan); implies -l platform"
+).option("--region <region>", "platform proxy egress region (auto | jp; jp needs a paid plan); implies -l platform").option("--screenshot", "platform browser engines: also capture a screenshot (expiring URL)").option("--respect-robots", "platform loader: honor the target site's robots.txt rules (off by default)").option("--api-key <key>", `platform API key (defaults to $${API_KEY_ENV})`).option("--platform-url <url>", `platform base URL (defaults to $${PLATFORM_URL_ENV} or the hosted instance)`).option("-i, --interactive", "guided prompt flow (the pre-v3 default)").option("-d, --debug", "verbose logs on stderr").action(async (source, flags) => {
+  rememberPlatformUrl(flags);
   if (flags.interactive) {
     await interactiveCommand(source, flags);
     return;
@@ -3789,6 +4536,46 @@ program.argument("[source]", "URL or local HTML file to convert").option("-o, --
   }
   await runCommand(resolveRunOptions(source, flags));
 });
+var collect = (value, previous = []) => [...previous, value];
+var addPlatformJobOptions = (command) => command.requiredOption("-o, --output <dir>", "directory for the Markdown files (one per page; created if missing)").option(
+  "--engine <engine>",
+  "platform engine (auto | fetch | browser | proxy-fetch | proxy-browser; proxy-* need a paid plan)"
+).option("--region <region>", "proxy egress region (auto | jp); jp runs on the paid proxy tier").option("--extractor <extractor>", `main-content extraction preset (${EXTRACTORS.join(" | ")})`).option("--frontmatter", "prepend YAML front matter built from each page's metadata").option("--json", "print a JSON envelope {jobId, pages[], failures[], ...} to stdout instead of file paths").option("--timeout <seconds>", "stop waiting after this long (the job keeps running); default 1800").option("--api-key <key>", `platform API key (defaults to $${API_KEY_ENV})`).option("--platform-url <url>", `platform base URL (defaults to $${PLATFORM_URL_ENV} or the hosted instance)`).option("-d, --debug", "verbose logs on stderr");
+var JOB_OUTPUT_HELP = `
+Output:
+  One .md per converted page under --output, mirroring URL paths (/docs/intro \u2192 docs/intro.md).
+  stdout lists the written files (or the --json envelope); progress and the summary go to stderr.
+  Exit 1 when the job failed or converted no pages; failed pages are listed and never billed.
+`;
+addPlatformJobOptions(
+  program.command("crawl").description("crawl a site on the hosted platform and write one Markdown file per page").argument("<url>", "seed URL; only same-origin links are followed")
+).option("--max-depth <n>", "link depth from the seed (0-5, default 2)").option("--limit <n>", "maximum pages (1-500, default 50)").option("--include <regex>", "only crawl pathnames matching this regex (repeatable)", collect).option("--exclude <regex>", "skip pathnames matching this regex (repeatable)", collect).option(
+  "--sitemap <mode>",
+  "use the site's sitemaps: skip (default) | include (add their URLs) | only (no link following)"
+).option("--respect-robots", "honor robots.txt (the default for crawls)").option("--no-respect-robots", "do not read robots.txt").option("--llms-txt", "also write llms.txt (page index) and llms-full.txt (all pages) per llmstxt.org").addHelpText(
+  "after",
+  `${JOB_OUTPUT_HELP}
+Examples:
+  $ webforai crawl https://docs.example.com -o docs --limit 100 --include '^/docs' --llms-txt
+  $ webforai crawl https://blog.example.com -o blog --sitemap only --limit 500
+`
+).action(async (url, flags) => {
+  rememberPlatformUrl(flags);
+  await runJobCommand(resolveJobOptions("crawl", [url], flags));
+});
+addPlatformJobOptions(
+  program.command("batch").description("convert many URLs on the hosted platform, one Markdown file per page").argument("[urls...]", "URLs to convert (up to 100 per job)")
+).option("-f, --file <path>", "read URLs from a file, one per line ('-' for stdin; # comments allowed)").option("--respect-robots", "honor each site's robots.txt (off by default for batches)").addHelpText(
+  "after",
+  `${JOB_OUTPUT_HELP}
+Examples:
+  $ webforai batch https://a.example/post https://b.example/news -o pages
+  $ cat urls.txt | webforai batch --file - -o pages --json
+`
+).action(async (urls, flags) => {
+  rememberPlatformUrl(flags);
+  await runJobCommand(resolveJobOptions("batch", urls, flags));
+});
 program.command("skill").description("print the Agent Skill (SKILL.md) for this CLI, or install it via the skills CLI").option("--install", "install through `npx skills add` (interactive agent picker; flags below forward to it)").option("-g, --global", "forward to skills add: install user-level instead of project-level").option("-a, --agent <agents>", "forward to skills add: agents to install to ('*' for all)").option("-y, --yes", "forward to skills add: skip confirmation prompts").option("--copy", "forward to skills add: copy files instead of symlinking").option("--all", "forward to skills add: all skills, all agents, no prompts").option("--dir <path>", "offline fallback: write <path>/webforai/SKILL.md directly, without the skills CLI").addHelpText("after", "\nAlso installable straight from the repository: npx skills add inaridiy/webforai\n").action(async (flags) => {
   await skillCommand(flags);
 });
@@ -3800,8 +4587,19 @@ var fail = (error) => {
     console.error(`error: ${error.message}`);
     process.exit(2);
   }
+  if (error instanceof PlatformApiError) {
+    const retry = error.retryAfter === void 0 ? "" : ` (retryAfter: ${error.retryAfter}s)`;
+    console.error(`error: ${error.code}: ${error.message}${retry}`);
+    if (DASHBOARD_ERRORS.has(error.code) && !error.message.includes("/dashboard")) {
+      console.error(`hint: manage API keys and credits at ${platformDashboardUrl(activePlatformUrl)}`);
+    }
+    process.exit(1);
+  }
+  if (error instanceof JobIncompleteError) {
+    console.error(`error: ${error.message}`);
+    process.exit(1);
+  }
   console.error(error instanceof Error ? `error: ${error.message}` : `error: ${String(error)}`);
   process.exit(1);
 };
-program.exitOverride();
 program.parseAsync().catch(fail);