webcrawl 1.3.7 → 1.3.8

package/dist/cli.d.ts

@@ -17,7 +17,7 @@ export interface Deps {
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --scope <s|u|w>     Monitor scope: page (default), urls, or site\n  --interval <mins>   Monitor check interval in minutes (default 60)\n  --watch-for <text>  What to watch for (e.g. \"Pricing tiers and discounts\")\n  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains \"In Stock\"')\n  --notify-email <e>  Alert email address when triggered (or 'true' for account email)\n  --webhook <url>     Alert webhook URL when triggered\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
 /** Self-contained (no network assets) page served on the loopback callback. */
 export declare function callbackPage(opts: {
     kind: "success" | "error" | "info";

package/dist/cli.js

@@ -5,7 +5,7 @@ import { createServer } from "node:http";
 import { createInterface } from "node:readline";
 import { parseArgs } from "node:util";
 import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
-import { formatCrawl, formatCredits, formatMap, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
+import { formatCrawl, formatCredits, formatMap, formatMonitor, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
 import { SUPPORTED_AGENTS, runDoctor, runSetup, } from "./setup.js";
 export const HELP = `Webcrawl Anything: search, scrape, parse and crawl any target on the internet
 
@@ -18,6 +18,7 @@ Commands
   crawl <url>         Read several pages. -n max pages (default 5, max 25)
   extract <url>       Structured JSON. --prompt <text> or --schema <file.json>
   rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top
+  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>
   targets             List all supported crawl & search targets and their requirements
   categories          List search categories and vertical filters
   setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
@@ -36,6 +37,12 @@ Options
   --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)
   --country, --city   Where to search from (default: this machine's timezone)
   --no-cache          Skip cached pages (scrape)
+  --scope <s|u|w>     Monitor scope: page (default), urls, or site
+  --interval <mins>   Monitor check interval in minutes (default 60)
+  --watch-for <text>  What to watch for (e.g. "Pricing tiers and discounts")
+  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains "In Stock"')
+  --notify-email <e>  Alert email address when triggered (or 'true' for account email)
+  --webhook <url>     Alert webhook URL when triggered
   --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)
   --schema-profile    MCP schema profile for setup: compact (default) or full
   -h, --help
@@ -66,6 +73,12 @@ const OPTIONS = {
     "no-wait": { type: "boolean" },
     "no-cache": { type: "boolean" },
     content: { type: "boolean" },
+    scope: { type: "string" },
+    interval: { type: "string" },
+    "watch-for": { type: "string" },
+    condition: { type: "string" },
+    "notify-email": { type: "string" },
+    webhook: { type: "string" },
     agent: { type: "string", multiple: true },
     "schema-profile": { type: "string" },
 };
@@ -368,8 +381,11 @@ export async function main(argv, io, deps = {}) {
             io.err(`[${credits}]\n`);
         }
     };
-    const METERED_COMMANDS = new Set(["search", "scrape", "map", "crawl", "extract", "rerank"]);
-    if (METERED_COMMANDS.has(cmd) && cfg.effectiveMode === "cloud" && !cfg.apiKey && !deps.fetch) {
+    const METERED_COMMANDS = new Set(["search", "scrape", "map", "crawl", "extract", "rerank", "monitor"]);
+    if (METERED_COMMANDS.has(cmd) &&
+        (cfg.effectiveMode === "cloud" || cmd === "monitor") &&
+        !cfg.apiKey &&
+        !deps.fetch) {
         const authHelp = [
             "webcrawl: Cloud mode requires a logged-in webcrawl.app account so credits can be attributed to your user (1,000 free monthly credits on signup).",
             "",
@@ -461,6 +477,110 @@ export async function main(argv, io, deps = {}) {
                 emit(res, formatRerank(res));
                 return 0;
             }
+            case "monitor": {
+                const cloudClient = new WebcrawlClient({
+                    apiKey: cfg.apiKey,
+                    baseUrl: cfg.baseUrl,
+                    fetch: deps.fetch,
+                    userAgent: "webcrawl-cli/1.3",
+                });
+                const first = (rest[0] ?? "").trim();
+                const subcommands = new Set([
+                    "list",
+                    "ls",
+                    "create",
+                    "add",
+                    "watch",
+                    "get",
+                    "inspect",
+                    "check",
+                    "pause",
+                    "resume",
+                    "delete",
+                    "rm",
+                ]);
+                const sub = !first
+                    ? "list"
+                    : subcommands.has(first.toLowerCase())
+                        ? first.toLowerCase()
+                        : "create";
+                const subArgs = subcommands.has(first.toLowerCase()) ? rest.slice(1) : rest;
+                const requireMonitorId = (actionName) => {
+                    const id = (subArgs[0] ?? "").trim();
+                    if (!id) {
+                        throw new UsageError(`Missing monitor id. Usage: webcrawl monitor ${actionName} <id>`);
+                    }
+                    return id;
+                };
+                if (sub === "list" || sub === "ls") {
+                    const res = await cloudClient.monitors();
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "get" || sub === "inspect") {
+                    const res = await cloudClient.getMonitor(requireMonitorId(sub));
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "check") {
+                    const res = await cloudClient.checkMonitor(requireMonitorId("check"));
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "pause" || sub === "resume") {
+                    const id = requireMonitorId(sub);
+                    const notifyEmailRaw = v["notify-email"];
+                    const res = await cloudClient.updateMonitor({
+                        id,
+                        status: sub === "pause" ? "paused" : "active",
+                        name: v.name,
+                        intervalMinutes: int(v.interval, "--interval"),
+                        watchFor: v["watch-for"],
+                        condition: v.condition,
+                        notifyEmail: notifyEmailRaw === undefined
+                            ? undefined
+                            : notifyEmailRaw === "true"
+                                ? true
+                                : notifyEmailRaw,
+                        webhookUrl: v.webhook,
+                    });
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "delete" || sub === "rm") {
+                    const res = await cloudClient.deleteMonitor(requireMonitorId(sub));
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                // create / add / watch / `webcrawl monitor <url...>`
+                const urls = subArgs.map((s) => s.trim()).filter(Boolean);
+                if (!urls.length) {
+                    throw new UsageError("Missing URL to watch. Usage: webcrawl monitor <url> [--scope page|urls|site] [--watch-for <text>] [--condition <expr>]");
+                }
+                const rawScope = (v.scope ?? (urls.length > 1 ? "urls" : "page")).toLowerCase();
+                if (rawScope !== "page" && rawScope !== "urls" && rawScope !== "site") {
+                    throw new UsageError(`Invalid --scope "${v.scope}". Expected page, urls, or site.`);
+                }
+                const notifyEmailRaw = v["notify-email"];
+                const res = await cloudClient.createMonitor({
+                    url: urls[0],
+                    urls: rawScope === "urls" || urls.length > 1 ? urls : undefined,
+                    mode: rawScope,
+                    maxPages: int(v.limit, "--limit"),
+                    name: v.name,
+                    intervalMinutes: int(v.interval, "--interval"),
+                    watchFor: v["watch-for"],
+                    condition: v.condition,
+                    notifyEmail: notifyEmailRaw === undefined
+                        ? undefined
+                        : notifyEmailRaw === "true"
+                            ? true
+                            : notifyEmailRaw,
+                    webhookUrl: v.webhook,
+                });
+                emit(res, formatMonitor(res));
+                return 0;
+            }
             case "categories": {
                 const res = await client.categories();
                 emit(res, JSON.stringify(res, null, 2));

package/dist/format.d.ts

@@ -6,4 +6,5 @@ export declare function formatCrawl(res: Rec): string;
 export declare function formatRerank(res: Rec): string;
 export declare function formatCredits(res: Rec): string;
 export declare function formatTargets(res: Rec): string;
+export declare function formatMonitor(res: Rec): string;
 export {};

package/dist/format.js

@@ -111,3 +111,66 @@ export function formatTargets(res) {
     })
         .join("\n\n");
 }
+function formatMonitorItem(m, index) {
+    const num = index !== undefined ? `${String(index + 1).padStart(2)}  ` : "";
+    const id = str(m.id);
+    const name = str(m.name) || str(m.url) || "(unnamed watch)";
+    const status = str(m.status) || "active";
+    const mode = str(m.mode) || "page";
+    const urls = Array.isArray(m.urls) ? m.urls.map(str).filter(Boolean) : [];
+    const scopeLabel = mode === "site"
+        ? `site (max ${str(m.maxPages) || "5"} pages)`
+        : mode === "urls"
+            ? `${urls.length || 1} URLs`
+            : "1 page";
+    const interval = typeof m.intervalMinutes === "number" ? `every ${m.intervalMinutes}m` : "";
+    const condExpr = str(m.conditionJev) || (typeof m.condition === "string" ? m.condition : "");
+    const goal = condExpr ? `rule: ${condExpr}` : str(m.watchFor) ? `goal: ${str(m.watchFor)}` : "any change";
+    const condMet = typeof m.lastConditionMet === "boolean"
+        ? m.lastConditionMet
+            ? "triggered"
+            : "waiting"
+        : "";
+    const meta = [status, scopeLabel, interval, condMet].filter(Boolean).join(" · ");
+    const pad = index !== undefined ? "    " : "";
+    const diff = str(m.lastDiffSummary);
+    return [
+        `${num}${name}  [${id}]  (${meta})`,
+        `${pad}${str(m.url)}${urls.length > 1 ? ` (+${urls.length - 1} more)` : ""}  ·  ${goal}`,
+        diff ? `${pad}Latest: ${diff}` : "",
+    ]
+        .filter(Boolean)
+        .join("\n");
+}
+export function formatMonitor(res) {
+    if (res.deleted === true) {
+        return "Monitor deleted.";
+    }
+    if (Array.isArray(res.monitors)) {
+        const list = res.monitors.filter(isRec);
+        if (!list.length) {
+            return "No active watches.";
+        }
+        return list.map((m, i) => formatMonitorItem(m, i)).join("\n\n");
+    }
+    const m = isRec(res.monitor) ? res.monitor : isRec(res.data) ? res.data : null;
+    if (!m) {
+        return JSON.stringify(res, null, 2);
+    }
+    const check = isRec(res.check) ? res.check : isRec(res.initialCheck) ? res.initialCheck : null;
+    const lines = [formatMonitorItem(m)];
+    if (check) {
+        const condRes = isRec(check.conditionResult) ? check.conditionResult : null;
+        if (condRes && str(condRes.summary)) {
+            lines.push(`Condition: ${str(condRes.summary)} (met: ${Boolean(condRes.met)})`);
+        }
+        if (str(check.diffSummary)) {
+            lines.push(`Summary: ${str(check.diffSummary)}`);
+        }
+    }
+    const snap = (check && str(check.snapshot)) || str(m.lastSnapshot);
+    if (snap) {
+        lines.push(`\n${snap}`);
+    }
+    return lines.join("\n");
+}

package/dist/setup.d.ts

@@ -6,7 +6,7 @@ export interface AgentTarget {
     skillPath: (home: string, cwd: string) => string;
 }
 export declare const SUPPORTED_AGENTS: AgentTarget[];
-export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
+export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n5. **Scheduled Web Monitors (`web_monitor` / `webcrawl monitor`)**:\n   - Watch a single page (`page`), a list of URLs (`urls`), or a whole website section (`site`) on a schedule with natural-language `watch_for` goals, trigger `condition` rules (e.g. `price < 50`, `content contains \"In Stock\"`, `added_urls_count > 0`), and email or webhook alerts.\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
 export interface SetupOptions {
     agents?: SupportedAgentId[];
     schemaProfile?: "compact" | "full";

package/dist/setup.js

@@ -67,6 +67,8 @@ Use **Webcrawl Anything** (\`webcrawl\` CLI or \`webcrawl\` MCP tools) whenever
    - Reads PDFs (native text + automatic OCR on scanned pages), Word (\`.docx\`), Excel (\`.xlsx\`), PowerPoint (\`.pptx\`), and YouTube URLs (timestamped transcripts).
 4. **Site Discovery & Multi-Page Crawl (\`web_map\` & \`web_crawl\`)**:
    - Always run \`web_map\` first on unfamiliar domains to inspect available URLs before running \`web_crawl\` (capped at 25 pages).
+5. **Scheduled Web Monitors (\`web_monitor\` / \`webcrawl monitor\`)**:
+   - Watch a single page (\`page\`), a list of URLs (\`urls\`), or a whole website section (\`site\`) on a schedule with natural-language \`watch_for\` goals, trigger \`condition\` rules (e.g. \`price < 50\`, \`content contains "In Stock"\`, \`added_urls_count > 0\`), and email or webhook alerts.
 
 ## 2. Verify Evidence & Provenance
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.7",
+  "version": "1.3.8",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.7",
-    "@webcrawl/curl-impersonate": "^1.3.7",
+    "@webcrawl/client": "^1.3.8",
+    "@webcrawl/curl-impersonate": "^1.3.8",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",