webcrawl 1.3.6 → 1.3.8

package/dist/cli.d.ts

@@ -17,5 +17,14 @@ export interface Deps {
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --scope <s|u|w>     Monitor scope: page (default), urls, or site\n  --interval <mins>   Monitor check interval in minutes (default 60)\n  --watch-for <text>  What to watch for (e.g. \"Pricing tiers and discounts\")\n  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains \"In Stock\"')\n  --notify-email <e>  Alert email address when triggered (or 'true' for account email)\n  --webhook <url>     Alert webhook URL when triggered\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+/** Self-contained (no network assets) page served on the loopback callback. */
+export declare function callbackPage(opts: {
+    kind: "success" | "error" | "info";
+    title: string;
+    body: string;
+    email?: string;
+    credits?: string;
+    tryIt?: string;
+}): string;
 export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;

package/dist/cli.js

@@ -5,7 +5,7 @@ import { createServer } from "node:http";
 import { createInterface } from "node:readline";
 import { parseArgs } from "node:util";
 import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
-import { formatCrawl, formatCredits, formatMap, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
+import { formatCrawl, formatCredits, formatMap, formatMonitor, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
 import { SUPPORTED_AGENTS, runDoctor, runSetup, } from "./setup.js";
 export const HELP = `Webcrawl Anything: search, scrape, parse and crawl any target on the internet
 
@@ -18,6 +18,7 @@ Commands
   crawl <url>         Read several pages. -n max pages (default 5, max 25)
   extract <url>       Structured JSON. --prompt <text> or --schema <file.json>
   rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top
+  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>
   targets             List all supported crawl & search targets and their requirements
   categories          List search categories and vertical filters
   setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
@@ -36,6 +37,12 @@ Options
   --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)
   --country, --city   Where to search from (default: this machine's timezone)
   --no-cache          Skip cached pages (scrape)
+  --scope <s|u|w>     Monitor scope: page (default), urls, or site
+  --interval <mins>   Monitor check interval in minutes (default 60)
+  --watch-for <text>  What to watch for (e.g. "Pricing tiers and discounts")
+  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains "In Stock"')
+  --notify-email <e>  Alert email address when triggered (or 'true' for account email)
+  --webhook <url>     Alert webhook URL when triggered
   --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)
   --schema-profile    MCP schema profile for setup: compact (default) or full
   -h, --help
@@ -66,6 +73,12 @@ const OPTIONS = {
     "no-wait": { type: "boolean" },
     "no-cache": { type: "boolean" },
     content: { type: "boolean" },
+    scope: { type: "string" },
+    interval: { type: "string" },
+    "watch-for": { type: "string" },
+    condition: { type: "string" },
+    "notify-email": { type: "string" },
+    webhook: { type: "string" },
     agent: { type: "string", multiple: true },
     "schema-profile": { type: "string" },
 };
@@ -112,6 +125,52 @@ function openBrowserUrl(url) {
         // ignore
     }
 }
+const esc = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+/** Self-contained (no network assets) page served on the loopback callback. */
+export function callbackPage(opts) {
+    const tone = opts.kind === "success" ? "#157f3b" : opts.kind === "error" ? "#b42318" : "#6236e8";
+    const icon = opts.kind === "success"
+        ? `<path class="mk" d="M20 33l8 8 16-17"/>`
+        : opts.kind === "error"
+            ? `<path class="mk" d="M23 23l18 18M41 23L23 41"/>`
+            : `<path class="mk" d="M32 22v14M32 42v.5"/>`;
+    const chips = [opts.email ? `<span class="chip">${esc(opts.email)}</span>` : "", opts.credits ? `<span class="chip hot">${esc(opts.credits)}</span>` : ""].join("");
+    const dots = Array.from({ length: 14 }, (_, i) => `<i style="--a:${i * 25.7}deg;--d:${70 + (i % 3) * 18}px;--t:${(i % 5) * 0.06}s"></i>`).join("");
+    return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>${esc(opts.title)}</title>
+<style>
+:root{--tone:${tone}}*{box-sizing:border-box}
+body{margin:0;min-height:100vh;display:grid;place-items:center;padding:24px;color:#1b1631;font:15px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Inter,sans-serif;background:#fbfaf8}
+main{width:100%;max-width:440px;text-align:center;background:#fff;border:1px solid #e2ddd5;border-radius:16px;padding:36px 32px 28px;box-shadow:0 1px 2px #1b16310d,0 12px 32px #1b16310f;animation:rise .5s cubic-bezier(.2,.8,.2,1) both}
+.brand{display:flex;align-items:center;justify-content:center;gap:8px;font-weight:600;letter-spacing:-.035em;color:#1b1631;margin-bottom:26px}.brand svg{width:26px;height:26px}
+.badge{position:relative;width:96px;height:96px;margin:0 auto 18px}
+.badge svg.ring{width:96px;height:96px;overflow:visible}
+.ring circle{fill:color-mix(in srgb,var(--tone) 12%,transparent);stroke:var(--tone);stroke-width:2.5;stroke-dasharray:190;stroke-dashoffset:190;animation:draw .7s .1s ease-out forwards}
+.mk{fill:none;stroke:var(--tone);stroke-width:4.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:60;stroke-dashoffset:60;animation:draw .45s .6s ease-out forwards}
+.burst{position:absolute;inset:0;pointer-events:none}.burst i{position:absolute;left:50%;top:50%;width:6px;height:6px;margin:-3px;border-radius:50%;background:var(--tone);opacity:0;transform:rotate(var(--a)) translateY(0);animation:pop .9s calc(.7s + var(--t)) ease-out forwards}
+h1{font-size:22px;margin:0 0 6px;letter-spacing:-.02em}p{margin:0 0 18px;color:#4a4460}
+.chips{display:flex;gap:8px;justify-content:center;flex-wrap:wrap;margin-bottom:20px}
+.chip{font-size:12.5px;padding:4px 10px;border-radius:999px;background:#f6f4f0;border:1px solid #e2ddd5;color:#4a4460}.chip.hot{background:#f1ecff;border-color:#d9ccff;color:#4f24cf}
+.try{text-align:left;background:#f6f4f0;border:1px solid #efebe5;border-radius:10px;padding:12px 14px;margin-bottom:18px}
+.trow{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}.trow small{margin:0!important}
+.copy{font:600 12px/1 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;padding:6px 10px;border-radius:7px;border:1px solid #e2ddd5;background:#fff;color:#4a4460;cursor:pointer}.copy:hover{border-color:#6236e8;color:#4f24cf}
+.try code{user-select:all;-webkit-user-select:all;cursor:text}
+.try small{display:block;color:#5e5873;margin-bottom:4px;font-size:12px;text-transform:uppercase;letter-spacing:.08em}
+.try code{font:13px/1.4 ui-monospace,SFMono-Regular,Menlo,monospace;color:#4f24cf;word-break:break-all}.try code:before{content:"$ ";color:#7d7690}
+.foot{font-size:13px;color:#5e5873;margin:0}
+@keyframes rise{from{opacity:0;transform:translateY(14px) scale(.98)}}
+@keyframes draw{to{stroke-dashoffset:0}}
+@keyframes pop{0%{opacity:1}100%{opacity:0;transform:rotate(var(--a)) translateY(calc(var(--d) * -1)) scale(.3)}}
+@media (prefers-reduced-motion:reduce){*{animation-duration:.01s!important;animation-delay:0s!important}}
+</style></head><body><main>
+<div class="brand"><svg viewBox="0 0 32 32" aria-hidden="true"><rect width="32" height="32" rx="7" fill="#1b1631"/><g stroke-linejoin="round" stroke-linecap="round" fill="none" stroke-width="3"><path d="M5 21.5L16 27.5L27 21.5" stroke="#8a80b8"/><path d="M5 16L16 22L27 16" stroke="#c9b8ff"/></g><path d="M16 4.5L27 10.5L16 16.5L5 10.5Z" fill="#8b6cff" stroke="#8b6cff" stroke-width="2" stroke-linejoin="round"/></svg>Webcrawl</div>
+<div class="badge"><svg class="ring" viewBox="0 0 64 64" aria-hidden="true"><circle cx="32" cy="32" r="30"/>${icon}</svg>${opts.kind === "success" ? `<div class="burst">${dots}</div>` : ""}</div>
+<h1>${esc(opts.title)}</h1><p>${opts.body}</p>
+${chips ? `<div class="chips">${chips}</div>` : ""}
+${opts.tryIt ? `<div class="try"><div class="trow"><small>Try it now</small><button type="button" class="copy" id="cp">Copy</button></div><code id="cmd">${esc(opts.tryIt)}</code></div>
+<script>(function(){var b=document.getElementById("cp"),c=document.getElementById("cmd");if(!b||!c)return;var t=c.textContent;function done(){b.textContent="Copied";setTimeout(function(){b.textContent="Copy"},1500)}function fb(){var r=document.createRange();r.selectNodeContents(c);var s=getSelection();s.removeAllRanges();s.addRange(r);try{document.execCommand("copy");done()}catch(e){}}b.onclick=function(){if(navigator.clipboard&&navigator.clipboard.writeText){navigator.clipboard.writeText(t).then(done,fb)}else{fb()}}})()</script>` : ""}
+<p class="foot">You can close this tab and return to your terminal.</p>
+</main></body></html>`;
+}
 async function runWebOAuthLogin(baseUrl, fetchImpl, io, openUrl = openBrowserUrl) {
     const verifier = crypto.randomBytes(32).toString("base64url");
     const challenge = crypto.createHash("sha256").update(verifier).digest("base64url");
@@ -167,14 +226,14 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io, openUrl = openBrowserUrl
                         if (handled) {
                             // Browser retry / prefetch of the same callback: the code is single-use, so never exchange it twice.
                             res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
-                            res.end("<h3>Already received. You can close this tab and return to your terminal.</h3>");
+                            res.end(callbackPage({ kind: "info", title: "Already received", body: "This sign-in link was already used. Your terminal has everything it needs." }));
                             return;
                         }
                         const oauthError = u.searchParams.get("error");
                         if (oauthError && returnedState === state) {
                             handled = true;
                             res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
-                            res.end("<h3>Authorization was cancelled. You can close this tab.</h3>");
+                            res.end(callbackPage({ kind: "error", title: "Sign-in cancelled", body: "You chose not to connect the CLI. Run <b>webcrawl login</b> whenever you want to try again." }));
                             clearTimeout(timeout);
                             server.close();
                             reject(new Error(`OAuth authorization failed: ${oauthError}.`));
@@ -183,7 +242,7 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io, openUrl = openBrowserUrl
                         if (!code || returnedState !== state) {
                             // Stray or forged request: reject it but keep waiting for the real callback.
                             res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
-                            res.end("<h3>Authorization failed or state mismatch.</h3>");
+                            res.end(callbackPage({ kind: "error", title: "Link not recognised", body: "This sign-in link does not match the one your terminal is waiting for. Run <b>webcrawl login</b> again for a fresh link." }));
                             return;
                         }
                         handled = true;
@@ -202,7 +261,7 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io, openUrl = openBrowserUrl
                             const detail = (await tokenRes.json().catch(() => ({})));
                             const reason = detail.error_description || detail.error;
                             res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
-                            res.end("<h3>Webcrawl sign-in failed. Return to your terminal for details.</h3>");
+                            res.end(callbackPage({ kind: "error", title: "Sign-in failed", body: "Webcrawl could not finish signing you in. Your terminal has the details. Run <b>webcrawl login</b> to try again." }));
                             throw new Error(`OAuth token exchange failed (HTTP ${tokenRes.status}${reason ? `: ${reason}` : ""}). Run \`webcrawl login\` again, or use \`webcrawl login --email <you@domain.com>\`.`);
                         }
                         const tokenData = (await tokenRes.json());
@@ -224,7 +283,17 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io, openUrl = openBrowserUrl
                         }
                         const meRes = await authedClient.me().catch(() => undefined);
                         res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
-                        res.end("<html><body style='font-family:system-ui;padding:2rem'><h2>✓ Webcrawl CLI authenticated!</h2><p>Your account and 1,000 monthly credits are now linked. You can close this tab and return to your terminal or agent.</p></body></html>");
+                        const u2 = meRes?.user;
+                        res.end(callbackPage({
+                            kind: "success",
+                            title: "You're connected",
+                            body: "Webcrawl CLI is signed in. Search, read and crawl the web from your terminal or your agent.",
+                            email: u2?.email,
+                            credits: u2?.creditsRemaining !== undefined && u2?.creditsLimit !== undefined
+                                ? `${u2.creditsRemaining.toLocaleString("en-US")} / ${u2.creditsLimit.toLocaleString("en-US")} credits`
+                                : undefined,
+                            tryIt: 'webcrawl search "best ramen in shibuya"',
+                        }));
                         clearTimeout(timeout);
                         server.close();
                         resolve({
@@ -312,8 +381,11 @@ export async function main(argv, io, deps = {}) {
             io.err(`[${credits}]\n`);
         }
     };
-    const METERED_COMMANDS = new Set(["search", "scrape", "map", "crawl", "extract", "rerank"]);
-    if (METERED_COMMANDS.has(cmd) && cfg.effectiveMode === "cloud" && !cfg.apiKey && !deps.fetch) {
+    const METERED_COMMANDS = new Set(["search", "scrape", "map", "crawl", "extract", "rerank", "monitor"]);
+    if (METERED_COMMANDS.has(cmd) &&
+        (cfg.effectiveMode === "cloud" || cmd === "monitor") &&
+        !cfg.apiKey &&
+        !deps.fetch) {
         const authHelp = [
             "webcrawl: Cloud mode requires a logged-in webcrawl.app account so credits can be attributed to your user (1,000 free monthly credits on signup).",
             "",
@@ -405,6 +477,110 @@ export async function main(argv, io, deps = {}) {
                 emit(res, formatRerank(res));
                 return 0;
             }
+            case "monitor": {
+                const cloudClient = new WebcrawlClient({
+                    apiKey: cfg.apiKey,
+                    baseUrl: cfg.baseUrl,
+                    fetch: deps.fetch,
+                    userAgent: "webcrawl-cli/1.3",
+                });
+                const first = (rest[0] ?? "").trim();
+                const subcommands = new Set([
+                    "list",
+                    "ls",
+                    "create",
+                    "add",
+                    "watch",
+                    "get",
+                    "inspect",
+                    "check",
+                    "pause",
+                    "resume",
+                    "delete",
+                    "rm",
+                ]);
+                const sub = !first
+                    ? "list"
+                    : subcommands.has(first.toLowerCase())
+                        ? first.toLowerCase()
+                        : "create";
+                const subArgs = subcommands.has(first.toLowerCase()) ? rest.slice(1) : rest;
+                const requireMonitorId = (actionName) => {
+                    const id = (subArgs[0] ?? "").trim();
+                    if (!id) {
+                        throw new UsageError(`Missing monitor id. Usage: webcrawl monitor ${actionName} <id>`);
+                    }
+                    return id;
+                };
+                if (sub === "list" || sub === "ls") {
+                    const res = await cloudClient.monitors();
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "get" || sub === "inspect") {
+                    const res = await cloudClient.getMonitor(requireMonitorId(sub));
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "check") {
+                    const res = await cloudClient.checkMonitor(requireMonitorId("check"));
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "pause" || sub === "resume") {
+                    const id = requireMonitorId(sub);
+                    const notifyEmailRaw = v["notify-email"];
+                    const res = await cloudClient.updateMonitor({
+                        id,
+                        status: sub === "pause" ? "paused" : "active",
+                        name: v.name,
+                        intervalMinutes: int(v.interval, "--interval"),
+                        watchFor: v["watch-for"],
+                        condition: v.condition,
+                        notifyEmail: notifyEmailRaw === undefined
+                            ? undefined
+                            : notifyEmailRaw === "true"
+                                ? true
+                                : notifyEmailRaw,
+                        webhookUrl: v.webhook,
+                    });
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                if (sub === "delete" || sub === "rm") {
+                    const res = await cloudClient.deleteMonitor(requireMonitorId(sub));
+                    emit(res, formatMonitor(res));
+                    return 0;
+                }
+                // create / add / watch / `webcrawl monitor <url...>`
+                const urls = subArgs.map((s) => s.trim()).filter(Boolean);
+                if (!urls.length) {
+                    throw new UsageError("Missing URL to watch. Usage: webcrawl monitor <url> [--scope page|urls|site] [--watch-for <text>] [--condition <expr>]");
+                }
+                const rawScope = (v.scope ?? (urls.length > 1 ? "urls" : "page")).toLowerCase();
+                if (rawScope !== "page" && rawScope !== "urls" && rawScope !== "site") {
+                    throw new UsageError(`Invalid --scope "${v.scope}". Expected page, urls, or site.`);
+                }
+                const notifyEmailRaw = v["notify-email"];
+                const res = await cloudClient.createMonitor({
+                    url: urls[0],
+                    urls: rawScope === "urls" || urls.length > 1 ? urls : undefined,
+                    mode: rawScope,
+                    maxPages: int(v.limit, "--limit"),
+                    name: v.name,
+                    intervalMinutes: int(v.interval, "--interval"),
+                    watchFor: v["watch-for"],
+                    condition: v.condition,
+                    notifyEmail: notifyEmailRaw === undefined
+                        ? undefined
+                        : notifyEmailRaw === "true"
+                            ? true
+                            : notifyEmailRaw,
+                    webhookUrl: v.webhook,
+                });
+                emit(res, formatMonitor(res));
+                return 0;
+            }
             case "categories": {
                 const res = await client.categories();
                 emit(res, JSON.stringify(res, null, 2));

package/dist/format.d.ts

@@ -6,4 +6,5 @@ export declare function formatCrawl(res: Rec): string;
 export declare function formatRerank(res: Rec): string;
 export declare function formatCredits(res: Rec): string;
 export declare function formatTargets(res: Rec): string;
+export declare function formatMonitor(res: Rec): string;
 export {};

package/dist/format.js

@@ -111,3 +111,66 @@ export function formatTargets(res) {
     })
         .join("\n\n");
 }
+function formatMonitorItem(m, index) {
+    const num = index !== undefined ? `${String(index + 1).padStart(2)}  ` : "";
+    const id = str(m.id);
+    const name = str(m.name) || str(m.url) || "(unnamed watch)";
+    const status = str(m.status) || "active";
+    const mode = str(m.mode) || "page";
+    const urls = Array.isArray(m.urls) ? m.urls.map(str).filter(Boolean) : [];
+    const scopeLabel = mode === "site"
+        ? `site (max ${str(m.maxPages) || "5"} pages)`
+        : mode === "urls"
+            ? `${urls.length || 1} URLs`
+            : "1 page";
+    const interval = typeof m.intervalMinutes === "number" ? `every ${m.intervalMinutes}m` : "";
+    const condExpr = str(m.conditionJev) || (typeof m.condition === "string" ? m.condition : "");
+    const goal = condExpr ? `rule: ${condExpr}` : str(m.watchFor) ? `goal: ${str(m.watchFor)}` : "any change";
+    const condMet = typeof m.lastConditionMet === "boolean"
+        ? m.lastConditionMet
+            ? "triggered"
+            : "waiting"
+        : "";
+    const meta = [status, scopeLabel, interval, condMet].filter(Boolean).join(" · ");
+    const pad = index !== undefined ? "    " : "";
+    const diff = str(m.lastDiffSummary);
+    return [
+        `${num}${name}  [${id}]  (${meta})`,
+        `${pad}${str(m.url)}${urls.length > 1 ? ` (+${urls.length - 1} more)` : ""}  ·  ${goal}`,
+        diff ? `${pad}Latest: ${diff}` : "",
+    ]
+        .filter(Boolean)
+        .join("\n");
+}
+export function formatMonitor(res) {
+    if (res.deleted === true) {
+        return "Monitor deleted.";
+    }
+    if (Array.isArray(res.monitors)) {
+        const list = res.monitors.filter(isRec);
+        if (!list.length) {
+            return "No active watches.";
+        }
+        return list.map((m, i) => formatMonitorItem(m, i)).join("\n\n");
+    }
+    const m = isRec(res.monitor) ? res.monitor : isRec(res.data) ? res.data : null;
+    if (!m) {
+        return JSON.stringify(res, null, 2);
+    }
+    const check = isRec(res.check) ? res.check : isRec(res.initialCheck) ? res.initialCheck : null;
+    const lines = [formatMonitorItem(m)];
+    if (check) {
+        const condRes = isRec(check.conditionResult) ? check.conditionResult : null;
+        if (condRes && str(condRes.summary)) {
+            lines.push(`Condition: ${str(condRes.summary)} (met: ${Boolean(condRes.met)})`);
+        }
+        if (str(check.diffSummary)) {
+            lines.push(`Summary: ${str(check.diffSummary)}`);
+        }
+    }
+    const snap = (check && str(check.snapshot)) || str(m.lastSnapshot);
+    if (snap) {
+        lines.push(`\n${snap}`);
+    }
+    return lines.join("\n");
+}

package/dist/setup.d.ts

@@ -6,7 +6,7 @@ export interface AgentTarget {
     skillPath: (home: string, cwd: string) => string;
 }
 export declare const SUPPORTED_AGENTS: AgentTarget[];
-export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
+export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n5. **Scheduled Web Monitors (`web_monitor` / `webcrawl monitor`)**:\n   - Watch a single page (`page`), a list of URLs (`urls`), or a whole website section (`site`) on a schedule with natural-language `watch_for` goals, trigger `condition` rules (e.g. `price < 50`, `content contains \"In Stock\"`, `added_urls_count > 0`), and email or webhook alerts.\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
 export interface SetupOptions {
     agents?: SupportedAgentId[];
     schemaProfile?: "compact" | "full";

package/dist/setup.js

@@ -67,6 +67,8 @@ Use **Webcrawl Anything** (\`webcrawl\` CLI or \`webcrawl\` MCP tools) whenever
    - Reads PDFs (native text + automatic OCR on scanned pages), Word (\`.docx\`), Excel (\`.xlsx\`), PowerPoint (\`.pptx\`), and YouTube URLs (timestamped transcripts).
 4. **Site Discovery & Multi-Page Crawl (\`web_map\` & \`web_crawl\`)**:
    - Always run \`web_map\` first on unfamiliar domains to inspect available URLs before running \`web_crawl\` (capped at 25 pages).
+5. **Scheduled Web Monitors (\`web_monitor\` / \`webcrawl monitor\`)**:
+   - Watch a single page (\`page\`), a list of URLs (\`urls\`), or a whole website section (\`site\`) on a schedule with natural-language \`watch_for\` goals, trigger \`condition\` rules (e.g. \`price < 50\`, \`content contains "In Stock"\`, \`added_urls_count > 0\`), and email or webhook alerts.
 
 ## 2. Verify Evidence & Provenance
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.6",
+  "version": "1.3.8",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.6",
-    "@webcrawl/curl-impersonate": "^1.3.6",
+    "@webcrawl/client": "^1.3.8",
+    "@webcrawl/curl-impersonate": "^1.3.8",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",