webcrawl 1.3.10 → 1.4.1

package/dist/core/local-engine.d.ts

@@ -32,4 +32,9 @@ export interface LocalEngineBackend {
     targets(): Promise<Record<string, unknown>>;
     usage(): Promise<Record<string, unknown>>;
 }
+/**
+ * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
+ * Matching is a case-insensitive substring of the URL. Exported for unit tests.
+ */
+export declare function filterMappedLinks(links: string[], search?: string, limit?: number): string[];
 export declare function createLocalEngineClient(): LocalEngineBackend;

package/dist/core/local-engine.js

@@ -29,6 +29,20 @@ function safeDomainOf(url) {
         return "";
     }
 }
+/**
+ * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
+ * Matching is a case-insensitive substring of the URL. Exported for unit tests.
+ */
+export function filterMappedLinks(links, search, limit = 100) {
+    const cap = Math.max(1, limit);
+    const needle = search?.trim().toLowerCase();
+    const filtered = needle
+        ? links.filter((l) => l.toLowerCase().includes(needle))
+        : links;
+    return filtered.slice(0, cap);
+}
+/** How many URLs to discover before applying `--filter`, so a small `-n` cannot hide matches. */
+const MAP_FILTER_DISCOVER_LIMIT = 5000;
 export function createLocalEngineClient() {
     return {
         baseUrl: "local://engine",
@@ -369,13 +383,15 @@ export function createLocalEngineClient() {
         },
         async map(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
+            const limit = Math.max(1, p.limit ?? 100);
+            const needle = p.search?.trim();
+            // Discover first (widely when filtering), then apply search and limit so
+            // `map --filter webhooks -n 10` can still find /webhooks/* URLs that are
+            // not among the first 10 sitemap entries (#11).
             const mapped = await mapWebsiteUrls(targetUrl, {
-                limit: p.limit ?? 100,
+                limit: needle ? Math.max(limit, MAP_FILTER_DISCOVER_LIMIT) : limit,
             });
-            const needle = p.search?.trim().toLowerCase();
-            const links = needle
-                ? mapped.links.filter((l) => l.toLowerCase().includes(needle))
-                : mapped.links;
+            const links = filterMappedLinks(mapped.links, needle, limit);
             return {
                 success: true,
                 mode: "local",

package/dist/setup.d.ts

@@ -6,7 +6,7 @@ export interface AgentTarget {
     skillPath: (home: string, cwd: string) => string;
 }
 export declare const SUPPORTED_AGENTS: AgentTarget[];
-export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n5. **Scheduled Web Monitors (`web_monitor` / `webcrawl monitor`)**:\n   - Watch a single page (`page`), a list of URLs (`urls`), or a whole website section (`site`) on a schedule with natural-language `watch_for` goals, trigger `condition` rules (e.g. `price < 50`, `content contains \"In Stock\"`, `added_urls_count > 0`), and email or webhook alerts.\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
+export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_read` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n5. **Scheduled Web Monitors (`web_monitor` / `webcrawl monitor`)**:\n   - Watch a single page (`page`), a list of URLs (`urls`), or a whole website section (`site`) on a schedule with natural-language `watch_for` goals, trigger `condition` rules (e.g. `price < 50`, `content contains \"In Stock\"`, `added_urls_count > 0`), and email or webhook alerts.\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
 export interface SetupOptions {
     agents?: SupportedAgentId[];
     schemaProfile?: "compact" | "full";

package/dist/setup.js

@@ -61,7 +61,7 @@ Use **Webcrawl Anything** (\`webcrawl\` CLI or \`webcrawl\` MCP tools) whenever
      - \`security\`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.
      - \`academic\`: ArXiv and research papers.
      - \`tech\` / \`news\` / \`general\`: General and developer search.
-2. **Page & Developer Target Inspection (\`web_scrape\` / \`webcrawl scrape <url>\`)**:
+2. **Page & Developer Target Inspection (\`web_read\` / \`webcrawl scrape <url>\`)**:
    - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or \`llms.txt\` manifest.
 3. **Binary Documents & Media (\`web_parse\` / \`webcrawl scrape <pdf-or-youtube-url>\`)**:
    - Reads PDFs (native text + automatic OCR on scanned pages), Word (\`.docx\`), Excel (\`.xlsx\`), PowerPoint (\`.pptx\`), and YouTube URLs (timestamped transcripts).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.10",
+  "version": "1.4.1",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.10",
-    "@webcrawl/curl-impersonate": "^1.3.10",
+    "@webcrawl/client": "^1.4.1",
+    "@webcrawl/curl-impersonate": "^1.4.1",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",