webcrawl 1.3.9 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/setup.d.ts +1 -1
  2. package/dist/setup.js +1 -1
  3. package/package.json +3 -3
package/dist/setup.d.ts CHANGED
@@ -6,7 +6,7 @@ export interface AgentTarget {
6
6
  skillPath: (home: string, cwd: string) => string;
7
7
  }
8
8
  export declare const SUPPORTED_AGENTS: AgentTarget[];
9
- export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n - Pass `category` when the query belongs to a vertical:\n - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n - `flights`: IATA routes, fares, airlines, stops, cabin class.\n - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n - `academic`: ArXiv and research papers.\n - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n5. **Scheduled Web Monitors (`web_monitor` / `webcrawl monitor`)**:\n - Watch a single page (`page`), a list of URLs (`urls`), or a whole website section (`site`) on a schedule with natural-language `watch_for` goals, trigger `condition` rules (e.g. `price < 50`, `content contains \"In Stock\"`, `added_urls_count > 0`), and email or webhook alerts.\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
9
+ export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n - Pass `category` when the query belongs to a vertical:\n - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n - `flights`: IATA routes, fares, airlines, stops, cabin class.\n - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n - `academic`: ArXiv and research papers.\n - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_read` / `webcrawl scrape <url>`)**:\n - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n5. **Scheduled Web Monitors (`web_monitor` / `webcrawl monitor`)**:\n - Watch a single page (`page`), a list of URLs (`urls`), or a whole website section (`site`) on a schedule with natural-language `watch_for` goals, trigger `condition` rules (e.g. `price < 50`, `content contains \"In Stock\"`, `added_urls_count > 0`), and email or webhook alerts.\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
10
10
  export interface SetupOptions {
11
11
  agents?: SupportedAgentId[];
12
12
  schemaProfile?: "compact" | "full";
package/dist/setup.js CHANGED
@@ -61,7 +61,7 @@ Use **Webcrawl Anything** (\`webcrawl\` CLI or \`webcrawl\` MCP tools) whenever
61
61
  - \`security\`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.
62
62
  - \`academic\`: ArXiv and research papers.
63
63
  - \`tech\` / \`news\` / \`general\`: General and developer search.
64
- 2. **Page & Developer Target Inspection (\`web_scrape\` / \`webcrawl scrape <url>\`)**:
64
+ 2. **Page & Developer Target Inspection (\`web_read\` / \`webcrawl scrape <url>\`)**:
65
65
  - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or \`llms.txt\` manifest.
66
66
  3. **Binary Documents & Media (\`web_parse\` / \`webcrawl scrape <pdf-or-youtube-url>\`)**:
67
67
  - Reads PDFs (native text + automatic OCR on scanned pages), Word (\`.docx\`), Excel (\`.xlsx\`), PowerPoint (\`.pptx\`), and YouTube URLs (timestamped transcripts).
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "webcrawl",
3
- "version": "1.3.9",
3
+ "version": "1.4.0",
4
4
  "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -127,8 +127,8 @@
127
127
  "@ai-sdk/gateway": "^4.0.123",
128
128
  "@firecrawl/pdf-inspector": "^1.25.2",
129
129
  "@vercel/sandbox": "^1.0.0",
130
- "@webcrawl/client": "^1.3.9",
131
- "@webcrawl/curl-impersonate": "^1.3.9",
130
+ "@webcrawl/client": "^1.4.0",
131
+ "@webcrawl/curl-impersonate": "^1.4.0",
132
132
  "ai": "^7.0.150",
133
133
  "impit": "^0.14.7",
134
134
  "mammoth": "^1.13.0",