webcrawl 1.4.0 → 1.4.1

package/dist/core/local-engine.d.ts

@@ -32,4 +32,9 @@ export interface LocalEngineBackend {
     targets(): Promise<Record<string, unknown>>;
     usage(): Promise<Record<string, unknown>>;
 }
+/**
+ * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
+ * Matching is a case-insensitive substring of the URL. Exported for unit tests.
+ */
+export declare function filterMappedLinks(links: string[], search?: string, limit?: number): string[];
 export declare function createLocalEngineClient(): LocalEngineBackend;

package/dist/core/local-engine.js

@@ -29,6 +29,20 @@ function safeDomainOf(url) {
         return "";
     }
 }
+/**
+ * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
+ * Matching is a case-insensitive substring of the URL. Exported for unit tests.
+ */
+export function filterMappedLinks(links, search, limit = 100) {
+    const cap = Math.max(1, limit);
+    const needle = search?.trim().toLowerCase();
+    const filtered = needle
+        ? links.filter((l) => l.toLowerCase().includes(needle))
+        : links;
+    return filtered.slice(0, cap);
+}
+/** How many URLs to discover before applying `--filter`, so a small `-n` cannot hide matches. */
+const MAP_FILTER_DISCOVER_LIMIT = 5000;
 export function createLocalEngineClient() {
     return {
         baseUrl: "local://engine",
@@ -369,13 +383,15 @@ export function createLocalEngineClient() {
         },
         async map(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
+            const limit = Math.max(1, p.limit ?? 100);
+            const needle = p.search?.trim();
+            // Discover first (widely when filtering), then apply search and limit so
+            // `map --filter webhooks -n 10` can still find /webhooks/* URLs that are
+            // not among the first 10 sitemap entries (#11).
             const mapped = await mapWebsiteUrls(targetUrl, {
-                limit: p.limit ?? 100,
+                limit: needle ? Math.max(limit, MAP_FILTER_DISCOVER_LIMIT) : limit,
             });
-            const needle = p.search?.trim().toLowerCase();
-            const links = needle
-                ? mapped.links.filter((l) => l.toLowerCase().includes(needle))
-                : mapped.links;
+            const links = filterMappedLinks(mapped.links, needle, limit);
             return {
                 success: true,
                 mode: "local",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.0",
-    "@webcrawl/curl-impersonate": "^1.4.0",
+    "@webcrawl/client": "^1.4.1",
+    "@webcrawl/curl-impersonate": "^1.4.1",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",