recker 1.0.97 → 1.0.98

package/dist/browser/scrape/spider.d.ts

@@ -15,6 +15,7 @@ export interface SpiderOptions {
     concurrency?: number;
     timeout?: number;
     delay?: number;
+    allowedDomains?: string[];
     exclude?: RegExp[];
     include?: RegExp[];
     userAgent?: string;

package/dist/browser/scrape/spider.js

@@ -115,23 +115,34 @@ function getRootDomain(hostname) {
     }
     return parts.slice(-2).join('.');
 }
-function shouldCrawl(url, baseHost, options, baseRootDomain) {
+function shouldCrawl(url, baseHost, options, baseRootDomain, allowedDomains) {
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
         const hostname = parsed.hostname.replace(/^www\./, '');
-        const sameDomain = options.sameDomain;
-        if (sameDomain === 'subdomain') {
-            const pageRoot = getRootDomain(hostname);
-            const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
-            if (pageRoot !== rootDomain)
+        if (allowedDomains) {
+            if (allowedDomains.exact.has(hostname)) {
+            }
+            else if (allowedDomains.wildcards.some(w => hostname === w || hostname.endsWith('.' + w))) {
+            }
+            else {
                 return false;
+            }
         }
-        else if (sameDomain !== false) {
-            if (hostname !== baseHost)
-                return false;
+        else {
+            const sameDomain = options.sameDomain;
+            if (sameDomain === 'subdomain') {
+                const pageRoot = getRootDomain(hostname);
+                const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
+                if (pageRoot !== rootDomain)
+                    return false;
+            }
+            else if (sameDomain !== false) {
+                if (hostname !== baseHost)
+                    return false;
+            }
         }
         const skipExtensions = [
             '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
@@ -290,6 +301,20 @@ export class Spider {
             useSitemap: options.useSitemap ?? false,
             transport: options.transport ?? 'auto',
             sitemapUrl: options.sitemapUrl,
+            allowedDomains: options.allowedDomains ? (() => {
+                const exact = new Set();
+                const wildcards = [];
+                for (const d of options.allowedDomains) {
+                    const lower = d.toLowerCase().replace(/^www\./, '');
+                    if (lower.startsWith('*.')) {
+                        wildcards.push(lower.slice(2));
+                    }
+                    else {
+                        exact.add(lower);
+                    }
+                }
+                return { exact, wildcards };
+            })() : undefined,
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
@@ -1113,7 +1138,7 @@ export class Spider {
                     if (!link.href)
                         continue;
                     const normalized = normalizeUrl(link.href);
-                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain))
+                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain, this.options.allowedDomains))
                         continue;
                     candidateUrls.push(normalized);
                     candidates.push({ url: normalized, depth: item.depth + 1 });

package/dist/scrape/spider.d.ts

@@ -15,6 +15,7 @@ export interface SpiderOptions {
     concurrency?: number;
     timeout?: number;
     delay?: number;
+    allowedDomains?: string[];
     exclude?: RegExp[];
     include?: RegExp[];
     userAgent?: string;

package/dist/scrape/spider.js

@@ -115,23 +115,34 @@ function getRootDomain(hostname) {
     }
     return parts.slice(-2).join('.');
 }
-function shouldCrawl(url, baseHost, options, baseRootDomain) {
+function shouldCrawl(url, baseHost, options, baseRootDomain, allowedDomains) {
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
         const hostname = parsed.hostname.replace(/^www\./, '');
-        const sameDomain = options.sameDomain;
-        if (sameDomain === 'subdomain') {
-            const pageRoot = getRootDomain(hostname);
-            const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
-            if (pageRoot !== rootDomain)
+        if (allowedDomains) {
+            if (allowedDomains.exact.has(hostname)) {
+            }
+            else if (allowedDomains.wildcards.some(w => hostname === w || hostname.endsWith('.' + w))) {
+            }
+            else {
                 return false;
+            }
         }
-        else if (sameDomain !== false) {
-            if (hostname !== baseHost)
-                return false;
+        else {
+            const sameDomain = options.sameDomain;
+            if (sameDomain === 'subdomain') {
+                const pageRoot = getRootDomain(hostname);
+                const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
+                if (pageRoot !== rootDomain)
+                    return false;
+            }
+            else if (sameDomain !== false) {
+                if (hostname !== baseHost)
+                    return false;
+            }
         }
         const skipExtensions = [
             '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
@@ -290,6 +301,20 @@ export class Spider {
             useSitemap: options.useSitemap ?? false,
             transport: options.transport ?? 'auto',
             sitemapUrl: options.sitemapUrl,
+            allowedDomains: options.allowedDomains ? (() => {
+                const exact = new Set();
+                const wildcards = [];
+                for (const d of options.allowedDomains) {
+                    const lower = d.toLowerCase().replace(/^www\./, '');
+                    if (lower.startsWith('*.')) {
+                        wildcards.push(lower.slice(2));
+                    }
+                    else {
+                        exact.add(lower);
+                    }
+                }
+                return { exact, wildcards };
+            })() : undefined,
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
@@ -1113,7 +1138,7 @@ export class Spider {
                     if (!link.href)
                         continue;
                     const normalized = normalizeUrl(link.href);
-                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain))
+                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain, this.options.allowedDomains))
                         continue;
                     candidateUrls.push(normalized);
                     candidates.push({ url: normalized, depth: item.depth + 1 });

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.97';
+const VERSION = '1.0.98';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.97",
+  "version": "1.0.98",
   "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",