webcontext-ai 1.0.0

package/dist/utils/scheduler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../../src/utils/scheduler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5D;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,IAAI,CAA0C;IAEtD,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI;IAYjH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAKxB,SAAS,IAAI,IAAI;IAKjB,QAAQ,IAAI,MAAM,EAAE;IAEpB;;;;OAIG;IACH,OAAO,CAAC,cAAc;CAmBvB"}

package/dist/utils/scheduler.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlScheduler = void 0;
+/**
+ * Simple cron-like scheduler for periodic re-crawling.
+ * Uses setInterval with parsed cron expressions for basic scheduling.
+ */
+class CrawlScheduler {
+    jobs = new Map();
+    schedule(id, config, executor) {
+        this.cancel(id);
+        const interval = this.cronToInterval(config.cron);
+        const timer = setInterval(async () => {
+            for (const url of config.urls) {
+                const result = await executor(url, config.options);
+                config.onComplete?.(result);
+            }
+        }, interval);
+        this.jobs.set(id, timer);
+    }
+    cancel(id) {
+        const timer = this.jobs.get(id);
+        if (timer) {
+            clearInterval(timer);
+            this.jobs.delete(id);
+        }
+    }
+    cancelAll() {
+        for (const timer of this.jobs.values())
+            clearInterval(timer);
+        this.jobs.clear();
+    }
+    listJobs() { return [...this.jobs.keys()]; }
+    /**
+     * Parse a cron expression into a millisecond interval.
+     * Supports: *\/N for minutes/hours, day-of-week specific (runs daily),
+     * and common patterns. Falls back to 1 hour for unsupported expressions.
+     */
+    cronToInterval(cron) {
+        const parts = cron.trim().split(/\s+/);
+        if (parts.length < 5)
+            return 60 * 60 * 1000; // fallback: 1 hour
+        const [minute, hour, dayOfMonth, , dayOfWeek] = parts;
+        // */N minutes (e.g., "*/5 * * * *" = every 5 min)
+        if (minute.startsWith('*/'))
+            return parseInt(minute.slice(2)) * 60 * 1000;
+        // */N hours (e.g., "0 */2 * * *" = every 2 hours)
+        if (hour.startsWith('*/'))
+            return parseInt(hour.slice(2)) * 60 * 60 * 1000;
+        // Daily at specific time (e.g., "0 9 * * *" or "0 9 * * MON")
+        if (minute !== '*' && hour !== '*' && dayOfMonth === '*')
+            return 24 * 60 * 60 * 1000;
+        // Every hour (e.g., "0 * * * *")
+        if (minute !== '*' && hour === '*')
+            return 60 * 60 * 1000;
+        // Every minute ("* * * * *")
+        if (minute === '*')
+            return 60 * 1000;
+        return 60 * 60 * 1000; // fallback: 1 hour
+    }
+}
+exports.CrawlScheduler = CrawlScheduler;
+//# sourceMappingURL=scheduler.js.map

package/dist/utils/scheduler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../../src/utils/scheduler.ts"],"names":[],"mappings":";;;AAEA;;;GAGG;AACH,MAAa,cAAc;IACjB,IAAI,GAAgC,IAAI,GAAG,EAAE,CAAC;IAEtD,QAAQ,CAAC,EAAU,EAAE,MAAsB,EAAE,QAA6D;QACxG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACnC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBACnD,MAAM,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC,EAAE,QAAQ,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,CAAC,EAAU;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,KAAK,EAAE,CAAC;YAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAAC,CAAC;IAC5D,CAAC;IAED,SAAS;QACP,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,aAAa,CAAC,KAAK,CAAC,CAAC;QAC7D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED,QAAQ,KAAe,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtD;;;;OAIG;IACK,cAAc,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,mBAAmB;QAEhE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,AAAD,EAAG,SAAS,CAAC,GAAG,KAAK,CAAC;QAEtD,kDAAkD;QAClD,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QAC1E,kDAAkD;QAClD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC3E,8DAA8D;QAC9D,IAAI,MAAM,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACrF,iCAAiC;QACjC,IAAI,MAAM,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;YAAE,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC1D,6BAA6B;QAC7B,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QAErC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,mBAAmB;IAC5C,CAAC;CACF;AAnDD,wCAmDC"}

package/dist/utils/sitemap.d.ts

@@ -0,0 +1,17 @@
+import { SitemapEntry } from '../core/types';
+/**
+ * Parse sitemap.xml and sitemap index files.
+ * Supports: standard sitemaps, sitemap indexes, gzipped sitemaps.
+ */
+export declare class SitemapParser {
+    private userAgent;
+    constructor(userAgent?: string);
+    /** Parse a sitemap URL, handling both sitemap indexes and regular sitemaps */
+    parse(sitemapUrl: string): Promise<SitemapEntry[]>;
+    /** Discover sitemap URL from robots.txt or common locations */
+    discover(baseUrl: string): Promise<string | null>;
+    private fetchXml;
+    private parseEntries;
+    private parseSitemapIndex;
+}
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/utils/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/utils/sitemap.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAyB;IAIhD,8EAA8E;IACxE,KAAK,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAYxD,+DAA+D;IACzD,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;YA4BzC,QAAQ;IAUtB,OAAO,CAAC,YAAY;IAuBpB,OAAO,CAAC,iBAAiB;CAW1B"}

package/dist/utils/sitemap.js

@@ -0,0 +1,118 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SitemapParser = void 0;
+const cheerio = __importStar(require("cheerio"));
+/**
+ * Parse sitemap.xml and sitemap index files.
+ * Supports: standard sitemaps, sitemap indexes, gzipped sitemaps.
+ */
+class SitemapParser {
+    userAgent;
+    constructor(userAgent = 'WebContext/1.0') {
+        this.userAgent = userAgent;
+    }
+    /** Parse a sitemap URL, handling both sitemap indexes and regular sitemaps */
+    async parse(sitemapUrl) {
+        const xml = await this.fetchXml(sitemapUrl);
+        const sitemapUrls = this.parseSitemapIndex(xml);
+        if (sitemapUrls.length > 0) {
+            const results = await Promise.all(sitemapUrls.map((url) => this.parse(url)));
+            return results.flat();
+        }
+        return this.parseEntries(xml);
+    }
+    /** Discover sitemap URL from robots.txt or common locations */
+    async discover(baseUrl) {
+        const base = baseUrl.replace(/\/$/, '');
+        try {
+            const res = await fetch(`${base}/robots.txt`, {
+                headers: { 'User-Agent': this.userAgent },
+            });
+            if (res.ok) {
+                const text = await res.text();
+                const match = text.match(/^Sitemap:\s*(.+)$/im);
+                if (match)
+                    return match[1].trim();
+            }
+        }
+        catch { }
+        const commonPaths = ['/sitemap.xml', '/sitemap_index.xml'];
+        for (const path of commonPaths) {
+            try {
+                const res = await fetch(`${base}${path}`, {
+                    method: 'HEAD',
+                    headers: { 'User-Agent': this.userAgent },
+                });
+                if (res.ok)
+                    return `${base}${path}`;
+            }
+            catch { }
+        }
+        return null;
+    }
+    async fetchXml(url) {
+        const res = await fetch(url, {
+            headers: { 'User-Agent': this.userAgent },
+        });
+        if (!res.ok) {
+            throw new Error(`Failed to fetch sitemap: ${url} (${res.status})`);
+        }
+        return res.text();
+    }
+    parseEntries(xml) {
+        const $ = cheerio.load(xml, { xmlMode: true });
+        const entries = [];
+        $('url').each((_, el) => {
+            const loc = $(el).find('loc').text().trim();
+            if (!loc)
+                return;
+            const entry = { url: loc };
+            const lastmod = $(el).find('lastmod').text().trim();
+            const changefreq = $(el).find('changefreq').text().trim();
+            const priority = $(el).find('priority').text().trim();
+            if (lastmod)
+                entry.lastmod = lastmod;
+            if (changefreq)
+                entry.changefreq = changefreq;
+            if (priority)
+                entry.priority = parseFloat(priority);
+            entries.push(entry);
+        });
+        return entries;
+    }
+    parseSitemapIndex(xml) {
+        const $ = cheerio.load(xml, { xmlMode: true });
+        const urls = [];
+        $('sitemapindex sitemap loc').each((_, el) => {
+            const loc = $(el).text().trim();
+            if (loc)
+                urls.push(loc);
+        });
+        return urls;
+    }
+}
+exports.SitemapParser = SitemapParser;
+//# sourceMappingURL=sitemap.js.map

package/dist/utils/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/utils/sitemap.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAmC;AAGnC;;;GAGG;AACH,MAAa,aAAa;IAChB,SAAS,CAAS;IAE1B,YAAY,YAAoB,gBAAgB;QAC9C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,8EAA8E;IAC9E,KAAK,CAAC,KAAK,CAAC,UAAkB;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAEhD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7E,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC;QAED,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,QAAQ,CAAC,OAAe;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,aAAa,EAAE;gBAC5C,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;aAC1C,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAChD,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACpC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,MAAM,WAAW,GAAG,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAC;QAC3D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE;oBACxC,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;iBAC1C,CAAC,CAAC;gBACH,IAAI,GAAG,CAAC,EAAE;oBAAE,OAAO,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAW;QAChC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACtB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,MAAM,KAAK,GAAiB,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1D,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAEtD,IAAI,OAAO;gBAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;YACrC,IAAI,UAAU;gBAAE,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;YAC9C,IAAI,QAAQ;gBAAE,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YAEpD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,iBAAiB,CAAC,GAAW;QACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3C,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,GAAG;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AA7FD,sCA6FC"}

package/dist/utils/validation.d.ts

@@ -0,0 +1,142 @@
+import { z } from 'zod';
+import { CrawlOptions, WebContextConfig } from '../core/types';
+/** Zod schemas for input validation */
+export declare const urlSchema: z.ZodString;
+export declare const crawlOptionsSchema: z.ZodObject<{
+    url: z.ZodString;
+    depth: z.ZodOptional<z.ZodNumber>;
+    maxPages: z.ZodOptional<z.ZodNumber>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+    delay: z.ZodOptional<z.ZodNumber>;
+    respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+    includeSitemap: z.ZodOptional<z.ZodBoolean>;
+    includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    url: z.ZodString;
+    depth: z.ZodOptional<z.ZodNumber>;
+    maxPages: z.ZodOptional<z.ZodNumber>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+    delay: z.ZodOptional<z.ZodNumber>;
+    respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+    includeSitemap: z.ZodOptional<z.ZodBoolean>;
+    includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    url: z.ZodString;
+    depth: z.ZodOptional<z.ZodNumber>;
+    maxPages: z.ZodOptional<z.ZodNumber>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+    delay: z.ZodOptional<z.ZodNumber>;
+    respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+    includeSitemap: z.ZodOptional<z.ZodBoolean>;
+    includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, z.ZodTypeAny, "passthrough">>;
+export declare const webContextConfigSchema: z.ZodObject<{
+    baseUrl: z.ZodString;
+    outputDir: z.ZodOptional<z.ZodString>;
+    crawlOptions: z.ZodOptional<z.ZodObject<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, z.ZodTypeAny, "passthrough">>>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    baseUrl: z.ZodString;
+    outputDir: z.ZodOptional<z.ZodString>;
+    crawlOptions: z.ZodOptional<z.ZodObject<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, z.ZodTypeAny, "passthrough">>>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    baseUrl: z.ZodString;
+    outputDir: z.ZodOptional<z.ZodString>;
+    crawlOptions: z.ZodOptional<z.ZodObject<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        url: z.ZodString;
+        depth: z.ZodOptional<z.ZodNumber>;
+        maxPages: z.ZodOptional<z.ZodNumber>;
+        timeout: z.ZodOptional<z.ZodNumber>;
+        delay: z.ZodOptional<z.ZodNumber>;
+        respectRobotsTxt: z.ZodOptional<z.ZodBoolean>;
+        includeSitemap: z.ZodOptional<z.ZodBoolean>;
+        includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, z.ZodTypeAny, "passthrough">>>;
+}, z.ZodTypeAny, "passthrough">>;
+export declare function validateUrl(url: string): string;
+export declare function validateCrawlOptions(options: unknown): CrawlOptions;
+export declare function validateConfig(config: unknown): WebContextConfig;
+//# sourceMappingURL=validation.d.ts.map

package/dist/utils/validation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/utils/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAE/D,uCAAuC;AACvC,eAAO,MAAM,SAAS,aAAuC,CAAC;AAE9D,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAUf,CAAC;AAEjB,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAInB,CAAC;AAEjB,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,YAAY,CAEnE;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAEhE"}

package/dist/utils/validation.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateConfig = exports.validateCrawlOptions = exports.validateUrl = exports.webContextConfigSchema = exports.crawlOptionsSchema = exports.urlSchema = void 0;
+const zod_1 = require("zod");
+/** Zod schemas for input validation */
+exports.urlSchema = zod_1.z.string().url('Invalid URL format');
+exports.crawlOptionsSchema = zod_1.z.object({
+    url: exports.urlSchema,
+    depth: zod_1.z.number().int().min(0, 'Depth must be >= 0').max(10, 'Depth must be <= 10').optional(),
+    maxPages: zod_1.z.number().int().min(1, 'maxPages must be >= 1').max(10000, 'maxPages must be <= 10000').optional(),
+    timeout: zod_1.z.number().int().min(1000, 'Timeout must be >= 1000ms').max(120000, 'Timeout must be <= 120000ms').optional(),
+    delay: zod_1.z.number().int().min(0, 'Delay must be >= 0').max(60000, 'Delay must be <= 60000ms').optional(),
+    respectRobotsTxt: zod_1.z.boolean().optional(),
+    includeSitemap: zod_1.z.boolean().optional(),
+    includePatterns: zod_1.z.array(zod_1.z.string()).optional(),
+    excludePatterns: zod_1.z.array(zod_1.z.string()).optional(),
+}).passthrough();
+exports.webContextConfigSchema = zod_1.z.object({
+    baseUrl: exports.urlSchema,
+    outputDir: zod_1.z.string().min(1, 'Output directory is required').optional(),
+    crawlOptions: exports.crawlOptionsSchema.optional(),
+}).passthrough();
+function validateUrl(url) {
+    return exports.urlSchema.parse(url);
+}
+exports.validateUrl = validateUrl;
+function validateCrawlOptions(options) {
+    return exports.crawlOptionsSchema.parse(options);
+}
+exports.validateCrawlOptions = validateCrawlOptions;
+function validateConfig(config) {
+    return exports.webContextConfigSchema.parse(config);
+}
+exports.validateConfig = validateConfig;
+//# sourceMappingURL=validation.js.map

package/dist/utils/validation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validation.js","sourceRoot":"","sources":["../../src/utils/validation.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAGxB,uCAAuC;AAC1B,QAAA,SAAS,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;AAEjD,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,GAAG,EAAE,iBAAS;IACd,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,qBAAqB,CAAC,CAAC,QAAQ,EAAE;IAC9F,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC,QAAQ,EAAE;IAC7G,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC,QAAQ,EAAE;IACtH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC,QAAQ,EAAE;IACtG,gBAAgB,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACxC,cAAc,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC/C,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAChD,CAAC,CAAC,WAAW,EAAE,CAAC;AAEJ,QAAA,sBAAsB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC7C,OAAO,EAAE,iBAAS;IAClB,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,8BAA8B,CAAC,CAAC,QAAQ,EAAE;IACvE,YAAY,EAAE,0BAAkB,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC,WAAW,EAAE,CAAC;AAEjB,SAAgB,WAAW,CAAC,GAAW;IACrC,OAAO,iBAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AAC9B,CAAC;AAFD,kCAEC;AAED,SAAgB,oBAAoB,CAAC,OAAgB;IACnD,OAAO,0BAAkB,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC;AAC3D,CAAC;AAFD,oDAEC;AAED,SAAgB,cAAc,CAAC,MAAe;IAC5C,OAAO,8BAAsB,CAAC,KAAK,CAAC,MAAM,CAAqB,CAAC;AAClE,CAAC;AAFD,wCAEC"}

package/dist/utils/webhook.d.ts

@@ -0,0 +1,21 @@
+import { CrawlResult, ContentDiff } from '../core/types';
+export interface WebhookConfig {
+    url: string;
+    secret?: string;
+    events: Array<'crawl.complete' | 'crawl.error' | 'content.changed'>;
+    headers?: Record<string, string>;
+}
+/**
+ * Webhook notification system for crawl events.
+ * Sends POST requests to configured URLs when events occur.
+ */
+export declare class WebhookNotifier {
+    private configs;
+    register(config: WebhookConfig): void;
+    unregister(url: string): void;
+    notifyCrawlComplete(result: CrawlResult): Promise<void>;
+    notifyCrawlError(url: string, error: string): Promise<void>;
+    notifyContentChanged(diffs: ContentDiff[]): Promise<void>;
+    private send;
+}
+//# sourceMappingURL=webhook.d.ts.map

package/dist/utils/webhook.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webhook.d.ts","sourceRoot":"","sources":["../../src/utils/webhook.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEzD,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,aAAa,GAAG,iBAAiB,CAAC,CAAC;IACpE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAuB;IAEtC,QAAQ,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAIrC,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIvB,mBAAmB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvD,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU3D,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAkBjD,IAAI;CAwBnB"}

package/dist/utils/webhook.js

@@ -0,0 +1,108 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebhookNotifier = void 0;
+/**
+ * Webhook notification system for crawl events.
+ * Sends POST requests to configured URLs when events occur.
+ */
+class WebhookNotifier {
+    configs = [];
+    register(config) {
+        this.configs.push(config);
+    }
+    unregister(url) {
+        this.configs = this.configs.filter(c => c.url !== url);
+    }
+    async notifyCrawlComplete(result) {
+        const subscribers = this.configs.filter(c => c.events.includes('crawl.complete'));
+        const payload = {
+            event: 'crawl.complete',
+            timestamp: new Date().toISOString(),
+            data: {
+                source: result.context.source,
+                pagesProcessed: result.stats.pagesProcessed,
+                totalTokens: result.stats.totalTokens,
+                duration: result.stats.duration,
+                errors: result.stats.errors.length,
+            },
+        };
+        await this.send(subscribers, payload);
+    }
+    async notifyCrawlError(url, error) {
+        const subscribers = this.configs.filter(c => c.events.includes('crawl.error'));
+        const payload = {
+            event: 'crawl.error',
+            timestamp: new Date().toISOString(),
+            data: { url, error },
+        };
+        await this.send(subscribers, payload);
+    }
+    async notifyContentChanged(diffs) {
+        if (!diffs.length)
+            return;
+        const subscribers = this.configs.filter(c => c.events.includes('content.changed'));
+        const payload = {
+            event: 'content.changed',
+            timestamp: new Date().toISOString(),
+            data: {
+                changedPages: diffs.length,
+                diffs: diffs.map(d => ({
+                    url: d.url,
+                    addedSections: d.addedSections,
+                    removedSections: d.removedSections,
+                })),
+            },
+        };
+        await this.send(subscribers, payload);
+    }
+    async send(subscribers, payload) {
+        const promises = subscribers.map(async (config) => {
+            try {
+                const headers = {
+                    'Content-Type': 'application/json',
+                    ...config.headers,
+                };
+                if (config.secret) {
+                    const { createHmac } = await Promise.resolve().then(() => __importStar(require('crypto')));
+                    const signature = createHmac('sha256', config.secret)
+                        .update(JSON.stringify(payload))
+                        .digest('hex');
+                    headers['X-Webhook-Signature'] = signature;
+                }
+                await fetch(config.url, {
+                    method: 'POST',
+                    headers,
+                    body: JSON.stringify(payload),
+                    signal: AbortSignal.timeout(10000),
+                });
+            }
+            catch { }
+        });
+        await Promise.allSettled(promises);
+    }
+}
+exports.WebhookNotifier = WebhookNotifier;
+//# sourceMappingURL=webhook.js.map

package/dist/utils/webhook.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webhook.js","sourceRoot":"","sources":["../../src/utils/webhook.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AASA;;;GAGG;AACH,MAAa,eAAe;IAClB,OAAO,GAAoB,EAAE,CAAC;IAEtC,QAAQ,CAAC,MAAqB;QAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,UAAU,CAAC,GAAW;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,MAAmB;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,gBAAgB;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE;gBACJ,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;gBAC7B,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc;gBAC3C,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;gBACrC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;gBAC/B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;aACnC;SACF,CAAC;QACF,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,KAAa;QAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,aAAa;YACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE;SACrB,CAAC;QACF,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,KAAoB;QAC7C,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,OAAO;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACnF,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,iBAAiB;YACxB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE;gBACJ,YAAY,EAAE,KAAK,CAAC,MAAM;gBAC1B,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACrB,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,aAAa,EAAE,CAAC,CAAC,aAAa;oBAC9B,eAAe,EAAE,CAAC,CAAC,eAAe;iBACnC,CAAC,CAAC;aACJ;SACF,CAAC;QACF,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,WAA4B,EAAE,OAAY;QAC3D,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAChD,IAAI,CAAC;gBACH,MAAM,OAAO,GAA2B;oBACtC,cAAc,EAAE,kBAAkB;oBAClC,GAAG,MAAM,CAAC,OAAO;iBAClB,CAAC;gBACF,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClB,MAAM,EAAE,UAAU,EAAE,GAAG,wDAAa,QAAQ,GAAC,CAAC;oBAC9C,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC;yBAClD,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;yBAC/B,MAAM,CAAC,KAAK,CAAC,CAAC;oBACjB,OAAO,CAAC,qBAAqB,CAAC,GAAG,SAAS,CAAC;gBAC7C,CAAC;gBACD,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE;oBACtB,MAAM,EAAE,MAAM;oBACd,OAAO;oBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;oBAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;iBACnC,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;CACF;AA/ED,0CA+EC"}

package/package.json

@@ -0,0 +1,109 @@
+{
+  "name": "webcontext-ai",
+  "version": "1.0.0",
+  "description": "Turn any web content into clean AI-ready context — with crawling, chunking, semantic search, and MCP tools",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "webcontext": "./dist/cli/index.js",
+    "webcontext-mcp": "./dist/mcp-server.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.js"
+    },
+    "./sdk/client": {
+      "types": "./dist/sdk/client.d.ts",
+      "require": "./dist/sdk/client.js"
+    },
+    "./sdk/mcp": {
+      "types": "./dist/sdk/mcp.d.ts",
+      "require": "./dist/sdk/mcp.js"
+    },
+    "./search": {
+      "types": "./dist/search/vector.d.ts",
+      "require": "./dist/search/vector.js"
+    },
+    "./utils": {
+      "types": "./dist/utils/index.d.ts",
+      "require": "./dist/utils/index.js"
+    },
+    "./export": {
+      "types": "./dist/export/index.d.ts",
+      "require": "./dist/export/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build",
+    "dev": "ts-node src/index.ts",
+    "cli": "ts-node src/cli/index.ts",
+    "start": "node dist/cli/index.js",
+    "serve": "node dist/cli/index.js serve",
+    "test": "jest",
+    "lint": "eslint src/"
+  },
+  "keywords": [
+    "web-scraping",
+    "ai-context",
+    "llm",
+    "rag",
+    "markdown",
+    "crawling",
+    "web-crawler",
+    "documentation",
+    "firecrawl",
+    "mcp",
+    "vector-search",
+    "sitemap",
+    "tfidf",
+    "semantic-search",
+    "mcp-server",
+    "langchain",
+    "pdf-extraction",
+    "github",
+    "deduplication"
+  ],
+  "author": "sumeethmoolya",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Sumeeth-24/webScrapper-ai.git"
+  },
+  "homepage": "https://github.com/Sumeeth-24/webScrapper-ai#readme",
+  "license": "MIT",
+  "dependencies": {
+    "turndown": "7.2.0",
+    "cheerio": "1.0.0-rc.12",
+    "tiktoken": "1.0.15",
+    "commander": "12.1.0",
+    "ora": "5.4.1",
+    "chalk": "4.1.2",
+    "p-queue": "6.6.2",
+    "robots-parser": "3.0.1",
+    "lru-cache": "10.2.2",
+    "zod": "3.23.8",
+    "express": "4.19.2",
+    "cors": "2.8.5"
+  },
+  "optionalDependencies": {
+    "playwright": "1.44.0",
+    "pdf-parse": "1.1.1"
+  },
+  "devDependencies": {
+    "typescript": "5.4.5",
+    "@types/node": "20.12.12",
+    "@types/turndown": "5.0.4",
+    "@types/express": "4.17.21",
+    "@types/cors": "2.8.17",
+    "jest": "29.7.0",
+    "ts-jest": "29.1.4",
+    "ts-node": "10.9.2",
+    "eslint": "9.3.0"
+  }
+}