webcontext-ai 1.0.0

package/dist/mcp-server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-server.js","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":";;;AACA;;;;;;;GAOG;AACH,mCAA2C;AAE3C,MAAM,KAAK,GAAG,IAAA,oBAAc,EAAC;IAC3B,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE;IACvE,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACnG,SAAS,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE;CAClD,CAAC,CAAC;AAEH,oCAAoC;AACpC,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,OAAO;IAChB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,WAAW,EAAE,CAAC,CAAC,WAAW;KAC3B,CAAC,CAAC;CACJ,CAAC;AAEF,SAAS,YAAY,CAAC,EAAmB,EAAE,MAAW;IACpD,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,SAAS,CAAC,EAA0B,EAAE,IAAY,EAAE,OAAe;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,OAAY;IACvC,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAEvC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,YAAY;YACf,YAAY,CAAC,EAAE,EAAE;gBACf,eAAe,EAAE,YAAY;gBAC7B,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3B,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE;aAC3D,CAAC,CAAC;YACH,MAAM;QAER,KAAK,YAAY;YACf,YAAY,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1C,MAAM;QAER,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,SAAS,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,iBAAiB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACtD,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAC1D,YAAY,CAAC,EAAE,EAAE;oBACf,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBACnE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,YAAY,CAAC,EAAE,EAAE;oBACf,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC1D,OAAO,EAAE,IAAI;iBACd,CAAC,CAAC;YACL,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,2BAA2B;YAC9B,uCAAuC;YACvC,MAAM;QAER;YACE,IAAI,EAAE;gBAAE,SAAS,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,qBAAqB,MAAM,EAAE,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,mDAAmD;AACnD,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;AACnC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;IACzC,MAAM,IAAI,KAAK,CAAC;IAEhB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC7C,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,MAAM;QAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;YACrC,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,GAAG,aAAa;YAAE,MAAM;QAErD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;QAChE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC;QAEjD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC"}

package/dist/sdk/client.d.ts

@@ -0,0 +1,48 @@
+import { CrawlOptions, CrawlResult, ContentChunk, SearchResult, WebContextConfig, MetricsData } from '../core/types';
+/**
+ * WebContext SDK Client - connects to a running WebContext server or runs locally.
+ */
+export declare class WebContextClient {
+    private baseUrl?;
+    private local?;
+    constructor(options?: {
+        serverUrl?: string;
+        config?: WebContextConfig;
+    });
+    extract(url: string, options?: Partial<CrawlOptions>): Promise<CrawlResult>;
+    crawl(url: string, options?: Partial<CrawlOptions>): Promise<CrawlResult>;
+    toMarkdown(url: string): Promise<string>;
+    toChunks(url: string, options?: Partial<CrawlOptions>): Promise<ContentChunk[]>;
+    toContext(url: string, maxTokens?: number): Promise<string>;
+    search(url: string, query: string, topK?: number): Promise<SearchResult[]>;
+    getMetrics(): Promise<MetricsData>;
+    scheduleJob(id: string, config: {
+        cron: string;
+        urls: string[];
+        options?: Partial<CrawlOptions>;
+    }): Promise<{
+        id: string;
+        status: string;
+    }>;
+    cancelJob(id: string): Promise<{
+        id: string;
+        status: string;
+    }>;
+    private post;
+    private get;
+}
+/**
+ * LangChain-compatible document loader.
+ */
+export declare class WebContextLoader {
+    private client;
+    constructor(options?: {
+        serverUrl?: string;
+        config?: WebContextConfig;
+    });
+    load(url: string, options?: Partial<CrawlOptions>): Promise<Array<{
+        pageContent: string;
+        metadata: Record<string, any>;
+    }>>;
+}
+//# sourceMappingURL=client.d.ts.map

package/dist/sdk/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/sdk/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAKrH;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,KAAK,CAAC,CAAa;gBAEf,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,gBAAgB,CAAA;KAAO;IAQrE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAM/E,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAM7E,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOxC,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAOnF,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAOzD,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,SAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAOrE,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAKlC,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAK3I,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;YAMtD,IAAI;YAaJ,GAAG;CAKlB;AAED;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAmB;gBAErB,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,gBAAgB,CAAA;KAAO;IAIrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAAE,CAAC,CAAC;CAcrI"}

package/dist/sdk/client.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebContextLoader = exports.WebContextClient = void 0;
+const index_1 = require("../index");
+const zod_1 = require("zod");
+const urlSchema = zod_1.z.string().url();
+/**
+ * WebContext SDK Client - connects to a running WebContext server or runs locally.
+ */
+class WebContextClient {
+    baseUrl;
+    local;
+    constructor(options = {}) {
+        if (options.serverUrl) {
+            this.baseUrl = options.serverUrl.replace(/\/$/, '');
+        }
+        else {
+            this.local = new index_1.WebContext(options.config);
+        }
+    }
+    async extract(url, options = {}) {
+        urlSchema.parse(url);
+        if (this.local)
+            return this.local.extract(url, options);
+        return this.post('/extract', { url, options });
+    }
+    async crawl(url, options = {}) {
+        urlSchema.parse(url);
+        if (this.local)
+            return this.local.crawlDocs(url, options);
+        return this.post('/crawl', { url, options });
+    }
+    async toMarkdown(url) {
+        urlSchema.parse(url);
+        if (this.local)
+            return this.local.toMarkdown(url);
+        const res = await this.post('/extract', { url });
+        return res.markdown;
+    }
+    async toChunks(url, options = {}) {
+        urlSchema.parse(url);
+        if (this.local)
+            return this.local.toChunks(url, options);
+        const res = await this.post('/chunks', { url, options });
+        return res.chunks;
+    }
+    async toContext(url, maxTokens = 8000) {
+        urlSchema.parse(url);
+        if (this.local)
+            return this.local.toContext(url, { maxTokens });
+        const res = await this.post('/context', { url, maxTokens });
+        return res.context;
+    }
+    async search(url, query, topK = 5) {
+        urlSchema.parse(url);
+        if (this.local)
+            return this.local.search(url, query, topK);
+        const res = await this.post('/search', { url, query, topK });
+        return res.results;
+    }
+    async getMetrics() {
+        if (!this.baseUrl)
+            throw new Error('Metrics only available in remote mode');
+        return this.get('/metrics');
+    }
+    async scheduleJob(id, config) {
+        config.urls.forEach(u => urlSchema.parse(u));
+        return this.post('/schedule', { id, ...config });
+    }
+    async cancelJob(id) {
+        const response = await fetch(`${this.baseUrl}/schedule/${id}`, { method: 'DELETE' });
+        if (!response.ok)
+            throw new Error(`HTTP ${response.status}`);
+        return response.json();
+    }
+    async post(path, body) {
+        const response = await fetch(`${this.baseUrl}${path}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(body),
+        });
+        if (!response.ok) {
+            const err = await response.json().catch(() => ({ error: response.statusText }));
+            throw new Error(err.error || `HTTP ${response.status}`);
+        }
+        return response.json();
+    }
+    async get(path) {
+        const response = await fetch(`${this.baseUrl}${path}`);
+        if (!response.ok)
+            throw new Error(`HTTP ${response.status}`);
+        return response.json();
+    }
+}
+exports.WebContextClient = WebContextClient;
+/**
+ * LangChain-compatible document loader.
+ */
+class WebContextLoader {
+    client;
+    constructor(options = {}) {
+        this.client = new WebContextClient(options);
+    }
+    async load(url, options = {}) {
+        const chunks = await this.client.toChunks(url, options);
+        return chunks.map(chunk => ({
+            pageContent: chunk.content,
+            metadata: {
+                source: chunk.metadata.sourceUrl,
+                title: chunk.metadata.title,
+                headingPath: chunk.metadata.headingPath,
+                chunkIndex: chunk.metadata.chunkIndex,
+                hasCode: chunk.metadata.hasCode,
+                language: chunk.metadata.language,
+            },
+        }));
+    }
+}
+exports.WebContextLoader = WebContextLoader;
+//# sourceMappingURL=client.js.map

package/dist/sdk/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/sdk/client.ts"],"names":[],"mappings":";;;AAAA,oCAAsC;AAGtC,6BAAwB;AAExB,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;AAEnC;;GAEG;AACH,MAAa,gBAAgB;IACnB,OAAO,CAAU;IACjB,KAAK,CAAc;IAE3B,YAAY,UAA6D,EAAE;QACzE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,GAAG,IAAI,kBAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC5D,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC1D,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1D,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAW;QAC1B,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACjD,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC7D,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACzD,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,IAAI;QAC3C,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5D,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,KAAa,EAAE,IAAI,GAAG,CAAC;QAC/C,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAU,EAAE,MAAyE;QACrG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAAU;QACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,aAAa,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,IAAS;QACxC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,EAAE;YACrD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,GAAG,CAAC,IAAY;QAC5B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;CACF;AAtFD,4CAsFC;AAED;;GAEG;AACH,MAAa,gBAAgB;IACnB,MAAM,CAAmB;IAEjC,YAAY,UAA6D,EAAE;QACzE,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,UAAiC,EAAE;QACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1B,WAAW,EAAE,KAAK,CAAC,OAAO;YAC1B,QAAQ,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;gBAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;gBACvC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACrC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;gBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;aAClC;SACF,CAAC,CAAC,CAAC;IACN,CAAC;CACF;AArBD,4CAqBC"}

package/dist/sdk/mcp.d.ts

@@ -0,0 +1,12 @@
+/**
+ * MCP (Model Context Protocol) Server for WebContext.
+ * Exposes web extraction as tools for AI agents.
+ */
+export interface MCPTool {
+    name: string;
+    description: string;
+    inputSchema: Record<string, any>;
+    handler: (input: any) => Promise<any>;
+}
+export declare function createMCPTools(config?: any): MCPTool[];
+//# sourceMappingURL=mcp.d.ts.map

package/dist/sdk/mcp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../src/sdk/mcp.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;CACvC;AAED,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,GAAG,GAAG,OAAO,EAAE,CA4ItD"}

package/dist/sdk/mcp.js

@@ -0,0 +1,146 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMCPTools = void 0;
+const index_1 = require("../index");
+function createMCPTools(config) {
+    const wc = new index_1.WebContext(config);
+    return [
+        {
+            name: 'webcontext_extract',
+            description: 'Extract clean content from a web URL. Returns markdown with code blocks preserved.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'URL to extract content from' },
+                    focus: { type: 'string', enum: ['full', 'article', 'code', 'api', 'readme'], description: 'Content focus mode' },
+                    maxTokens: { type: 'number', description: 'Maximum tokens in output' },
+                },
+                required: ['url'],
+            },
+            handler: async (input) => {
+                const context = await wc.toContext(input.url, {
+                    focusMode: input.focus || 'full',
+                    maxTokens: input.maxTokens,
+                });
+                return { content: context };
+            },
+        },
+        {
+            name: 'webcontext_crawl',
+            description: 'Crawl a documentation site and return structured content.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'Base URL to crawl' },
+                    depth: { type: 'number', description: 'Crawl depth (default: 2)' },
+                    maxPages: { type: 'number', description: 'Max pages (default: 20)' },
+                },
+                required: ['url'],
+            },
+            handler: async (input) => {
+                const result = await wc.crawlDocs(input.url, {
+                    depth: input.depth ?? 2,
+                    maxPages: input.maxPages ?? 20,
+                });
+                return {
+                    pages: result.pages.map(p => ({ title: p.title, url: p.url, markdown: p.markdown.slice(0, 2000) })),
+                    stats: result.stats,
+                };
+            },
+        },
+        {
+            name: 'webcontext_search',
+            description: 'Extract content from a URL and perform semantic search for specific information.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'URL to search within' },
+                    query: { type: 'string', description: 'Search query to find relevant sections' },
+                    topK: { type: 'number', description: 'Number of results (default: 5)' },
+                },
+                required: ['url', 'query'],
+            },
+            handler: async (input) => {
+                const results = await wc.search(input.url, input.query, input.topK ?? 5);
+                return { results: results.map(r => ({ content: r.chunk.content, score: r.score, metadata: r.chunk.metadata })) };
+            },
+        },
+        {
+            name: 'webcontext_chunk',
+            description: 'Get RAG-ready content chunks from a URL.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'URL to chunk' },
+                    maxTokens: { type: 'number', description: 'Max tokens per chunk (default: 1500)' },
+                },
+                required: ['url'],
+            },
+            handler: async (input) => {
+                const chunks = await wc.toChunks(input.url);
+                return { chunks: chunks.map(c => ({ id: c.id, content: c.content, tokens: c.tokens, metadata: c.metadata })), count: chunks.length };
+            },
+        },
+        {
+            name: 'webcontext_summarize',
+            description: 'Get an extractive summary of a web page.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'URL to summarize' },
+                },
+                required: ['url'],
+            },
+            handler: async (input) => {
+                const result = await wc.extract(input.url);
+                return {
+                    title: result.pages[0]?.title,
+                    summary: result.context.summary,
+                    pageCount: result.stats.pagesProcessed,
+                    totalTokens: result.stats.totalTokens,
+                };
+            },
+        },
+        {
+            name: 'webcontext_github',
+            description: 'Extract README and docs from a GitHub repository.',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'GitHub repository URL (e.g., https://github.com/user/repo)' },
+                    includeDocs: { type: 'boolean', description: 'Also extract /docs folder (default: true)' },
+                },
+                required: ['url'],
+            },
+            handler: async (input) => {
+                const result = await wc.extractGitHub(input.url, { depth: input.includeDocs !== false ? 1 : 0 });
+                return {
+                    pages: result.pages.map(p => ({ title: p.title, url: p.url, markdown: p.markdown.slice(0, 3000) })),
+                    stats: result.stats,
+                };
+            },
+        },
+        {
+            name: 'webcontext_pdf',
+            description: 'Extract text content from a PDF file (URL or local path).',
+            inputSchema: {
+                type: 'object',
+                properties: {
+                    source: { type: 'string', description: 'PDF URL or local file path' },
+                },
+                required: ['source'],
+            },
+            handler: async (input) => {
+                const result = await wc.extractPdf(input.source);
+                return {
+                    title: result.pages[0]?.title,
+                    markdown: result.pages[0]?.markdown.slice(0, 5000),
+                    chunks: result.context.chunks.length,
+                    totalTokens: result.stats.totalTokens,
+                };
+            },
+        },
+    ];
+}
+exports.createMCPTools = createMCPTools;
+//# sourceMappingURL=mcp.js.map

package/dist/sdk/mcp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../../src/sdk/mcp.ts"],"names":[],"mappings":";;;AAAA,oCAAsC;AAetC,SAAgB,cAAc,CAAC,MAAY;IACzC,MAAM,EAAE,GAAG,IAAI,kBAAU,CAAC,MAAM,CAAC,CAAC;IAElC,OAAO;QACL;YACE,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,oFAAoF;YACjG,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE;oBACnE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,WAAW,EAAE,oBAAoB,EAAE;oBAChH,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0BAA0B,EAAE;iBACvE;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB;YACD,OAAO,EAAE,KAAK,EAAE,KAA0D,EAAE,EAAE;gBAC5E,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC5C,SAAS,EAAG,KAAK,CAAC,KAAa,IAAI,MAAM;oBACzC,SAAS,EAAE,KAAK,CAAC,SAAS;iBAC3B,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;YAC9B,CAAC;SACF;QACD;YACE,IAAI,EAAE,kBAAkB;YACxB,WAAW,EAAE,2DAA2D;YACxE,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE;oBACzD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0BAA0B,EAAE;oBAClE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,yBAAyB,EAAE;iBACrE;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB;YACD,OAAO,EAAE,KAAK,EAAE,KAAyD,EAAE,EAAE;gBAC3E,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC3C,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,CAAC;oBACvB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE;iBAC/B,CAAC,CAAC;gBACH,OAAO;oBACL,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBACnG,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB,CAAC;YACJ,CAAC;SACF;QACD;YACE,IAAI,EAAE,mBAAmB;YACzB,WAAW,EAAE,kFAAkF;YAC/F,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE;oBAC5D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wCAAwC,EAAE;oBAChF,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gCAAgC,EAAE;iBACxE;gBACD,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;aAC3B;YACD,OAAO,EAAE,KAAK,EAAE,KAAoD,EAAE,EAAE;gBACtE,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBACzE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC;YACnH,CAAC;SACF;QACD;YACE,IAAI,EAAE,kBAAkB;YACxB,WAAW,EAAE,0CAA0C;YACvD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;oBACpD,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sCAAsC,EAAE;iBACnF;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB;YACD,OAAO,EAAE,KAAK,EAAE,KAA0C,EAAE,EAAE;gBAC5D,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YACvI,CAAC;SACF;QACD;YACE,IAAI,EAAE,sBAAsB;YAC5B,WAAW,EAAE,0CAA0C;YACvD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE;iBACzD;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB;YACD,OAAO,EAAE,KAAK,EAAE,KAAsB,EAAE,EAAE;gBACxC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3C,OAAO;oBACL,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;oBAC7B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO;oBAC/B,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc;oBACtC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;iBACtC,CAAC;YACJ,CAAC;SACF;QACD;YACE,IAAI,EAAE,mBAAmB;YACzB,WAAW,EAAE,mDAAmD;YAChE,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4DAA4D,EAAE;oBAClG,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,2CAA2C,EAAE;iBAC3F;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB;YACD,OAAO,EAAE,KAAK,EAAE,KAA6C,EAAE,EAAE;gBAC/D,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjG,OAAO;oBACL,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBACnG,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB,CAAC;YACJ,CAAC;SACF;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,2DAA2D;YACxE,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4BAA4B,EAAE;iBACtE;gBACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;aACrB;YACD,OAAO,EAAE,KAAK,EAAE,KAAyB,EAAE,EAAE;gBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjD,OAAO;oBACL,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;oBAC7B,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;oBAClD,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM;oBACpC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;iBACtC,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC;AA5ID,wCA4IC"}

package/dist/sdk/server.d.ts

@@ -0,0 +1,5 @@
+/**
+ * REST API server for WebContext.
+ */
+export declare function startServer(port?: number): void;
+//# sourceMappingURL=server.d.ts.map

package/dist/sdk/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/sdk/server.ts"],"names":[],"mappings":"AASA;;GAEG;AACH,wBAAgB,WAAW,CAAC,IAAI,GAAE,MAAa,GAAG,IAAI,CAkIrD"}

package/dist/sdk/server.js

@@ -0,0 +1,158 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startServer = void 0;
+const express_1 = __importDefault(require("express"));
+const cors_1 = __importDefault(require("cors"));
+const index_1 = require("../index");
+const metrics_1 = require("../utils/metrics");
+const scheduler_1 = require("../utils/scheduler");
+const vector_1 = require("../search/vector");
+const validation_1 = require("../utils/validation");
+/**
+ * REST API server for WebContext.
+ */
+function startServer(port = 3456) {
+    const app = (0, express_1.default)();
+    app.use((0, cors_1.default)());
+    app.use(express_1.default.json());
+    const wc = new index_1.WebContext({ metrics: true });
+    const metrics = new metrics_1.MetricsCollector();
+    const scheduler = new scheduler_1.CrawlScheduler();
+    const vector = new vector_1.VectorSearch();
+    // Request logging
+    app.use((req, res, next) => {
+        const start = Date.now();
+        res.on('finish', () => {
+            console.log(`${req.method} ${req.path} ${res.statusCode} ${Date.now() - start}ms`);
+        });
+        next();
+    });
+    app.get('/health', (_req, res) => {
+        res.json({ status: 'ok', version: '1.0.0' });
+    });
+    // Extract single URL
+    app.post('/extract', async (req, res, next) => {
+        try {
+            const { url, options = {} } = req.body;
+            if (!url)
+                return res.status(400).json({ error: 'url is required' });
+            (0, validation_1.validateUrl)(url);
+            const result = await wc.extract(url, options);
+            res.json({
+                markdown: result.pages.map(p => p.markdown).join('\n\n---\n\n'),
+                metadata: result.pages[0]?.metadata,
+                stats: result.stats,
+            });
+        }
+        catch (err) {
+            next(err);
+        }
+    });
+    // Crawl documentation site
+    app.post('/crawl', async (req, res, next) => {
+        try {
+            const { url, options = {} } = req.body;
+            if (!url)
+                return res.status(400).json({ error: 'url is required' });
+            (0, validation_1.validateUrl)(url);
+            const result = await wc.crawlDocs(url, options);
+            res.json({ context: result.context, stats: result.stats });
+        }
+        catch (err) {
+            next(err);
+        }
+    });
+    // Generate LLM context
+    app.post('/context', async (req, res, next) => {
+        try {
+            const { url, maxTokens = 8000, focusMode = 'full' } = req.body;
+            if (!url)
+                return res.status(400).json({ error: 'url is required' });
+            (0, validation_1.validateUrl)(url);
+            const context = await wc.toContext(url, { focusMode, maxTokens });
+            res.json({ context, tokens: Math.ceil(context.length / 4) });
+        }
+        catch (err) {
+            next(err);
+        }
+    });
+    // Get chunks for RAG
+    app.post('/chunks', async (req, res, next) => {
+        try {
+            const { url, options = {} } = req.body;
+            if (!url)
+                return res.status(400).json({ error: 'url is required' });
+            (0, validation_1.validateUrl)(url);
+            const chunks = await wc.toChunks(url, options);
+            res.json({ chunks, count: chunks.length, totalTokens: chunks.reduce((s, c) => s + c.tokens, 0) });
+        }
+        catch (err) {
+            next(err);
+        }
+    });
+    // Semantic search
+    app.post('/search', async (req, res, next) => {
+        try {
+            const { url, query, topK = 5 } = req.body;
+            if (!url || !query)
+                return res.status(400).json({ error: 'url and query are required' });
+            (0, validation_1.validateUrl)(url);
+            const chunks = await wc.toChunks(url);
+            vector.index(chunks);
+            const results = vector.search(query, topK);
+            res.json({ results, count: results.length });
+        }
+        catch (err) {
+            next(err);
+        }
+    });
+    // Metrics
+    app.get('/metrics', (_req, res) => {
+        const wcMetrics = wc.getMetrics();
+        res.json(wcMetrics || metrics.getMetrics());
+    });
+    // Schedule crawl
+    app.post('/schedule', (req, res, next) => {
+        try {
+            const { id, cron, urls, options = {} } = req.body;
+            if (!id || !cron || !urls?.length) {
+                return res.status(400).json({ error: 'id, cron, and urls are required' });
+            }
+            urls.forEach((u) => (0, validation_1.validateUrl)(u));
+            scheduler.schedule(id, { cron, urls, options }, (url, opts) => wc.crawlDocs(url, opts));
+            res.json({ id, status: 'scheduled' });
+        }
+        catch (err) {
+            next(err);
+        }
+    });
+    // Cancel scheduled job
+    app.delete('/schedule/:id', (req, res) => {
+        scheduler.cancel(req.params.id);
+        res.json({ id: req.params.id, status: 'cancelled' });
+    });
+    // List scheduled jobs
+    app.get('/schedule', (_req, res) => {
+        res.json({ jobs: scheduler.listJobs() });
+    });
+    // Error handling middleware
+    app.use((err, _req, res, _next) => {
+        const status = err.status || 500;
+        res.status(status).json({ error: err.message || 'Internal server error' });
+    });
+    app.listen(port, () => {
+        console.log(`🌐 WebContext API server running on http://localhost:${port}`);
+        console.log(`   POST /extract  - Extract content from URL`);
+        console.log(`   POST /crawl    - Crawl documentation site`);
+        console.log(`   POST /context  - Generate LLM-ready context`);
+        console.log(`   POST /chunks   - Get RAG-ready chunks`);
+        console.log(`   POST /search   - Semantic search within content`);
+        console.log(`   GET  /metrics  - View metrics`);
+        console.log(`   POST /schedule - Schedule recurring crawls`);
+    });
+}
+exports.startServer = startServer;
+//# sourceMappingURL=server.js.map

package/dist/sdk/server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/sdk/server.ts"],"names":[],"mappings":";;;;;;AAAA,sDAAmE;AACnE,gDAAwB;AACxB,oCAAsC;AAEtC,8CAAoD;AACpD,kDAAoD;AACpD,6CAAgD;AAChD,oDAAkD;AAElD;;GAEG;AACH,SAAgB,WAAW,CAAC,OAAe,IAAI;IAC7C,MAAM,GAAG,GAAG,IAAA,iBAAO,GAAE,CAAC;IACtB,GAAG,CAAC,GAAG,CAAC,IAAA,cAAI,GAAE,CAAC,CAAC;IAChB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAExB,MAAM,EAAE,GAAG,IAAI,kBAAU,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,0BAAgB,EAAE,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,0BAAc,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,qBAAY,EAAE,CAAC;IAElC,kBAAkB;IAClB,GAAG,CAAC,GAAG,CAAC,CAAC,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YACpB,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QACH,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;QAClD,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC7E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YACvC,IAAI,CAAC,GAAG;gBAAE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACpE,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;YACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,OAAgC,CAAC,CAAC;YACvE,GAAG,CAAC,IAAI,CAAC;gBACP,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;gBAC/D,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ;gBACnC,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC3E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YACvC,IAAI,CAAC,GAAG;gBAAE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACpE,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;YACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAgC,CAAC,CAAC;YACzE,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,uBAAuB;IACvB,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC7E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,SAAS,GAAG,IAAI,EAAE,SAAS,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YAC/D,IAAI,CAAC,GAAG;gBAAE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACpE,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;YACjB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;YAClE,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC5E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YACvC,IAAI,CAAC,GAAG;gBAAE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACpE,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;YACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC/C,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,kBAAkB;IAClB,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC5E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YAC1C,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK;gBAAE,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC,CAAC;YACzF,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;YACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,UAAU;IACV,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;QACnD,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC;QAClC,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,iBAAiB;IACjB,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QACxE,IAAI,CAAC;YACH,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YAClD,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAClC,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iCAAiC,EAAE,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,IAAA,wBAAW,EAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YACxF,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,uBAAuB;IACvB,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,GAAY,EAAE,GAAa,EAAE,EAAE;QAC1D,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,sBAAsB;IACtB,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;QACpD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,4BAA4B;IAC5B,GAAG,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,IAAa,EAAE,GAAa,EAAE,KAAmB,EAAE,EAAE;QACtE,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC;QACjC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,IAAI,uBAAuB,EAAE,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QACpB,OAAO,CAAC,GAAG,CAAC,wDAAwD,IAAI,EAAE,CAAC,CAAC;QAC5E,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC;AAlID,kCAkIC"}

package/dist/search/vector.d.ts

@@ -0,0 +1,26 @@
+import { ContentChunk, SearchResult } from '../core/types';
+/**
+ * Simple in-memory vector search for content chunks.
+ * Uses TF-IDF based embeddings for local semantic search without external dependencies.
+ */
+export declare class VectorSearch {
+    private embeddings;
+    private vocabulary;
+    private idfScores;
+    private docFrequency;
+    /** Index chunks for search */
+    index(chunks: ContentChunk[]): void;
+    /** Search indexed chunks by query */
+    search(query: string, topK?: number): SearchResult[];
+    /** Add a single chunk to the index and recompute IDF from actual document frequencies */
+    addChunk(chunk: ContentChunk): void;
+    /** Clear the index */
+    clear(): void;
+    /** Get index size */
+    get size(): number;
+    private tokenize;
+    private computeTfIdf;
+    private cosineSimilarity;
+    private buildVocabulary;
+}
+//# sourceMappingURL=vector.d.ts.map

package/dist/search/vector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAiB,YAAY,EAAmB,MAAM,eAAe,CAAC;AAmB3F;;;GAGG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAyB;IAC3C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,YAAY,CAAkC;IAEtD,8BAA8B;IAC9B,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI;IAenC,qCAAqC;IACrC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,YAAY,EAAE;IAiBvD,yFAAyF;IACzF,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IA6BnC,sBAAsB;IACtB,KAAK,IAAI,IAAI;IAOb,qBAAqB;IACrB,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,YAAY;IAmBpB,OAAO,CAAC,gBAAgB;IAgBxB,OAAO,CAAC,eAAe;CAgBxB"}