vectra 0.12.2 → 0.14.0

package/lib/WebFetcher.js

@@ -0,0 +1,238 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebFetcher = void 0;
+const cheerio = __importStar(require("cheerio"));
+const turndown_1 = __importDefault(require("turndown"));
+const ALLOWED_CONTENT_TYPES = [
+    "text/html",
+    "application/json",
+    "application/xml",
+    "application/javascript",
+    "text/plain",
+];
+const DEFAULT_HEADERS = {
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, sdch, br",
+    "Accept-Language": "en-US,en;q=0.8,ms;q=0.6",
+    "Alt-Used": "LEAVE-THIS-KEY-SET-BY-TOOL",
+    Connection: "keep-alive",
+    Host: "LEAVE-THIS-KEY-SET-BY-TOOL",
+    Referer: "https://www.google.com/",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "cross-site",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
+};
+class WebFetcher {
+    constructor(config) {
+        this._config = Object.assign({
+            htmlToMarkdown: true,
+            summarizeHtml: false,
+        }, config);
+    }
+    fetch(uri, onDocument) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            // Clone headers to avoid mutating the original
+            const headers = Object.assign({}, DEFAULT_HEADERS, this._config.headers);
+            // get hostname from url
+            const host = new URL(uri).hostname;
+            headers['Host'] = host;
+            headers['Alt-Used'] = host;
+            // Fetch page and check for errors
+            const response = yield fetch(uri, Object.assign(Object.assign({}, this._config.requestConfig), { method: 'GET', headers }));
+            if (response.status >= 400) {
+                throw new Error(`Site returned an HTTP status of ${response.status}`);
+            }
+            // Check for valid content type
+            const contentType = (_a = response.headers.get('content-type')) !== null && _a !== void 0 ? _a : '';
+            const contentTypeArray = contentType.split(';');
+            if (!contentTypeArray[0] || !ALLOWED_CONTENT_TYPES.includes(contentTypeArray[0])) {
+                throw new Error(`Site returned an invalid content type of ${contentType}`);
+            }
+            // Convert content type to doc type
+            const docType = contentTypeArray[0] != 'text/plain' ? contentTypeArray[0].split('/')[1] : undefined;
+            if (docType == 'html' && this._config.htmlToMarkdown) {
+                const data = yield response.text();
+                const text = this.htmlToMarkdown(data, uri);
+                return yield onDocument(uri, text, 'md');
+            }
+            else {
+                const text = yield response.text();
+                return yield onDocument(uri, text, docType);
+            }
+        });
+    }
+    htmlToMarkdown(html, baseUrl) {
+        var _a;
+        // Parse HTML
+        const $ = cheerio.load(html, { scriptingEnabled: true });
+        // Remove scripts and convert relative links to absolute
+        $("script").remove();
+        $("a").each((_, elem) => {
+            const $el = $(elem);
+            const href = $el.attr("href");
+            if (href && !href.startsWith("http")) {
+                try {
+                    $el.attr("href", new URL(href, baseUrl).toString());
+                }
+                catch (_a) {
+                    // ignore bad URLs
+                }
+            }
+        });
+        // Convert to markdown
+        const body = (_a = $("body").html()) !== null && _a !== void 0 ? _a : "";
+        const turndownService = new turndown_1.default({
+            hr: "\n\n---\n\n",
+        });
+        convertTables(turndownService);
+        const md = turndownService.turndown(body);
+        // Remove any overly long header text
+        const firstSpace = md.indexOf(" ");
+        const firstNewline = md.indexOf("\n");
+        const candidates = [firstSpace, firstNewline].filter((i) => i >= 0);
+        const contentStart = candidates.length ? Math.min(...candidates) : -1;
+        if (contentStart > 64) {
+            return md.slice(contentStart);
+        }
+        else {
+            return md;
+        }
+    }
+}
+exports.WebFetcher = WebFetcher;
+function convertTables(turndownService) {
+    turndownService.addRule("tableCell", {
+        filter: ["th", "td"],
+        replacement: function (content, node) {
+            return cell(content, node);
+        },
+    });
+    turndownService.addRule("tableRow", {
+        filter: "tr",
+        replacement: function (content, node) {
+            var _a, _b;
+            let borderCells = "";
+            const alignMap = { left: ":--", right: "--:", center: ":-:" };
+            if (isHeadingRow(node)) {
+                const children = node.childNodes;
+                for (let i = 0; i < children.length; i++) {
+                    let border = "---";
+                    const align = (((_b = (_a = children[i]).getAttribute) === null || _b === void 0 ? void 0 : _b.call(_a, "align")) || "").toLowerCase();
+                    if (align)
+                        border = alignMap[align] || border;
+                    borderCells += cell(border, children[i]);
+                }
+            }
+            return "\n" + content + (borderCells ? "\n" + borderCells : "");
+        },
+    });
+    turndownService.addRule("table", {
+        filter: ["table"],
+        replacement: function (content) {
+            // Ensure there are no blank lines
+            content = content.replace("\n\n", "\n");
+            return "\n\n" + content + "\n\n";
+        },
+    });
+    turndownService.addRule("tableSection", {
+        filter: ["thead", "tbody", "tfoot"],
+        replacement: function (content) {
+            return content;
+        },
+    });
+}
+const indexOf = Array.prototype.indexOf;
+const every = Array.prototype.every;
+// A tr is a heading row if:
+// - the parent is a THEAD
+// - or if its the first child of the TABLE or the first TBODY (possibly
+//   following a blank THEAD)
+// - and every cell is a TH
+function isHeadingRow(tr) {
+    const parentNode = tr.parentNode;
+    return (parentNode.nodeName === "THEAD" ||
+        (parentNode.firstChild === tr &&
+            (parentNode.nodeName === "TABLE" || isFirstTbody(parentNode)) &&
+            every.call(parentNode ? tr.childNodes : [], function (n) {
+                return n.nodeName === "TH";
+            })));
+}
+function isFirstTbody(element) {
+    const previousSibling = element.previousSibling;
+    return (element.nodeName === "TBODY" &&
+        (!previousSibling ||
+            (previousSibling.nodeName === "THEAD" && /^\s*$/i.test(previousSibling.textContent))));
+}
+function cell(content, node) {
+    const index = indexOf.call(node.parentNode.childNodes, node);
+    let prefix = " ";
+    if (index === 0) {
+        prefix = "| ";
+    }
+    return cleanContent(prefix + content + " |");
+}
+function cleanContent(content) {
+    let output = "";
+    const chars = ["\n", "\r", "\t", "\f", "\v", "\u00a0", "\u2028", "\u2029", " "];
+    for (let i = 0; i < content.length; i++) {
+        if (chars.includes(content[i])) {
+            if (output[output.length - 1] !== " ") {
+                output += " ";
+            }
+            continue;
+        }
+        else {
+            output += content[i];
+        }
+    }
+    return output;
+}
+//# sourceMappingURL=WebFetcher.js.map

package/lib/WebFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.js","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,iDAAmC;AACnC,wDAAuC;AAEvC,MAAM,qBAAqB,GAAG;IAC5B,WAAW;IACX,kBAAkB;IAClB,iBAAiB;IACjB,wBAAwB;IACxB,YAAY;CACb,CAAC;AAEF,MAAM,eAAe,GAA2B;IAC9C,MAAM,EAAE,uFAAuF;IAC/F,iBAAiB,EAAE,yBAAyB;IAC5C,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,4BAA4B;IACxC,UAAU,EAAE,YAAY;IACxB,IAAI,EAAE,4BAA4B;IAClC,OAAO,EAAE,yBAAyB;IAClC,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,YAAY;IAC9B,2BAA2B,EAAE,GAAG;IAChC,YAAY,EAAE,0GAA0G;CACzH,CAAC;AASF,MAAa,UAAU;IAGrB,YAAmB,MAAkC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAC1B;YACE,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK;SACD,EACrB,MAAM,CACP,CAAC;IACJ,CAAC;IAEY,KAAK,CAChB,GAAW,EACX,UAA6E;;;YAE7E,+CAA+C;YAC/C,MAAM,OAAO,GAA2B,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAEjG,wBAAwB;YACxB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;YAE3B,kCAAkC;YAClC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,kCAC3B,IAAI,CAAC,OAAO,CAAC,aAAa,KAC7B,MAAM,EAAE,KAAK,EACb,OAAO,IACP,CAAC;YACH,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACxE,CAAC;YAED,+BAA+B;YAC/B,MAAM,WAAW,GAAG,MAAA,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,mCAAI,EAAE,CAAC;YAC/D,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjF,MAAM,IAAI,KAAK,CAAC,4CAA4C,WAAW,EAAE,CAAC,CAAC;YAC7E,CAAC;YAED,mCAAmC;YACnC,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACpG,IAAI,OAAO,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC5C,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;KAAA;IAEO,cAAc,CAAC,IAAY,EAAE,OAAe;;QAClD,aAAa;QACb,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzD,wDAAwD;QACxD,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACrB,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACtB,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtD,CAAC;gBAAC,WAAM,CAAC;oBACP,kBAAkB;gBACpB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,sBAAsB;QACtB,MAAM,IAAI,GAAG,MAAA,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,mCAAI,EAAE,CAAC;QACpC,MAAM,eAAe,GAAG,IAAI,kBAAe,CAAC;YAC1C,EAAE,EAAE,aAAa;SAClB,CAAC,CAAC;QACH,aAAa,CAAC,eAAe,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE1C,qCAAqC;QACrC,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACpE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,IAAI,YAAY,GAAG,EAAE,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AA5FD,gCA4FC;AAED,SAAS,aAAa,CAAC,eAAgC;IACrD,eAAe,CAAC,OAAO,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,WAAW,EAAE,UAAU,OAAe,EAAE,IAAS;YAC/C,OAAO,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7B,CAAC;KACF,CAAC,CAAC;IAEH,eAAe,CAAC,OAAO,CAAC,UAAU,EAAE;QAClC,MAAM,EAAE,IAAI;QACZ,WAAW,EAAE,UAAU,OAAe,EAAE,IAAS;;YAC/C,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,MAAM,QAAQ,GAA2B,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAEtF,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,MAAM,QAAQ,GAAI,IAAY,CAAC,UAAU,CAAC;gBAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACzC,IAAI,MAAM,GAAG,KAAK,CAAC;oBACnB,MAAM,KAAK,GAAW,CAAC,CAAA,MAAA,MAAA,QAAQ,CAAC,CAAC,CAAC,EAAC,YAAY,mDAAG,OAAO,CAAC,KAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;oBAChF,IAAI,KAAK;wBAAE,MAAM,GAAI,QAAgB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;oBACvD,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YACD,OAAO,IAAI,GAAG,OAAO,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC;KACF,CAAC,CAAC;IAEH,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE;QAC/B,MAAM,EAAE,CAAC,OAAO,CAAC;QACjB,WAAW,EAAE,UAAU,OAAe;YACpC,kCAAkC;YAClC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxC,OAAO,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;QACnC,CAAC;KACF,CAAC,CAAC;IAEH,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE;QACtC,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;QACnC,WAAW,EAAE,UAAU,OAAe;YACpC,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC;AACxC,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC;AAEpC,4BAA4B;AAC5B,0BAA0B;AAC1B,wEAAwE;AACxE,6BAA6B;AAC7B,2BAA2B;AAC3B,SAAS,YAAY,CAAC,EAAO;IAC3B,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC;IACjC,OAAO,CACL,UAAU,CAAC,QAAQ,KAAK,OAAO;QAC/B,CAAC,UAAU,CAAC,UAAU,KAAK,EAAE;YAC3B,CAAC,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,UAAU,CAAM;gBAC1D,OAAO,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC;YAC7B,CAAC,CAAC,CAAC,CACN,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,OAAY;IAChC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,OAAO,CACL,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,eAAe;YACf,CAAC,eAAe,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,CACxF,CAAC;AACJ,CAAC;AAED,SAAS,IAAI,CAAC,OAAe,EAAE,IAAS;IACtC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAC7D,IAAI,MAAM,GAAG,GAAG,CAAC;IACjB,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,YAAY,CAAC,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,KAAK,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IAChF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,CAAC;YAChB,CAAC;YACD,SAAS;QACX,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/lib/WebFetcher.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=WebFetcher.spec.d.ts.map

package/lib/WebFetcher.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.spec.d.ts","sourceRoot":"","sources":["../src/WebFetcher.spec.ts"],"names":[],"mappings":""}

package/lib/WebFetcher.spec.js

@@ -0,0 +1,263 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const mocha_1 = require("mocha");
+const sinon_1 = __importDefault(require("sinon"));
+const WebFetcher_1 = require("./WebFetcher");
+(0, mocha_1.describe)("WebFetcher", () => {
+    let sandbox;
+    let fetchStub;
+    let lastUrl;
+    let lastInit;
+    function makeFetchResponse(status, data, contentType) {
+        return {
+            status,
+            statusText: status >= 400 ? `HTTP ${status}` : 'OK',
+            ok: status >= 200 && status < 300,
+            headers: new Headers({ 'content-type': contentType }),
+            text: () => __awaiter(this, void 0, void 0, function* () { return data; }),
+            json: () => __awaiter(this, void 0, void 0, function* () { return JSON.parse(data); }),
+        };
+    }
+    (0, mocha_1.beforeEach)(() => {
+        sandbox = sinon_1.default.createSandbox();
+        fetchStub = sandbox.stub(globalThis, 'fetch').callsFake((input, init) => __awaiter(void 0, void 0, void 0, function* () {
+            lastUrl = typeof input === 'string' ? input : input.url;
+            lastInit = init;
+            return makeFetchResponse(200, 'ok', 'text/plain');
+        }));
+        lastUrl = undefined;
+        lastInit = undefined;
+    });
+    (0, mocha_1.afterEach)(() => {
+        sandbox.restore();
+    });
+    (0, mocha_1.it)("constructor uses defaults and merges config", () => {
+        const fetcherDefault = new WebFetcher_1.WebFetcher();
+        node_assert_1.default.strictEqual(fetcherDefault["_config"].htmlToMarkdown, true);
+        node_assert_1.default.strictEqual(fetcherDefault["_config"].summarizeHtml, false);
+        const custom = {
+            htmlToMarkdown: false,
+            summarizeHtml: true,
+            headers: { Accept: "custom/type", "User-Agent": "custom-agent" },
+            requestConfig: { signal: AbortSignal.timeout(123) },
+        };
+        const fetcher = new WebFetcher_1.WebFetcher(custom);
+        node_assert_1.default.strictEqual(fetcher["_config"].htmlToMarkdown, false);
+        node_assert_1.default.strictEqual(fetcher["_config"].summarizeHtml, true);
+        node_assert_1.default.deepStrictEqual(fetcher["_config"].headers, {
+            Accept: "custom/type",
+            "User-Agent": "custom-agent",
+        });
+        node_assert_1.default.ok(fetcher["_config"].requestConfig);
+    });
+    (0, mocha_1.it)("throws on HTTP error (>=400)", () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(404, '<html></html>', 'text/html'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        yield node_assert_1.default.rejects(() => fetcher.fetch("https://example.com/404", () => __awaiter(void 0, void 0, void 0, function* () { return true; })), /Site returned an HTTP status of 404/);
+    }));
+    (0, mocha_1.it)("throws on invalid content-type", () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, '…', 'image/png'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        yield node_assert_1.default.rejects(() => fetcher.fetch("https://example.com/img", () => __awaiter(void 0, void 0, void 0, function* () { return true; })), /Site returned an invalid content type of image\/png/);
+    }));
+    (0, mocha_1.it)("handles text/html with htmlToMarkdown=true with tables, alignments, and link rewriting", () => __awaiter(void 0, void 0, void 0, function* () {
+        const html = `
+      <html><body>
+        <script>evil()</script>
+        <a href="/rel">relative</a>
+        <a href="//cdn.example.com/lib.js">proto-rel</a>
+        <table>
+          <thead>
+            <tr>
+              <th align="left">H1</th>
+              <th align="right">H2</th>
+              <th align="center">H3</th>
+            </tr>
+          </thead>
+          <tbody>
+            <tr>
+              <td>Cell&nbsp;1</td>
+              <td>Cell\t2</td>
+              <td>Cell\n3</td>
+            </tr>
+          </tbody>
+        </table>
+        <table>
+          <tbody>
+            <tr><th>A</th><th>B</th></tr>
+            <tr><td>a</td><td>b</td></tr>
+          </tbody>
+        </table>
+      </body></html>
+    `;
+        fetchStub.resolves(makeFetchResponse(200, html, 'text/html'));
+        const fetcher = new WebFetcher_1.WebFetcher({ htmlToMarkdown: true });
+        const calls = [];
+        const onDocument = (uri, text, docType) => __awaiter(void 0, void 0, void 0, function* () {
+            calls.push([uri, text, docType]);
+            return true;
+        });
+        const ok = yield fetcher.fetch("https://example.com/page", onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(calls.length, 1);
+        const [uri, md, docType] = calls[0];
+        node_assert_1.default.strictEqual(uri, "https://example.com/page");
+        node_assert_1.default.strictEqual(docType, "md");
+        node_assert_1.default.ok(!String(md).includes("<script>"));
+        node_assert_1.default.ok(String(md).includes("https://example.com/rel"));
+        node_assert_1.default.ok(String(md).includes("https://cdn.example.com/lib.js"));
+        node_assert_1.default.ok(String(md).includes("| H1 | H2 | H3 |"));
+        node_assert_1.default.ok(String(md).includes("| :-- | --: | :-: |"));
+        node_assert_1.default.ok(String(md).includes("| Cell 1 | Cell 2 | Cell 3 |"));
+        node_assert_1.default.ok(String(md).includes("| A | B |"));
+    }));
+    (0, mocha_1.it)("handles text/html with htmlToMarkdown=false (passes raw html, docType 'html')", () => __awaiter(void 0, void 0, void 0, function* () {
+        const html = "<html><body><p>content</p></body></html>";
+        fetchStub.resolves(makeFetchResponse(200, html, 'text/html'));
+        const fetcher = new WebFetcher_1.WebFetcher({ htmlToMarkdown: false });
+        const calls = [];
+        const onDocument = (uri, text, docType) => __awaiter(void 0, void 0, void 0, function* () {
+            calls.push([uri, text, docType]);
+            return true;
+        });
+        const ok = yield fetcher.fetch("https://example.com/raw", onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(calls.length, 1);
+        const [_, text, docType] = calls[0];
+        node_assert_1.default.strictEqual(docType, "html");
+        node_assert_1.default.strictEqual(text, html);
+    }));
+    (0, mocha_1.it)("handles application/json; charset=… (docType 'json')", () => __awaiter(void 0, void 0, void 0, function* () {
+        const json = '{"a":1}';
+        fetchStub.resolves(makeFetchResponse(200, json, 'application/json; charset=utf-8'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const calls = [];
+        const onDocument = (...args) => __awaiter(void 0, void 0, void 0, function* () {
+            calls.push(args);
+            return true;
+        });
+        const ok = yield fetcher.fetch("https://example.com/data", onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(calls[0][2], "json");
+        node_assert_1.default.strictEqual(calls[0][1], json);
+    }));
+    (0, mocha_1.it)("handles application/xml (docType 'xml')", () => __awaiter(void 0, void 0, void 0, function* () {
+        const xml = "<root/>";
+        fetchStub.resolves(makeFetchResponse(200, xml, 'application/xml'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const calls = [];
+        const onDocument = (...args) => __awaiter(void 0, void 0, void 0, function* () {
+            calls.push(args);
+            return true;
+        });
+        const ok = yield fetcher.fetch("https://example.com/xml", onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(calls[0][2], "xml");
+        node_assert_1.default.strictEqual(calls[0][1], xml);
+    }));
+    (0, mocha_1.it)("handles application/javascript (docType 'javascript')", () => __awaiter(void 0, void 0, void 0, function* () {
+        const js = "console.log('hi');";
+        fetchStub.resolves(makeFetchResponse(200, js, 'application/javascript'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const calls = [];
+        const onDocument = (...args) => __awaiter(void 0, void 0, void 0, function* () {
+            calls.push(args);
+            return true;
+        });
+        const ok = yield fetcher.fetch("https://example.com/app.js", onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(calls[0][2], "javascript");
+        node_assert_1.default.strictEqual(calls[0][1], js);
+    }));
+    (0, mocha_1.it)("handles text/plain (docType undefined)", () => __awaiter(void 0, void 0, void 0, function* () {
+        const text = "plain";
+        fetchStub.resolves(makeFetchResponse(200, text, 'text/plain'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const calls = [];
+        const onDocument = (...args) => __awaiter(void 0, void 0, void 0, function* () {
+            calls.push(args);
+            return true;
+        });
+        const ok = yield fetcher.fetch("https://example.com/txt", onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(calls[0][2], undefined);
+        node_assert_1.default.strictEqual(calls[0][1], text);
+    }));
+    (0, mocha_1.it)("sets Host and Alt-Used to request hostname; merges and does not mutate caller headers", () => __awaiter(void 0, void 0, void 0, function* () {
+        let capturedUrl;
+        let capturedInit;
+        fetchStub.callsFake((url, init) => __awaiter(void 0, void 0, void 0, function* () {
+            capturedUrl = url;
+            capturedInit = init;
+            return makeFetchResponse(200, 'ok', 'text/plain');
+        }));
+        const userHeaders = {
+            Accept: "custom/type",
+            "User-Agent": "custom-agent",
+        };
+        const fetcher = new WebFetcher_1.WebFetcher({ headers: userHeaders });
+        const onDocument = () => __awaiter(void 0, void 0, void 0, function* () { return true; });
+        const url = "https://host.example.com/path?q=1";
+        const ok = yield fetcher.fetch(url, onDocument);
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(capturedUrl, url);
+        node_assert_1.default.ok(capturedInit);
+        node_assert_1.default.ok(capturedInit.headers);
+        const headers = capturedInit.headers;
+        node_assert_1.default.strictEqual(headers.Host, "host.example.com");
+        node_assert_1.default.strictEqual(headers["Alt-Used"], "host.example.com");
+        node_assert_1.default.strictEqual(headers.Accept, "custom/type");
+        node_assert_1.default.strictEqual(headers["User-Agent"], "custom-agent");
+        node_assert_1.default.strictEqual(userHeaders.Host, undefined);
+        node_assert_1.default.strictEqual(userHeaders["Alt-Used"], undefined);
+    }));
+    (0, mocha_1.it)("merges requestConfig options into fetch call", () => __awaiter(void 0, void 0, void 0, function* () {
+        let capturedInit;
+        fetchStub.callsFake((_url, init) => __awaiter(void 0, void 0, void 0, function* () {
+            capturedInit = init;
+            return makeFetchResponse(200, 'ok', 'text/plain');
+        }));
+        const fetcher = new WebFetcher_1.WebFetcher({
+            requestConfig: { keepalive: true },
+        });
+        const ok = yield fetcher.fetch("https://example.com", () => __awaiter(void 0, void 0, void 0, function* () { return true; }));
+        node_assert_1.default.strictEqual(ok, true);
+        node_assert_1.default.strictEqual(capturedInit.keepalive, true);
+    }));
+    (0, mocha_1.it)("htmlToMarkdown trims overly long header text when first space/newline index > 64", () => {
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const longText = "a".repeat(70) + " rest\nmore";
+        const html = `<html><body><p>${longText}</p></body></html>`;
+        const md = fetcher["htmlToMarkdown"](html, "https://example.com");
+        node_assert_1.default.ok(!md.startsWith("a".repeat(70)));
+        node_assert_1.default.ok(md.includes("rest"));
+    });
+    (0, mocha_1.it)("htmlToMarkdown leaves short content unchanged", () => {
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const html = `<html><body><p>short text</p></body></html>`;
+        const md = fetcher["htmlToMarkdown"](html, "https://example.com");
+        node_assert_1.default.ok(md.includes("short text"));
+    });
+    (0, mocha_1.it)("propagates onDocument return value (true/false)", () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, 'ok', 'text/plain'));
+        const fetcher = new WebFetcher_1.WebFetcher();
+        const yes = yield fetcher.fetch("https://example.com/yes", () => __awaiter(void 0, void 0, void 0, function* () { return true; }));
+        const no = yield fetcher.fetch("https://example.com/no", () => __awaiter(void 0, void 0, void 0, function* () { return false; }));
+        node_assert_1.default.strictEqual(yes, true);
+        node_assert_1.default.strictEqual(no, false);
+    }));
+});
+//# sourceMappingURL=WebFetcher.spec.js.map

package/lib/WebFetcher.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.spec.js","sourceRoot":"","sources":["../src/WebFetcher.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8DAAiC;AACjC,iCAA4D;AAC5D,kDAA0B;AAC1B,6CAA0C;AAE1C,IAAA,gBAAQ,EAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,IAAI,OAA2B,CAAC;IAChC,IAAI,SAA0B,CAAC;IAE/B,IAAI,OAA2B,CAAC;IAChC,IAAI,QAAiC,CAAC;IAEtC,SAAS,iBAAiB,CAAC,MAAc,EAAE,IAAY,EAAE,WAAmB;QAC1E,OAAO;YACL,MAAM;YACN,UAAU,EAAE,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI;YACnD,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;YACjC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;YACrD,IAAI,EAAE,GAAS,EAAE,gDAAC,OAAA,IAAI,CAAA,GAAA;YACtB,IAAI,EAAE,GAAS,EAAE,gDAAC,OAAA,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA,GAAA;SACvB,CAAC;IAChB,CAAC;IAED,IAAA,kBAAU,EAAC,GAAG,EAAE;QACd,OAAO,GAAG,eAAK,CAAC,aAAa,EAAE,CAAC;QAChC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAO,KAAU,EAAE,IAAU,EAAE,EAAE;YACvF,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;YACxD,QAAQ,GAAG,IAAI,CAAC;YAChB,OAAO,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,CAAC,CAAA,CAAC,CAAC;QACH,OAAO,GAAG,SAAS,CAAC;QACpB,QAAQ,GAAG,SAAS,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,IAAA,iBAAS,EAAC,GAAG,EAAE;QACb,OAAO,CAAC,OAAO,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,cAAc,GAAG,IAAI,uBAAU,EAAE,CAAC;QACxC,qBAAM,CAAC,WAAW,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACnE,qBAAM,CAAC,WAAW,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG;YACb,cAAc,EAAE,KAAK;YACrB,aAAa,EAAE,IAAI;YACnB,OAAO,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE;YAChE,aAAa,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;SACpD,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,uBAAU,CAAC,MAAM,CAAC,CAAC;QACvC,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QAC7D,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QAC3D,qBAAM,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE;YACjD,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,cAAc;SAC7B,CAAC,CAAC;QACH,qBAAM,CAAC,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,8BAA8B,EAAE,GAAS,EAAE;QAC5C,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,qBAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,GAAS,EAAE,kDAAC,OAAA,IAAI,CAAA,GAAA,CAAC,EAChE,qCAAqC,CACtC,CAAC;IACJ,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,gCAAgC,EAAE,GAAS,EAAE;QAC9C,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,qBAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,GAAS,EAAE,kDAAC,OAAA,IAAI,CAAA,GAAA,CAAC,EAChE,qDAAqD,CACtD,CAAC;IACJ,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,wFAAwF,EAAE,GAAS,EAAE;QACtG,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4BZ,CAAC;QACF,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAE9D,MAAM,OAAO,GAAG,IAAI,uBAAU,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,CAAO,GAAW,EAAE,IAAY,EAAE,OAAgB,EAAE,EAAE;YACvE,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC,CAAA,CAAC;QAEF,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAC;QACvE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAEpC,MAAM,CAAC,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,qBAAM,CAAC,WAAW,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;QACpD,qBAAM,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAElC,qBAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;QAC5C,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC,CAAC;QAC1D,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACjE,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;QACnD,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC;QACtD,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC,CAAC;QAC/D,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,+EAA+E,EAAE,GAAS,EAAE;QAC7F,MAAM,IAAI,GAAG,0CAA0C,CAAC;QACxD,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAE9D,MAAM,OAAO,GAAG,IAAI,uBAAU,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,CAAO,GAAW,EAAE,IAAY,EAAE,OAAgB,EAAE,EAAE;YACvE,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC,CAAA,CAAC;QAEF,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,UAAU,CAAC,CAAC;QACtE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAEpC,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,qBAAM,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACpC,qBAAM,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,sDAAsD,EAAE,GAAS,EAAE;QACpE,MAAM,IAAI,GAAG,SAAS,CAAC;QACvB,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,iCAAiC,CAAC,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,CAAO,GAAG,IAAW,EAAE,EAAE;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC,CAAA,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAC;QACvE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACxC,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACxC,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,yCAAyC,EAAE,GAAS,EAAE;QACvD,MAAM,GAAG,GAAG,SAAS,CAAC;QACtB,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,CAAO,GAAG,IAAW,EAAE,EAAE;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC,CAAA,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,UAAU,CAAC,CAAC;QACtE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACvC,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,uDAAuD,EAAE,GAAS,EAAE;QACrE,MAAM,EAAE,GAAG,oBAAoB,CAAC;QAChC,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,EAAE,EAAE,wBAAwB,CAAC,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,CAAO,GAAG,IAAW,EAAE,EAAE;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC,CAAA,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,UAAU,CAAC,CAAC;QACzE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC9C,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtC,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,wCAAwC,EAAE,GAAS,EAAE;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC;QACrB,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,CAAO,GAAG,IAAW,EAAE,EAAE;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC,CAAA,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,UAAU,CAAC,CAAC;QACtE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC3C,qBAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACxC,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,uFAAuF,EAAE,GAAS,EAAE;QACrG,IAAI,WAA+B,CAAC;QACpC,IAAI,YAAqC,CAAC;QAC1C,SAAS,CAAC,SAAS,CAAC,CAAO,GAAW,EAAE,IAAkB,EAAE,EAAE;YAC5D,WAAW,GAAG,GAAG,CAAC;YAClB,YAAY,GAAG,IAAI,CAAC;YACpB,OAAO,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,CAAC,CAAA,CAAC,CAAC;QAEH,MAAM,WAAW,GAA2B;YAC1C,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,cAAc;SAC7B,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,uBAAU,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAG,GAAS,EAAE,kDAAC,OAAA,IAAI,CAAA,GAAA,CAAC;QACpC,MAAM,GAAG,GAAG,mCAAmC,CAAC;QAChD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAChD,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAE7B,qBAAM,CAAC,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACrC,qBAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;QACxB,qBAAM,CAAC,EAAE,CAAC,YAAa,CAAC,OAAO,CAAC,CAAC;QAEjC,MAAM,OAAO,GAAG,YAAa,CAAC,OAAiC,CAAC;QAChE,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;QACrD,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAE5D,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAClD,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,cAAc,CAAC,CAAC;QAE1D,qBAAM,CAAC,WAAW,CAAE,WAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACzD,qBAAM,CAAC,WAAW,CAAE,WAAmB,CAAC,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,8CAA8C,EAAE,GAAS,EAAE;QAC5D,IAAI,YAAqC,CAAC;QAC1C,SAAS,CAAC,SAAS,CAAC,CAAO,IAAY,EAAE,IAAkB,EAAE,EAAE;YAC7D,YAAY,GAAG,IAAI,CAAC;YACpB,OAAO,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,CAAC,CAAA,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,uBAAU,CAAC;YAC7B,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,GAAS,EAAE,kDAAC,OAAA,IAAI,CAAA,GAAA,CAAC,CAAC;QACxE,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7B,qBAAM,CAAC,WAAW,CAAC,YAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,kFAAkF,EAAE,GAAG,EAAE;QAC1F,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC;QAChD,MAAM,IAAI,GAAG,kBAAkB,QAAQ,oBAAoB,CAAC;QAC5D,MAAM,EAAE,GAAI,OAAe,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QAC3E,qBAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,qBAAM,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,6CAA6C,CAAC;QAC3D,MAAM,EAAE,GAAI,OAAe,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;QAC3E,qBAAM,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,iDAAiD,EAAE,GAAS,EAAE;QAC/D,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,GAAS,EAAE,kDAAC,OAAA,IAAI,CAAA,GAAA,CAAC,CAAC;QAC7E,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAS,EAAE,kDAAC,OAAA,KAAK,CAAA,GAAA,CAAC,CAAC;QAC5E,qBAAM,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9B,qBAAM,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/lib/browser.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Browser entry point for Vectra.
+ *
+ * This module exports only browser-compatible components.
+ * Node-specific components (LocalFileStorage, FileFetcher, WebFetcher) are excluded.
+ *
+ * For browser usage:
+ * - Use `VirtualFileStorage` for in-memory storage (ephemeral)
+ * - Use `IndexedDBStorage` for persistent browser storage
+ * - Use `BrowserWebFetcher` for fetching web content
+ */
+export { LocalIndex, CreateIndexConfig } from './LocalIndex';
+export { LocalDocumentIndex, LocalDocumentIndexConfig, DocumentQueryOptions } from './LocalDocumentIndex';
+export { LocalDocument } from './LocalDocument';
+export { LocalDocumentResult } from './LocalDocumentResult';
+export { TextSplitter, TextSplitterConfig } from './TextSplitter';
+export { GPT3Tokenizer } from './GPT3Tokenizer';
+export { OpenAIEmbeddings, OpenAIEmbeddingsOptions, AzureOpenAIEmbeddingsOptions, OSSEmbeddingsOptions, BaseOpenAIEmbeddingsOptions } from './OpenAIEmbeddings';
+export { TransformersEmbeddings, TransformersEmbeddingsOptions } from './TransformersEmbeddings';
+export { TransformersTokenizer } from './TransformersTokenizer';
+export { ItemSelector } from './ItemSelector';
+export { EmbeddingsModel, EmbeddingsResponse, EmbeddingsResponseStatus, TextChunk, TextFetcher, IndexStats, IndexItem, MetadataFilter, MetadataTypes, QueryResult, Tokenizer, DocumentChunkMetadata, DocumentCatalogStats, DocumentTextSection, IndexData } from './types';
+export { FileStorage, FileDetails, ListFilesFilter } from './storage/FileStorage';
+export { FileStorageUtilities } from './storage/FileStorageUtilities';
+export { FileType } from './storage/FileType';
+export { VirtualFileStorage } from './storage/VirtualFileStorage';
+export { IndexedDBStorage } from './storage/IndexedDBStorage';
+export { BrowserWebFetcher, BrowserWebFetcherConfig } from './BrowserWebFetcher';
+export { pathUtils } from './utils/pathUtils';
+//# sourceMappingURL=browser.d.ts.map

package/lib/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAG1G,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAG5D,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,OAAO,EACH,gBAAgB,EAChB,uBAAuB,EACvB,4BAA4B,EAC5B,oBAAoB,EACpB,2BAA2B,EAC9B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAC;AACjG,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EACH,eAAe,EACf,kBAAkB,EAClB,wBAAwB,EACxB,SAAS,EACT,WAAW,EACX,UAAU,EACV,SAAS,EACT,cAAc,EACd,aAAa,EACb,WAAW,EACX,SAAS,EACT,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EACnB,SAAS,EACZ,MAAM,SAAS,CAAC;AAGjB,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAG9D,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAGjF,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}

package/lib/browser.js

@@ -0,0 +1,52 @@
+"use strict";
+/**
+ * Browser entry point for Vectra.
+ *
+ * This module exports only browser-compatible components.
+ * Node-specific components (LocalFileStorage, FileFetcher, WebFetcher) are excluded.
+ *
+ * For browser usage:
+ * - Use `VirtualFileStorage` for in-memory storage (ephemeral)
+ * - Use `IndexedDBStorage` for persistent browser storage
+ * - Use `BrowserWebFetcher` for fetching web content
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pathUtils = exports.BrowserWebFetcher = exports.IndexedDBStorage = exports.VirtualFileStorage = exports.FileStorageUtilities = exports.ItemSelector = exports.TransformersTokenizer = exports.TransformersEmbeddings = exports.OpenAIEmbeddings = exports.GPT3Tokenizer = exports.TextSplitter = exports.LocalDocumentResult = exports.LocalDocument = exports.LocalDocumentIndex = exports.LocalIndex = void 0;
+// Core index classes
+var LocalIndex_1 = require("./LocalIndex");
+Object.defineProperty(exports, "LocalIndex", { enumerable: true, get: function () { return LocalIndex_1.LocalIndex; } });
+var LocalDocumentIndex_1 = require("./LocalDocumentIndex");
+Object.defineProperty(exports, "LocalDocumentIndex", { enumerable: true, get: function () { return LocalDocumentIndex_1.LocalDocumentIndex; } });
+// Document classes
+var LocalDocument_1 = require("./LocalDocument");
+Object.defineProperty(exports, "LocalDocument", { enumerable: true, get: function () { return LocalDocument_1.LocalDocument; } });
+var LocalDocumentResult_1 = require("./LocalDocumentResult");
+Object.defineProperty(exports, "LocalDocumentResult", { enumerable: true, get: function () { return LocalDocumentResult_1.LocalDocumentResult; } });
+// Text processing
+var TextSplitter_1 = require("./TextSplitter");
+Object.defineProperty(exports, "TextSplitter", { enumerable: true, get: function () { return TextSplitter_1.TextSplitter; } });
+var GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+Object.defineProperty(exports, "GPT3Tokenizer", { enumerable: true, get: function () { return GPT3Tokenizer_1.GPT3Tokenizer; } });
+// Embeddings
+var OpenAIEmbeddings_1 = require("./OpenAIEmbeddings");
+Object.defineProperty(exports, "OpenAIEmbeddings", { enumerable: true, get: function () { return OpenAIEmbeddings_1.OpenAIEmbeddings; } });
+var TransformersEmbeddings_1 = require("./TransformersEmbeddings");
+Object.defineProperty(exports, "TransformersEmbeddings", { enumerable: true, get: function () { return TransformersEmbeddings_1.TransformersEmbeddings; } });
+var TransformersTokenizer_1 = require("./TransformersTokenizer");
+Object.defineProperty(exports, "TransformersTokenizer", { enumerable: true, get: function () { return TransformersTokenizer_1.TransformersTokenizer; } });
+// Query utilities
+var ItemSelector_1 = require("./ItemSelector");
+Object.defineProperty(exports, "ItemSelector", { enumerable: true, get: function () { return ItemSelector_1.ItemSelector; } });
+var FileStorageUtilities_1 = require("./storage/FileStorageUtilities");
+Object.defineProperty(exports, "FileStorageUtilities", { enumerable: true, get: function () { return FileStorageUtilities_1.FileStorageUtilities; } });
+var VirtualFileStorage_1 = require("./storage/VirtualFileStorage");
+Object.defineProperty(exports, "VirtualFileStorage", { enumerable: true, get: function () { return VirtualFileStorage_1.VirtualFileStorage; } });
+var IndexedDBStorage_1 = require("./storage/IndexedDBStorage");
+Object.defineProperty(exports, "IndexedDBStorage", { enumerable: true, get: function () { return IndexedDBStorage_1.IndexedDBStorage; } });
+// Browser-compatible fetcher
+var BrowserWebFetcher_1 = require("./BrowserWebFetcher");
+Object.defineProperty(exports, "BrowserWebFetcher", { enumerable: true, get: function () { return BrowserWebFetcher_1.BrowserWebFetcher; } });
+// Utilities
+var pathUtils_1 = require("./utils/pathUtils");
+Object.defineProperty(exports, "pathUtils", { enumerable: true, get: function () { return pathUtils_1.pathUtils; } });
+//# sourceMappingURL=browser.js.map

package/lib/browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAEH,qBAAqB;AACrB,2CAA6D;AAApD,wGAAA,UAAU,OAAA;AACnB,2DAA0G;AAAjG,wHAAA,kBAAkB,OAAA;AAE3B,mBAAmB;AACnB,iDAAgD;AAAvC,8GAAA,aAAa,OAAA;AACtB,6DAA4D;AAAnD,0HAAA,mBAAmB,OAAA;AAE5B,kBAAkB;AAClB,+CAAkE;AAAzD,4GAAA,YAAY,OAAA;AACrB,iDAAgD;AAAvC,8GAAA,aAAa,OAAA;AAEtB,aAAa;AACb,uDAM4B;AALxB,oHAAA,gBAAgB,OAAA;AAMpB,mEAAiG;AAAxF,gIAAA,sBAAsB,OAAA;AAC/B,iEAAgE;AAAvD,8HAAA,qBAAqB,OAAA;AAE9B,kBAAkB;AAClB,+CAA8C;AAArC,4GAAA,YAAY,OAAA;AAuBrB,uEAAsE;AAA7D,4HAAA,oBAAoB,OAAA;AAE7B,mEAAkE;AAAzD,wHAAA,kBAAkB,OAAA;AAC3B,+DAA8D;AAArD,oHAAA,gBAAgB,OAAA;AAEzB,6BAA6B;AAC7B,yDAAiF;AAAxE,sHAAA,iBAAiB,OAAA;AAE1B,YAAY;AACZ,+CAA8C;AAArC,sGAAA,SAAS,OAAA"}