vectra 0.12.2 → 0.12.3

package/lib/vectra-cli.js

@@ -0,0 +1,323 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = run;
+const fs = __importStar(require("fs/promises"));
+const yargs_1 = __importDefault(require("yargs/yargs"));
+const helpers_1 = require("yargs/helpers");
+const LocalDocumentIndex_1 = require("./LocalDocumentIndex");
+const WebFetcher_1 = require("./WebFetcher");
+const OpenAIEmbeddings_1 = require("./OpenAIEmbeddings");
+const internals_1 = require("./internals");
+const FileFetcher_1 = require("./FileFetcher");
+function run() {
+    return __awaiter(this, void 0, void 0, function* () {
+        // prettier-ignore
+        const args = yield (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
+            .scriptName('vectra')
+            .command('create <index>', `create a new local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            console.log(internals_1.Colorize.output(`creating index at ${folderPath}`));
+            yield index.createIndex({ version: 1, deleteIfExists: true });
+        }))
+            .command('delete <index>', `delete an existing local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            console.log(internals_1.Colorize.output(`deleting index at ${folderPath}`));
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            yield index.deleteIndex();
+        }))
+            .command('add <index>', `adds one or more web pages to an index`, (yargs) => {
+            return yargs
+                .option('keys', {
+                alias: 'k',
+                describe: 'path of a JSON file containing the model keys to use for generating embeddings',
+                type: 'string'
+            })
+                .option('uri', {
+                alias: 'u',
+                array: true,
+                describe: 'http/https link to a web page to add',
+                type: 'string'
+            })
+                .option('list', {
+                alias: 'l',
+                describe: 'path to a file containing a list of web pages to add',
+                type: 'string'
+            })
+                .option('cookie', {
+                alias: 'c',
+                describe: 'optional cookies to add to web fetch requests',
+                type: 'string'
+            })
+                .option('chunk-size', {
+                alias: 'cs',
+                describe: 'size of the generated chunks in tokens (defaults to 512)',
+                type: 'number',
+                default: 512
+            })
+                .check((argv) => {
+                if (Array.isArray(argv.uri) && argv.uri.length > 0) {
+                    return true;
+                }
+                else if (typeof argv.list == 'string' && argv.list.trim().length > 0) {
+                    return true;
+                }
+                else {
+                    throw new Error(`you must specify either one or more "--uri <link>" for the pages to add or a "--list <file path>" for a file containing the list of pages to add.`);
+                }
+            })
+                .demandOption(['keys']);
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            console.log(internals_1.Colorize.title('Adding Web Pages to Index'));
+            // Get embedding options
+            const options = JSON.parse(yield fs.readFile(args.keys, 'utf-8'));
+            if (options.apiKey && !options.model) {
+                options.model = 'text-embedding-ada-002';
+                options.maxTokens = 8000;
+            }
+            // Create embeddings
+            const embeddings = new OpenAIEmbeddings_1.OpenAIEmbeddings(options);
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({
+                folderPath,
+                embeddings,
+                chunkingConfig: {
+                    chunkSize: args.chunkSize
+                }
+            });
+            // Get list of url's
+            const uris = yield getItemList(args.uri, args.list, 'web page');
+            // Fetch documents
+            const fileFetcher = new FileFetcher_1.FileFetcher();
+            const webFetcher = args.cookie ? new WebFetcher_1.WebFetcher({ headers: { "cookie": args.cookie } }) : new WebFetcher_1.WebFetcher();
+            for (const path of uris) {
+                try {
+                    console.log(internals_1.Colorize.progress(`fetching ${path}`));
+                    const fetcher = path.startsWith('http') ? webFetcher : fileFetcher;
+                    yield fetcher.fetch(path, (uri, text, docType) => __awaiter(this, void 0, void 0, function* () {
+                        console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.progress(`indexing ${uri}`)));
+                        yield index.upsertDocument(uri, text, docType);
+                        console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.success(`added ${uri}`)));
+                        return true;
+                    }));
+                }
+                catch (err) {
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.error(`Error adding: ${path}\n${err.message}`)));
+                }
+            }
+        }))
+            .command('remove <index>', `removes one or more documents from an index`, (yargs) => {
+            return yargs
+                .option('uri', {
+                alias: 'u',
+                array: true,
+                describe: 'uri of a document to remove',
+                type: 'string'
+            })
+                .option('list', {
+                alias: 'l',
+                describe: 'path to a file containing a list of documents to remove',
+                type: 'string'
+            })
+                .check((argv) => {
+                if (Array.isArray(argv.uri) && argv.uri.length > 0) {
+                    return true;
+                }
+                else if (typeof argv.list == 'string' && argv.list.trim().length > 0) {
+                    return true;
+                }
+                else {
+                    throw new Error(`you must specify either one or more "--uri <link>" for the pages to add or a "--list <file path>" for a file containing the list of pages to add.`);
+                }
+            });
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            // Get list of uri's
+            const uris = yield getItemList(args.uri, args.list, 'document');
+            // Remove documents
+            for (const uri of uris) {
+                console.log(`removing ${uri}`);
+                yield index.deleteDocument(uri);
+            }
+        }))
+            .command('stats <index>', `prints the stats for a local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            const stats = yield index.getCatalogStats();
+            console.log(internals_1.Colorize.title('Index Stats'));
+            console.log(internals_1.Colorize.output(stats));
+        }))
+            .command('query <index> <query>', `queries a local index`, (yargs) => {
+            return yargs
+                .option('keys', {
+                alias: 'k',
+                describe: 'path of a JSON file containing the model keys to use for generating embeddings'
+            })
+                .option('document-count', {
+                alias: 'dc',
+                describe: 'max number of documents to return (defaults to 10)',
+                type: 'number',
+                default: 10
+            })
+                .option('chunk-count', {
+                alias: 'cc',
+                describe: 'max number of chunks to return (defaults to 50)',
+                type: 'number',
+                default: 50
+            })
+                .option('section-count', {
+                alias: 'sc',
+                describe: 'max number of document sections to render (defaults to 1)',
+                type: 'number',
+                default: 1
+            })
+                .option('tokens', {
+                alias: 't',
+                describe: 'max number of tokens to render for each document section (defaults to 2000)',
+                type: 'number',
+                default: 2000
+            })
+                .option('format', {
+                alias: 'f',
+                describe: `format of the rendered results. Defaults to 'sections'`,
+                choices: ['sections', 'stats', 'chunks'],
+                default: 'sections'
+            })
+                .option('overlap', {
+                alias: 'o',
+                describe: `whether to add overlapping chunks to sections.`,
+                type: 'boolean',
+                default: true
+            })
+                .option('bm25', {
+                alias: 'b',
+                describe: 'Use Okapi-bm25 keyword search alogrithm to perform hybrid search - semantic + keyword. Displayed in blue during search.',
+                type: 'boolean',
+                default: false
+            })
+                .demandOption(['keys']);
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            console.log(internals_1.Colorize.title('Querying Index'));
+            // Get embedding options
+            const options = JSON.parse(yield fs.readFile(args.keys, 'utf-8'));
+            if (options.apiKey && !options.model) {
+                options.model = 'text-embedding-ada-002';
+                options.maxTokens = 8000;
+            }
+            // Create embeddings
+            const embeddings = new OpenAIEmbeddings_1.OpenAIEmbeddings(options);
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({
+                folderPath,
+                embeddings
+            });
+            // Query index
+            const query = args.query;
+            const results = yield index.queryDocuments(query, {
+                maxDocuments: args.documentCount,
+                maxChunks: args.chunkCount,
+                isBm25: args.bm25,
+            });
+            // Render results
+            for (const result of results) {
+                console.log(internals_1.Colorize.output(result.uri));
+                console.log(internals_1.Colorize.value('score', result.score));
+                console.log(internals_1.Colorize.value('chunks', result.chunks.length));
+                if (args.format == 'sections') {
+                    const sections = yield result.renderSections(args.tokens, args.sectionCount, args.overlap);
+                    console.log(sections.length);
+                    for (let i = 0; i < sections.length; i++) {
+                        const section = sections[i];
+                        const isBm25 = sections[i].isBm25;
+                        console.log(isBm25);
+                        console.log(internals_1.Colorize.title(args.sectionCount == 1 ? 'Section' : `Section ${i + 1}`));
+                        console.log(internals_1.Colorize.value('score', section.score));
+                        console.log(internals_1.Colorize.value('tokens', section.tokenCount));
+                        console.log(internals_1.Colorize.output(section.text, isBm25));
+                    }
+                }
+                else if (args.format == 'chunks') {
+                    const text = yield result.loadText();
+                    for (let i = 0; i < result.chunks.length; i++) {
+                        const chunk = result.chunks[i];
+                        const startPos = chunk.item.metadata.startPos;
+                        const endPos = chunk.item.metadata.endPos;
+                        const isBm25 = Boolean(chunk.item.metadata.isBm25);
+                        console.log(internals_1.Colorize.title(`Chunk ${i + 1}`));
+                        console.log(internals_1.Colorize.value('score', chunk.score));
+                        console.log(internals_1.Colorize.value('startPos', startPos));
+                        console.log(internals_1.Colorize.value('endPos', endPos));
+                        console.log(internals_1.Colorize.output(text.substring(startPos, endPos + 1), isBm25));
+                    }
+                }
+            }
+        }))
+            .help()
+            .demandCommand()
+            .parseAsync();
+    });
+}
+function getItemList(items, listFile, uriType) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (Array.isArray(items) && items.length > 0) {
+            return items;
+        }
+        else if (typeof listFile == 'string' && listFile.trim().length > 0) {
+            const list = yield fs.readFile(listFile, 'utf-8');
+            return list.split('\n').map((item) => item.trim()).filter((item) => item.length > 0);
+        }
+        else {
+            throw new Error(`you must specify either one or more "--uri <${uriType}>" for the items or a "--list <file path>" for a file containing the items.`);
+        }
+    });
+}
+//# sourceMappingURL=vectra-cli.js.map

package/lib/vectra-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectra-cli.js","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AASA,kBA+PC;AAxQD,gDAAkC;AAClC,wDAAgC;AAChC,2CAAwC;AACxC,6DAA0D;AAC1D,6CAA0C;AAC1C,yDAAmI;AACnI,2CAAuC;AACvC,+CAA4C;AAE5C,SAAsB,GAAG;;QACrB,kBAAkB;QAClB,MAAM,IAAI,GAAG,MAAM,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;aAC1C,UAAU,CAAC,QAAQ,CAAC;aACpB,OAAO,CAAC,gBAAgB,EAAE,0BAA0B,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YACtE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,gCAAgC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,aAAa,EAAE,wCAAwC,EAAE,CAAC,KAAK,EAAE,EAAE;YACxE,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;gBAC1F,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,sCAAsC;gBAChD,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,sDAAsD;gBAChE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,+CAA+C;gBACzD,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,0DAA0D;gBACpE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,GAAG;aACf,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjD,OAAO,IAAI,CAAC;gBAChB,CAAC;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrE,OAAO,IAAI,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;gBACzK,CAAC;YACL,CAAC,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAEzD,wBAAwB;YACxB,MAAM,OAAO,GAA8E,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACvJ,IAAK,OAAmC,CAAC,MAAM,IAAI,CAAE,OAAmC,CAAC,KAAK,EAAE,CAAC;gBAC5F,OAAmC,CAAC,KAAK,GAAG,wBAAwB,CAAC;gBACrE,OAAmC,CAAC,SAAS,GAAG,IAAI,CAAC;YAC1D,CAAC;YAED,oBAAoB;YACpB,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,OAAO,CAAC,CAAC;YAEjD,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;gBACV,cAAc,EAAE;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;iBAC5B;aACJ,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,kBAAkB;YAClB,MAAM,WAAW,GAAG,IAAI,yBAAW,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,uBAAU,CAAC,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,EAAC,CAAC,CAAC,CAAC,CAAC,IAAI,uBAAU,EAAE,CAAC;YAC1G,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;oBACnD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;oBACnE,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAO,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;wBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;wBACxE,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;wBAC/C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;wBACpE,OAAO,IAAI,CAAC;oBAChB,CAAC,CAAA,CAAC,CAAC;gBACP,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACpB,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,KAAK,CAAC,iBAAiB,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC1G,CAAC;YACL,CAAC;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,6CAA6C,EAAE,CAAC,KAAK,EAAE,EAAE;YAChF,OAAO,KAAK;iBACP,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,6BAA6B;gBACvC,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,yDAAyD;gBACnE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjD,OAAO,IAAI,CAAC;gBAChB,CAAC;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrE,OAAO,IAAI,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;gBACzK,CAAC;YACL,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YAErD,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,mBAAmB;YACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;gBAC/B,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,eAAe,EAAE,oCAAoC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,uBAAuB,EAAE,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE;YACjE,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;aAC7F,CAAC;iBACD,MAAM,CAAC,gBAAgB,EAAE;gBACtB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,oDAAoD;gBAC9D,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,iDAAiD;gBAC3D,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,eAAe,EAAE;gBACrB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,2DAA2D;gBACrE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,CAAC;aACb,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,6EAA6E;gBACvF,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI;aAChB,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,wDAAwD;gBAClE,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;gBACxC,OAAO,EAAE,UAAU;aACtB,CAAC;iBACD,MAAM,CAAC,SAAS,EAAE;gBACf,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gDAAgD;gBAC1D,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,IAAI;aAChB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,yHAAyH;gBACnI,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;aACjB,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE9C,wBAAwB;YACxB,MAAM,OAAO,GAA8E,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACvJ,IAAK,OAAmC,CAAC,MAAM,IAAI,CAAE,OAAmC,CAAC,KAAK,EAAE,CAAC;gBAC5F,OAAmC,CAAC,KAAK,GAAG,wBAAwB,CAAC;gBACrE,OAAmC,CAAC,SAAS,GAAG,IAAI,CAAC;YAC1D,CAAC;YAED,oBAAoB;YACpB,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,OAAO,CAAC,CAAC;YAEjD,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;aACb,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE;gBAC9C,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,MAAM,EAAE,IAAI,CAAC,IAAe;aAC/B,CAAC,CAAC;YAEH,iBAAiB;YACjB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;oBAC5B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3F,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAClC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBACpB,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBACrF,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;wBACpD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;wBAC1D,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;oBACvD,CAAC;gBACL,CAAC;qBAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;oBACjC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;wBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;oBAC/E,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC,CAAA,CAAC;aACD,IAAI,EAAE;aACN,aAAa,EAAE;aACf,UAAU,EAAE,CAAC;IACtB,CAAC;CAAA;AAGD,SAAe,WAAW,CAAC,KAAe,EAAE,QAAgB,EAAE,OAAe;;QACzE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACjB,CAAC;aAAM,IAAI,OAAO,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzF,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,6EAA6E,CAAC,CAAA;QACxJ,CAAC;IACL,CAAC;CAAA"}

package/package.json

@@ -2,7 +2,7 @@
     "name": "vectra",
     "author": "Steven Ickman",
     "description": "A vector database that uses the local file system for storage.",
-    "version": "0.12.2",
+    "version": "0.12.3",
     "license": "MIT",
     "keywords": [
         "gpt"
@@ -47,6 +47,7 @@
         "@types/assert": "^1.5.11",
         "@types/mocha": "^10.0.10",
         "@types/node": "^22.15.11",
+        "@types/sinon": "^21.0.0",
         "@types/turndown": "^5.0.5",
         "@types/uuid": "10.0.0",
         "@types/yargs": "17.0.33",
@@ -55,6 +56,7 @@
         "nyc": "^17.1.0",
         "rimraf": "^5.0.1",
         "shx": "^0.4.0",
+        "sinon": "^21.0.1",
         "ts-mocha": "11.1.0",
         "ts-node": "^10.9.1",
         "typescript": "^5.8.3",