vectra 0.1.1 → 0.2.0

package/lib/vectra-cli.js

@@ -0,0 +1,276 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = void 0;
+const fs = __importStar(require("fs/promises"));
+const yargs_1 = __importDefault(require("yargs/yargs"));
+const helpers_1 = require("yargs/helpers");
+const LocalDocumentIndex_1 = require("./LocalDocumentIndex");
+const WebFetcher_1 = require("./WebFetcher");
+const OpenAIEmbeddings_1 = require("./OpenAIEmbeddings");
+const internals_1 = require("./internals");
+function run() {
+    return __awaiter(this, void 0, void 0, function* () {
+        // prettier-ignore
+        const args = yield (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
+            .command('create <index>', `create a new local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            console.log(internals_1.Colorize.output(`creating index at ${folderPath}`));
+            yield index.createIndex({ version: 1, deleteIfExists: true });
+        }))
+            .command('delete <index>', `delete an existing local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            console.log(internals_1.Colorize.output(`deleting index at ${folderPath}`));
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            yield index.deleteIndex();
+        }))
+            .command('add-web <index>', `adds one or more web pages to an index`, (yargs) => {
+            return yargs
+                .option('keys', {
+                alias: 'k',
+                describe: 'path of a JSON file containing the model keys to use for generating embeddings',
+                type: 'string'
+            })
+                .option('uri', {
+                alias: 'u',
+                array: true,
+                describe: 'http/https link to a web page to add',
+                type: 'string'
+            })
+                .option('list', {
+                alias: 'l',
+                describe: 'path to a file containing a list of web pages to add',
+                type: 'string'
+            })
+                .option('chunk-size', {
+                alias: 'cs',
+                describe: 'size of the generated chunks in tokens (defaults to 512)',
+                type: 'number',
+                default: 512
+            })
+                .check((argv) => {
+                if (Array.isArray(argv.uri) && argv.uri.length > 0) {
+                    return true;
+                }
+                else if (typeof argv.list == 'string' && argv.list.trim().length > 0) {
+                    return true;
+                }
+                else {
+                    throw new Error(`you must specify either one or more "--uri <link>" for the pages to add or a "--list <file path>" for a file containing the list of pages to add.`);
+                }
+            })
+                .demandOption(['keys']);
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            console.log(internals_1.Colorize.title('Adding Web Pages to Index'));
+            // Create embeddings
+            const keys = JSON.parse(yield fs.readFile(args.keys, 'utf-8'));
+            const embeddings = new OpenAIEmbeddings_1.OpenAIEmbeddings(Object.assign({ model: 'text-embedding-ada-002' }, keys));
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({
+                folderPath,
+                embeddings,
+                chunkingConfig: {
+                    chunkSize: args.chunkSize
+                }
+            });
+            // Get list of url's
+            const uris = yield getItemList(args.uri, args.list, 'web page');
+            // Fetch web pages
+            const fetcher = new WebFetcher_1.WebFetcher();
+            for (const uri of uris) {
+                try {
+                    console.log(internals_1.Colorize.progress(`fetching ${uri}`));
+                    const content = yield fetcher.fetch(uri);
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.progress(`indexing ${uri}`)));
+                    yield index.upsertDocument(uri, content);
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.success(`added ${uri}`)));
+                }
+                catch (err) {
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.error(`Error adding: ${uri}\n${err.message}`)));
+                }
+            }
+        }))
+            .command('remove <index>', `removes one or more documents from an index`, (yargs) => {
+            return yargs
+                .option('uri', {
+                alias: 'u',
+                array: true,
+                describe: 'uri of a document to remove',
+                type: 'string'
+            })
+                .option('list', {
+                alias: 'l',
+                describe: 'path to a file containing a list of documents to remove',
+                type: 'string'
+            })
+                .check((argv) => {
+                if (Array.isArray(argv.uri) && argv.uri.length > 0) {
+                    return true;
+                }
+                else if (typeof argv.list == 'string' && argv.list.trim().length > 0) {
+                    return true;
+                }
+                else {
+                    throw new Error(`you must specify either one or more "--uri <link>" for the pages to add or a "--list <file path>" for a file containing the list of pages to add.`);
+                }
+            });
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            // Get list of uri's
+            const uris = yield getItemList(args.uri, args.list, 'document');
+            // Remove documents
+            for (const uri of uris) {
+                console.log(`removing ${uri}`);
+                yield index.deleteDocument(uri);
+            }
+        }))
+            .command('stats <index>', `prints the stats for a local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            const stats = yield index.getCatalogStats();
+            console.log(internals_1.Colorize.title('Index Stats'));
+            console.log(internals_1.Colorize.output(stats));
+        }))
+            .command('query <index> <query>', `queries a local index`, (yargs) => {
+            return yargs
+                .option('keys', {
+                alias: 'k',
+                describe: 'path of a JSON file containing the model keys to use for generating embeddings'
+            })
+                .option('document-count', {
+                alias: 'dc',
+                describe: 'max number of documents to return (defaults to 10)',
+                type: 'count',
+                default: 10
+            })
+                .option('chunk-count', {
+                alias: 'cc',
+                describe: 'max number of chunks to return (defaults to 50)',
+                type: 'count',
+                default: 50
+            })
+                .option('section-count', {
+                alias: 'sc',
+                describe: 'max number of document sections to render (defaults to 1)',
+                type: 'count',
+                default: 1
+            })
+                .option('tokens', {
+                alias: 't',
+                describe: 'max number of tokens to render for each document section (defaults to 2000)',
+                type: 'count',
+                default: 2000
+            })
+                .option('format', {
+                alias: 'f',
+                describe: `format of the rendered results. Defaults to 'sections'`,
+                choices: ['sections', 'stats', 'chunks'],
+                default: 'sections'
+            })
+                .demandOption(['keys']);
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            console.log(internals_1.Colorize.title('Querying Index'));
+            // Create embeddings
+            const keys = JSON.parse(yield fs.readFile(args.keys, 'utf-8'));
+            const embeddings = new OpenAIEmbeddings_1.OpenAIEmbeddings(Object.assign({ model: 'text-embedding-ada-002' }, keys));
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({
+                folderPath,
+                embeddings
+            });
+            // Query index
+            const query = args.query;
+            const results = yield index.queryDocuments(query, {
+                maxDocuments: args.documentCount,
+                maxChunks: args.chunkCount,
+            });
+            // Render results
+            for (const result of results) {
+                console.log(internals_1.Colorize.output(result.uri));
+                console.log(internals_1.Colorize.value('score', result.score));
+                console.log(internals_1.Colorize.value('chunks', result.chunks.length));
+                if (args.format == 'sections') {
+                    const sections = yield result.renderSections(args.tokens, args.sectionCount);
+                    for (let i = 0; i < sections.length; i++) {
+                        const section = sections[i];
+                        console.log(internals_1.Colorize.title(args.sectionCount > 1 ? 'Section' : `Section ${1}`));
+                        console.log(internals_1.Colorize.value('score', section.score));
+                        console.log(internals_1.Colorize.value('tokens', section.tokenCount));
+                        console.log(internals_1.Colorize.output(section.text));
+                    }
+                }
+                else if (args.format == 'chunks') {
+                    const text = yield result.loadText();
+                    for (let i = 0; i < result.chunks.length; i++) {
+                        const chunk = result.chunks[i];
+                        const startPos = chunk.item.metadata.startPos;
+                        const endPos = chunk.item.metadata.endPos;
+                        console.log(internals_1.Colorize.title(`Chunk ${i + 1}`));
+                        console.log(internals_1.Colorize.value('score', chunk.score));
+                        console.log(internals_1.Colorize.value('startPos', startPos));
+                        console.log(internals_1.Colorize.value('endPos', endPos));
+                        console.log(internals_1.Colorize.output(text.substring(startPos, endPos + 1)));
+                    }
+                }
+            }
+        }))
+            .help()
+            .demandCommand()
+            .parseAsync();
+    });
+}
+exports.run = run;
+function getItemList(items, listFile, uriType) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (Array.isArray(items) && items.length > 0) {
+            return items;
+        }
+        else if (typeof listFile == 'string' && listFile.trim().length > 0) {
+            const list = yield fs.readFile(listFile, 'utf-8');
+            return list.split('\n').map((item) => item.trim()).filter((item) => item.length > 0);
+        }
+        else {
+            throw new Error(`you must specify either one or more "--uri <${uriType}>" for the items or a "--list <file path>" for a file containing the items.`);
+        }
+    });
+}
+//# sourceMappingURL=vectra-cli.js.map

package/lib/vectra-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectra-cli.js","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,wDAAgC;AAChC,2CAAwC;AACxC,6DAA0D;AAC1D,6CAA0C;AAC1C,yDAAsD;AACtD,2CAAuC;AAGvC,SAAsB,GAAG;;QACrB,kBAAkB;QAClB,MAAM,IAAI,GAAG,MAAM,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;aAC1C,OAAO,CAAC,gBAAgB,EAAE,0BAA0B,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YACtE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,gCAAgC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,iBAAiB,EAAE,wCAAwC,EAAE,CAAC,KAAK,EAAE,EAAE;YAC5E,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;gBAC1F,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,sCAAsC;gBAChD,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,sDAAsD;gBAChE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,0DAA0D;gBACpE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,GAAG;aACf,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChD,OAAO,IAAI,CAAC;iBACf;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpE,OAAO,IAAI,CAAC;iBACf;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;iBACxK;YACL,CAAC,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAEzD,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAElG,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;gBACV,cAAc,EAAE;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;iBAC5B;aACJ,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,kBAAkB;YAClB,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;YACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACpB,IAAI;oBACA,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC;oBAClD,MAAM,OAAO,GAAI,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;oBACxE,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;iBACvE;gBAAC,OAAO,GAAY,EAAE;oBACnB,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,KAAK,CAAC,iBAAiB,GAAG,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;iBACxG;aACJ;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,6CAA6C,EAAE,CAAC,KAAK,EAAE,EAAE;YAChF,OAAO,KAAK;iBACP,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,6BAA6B;gBACvC,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,yDAAyD;gBACnE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChD,OAAO,IAAI,CAAC;iBACf;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpE,OAAO,IAAI,CAAC;iBACf;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;iBACxK;YACL,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YAErD,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,mBAAmB;YACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACpB,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;gBAC/B,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aACnC;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,eAAe,EAAE,oCAAoC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,uBAAuB,EAAE,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE;YACjE,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;aAC7F,CAAC;iBACD,MAAM,CAAC,gBAAgB,EAAE;gBACtB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,oDAAoD;gBAC9D,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,iDAAiD;gBAC3D,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,eAAe,EAAE;gBACrB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,2DAA2D;gBACrE,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,CAAC;aACb,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,6EAA6E;gBACvF,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,IAAI;aAChB,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,wDAAwD;gBAClE,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;gBACxC,OAAO,EAAE,UAAU;aACtB,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE9C,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAElG,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;aACb,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE;gBAC9C,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,SAAS,EAAE,IAAI,CAAC,UAAU;aAC7B,CAAC,CAAC;YAEH,iBAAiB;YACjB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC1B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE;oBAC3B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBACtC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChF,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;wBACpD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;wBAC1D,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;qBAC9C;iBACJ;qBAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE;oBAChC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;qBACtE;iBACJ;aACJ;QACL,CAAC,CAAA,CAAC;aACD,IAAI,EAAE;aACN,aAAa,EAAE;aACf,UAAU,EAAE,CAAC;IACtB,CAAC;CAAA;AAxND,kBAwNC;AAGD,SAAe,WAAW,CAAC,KAAe,EAAE,QAAgB,EAAE,OAAe;;QACzE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,OAAO,KAAK,CAAC;SAChB;aAAM,IAAI,OAAO,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;YAClE,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;SACxF;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,6EAA6E,CAAC,CAAA;SACvJ;IACL,CAAC;CAAA"}

package/package.json

@@ -2,7 +2,7 @@
     "name": "vectra",
     "author": "Steven Ickman",
     "description": "A vector database that uses the local file system for storage.",
-    "version": "0.1.1",
+    "version": "0.2.0",
     "license": "MIT",
     "keywords": [
         "gpt"
@@ -16,6 +16,9 @@
     },
     "main": "./lib/index.js",
     "types": "./lib/index.d.ts",
+    "bin": {
+        "vectra": "./bin/vectra.js"
+    },
     "typesVersions": {
         "<3.9": {
             "*": [
@@ -24,13 +27,28 @@
         }
     },
     "dependencies": {
-        "uuid": "^8.3.2"
+        "axios": "^1.3.4",
+        "cheerio": "^1.0.0-rc.12",
+        "dotenv": "^8.2.0",
+        "gpt-3-encoder": "1.1.4",
+        "json-colorizer": "^2.2.2",
+        "openai": "^3.2.1",
+        "uuid": "^9.0.0",
+        "yargs": "^17.7.2"
     },
     "resolutions": {
     },
     "devDependencies": {
         "@types/node": "^14.14.31",
-        "@types/uuid": "^8.3.0"
+        "@types/mocha": "^8.2.0",
+        "@types/assert": "^1.5.3",
+        "@types/uuid": "9.0.1",
+        "@types/yargs": "17.0.24",
+        "mocha": "10.2.0",
+        "nyc": "^15.1.0",
+        "shx": "^0.3.2",
+        "ts-mocha": "10.0.0",
+        "typescript": "^4.2.3"
     },
     "scripts": {
         "build": "tsc -b",

package/src/GPT3Tokenizer.ts

@@ -0,0 +1,15 @@
+import { Tokenizer } from "./types";
+import { encode, decode } from "gpt-3-encoder";
+
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+export class GPT3Tokenizer implements Tokenizer {
+    public decode(tokens: number[]): string {
+        return decode(tokens);
+    }
+
+    public encode(text: string): number[] {
+        return encode(text);
+    }
+}

package/src/ItemSelector.ts

@@ -1,4 +1,4 @@
-import { MetadataFilter, MetadataTypes } from './LocalIndex';
+import { MetadataFilter, MetadataTypes } from './types';
 
 export class ItemSelector {
     /**
@@ -62,12 +62,12 @@ export class ItemSelector {
         for (const key in filter) {
             switch (key) {
                 case '$and':
-                    if (!filter[key].every((f) => this.select(metadata, f))) {
+                    if (!filter[key]!.every((f: MetadataFilter) => this.select(metadata, f))) {
                         return false;
                     }
                     break;
                 case '$or':
-                    if (!filter[key].some((f) => this.select(metadata, f))) {
+                    if (!filter[key]!.some((f: MetadataFilter) => this.select(metadata, f))) {
                         return false;
                     }
                     break;
@@ -120,32 +120,32 @@ export class ItemSelector {
                     }
                     break;
                 case '$gt':
-                    if (typeof value != 'number' || value <= filter[key]) {
+                    if (typeof value != 'number' || value <= filter[key]!) {
                         return false;
                     }
                     break;
                 case '$gte':
-                    if (typeof value != 'number' || value < filter[key]) {
+                    if (typeof value != 'number' || value < filter[key]!) {
                         return false;
                     }
                     break;
                 case '$lt':
-                    if (typeof value != 'number' || value >= filter[key]) {
+                    if (typeof value != 'number' || value >= filter[key]!) {
                         return false;
                     }
                     break;
                 case '$lte':
-                    if (typeof value != 'number' || value > filter[key]) {
+                    if (typeof value != 'number' || value > filter[key]!) {
                         return false;
                     }
                     break;
                 case '$in':
-                    if (typeof value == 'boolean' || !filter[key].includes(value)) {
+                    if (typeof value == 'boolean' || !filter[key]!.includes(value)) {
                         return false;
                     }
                     break;
                 case '$nin':
-                    if (typeof value == 'boolean' || filter[key].includes(value)) {
+                    if (typeof value == 'boolean' || filter[key]!.includes(value)) {
                         return false;
                     }
                     break;

package/src/LocalDocument.ts

@@ -0,0 +1,70 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { MetadataTypes } from './types';
+
+export class LocalDocument {
+    private readonly _folderPath: string;
+    private readonly _id: string;
+    private readonly _uri: string;
+    private _metadata: Record<string,MetadataTypes>|undefined;
+    private _text: string|undefined;
+
+    public constructor(folderPath: string, id: string, uri: string) {
+        this._folderPath = folderPath;
+        this._id = id;
+        this._uri = uri;
+    }
+
+    public get folderPath(): string {
+        return this._folderPath;
+    }
+
+    public get id(): string {
+        return this._id;
+    }
+
+    public get uri(): string {
+        return this._uri;
+    }
+
+    public async hasMetadata(): Promise<boolean> {
+        try {
+            await fs.access(path.join(this.folderPath, `${this.id}.json`));
+            return true;
+        } catch (err: unknown) {
+            return false;
+        }
+    }
+
+    public async loadMetadata(): Promise<Record<string,MetadataTypes>> {
+        if (this._metadata == undefined) {
+            let json: string;
+            try {
+                json = (await fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
+            } catch (err: unknown) {
+                throw new Error(`Error reading metadata for document "${this.uri}": ${(err as any).toString()}`);
+            }
+
+            try {
+                this._metadata = JSON.parse(json);
+            } catch (err: unknown) {
+                throw new Error(`Error parsing metadata for document "${this.uri}": ${(err as any).toString()}`);
+            }
+        }
+
+        return this._metadata!;
+    }
+
+    public async loadText(): Promise<string> {
+        if (this._text == undefined) {
+            try {
+                this._text = (await fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
+            } catch (err: unknown) {
+                throw new Error(`Error reading text file for document "${this.uri}": ${(err as any).toString()}`);
+            }
+        }
+
+        return this._text;
+    }
+
+}