vectra 0.9.0 → 0.10.0

package/README.md

@@ -1,5 +1,6 @@
 # Vectra
-Vectra is a local vector database for Node.js with features similar to [Pinecone](https://www.pinecone.io/) or [Qdrant](https://qdrant.tech/) but built using local files. Each Vectra index is a folder on disk. There's an `index.json` file in the folder that contains all the vectors for the index along with any indexed metadata.  When you create an index you can specify which metadata properties to index and only those fields will be stored in the `index.json` file. All of the other metadata for an item will be stored on disk in a separate file keyed by a GUID.
+
+Vectra is a local vector database for Node.js with features similar to [Pinecone](https://www.pinecone.io/) or [Qdrant](https://qdrant.tech/) but built using local files. Each Vectra index is a folder on disk. There's an `index.json` file in the folder that contains all the vectors for the index along with any indexed metadata. When you create an index you can specify which metadata properties to index and only those fields will be stored in the `index.json` file. All of the other metadata for an item will be stored on disk in a separate file keyed by a GUID.
 
 When queryng Vectra you'll be able to use the same subset of [Mongo DB query operators](https://www.mongodb.com/docs/manual/reference/operator/query/) that Pinecone supports and the results will be returned sorted by simularity. Every item in the index will first be filtered by metadata and then ranked for simularity. Even though every item is evaluated its all in memory so it should by nearly instantanious. Likely 1ms - 2ms for even a rather large index. Smaller indexes should be <1ms.
 
@@ -8,9 +9,10 @@ Keep in mind that your entire Vectra index is loaded into memory so it's not wel
 Pinecone style namespaces aren't directly supported but you could easily mimic them by creating a separate Vectra index (and folder) for each namespace.
 
 ## Other Language Bindings
+
 This repo contains the TypeScript/JavaScript binding for Vectra but other language bindings are being created. Since Vectra is file based, any language binding can be used to read or write a Vectra index. That means you can build a Vectra index using JS and then read it using Python.
 
-- [vectra-py](https://github.com/BMS-geodev/vectra-py) - Python version of Vectra.
+-   [vectra-py](https://github.com/BMS-geodev/vectra-py) - Python version of Vectra.
 
 ## Installation
 
@@ -31,7 +33,7 @@ const index = new LocalIndex(path.join(__dirname, '..', 'index'));
 Next, from inside an async function, create your index:
 
 ```typescript
-if (!await index.isIndexCreated()) {
+if (!(await index.isIndexCreated())) {
     await index.createIndex();
 }
 ```
@@ -39,26 +41,24 @@ if (!await index.isIndexCreated()) {
 Add some items to your index:
 
 ```typescript
-import { OpenAIApi, Configuration } from 'openai';
+import { OpenAI } from 'openai';
 
-const configuration = new Configuration({
+const openai = new OpenAI({
     apiKey: `<YOUR_KEY>`,
 });
 
-const api = new OpenAIApi(configuration);
-
 async function getVector(text: string) {
-    const response = await api.createEmbedding({
+    const response = await openai.embeddings.create({
         'model': 'text-embedding-ada-002',
         'input': text,
     });
-    return response.data.data[0].embedding;
+    return response.data[0].embedding;
 }
 
 async function addItem(text: string) {
     await index.insertItem({
         vector: await getVector(text),
-        metadata: { text }
+        metadata: { text },
     });
 }
 

package/package.json

@@ -2,7 +2,7 @@
     "name": "vectra",
     "author": "Steven Ickman",
     "description": "A vector database that uses the local file system for storage.",
-    "version": "0.9.0",
+    "version": "0.10.0",
     "license": "MIT",
     "keywords": [
         "gpt"
@@ -35,14 +35,14 @@
         "openai": "^3.2.1",
         "turndown": "^7.1.2",
         "uuid": "^9.0.0",
+        "wink-nlp": "^2.3.2",
         "yargs": "^17.7.2"
     },
-    "resolutions": {
-    },
+    "resolutions": {},
     "devDependencies": {
-        "@types/node": "^14.14.31",
-        "@types/mocha": "^8.2.0",
         "@types/assert": "^1.5.3",
+        "@types/mocha": "^8.2.0",
+        "@types/node": "^14.14.31",
         "@types/turndown": "^5.0.1",
         "@types/uuid": "9.0.1",
         "@types/yargs": "17.0.24",
@@ -50,7 +50,8 @@
         "nyc": "^15.1.0",
         "shx": "^0.3.2",
         "ts-mocha": "10.0.0",
-        "typescript": "^4.2.3"
+        "typescript": "^4.2.3",
+        "wink-bm25-text-search": "^3.1.2"
     },
     "scripts": {
         "build": "tsc -b",

package/src/ItemSelector.ts

@@ -149,7 +149,13 @@ export class ItemSelector {
                     }
                     break;
                 case '$nin':
-                    if (typeof value == 'boolean' || filter[key]!.includes(value)) {
+                    if (typeof value == 'boolean') {
+                        return false;
+                    }
+                    else if (typeof value == 'string' && filter[key]!.includes(value)) {
+                        return false;
+                    }
+                    else if (filter[key]!.some(val => typeof val == 'string' && val.includes(value as string))) {
                         return false;
                     }
                     break;

package/src/LocalDocumentIndex.ts

@@ -30,6 +30,12 @@ export interface DocumentQueryOptions {
      * Optional. Filter to apply to the document metadata.
      */
     filter?: MetadataFilter;
+
+    /**
+     * Optional. Turn on bm25 keyword search to perform hybrid search - semantic + keyword
+     */
+    isBm25?: boolean;
+
 }
 
 /**
@@ -378,7 +384,7 @@ export class LocalDocumentIndex extends LocalIndex<DocumentChunkMetadata> {
         }
 
         // Query index for chunks
-        const results = await this.queryItems(embeddings.output![0], options.maxChunks!, options.filter);
+        const results = await this.queryItems(embeddings.output![0], query, options.maxChunks!, options.filter, options.isBm25);
 
         // Group chunks by document
         const documentChunks: { [documentId: string]: QueryResult<DocumentChunkMetadata>[]; } = {};

package/src/LocalDocumentResult.ts

@@ -66,7 +66,8 @@ export class LocalDocumentResult extends LocalDocument {
                     startPos: startPos + offset,
                     endPos: startPos + offset + chunkLength - 1,
                     score: chunk.score,
-                    tokenCount: chunkLength
+                    tokenCount: chunkLength,
+                    isBm25: false
                 });
                 offset += chunkLength;
             }
@@ -103,7 +104,8 @@ export class LocalDocumentResult extends LocalDocument {
             return {
                 text: text,
                 tokenCount: section.tokenCount,
-                score: section.score
+                score: section.score,
+                isBm25: false,
             };
         });
     }
@@ -127,7 +129,8 @@ export class LocalDocumentResult extends LocalDocument {
             return [{
                 text,
                 tokenCount: length,
-                score: 1.0
+                score: 1.0,
+                isBm25: false,
             }];
         }
 
@@ -148,7 +151,8 @@ export class LocalDocumentResult extends LocalDocument {
                 startPos,
                 endPos,
                 score: chunk.score,
-                tokenCount: this._tokenizer.encode(chunkText).length
+                tokenCount: this._tokenizer.encode(chunkText).length,
+                isBm25: Boolean(chunk.item.metadata.isBm25),
             };
         }).filter(chunk => chunk.tokenCount <= maxTokens).sort((a, b) => a.startPos - b.startPos);
 
@@ -163,36 +167,63 @@ export class LocalDocumentResult extends LocalDocument {
             return [{
                 text: this._tokenizer.decode(tokens.slice(0, maxTokens)),
                 tokenCount: maxTokens,
-                score: topChunk.score
+                score: topChunk.score,
+                isBm25: false,
             }];
         }
 
-        // Generate sections
+        // Generate semantic sections
         const sections: Section[] = [];
         for (let i = 0; i < chunks.length; i++) {
             const chunk = chunks[i];
             let section = sections[sections.length - 1];
-            if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
-                section = {
-                    chunks: [],
-                    score: 0,
-                    tokenCount: 0
-                };
-                sections.push(section);
+            if (!chunk.isBm25) {
+                if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
+                    section = {
+                        chunks: [],
+                        score: 0,
+                        tokenCount: 0
+                    };
+                    sections.push(section);
+                }
+                section.chunks.push(chunk);
+                section.score += chunk.score;
+                section.tokenCount += chunk.tokenCount;
             }
-            section.chunks.push(chunk);
-            section.score += chunk.score;
-            section.tokenCount += chunk.tokenCount;
         }
 
+        // Generate bm25 sections
+        const bm25Sections: Section[] = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            let section = bm25Sections[bm25Sections.length - 1];
+            if (chunk.isBm25) {
+                if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
+                    section = {
+                        chunks: [],
+                        score: 0,
+                        tokenCount: 0
+                    };
+                    bm25Sections.push(section);
+                }
+                section.chunks.push(chunk);
+                section.score += chunk.score;
+                section.tokenCount += chunk.tokenCount;
+            }
+        }        
         // Normalize section scores
         sections.forEach(section => section.score /= section.chunks.length);
+        bm25Sections.forEach(section => section.score /= section.chunks.length);
 
         // Sort sections by score and limit to maxSections
         sections.sort((a, b) => b.score - a.score);
+        bm25Sections.sort((a, b) => b.score - a.score);
         if (sections.length > maxSections) {
             sections.splice(maxSections, sections.length - maxSections);
         }
+        if (bm25Sections.length > maxSections) {
+            bm25Sections.splice(maxSections, bm25Sections.length - maxSections);
+        }
 
         // Combine adjacent chunks of text
         sections.forEach(section => {
@@ -216,7 +247,8 @@ export class LocalDocumentResult extends LocalDocument {
                 startPos: -1,
                 endPos: -1,
                 score: 0,
-                tokenCount: this._tokenizer.encode('\n\n...\n\n').length
+                tokenCount: this._tokenizer.encode('\n\n...\n\n').length,
+                isBm25: false,
             };
             sections.forEach(section => {
                 // Insert connectors between chunks
@@ -242,7 +274,8 @@ export class LocalDocumentResult extends LocalDocument {
                             startPos: sectionStart - beforeBudget,
                             endPos: sectionStart - 1,
                             score: 0,
-                            tokenCount: beforeBudget
+                            tokenCount: beforeBudget,
+                            isBm25: false,
                         };
                         section.chunks.unshift(chunk);
                         section.tokenCount += chunk.tokenCount;
@@ -258,7 +291,8 @@ export class LocalDocumentResult extends LocalDocument {
                             startPos: sectionEnd + 1,
                             endPos: sectionEnd + afterBudget,
                             score: 0,
-                            tokenCount: afterBudget
+                            tokenCount: afterBudget,
+                            isBm25: false,
                         };
                         section.chunks.push(chunk);
                         section.tokenCount += chunk.tokenCount;
@@ -268,16 +302,29 @@ export class LocalDocumentResult extends LocalDocument {
             });
         }
 
-        // Return final rendered sections
-        return sections.map(section => {
+        const semanticDocTextSections = sections.map(section => {
+            let text = '';
+            section.chunks.forEach(chunk => text += chunk.text);
+            return {
+                text: text,
+                tokenCount: section.tokenCount,
+                score: section.score,
+                isBm25: false,
+            };
+        });
+        const bm25DocTextSections = bm25Sections.map(section => {
             let text = '';
             section.chunks.forEach(chunk => text += chunk.text);
             return {
                 text: text,
                 tokenCount: section.tokenCount,
-                score: section.score
+                score: section.score,
+                isBm25: true,
             };
         });
+
+        // Return final rendered sections
+        return [...semanticDocTextSections, ...bm25DocTextSections];
     }
 
     private encodeBeforeText(text: string, budget: number): number[] {
@@ -300,6 +347,7 @@ interface SectionChunk {
     endPos: number;
     score: number;
     tokenCount: number;
+    isBm25: boolean;
 }
 
 interface Section {

package/src/LocalIndex.ts

@@ -3,7 +3,11 @@ import * as path from 'path';
 import { v4 } from 'uuid';
 import { ItemSelector } from './ItemSelector';
 import { IndexItem, IndexStats, MetadataFilter, MetadataTypes, QueryResult } from './types';
-
+import { LocalDocument } from './LocalDocument';
+import { LocalDocumentIndex } from './LocalDocumentIndex';
+import bm25 from 'wink-bm25-text-search';
+import winkNLP from 'wink-nlp';
+import model from 'wink-eng-lite-web-model';
 export interface CreateIndexConfig {
     version: number;
     deleteIfExists?: boolean;
@@ -24,6 +28,8 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
 
     private _data?: IndexData;
     private _update?: IndexData;
+    //member fields for BM25
+    private _bm25Engine: any;
 
     /**
      * Creates a new instance of LocalIndex.
@@ -247,7 +253,7 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
      * @param filter Optional. Filter to apply.
      * @returns Similar items to the vector that matche the supplied filter.
      */
-    public async queryItems<TItemMetadata extends TMetadata = TMetadata>(vector: number[], topK: number, filter?: MetadataFilter): Promise<QueryResult<TItemMetadata>[]> {
+    public async queryItems<TItemMetadata extends TMetadata = TMetadata>(vector: number[], query: string, topK: number, filter?: MetadataFilter, isBm25?: boolean): Promise<QueryResult<TItemMetadata>[]> {
         await this.loadIndexData();
 
         // Filter items
@@ -285,6 +291,36 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
             }
         }
 
+        //Peform bm25 search only if enabled. Avoid duplicate chunks, which are already selected during semantic search.
+        if (isBm25) {
+            const itemSet = new Set();
+            for (const item of top) itemSet.add(item.item.id);
+            
+            this.setupbm25();
+
+            let currDoc;
+            let currDocTxt;
+            for (let i = 0; i < items.length; i++) {
+                if (!itemSet.has(items[i].id)) {
+                    const item = items[i];
+                    currDoc = new LocalDocument((this as unknown) as LocalDocumentIndex, item.metadata.documentId.toString(), '');
+                    currDocTxt = await currDoc.loadText();
+                    const startPos = item.metadata.startPos;
+                    const endPos = item.metadata.endPos;
+                    const chunkText = currDocTxt.substring(Number(startPos), Number(endPos) + 1);
+                    this._bm25Engine.addDoc({body: chunkText}, i);
+                }
+            }
+            this._bm25Engine.consolidate();
+            var results = await this.bm25Search(query, items, topK);
+            results.forEach((res: any) => {
+                top.push({
+                    item: Object.assign({}, {...items[res[0]], metadata: {...items[res[0]].metadata, isBm25: true}}) as any,
+                    score: res[1]
+                });
+            });
+            
+        }
         return top;
     }
 
@@ -350,7 +386,7 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
             }
 
             // Save remaining metadata to disk
-            metadataFile = `${v4}.json`;
+            metadataFile = `${v4()}.json`;
             const metadataPath = path.join(this._folderPath, metadataFile);
             await fs.writeFile(metadataPath, JSON.stringify(item.metadata));
         } else if (item.metadata) {
@@ -385,6 +421,37 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
             return newItem;
         }
     }
+
+    private async setupbm25(): Promise<any> {
+        this._bm25Engine = bm25();
+        const nlp = winkNLP( model );
+        const its = nlp.its;
+
+        const prepTask = function ( text: string ) {
+            const tokens: any[] = [];
+            nlp.readDoc(text)
+                .tokens()
+                // Use only words ignoring punctuations etc and from them remove stop words
+                .filter( (t: any) => ( t.out(its.type) === 'word' && !t.out(its.stopWordFlag) ) )
+                // Handle negation and extract stem of the word
+                .each( (t: any) => tokens.push( (t.out(its.negationFlag)) ? '!' + t.out(its.stem) : t.out(its.stem) ) );
+
+            return tokens;
+        };
+
+        this._bm25Engine.defineConfig( { fldWeights: { body: 1 } } );
+        // Step II: Define PrepTasks pipe.
+        this._bm25Engine.definePrepTasks( [ prepTask ] );
+    }
+
+    private async bm25Search(searchQuery: string, items: any, topK: number): Promise<any> {
+        var query = searchQuery;
+        // `results` is an array of [ doc-id, score ], sorted by score
+        var results = this._bm25Engine.search( query );
+
+        return results.slice(0, topK);
+    }
+
 }
 
 interface IndexData {

package/src/internals/Colorize.ts

@@ -16,9 +16,9 @@ export class Colorize {
         }
     }
 
-    public static output(output: object | string, quote: string = '', units: string = ''): string {
+    public static output(output: object | string, isBm25: boolean = false, quote: string = '', units: string = ''): string {
         if (typeof output === 'string') {
-            return `\x1b[32m${quote}${output}${quote}\x1b[0m`;
+            return isBm25 ? `\x1b[34m${quote}${output}${quote}\x1b[0m` : `\x1b[32m${quote}${output}${quote}\x1b[0m`;
         } else if (typeof output === 'object' && output !== null) {
             return colorizer(output, {
                 pretty: true,
@@ -54,7 +54,7 @@ export class Colorize {
     }
 
     public static value(field: string, value: any, units: string = ''): string {
-        return `${field}: ${Colorize.output(value, '"', units)}`;
+        return `${field}: ${Colorize.output(value, false, '"', units)}`;
     }
 
     public static warning(warning: string): string {

package/src/internals/wink-bm25-text-search.d.ts

@@ -0,0 +1,4 @@
+declare module 'wink-bm25-text-search' {
+    const bm25: any; 
+    export default bm25;
+  }

package/src/types.ts

@@ -172,4 +172,5 @@ export interface DocumentTextSection {
     text: string;
     tokenCount: number;
     score: number;
+    isBm25: boolean;
 }

package/src/vectra-cli.ts

@@ -191,6 +191,12 @@ export async function run() {
                     type: 'boolean',
                     default: true
                 })
+                .option('bm25', {
+                    alias: 'b',
+                    describe: 'Use Okapi-bm25 keyword search alogrithm to perform hybrid search - semantic + keyword. Displayed in blue during search.',
+                    type: 'boolean',
+                    default: false      
+                })
                 .demandOption(['keys']);
         }, async (args) => {
             console.log(Colorize.title('Querying Index'));
@@ -217,6 +223,7 @@ export async function run() {
             const results = await index.queryDocuments(query, {
                 maxDocuments: args.documentCount,
                 maxChunks: args.chunkCount,
+                isBm25: args.bm25 as boolean,
             });
 
             // Render results
@@ -226,12 +233,15 @@ export async function run() {
                 console.log(Colorize.value('chunks', result.chunks.length));
                 if (args.format == 'sections') {
                     const sections = await result.renderSections(args.tokens, args.sectionCount, args.overlap);
+                    console.log(sections.length);
                     for (let i = 0; i < sections.length; i++) {
                         const section = sections[i];
+                        const isBm25 = sections[i].isBm25;
+                        console.log(isBm25);
                         console.log(Colorize.title(args.sectionCount == 1 ? 'Section' : `Section ${i + 1}`));
                         console.log(Colorize.value('score', section.score));
                         console.log(Colorize.value('tokens', section.tokenCount));
-                        console.log(Colorize.output(section.text));
+                        console.log(Colorize.output(section.text, isBm25));
                     }
                 } else if (args.format == 'chunks') {
                     const text = await result.loadText();
@@ -239,11 +249,12 @@ export async function run() {
                         const chunk = result.chunks[i];
                         const startPos = chunk.item.metadata.startPos;
                         const endPos = chunk.item.metadata.endPos;
+                        const isBm25 = Boolean(chunk.item.metadata.isBm25);
                         console.log(Colorize.title(`Chunk ${i + 1}`));
                         console.log(Colorize.value('score', chunk.score));
                         console.log(Colorize.value('startPos', startPos));
                         console.log(Colorize.value('endPos', endPos));
-                        console.log(Colorize.output(text.substring(startPos, endPos + 1)));
+                        console.log(Colorize.output(text.substring(startPos, endPos + 1), isBm25));
                     }
                 }
             }

package/lib/FileFetcher.d.ts

@@ -1,5 +0,0 @@
-import { TextFetcher } from './types';
-export declare class FileFetcher implements TextFetcher {
-    fetch(uri: string, onDocument: (uri: string, text: string, docType?: string | undefined) => Promise<boolean>): Promise<boolean>;
-}
-//# sourceMappingURL=FileFetcher.d.ts.map

package/lib/FileFetcher.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"FileFetcher.d.ts","sourceRoot":"","sources":["../src/FileFetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAItC,qBAAa,WAAY,YAAW,WAAW;IAC9B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;CAyB/I"}

package/lib/FileFetcher.js

@@ -1,69 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FileFetcher = void 0;
-const fs = __importStar(require("fs/promises"));
-const path = __importStar(require("path"));
-class FileFetcher {
-    fetch(uri, onDocument) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Does path exist and is it a directory?
-            let isDirectory;
-            try {
-                const stat = yield fs.stat(uri);
-                isDirectory = stat.isDirectory();
-            }
-            catch (_a) {
-                return true;
-            }
-            // If directory, read all files and recurse
-            if (isDirectory) {
-                const files = yield fs.readdir(uri);
-                for (const file of files) {
-                    const filePath = path.join(uri, file);
-                    yield this.fetch(filePath, onDocument);
-                }
-                return true;
-            }
-            else {
-                // Read file and call onDocument
-                const text = yield fs.readFile(uri, 'utf8');
-                const parts = uri.split('.');
-                return yield onDocument(uri, text, parts.length > 0 ? parts[parts.length - 1].toLowerCase() : undefined);
-            }
-        });
-    }
-}
-exports.FileFetcher = FileFetcher;
-//# sourceMappingURL=FileFetcher.js.map

package/lib/FileFetcher.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"FileFetcher.js","sourceRoot":"","sources":["../src/FileFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,gDAAkC;AAClC,2CAA6B;AAE7B,MAAa,WAAW;IACP,KAAK,CAAC,GAAW,EAAE,UAAyF;;YACrH,yCAAyC;YACzC,IAAI,WAAoB,CAAC;YACzB,IAAI;gBACA,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;aACpC;YAAC,WAAM;gBACJ,OAAO,IAAI,CAAC;aACf;YAED,2CAA2C;YAC3C,IAAI,WAAW,EAAE;gBACb,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACtC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;iBAC1C;gBACD,OAAO,IAAI,CAAC;aACf;iBAAM;gBACH,gCAAgC;gBAChC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;aAC5G;QACL,CAAC;KAAA;CACJ;AA1BD,kCA0BC"}

package/lib/GPT3Tokenizer.d.ts

@@ -1,9 +0,0 @@
-import { Tokenizer } from "./types";
-/**
- * Tokenizer that uses GPT-3's encoder.
- */
-export declare class GPT3Tokenizer implements Tokenizer {
-    decode(tokens: number[]): string;
-    encode(text: string): number[];
-}
-//# sourceMappingURL=GPT3Tokenizer.d.ts.map

package/lib/GPT3Tokenizer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}

package/lib/GPT3Tokenizer.js

@@ -1,17 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GPT3Tokenizer = void 0;
-const gpt_3_encoder_1 = require("gpt-3-encoder");
-/**
- * Tokenizer that uses GPT-3's encoder.
- */
-class GPT3Tokenizer {
-    decode(tokens) {
-        return (0, gpt_3_encoder_1.decode)(tokens);
-    }
-    encode(text) {
-        return (0, gpt_3_encoder_1.encode)(text);
-    }
-}
-exports.GPT3Tokenizer = GPT3Tokenizer;
-//# sourceMappingURL=GPT3Tokenizer.js.map

package/lib/GPT3Tokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}