vectra 0.1.2 → 0.2.1

package/README.md

@@ -7,6 +7,11 @@ Keep in mind that your entire Vectra index is loaded into memory so it's not wel
 
 Pinecone style namespaces aren't directly supported but you could easily mimic them by creating a separate Vectra index (and folder) for each namespace.
 
+## Other Language Bindings
+This repo contains the TypeScript/JavaScript binding for Vectra but other language bindings are being created. Since Vectra is file based, any language binding can be used to read or write a Vectra index. That means you can build a Vectra index using JS and then read it using Python.
+
+- [vectra-py](https://github.com/BMS-geodev/vectra-py) - Python version of Vectra.
+
 ## Installation
 
 ```

package/bin/vectra.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+var app = require('../lib/vectra-cli.js');
+app.run();

package/lib/GPT3Tokenizer.d.ts

@@ -0,0 +1,9 @@
+import { Tokenizer } from "./types";
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+export declare class GPT3Tokenizer implements Tokenizer {
+    decode(tokens: number[]): string;
+    encode(text: string): number[];
+}
+//# sourceMappingURL=GPT3Tokenizer.d.ts.map

package/lib/GPT3Tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}

package/lib/GPT3Tokenizer.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GPT3Tokenizer = void 0;
+const gpt_3_encoder_1 = require("gpt-3-encoder");
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+class GPT3Tokenizer {
+    decode(tokens) {
+        return (0, gpt_3_encoder_1.decode)(tokens);
+    }
+    encode(text) {
+        return (0, gpt_3_encoder_1.encode)(text);
+    }
+}
+exports.GPT3Tokenizer = GPT3Tokenizer;
+//# sourceMappingURL=GPT3Tokenizer.js.map

package/lib/GPT3Tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}

package/lib/ItemSelector.d.ts

@@ -1,4 +1,4 @@
-import { MetadataFilter, MetadataTypes } from './LocalIndex';
+import { MetadataFilter, MetadataTypes } from './types';
 export declare class ItemSelector {
     /**
      * Returns the similarity between two vectors using the cosine similarity.

package/lib/ItemSelector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7D,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CAqDhC"}
+{"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExD,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CAqDhC"}

package/lib/ItemSelector.js.map

@@ -1 +1 @@
-{"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACpD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE;wBACjD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE;wBACjD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC3D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC1D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA3JD,oCA2JC"}
+{"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,KAAK,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACtE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC5D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC3D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA3JD,oCA2JC"}

package/lib/LocalDocument.d.ts

@@ -0,0 +1,16 @@
+import { MetadataTypes } from './types';
+export declare class LocalDocument {
+    private readonly _folderPath;
+    private readonly _id;
+    private readonly _uri;
+    private _metadata;
+    private _text;
+    constructor(folderPath: string, id: string, uri: string);
+    get folderPath(): string;
+    get id(): string;
+    get uri(): string;
+    hasMetadata(): Promise<boolean>;
+    loadMetadata(): Promise<Record<string, MetadataTypes>>;
+    loadText(): Promise<string>;
+}
+//# sourceMappingURL=LocalDocument.d.ts.map

package/lib/LocalDocument.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.d.ts","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,qBAAa,aAAa;IACtB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,KAAK,CAAmB;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAM9D,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED,IAAW,GAAG,IAAI,MAAM,CAEvB;IAEY,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/B,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,CAAC;IAmBrD,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAY3C"}

package/lib/LocalDocument.js

@@ -0,0 +1,99 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDocument = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+class LocalDocument {
+    constructor(folderPath, id, uri) {
+        this._folderPath = folderPath;
+        this._id = id;
+        this._uri = uri;
+    }
+    get folderPath() {
+        return this._folderPath;
+    }
+    get id() {
+        return this._id;
+    }
+    get uri() {
+        return this._uri;
+    }
+    hasMetadata() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield fs.access(path.join(this.folderPath, `${this.id}.json`));
+                return true;
+            }
+            catch (err) {
+                return false;
+            }
+        });
+    }
+    loadMetadata() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._metadata == undefined) {
+                let json;
+                try {
+                    json = (yield fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
+                }
+                catch (err) {
+                    throw new Error(`Error reading metadata for document "${this.uri}": ${err.toString()}`);
+                }
+                try {
+                    this._metadata = JSON.parse(json);
+                }
+                catch (err) {
+                    throw new Error(`Error parsing metadata for document "${this.uri}": ${err.toString()}`);
+                }
+            }
+            return this._metadata;
+        });
+    }
+    loadText() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._text == undefined) {
+                try {
+                    this._text = (yield fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
+                }
+                catch (err) {
+                    throw new Error(`Error reading text file for document "${this.uri}": ${err.toString()}`);
+                }
+            }
+            return this._text;
+        });
+    }
+}
+exports.LocalDocument = LocalDocument;
+//# sourceMappingURL=LocalDocument.js.map

package/lib/LocalDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.js","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAG7B,MAAa,aAAa;IAOtB,YAAmB,UAAkB,EAAE,EAAU,EAAE,GAAW;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAW,EAAE;QACT,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,IAAW,GAAG;QACV,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEY,WAAW;;YACpB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAEY,YAAY;;YACrB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE;gBAC7B,IAAI,IAAY,CAAC;gBACjB,IAAI;oBACA,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACxF;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;gBAED,IAAI;oBACA,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;iBACrC;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;aACJ;YAED,OAAO,IAAI,CAAC,SAAU,CAAC;QAC3B,CAAC;KAAA;IAEY,QAAQ;;YACjB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE;gBACzB,IAAI;oBACA,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC7F;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACrG;aACJ;YAED,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;KAAA;CAEJ;AAjED,sCAiEC"}

package/lib/LocalDocumentIndex.d.ts

@@ -0,0 +1,48 @@
+import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
+import { TextSplitterConfig } from "./TextSplitter";
+import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, DocumentCatalogStats } from "./types";
+import { LocalDocumentResult } from './LocalDocumentResult';
+import { LocalDocument } from './LocalDocument';
+export interface DocumentQueryOptions {
+    maxDocuments?: number;
+    maxChunks?: number;
+    filter?: MetadataFilter;
+}
+export interface LocalDocumentIndexConfig {
+    folderPath: string;
+    embeddings?: EmbeddingsModel;
+    tokenizer?: Tokenizer;
+    chunkingConfig?: Partial<TextSplitterConfig>;
+}
+export declare class LocalDocumentIndex extends LocalIndex {
+    private readonly _embeddings?;
+    private readonly _tokenizer;
+    private readonly _chunkingConfig?;
+    private _catalog?;
+    private _newCatalog?;
+    constructor(config: LocalDocumentIndexConfig);
+    /**
+     * Returns true if the document catalog exists.
+     */
+    isCatalogCreated(): Promise<boolean>;
+    getDocumentId(uri: string): Promise<string | undefined>;
+    getDocumentUri(documentId: string): Promise<string | undefined>;
+    createIndex(config?: CreateIndexConfig): Promise<void>;
+    deleteDocument(uri: string): Promise<void>;
+    getCatalogStats(): Promise<DocumentCatalogStats>;
+    /**
+     * Adds a document to the catalog.
+     * @remarks
+     * A new update is started if one is not already in progress. If an document with the same uri
+     * already exists, it will be replaced.
+     * @param item Item to insert
+     * @returns Inserted document
+     */
+    upsertDocument(uri: string, text: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument>;
+    queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]>;
+    beginUpdate(): Promise<void>;
+    cancelUpdate(): void;
+    endUpdate(): Promise<void>;
+    protected loadIndexData(): Promise<void>;
+}
+//# sourceMappingURL=LocalDocumentIndex.d.ts.map

package/lib/LocalDocumentIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAA0D,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD,MAAM,WAAW,oBAAoB;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAwB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAChD;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;gBAGnB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IASpC,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKvD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK/D,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C1C,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;;;;;OAOG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IA4G3G,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAqD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAavB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BjD"}

package/lib/LocalDocumentIndex.js

@@ -0,0 +1,367 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDocumentIndex = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const uuid_1 = require("uuid");
+const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+const LocalIndex_1 = require("./LocalIndex");
+const TextSplitter_1 = require("./TextSplitter");
+const LocalDocumentResult_1 = require("./LocalDocumentResult");
+const LocalDocument_1 = require("./LocalDocument");
+const EMBEDDINGS_BATCH_SIZE = 500;
+class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
+    constructor(config) {
+        var _a, _b;
+        super(config.folderPath);
+        this._embeddings = config.embeddings;
+        this._chunkingConfig = Object.assign({
+            keepSeparators: true,
+            chunkSize: 512,
+            chunkOverlap: 0,
+        }, config.chunkingConfig);
+        this._tokenizer = (_b = (_a = config.tokenizer) !== null && _a !== void 0 ? _a : this._chunkingConfig.tokenizer) !== null && _b !== void 0 ? _b : new GPT3Tokenizer_1.GPT3Tokenizer();
+        this._chunkingConfig.tokenizer = this._tokenizer;
+    }
+    /**
+     * Returns true if the document catalog exists.
+     */
+    isCatalogCreated() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield fs.access(path.join(this.folderPath, 'catalog.json'));
+                return true;
+            }
+            catch (err) {
+                return false;
+            }
+        });
+    }
+    getDocumentId(uri) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.loadIndexData();
+            return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.uriToId[uri];
+        });
+    }
+    getDocumentUri(documentId) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.loadIndexData();
+            return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.idToUri[documentId];
+        });
+    }
+    createIndex(config) {
+        const _super = Object.create(null, {
+            createIndex: { get: () => super.createIndex }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.createIndex.call(this, config);
+            yield this.loadIndexData();
+        });
+    }
+    deleteDocument(uri) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Lookup document ID
+            const documentId = yield this.getDocumentId(uri);
+            if (documentId == undefined) {
+                return;
+            }
+            // Delete document chunks from index and remove from catalog
+            yield this.beginUpdate();
+            try {
+                // Get list of chunks for document
+                const chunks = yield this.listItemsByMetadata({ documentId });
+                // Delete chunks
+                for (const chunk of chunks) {
+                    yield this.deleteItem(chunk.id);
+                }
+                // Remove entry from catalog
+                delete this._newCatalog.uriToId[uri];
+                delete this._newCatalog.idToUri[documentId];
+                this._newCatalog.count--;
+                // Commit changes
+                yield this.endUpdate();
+            }
+            catch (err) {
+                // Cancel update and raise error
+                this.cancelUpdate();
+                throw new Error(`Error deleting document "${uri}": ${err.toString()}`);
+            }
+            // Delete text file from disk
+            try {
+                yield fs.unlink(path.join(this.folderPath, `${documentId}.txt`));
+            }
+            catch (err) {
+                throw new Error(`Error removing text file for document "${uri}" from disk: ${err.toString()}`);
+            }
+            // Delete metadata file from disk
+            try {
+                yield fs.unlink(path.join(this.folderPath, `${documentId}.json`));
+            }
+            catch (err) {
+                // Ignore error
+            }
+        });
+    }
+    getCatalogStats() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const stats = yield this.getIndexStats();
+            return {
+                version: this._catalog.version,
+                documents: this._catalog.count,
+                chunks: stats.items,
+                metadata_config: stats.metadata_config
+            };
+        });
+    }
+    /**
+     * Adds a document to the catalog.
+     * @remarks
+     * A new update is started if one is not already in progress. If an document with the same uri
+     * already exists, it will be replaced.
+     * @param item Item to insert
+     * @returns Inserted document
+     */
+    upsertDocument(uri, text, metadata) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Ensure embeddings configured
+            if (!this._embeddings) {
+                throw new Error(`Embeddings model not configured.`);
+            }
+            // Check for existing document ID
+            let documentId = yield this.getDocumentId(uri);
+            if (documentId != undefined) {
+                // Delete existing document
+                yield this.deleteDocument(uri);
+            }
+            else {
+                // Generate new document ID
+                documentId = (0, uuid_1.v4)();
+            }
+            // Populate docType based on extension
+            const config = Object.assign({}, this._chunkingConfig);
+            const pos = uri.lastIndexOf('.');
+            if (pos >= 0) {
+                const ext = uri.substring(pos + 1).toLowerCase();
+                config.docType = ext;
+            }
+            // Split text into chunks
+            const splitter = new TextSplitter_1.TextSplitter(config);
+            const chunks = splitter.split(text);
+            // Break chunks into batches for embedding generation
+            const chunkBatches = [];
+            let currentBatch = [];
+            for (const chunk of chunks) {
+                currentBatch.push(chunk.text);
+                if (currentBatch.length >= EMBEDDINGS_BATCH_SIZE) {
+                    chunkBatches.push(currentBatch);
+                    currentBatch = [];
+                }
+            }
+            if (currentBatch.length > 0) {
+                chunkBatches.push(currentBatch);
+            }
+            // Generate embeddings for chunks
+            const embeddings = [];
+            for (const batch of chunkBatches) {
+                let response;
+                try {
+                    response = yield this._embeddings.createEmbeddings(batch);
+                }
+                catch (err) {
+                    throw new Error(`Error generating embeddings: ${err.toString()}`);
+                }
+                // Check for error
+                if (response.status != 'success') {
+                    throw new Error(`Error generating embeddings: ${response.message}`);
+                }
+                // Add embeddings to output
+                for (const embedding of response.output) {
+                    embeddings.push(embedding);
+                }
+            }
+            // Add document chunks to index
+            yield this.beginUpdate();
+            try {
+                // Add chunks to index
+                for (let i = 0; i < chunks.length; i++) {
+                    const chunk = chunks[i];
+                    const embedding = embeddings[i];
+                    const chunkMetadata = Object.assign({
+                        documentId,
+                        startPos: chunk.startPos,
+                        endPos: chunk.endPos,
+                    }, metadata);
+                    yield this.insertItem({
+                        id: (0, uuid_1.v4)(),
+                        metadata: chunkMetadata,
+                        vector: embedding,
+                    });
+                }
+                // Save metadata file to disk
+                if (metadata != undefined) {
+                    yield fs.writeFile(path.join(this.folderPath, `${documentId}.json`), JSON.stringify(metadata));
+                }
+                // Save text file to disk
+                yield fs.writeFile(path.join(this.folderPath, `${documentId}.txt`), text);
+                // Add entry to catalog
+                this._newCatalog.uriToId[uri] = documentId;
+                this._newCatalog.idToUri[documentId] = uri;
+                this._newCatalog.count++;
+                // Commit changes
+                yield this.endUpdate();
+            }
+            catch (err) {
+                // Cancel update and raise error
+                this.cancelUpdate();
+                throw new Error(`Error adding document "${uri}": ${err.toString()}`);
+            }
+            // Return document
+            return new LocalDocument_1.LocalDocument(this.folderPath, documentId, uri);
+        });
+    }
+    queryDocuments(query, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Ensure embeddings configured
+            if (!this._embeddings) {
+                throw new Error(`Embeddings model not configured.`);
+            }
+            // Ensure options are defined
+            options = Object.assign({
+                maxDocuments: 10,
+                maxChunks: 50,
+            }, options);
+            // Generate embeddings for query
+            let embeddings;
+            try {
+                embeddings = yield this._embeddings.createEmbeddings(query);
+            }
+            catch (err) {
+                throw new Error(`Error generating embeddings for query: ${err.toString()}`);
+            }
+            // Check for error
+            if (embeddings.status != 'success') {
+                throw new Error(`Error generating embeddings for query: ${embeddings.message}`);
+            }
+            // Query index for chunks
+            const results = yield this.queryItems(embeddings.output[0], options.maxChunks, options.filter);
+            // Group chunks by document
+            const documentChunks = {};
+            for (const result of results) {
+                const metadata = result.item.metadata;
+                if (documentChunks[metadata.documentId] == undefined) {
+                    documentChunks[metadata.documentId] = [];
+                }
+                documentChunks[metadata.documentId].push(result);
+            }
+            // Create a document result for each document
+            const documentResults = [];
+            for (const documentId in documentChunks) {
+                const chunks = documentChunks[documentId];
+                const uri = yield this.getDocumentUri(documentId);
+                const documentResult = new LocalDocumentResult_1.LocalDocumentResult(this.folderPath, documentId, uri, chunks, this._tokenizer);
+                documentResults.push(documentResult);
+            }
+            // Sort document results by score and return top results
+            return documentResults.sort((a, b) => b.score - a.score).slice(0, options.maxDocuments);
+        });
+    }
+    // Overrides
+    beginUpdate() {
+        const _super = Object.create(null, {
+            beginUpdate: { get: () => super.beginUpdate }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.beginUpdate.call(this);
+            this._newCatalog = Object.assign({}, this._catalog);
+        });
+    }
+    cancelUpdate() {
+        super.cancelUpdate();
+        this._newCatalog = undefined;
+    }
+    endUpdate() {
+        const _super = Object.create(null, {
+            endUpdate: { get: () => super.endUpdate }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.endUpdate.call(this);
+            try {
+                // Save catalog
+                yield fs.writeFile(path.join(this.folderPath, 'catalog.json'), JSON.stringify(this._newCatalog));
+                this._catalog = this._newCatalog;
+                this._newCatalog = undefined;
+            }
+            catch (err) {
+                throw new Error(`Error saving document catalog: ${err.toString()}`);
+            }
+        });
+    }
+    loadIndexData() {
+        const _super = Object.create(null, {
+            loadIndexData: { get: () => super.loadIndexData }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.loadIndexData.call(this);
+            if (this._catalog) {
+                return;
+            }
+            const catalogPath = path.join(this.folderPath, 'catalog.json');
+            if (yield this.isCatalogCreated()) {
+                // Load catalog
+                const buffer = yield fs.readFile(catalogPath);
+                this._catalog = JSON.parse(buffer.toString());
+            }
+            else {
+                try {
+                    // Initialize catalog
+                    this._catalog = {
+                        version: 1,
+                        count: 0,
+                        uriToId: {},
+                        idToUri: {},
+                    };
+                    yield fs.writeFile(catalogPath, JSON.stringify(this._catalog));
+                }
+                catch (err) {
+                    throw new Error(`Error creating document catalog: ${err.toString()}`);
+                }
+            }
+        });
+    }
+}
+exports.LocalDocumentIndex = LocalDocumentIndex;
+//# sourceMappingURL=LocalDocumentIndex.js.map

package/lib/LocalDocumentIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocumentIndex.js","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,mDAAgD;AAChD,6CAA6D;AAC7D,iDAAkE;AAElE,+DAA4D;AAC5D,mDAAgD;AAEhD,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAelC,MAAa,kBAAmB,SAAQ,uBAAU;IAQ9C,YAAmB,MAAgC;;QAC/C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,CAAC;SACI,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAA,MAAA,MAAM,CAAC,SAAS,mCAAI,IAAI,CAAC,eAAe,CAAC,SAAS,mCAAI,IAAI,6BAAa,EAAE,CAAC;QAC5F,IAAI,CAAC,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;IACrD,CAAC;IAED;;OAEG;IACU,gBAAgB;;YACzB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;gBAC5D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAEY,aAAa,CAAC,GAAW;;;YAClC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,GAAG,CAAC,CAAC;;KACtC;IAEY,cAAc,CAAC,UAAkB;;;YAC1C,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,UAAU,CAAC,CAAC;;KAC7C;IAEY,WAAW,CAAC,MAA0B;;;;;YAC/C,MAAM,OAAM,WAAW,YAAC,MAAM,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,CAAC;KAAA;IAEY,cAAc,CAAC,GAAW;;YACnC,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,OAAO;aACV;YAED,4DAA4D;YAC5D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,kCAAkC;gBAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAwB,EAAE,UAAU,EAAE,CAAC,CAAC;gBAErF,gBAAgB;gBAChB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBACxB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACnC;gBAED,4BAA4B;gBAC5B,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACnF;YAED,6BAA6B;YAC7B,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,CAAC,CAAC;aACpE;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,gBAAiB,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAC3G;YAED,iCAAiC;YACjC,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC,CAAC;aACrE;YAAC,OAAO,GAAY,EAAE;gBACnB,eAAe;aAClB;QACL,CAAC;KAAA;IAEY,eAAe;;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;YACxC,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,QAAS,CAAC,OAAO;gBAC/B,SAAS,EAAE,IAAI,CAAC,QAAS,CAAC,KAAK;gBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK;gBACnB,eAAe,EAAE,KAAK,CAAC,eAAe;aACzC,CAAC;QACN,CAAC;KAAA;IAED;;;;;;;OAOG;IACU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,QAAwC;;YAC3F,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,iCAAiC;YACjC,IAAI,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aAClC;iBAAM;gBACH,2BAA2B;gBAC3B,UAAU,GAAG,IAAA,SAAE,GAAE,CAAC;aACrB;YAED,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,EAAE;gBACV,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjD,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;aACxB;YAED,yBAAyB;YACzB,MAAM,QAAQ,GAAG,IAAI,2BAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,qDAAqD;YACrD,MAAM,YAAY,GAAe,EAAE,CAAC;YACpC,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBACxB,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9B,IAAI,YAAY,CAAC,MAAM,IAAI,qBAAqB,EAAE;oBAC9C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAChC,YAAY,GAAG,EAAE,CAAC;iBACrB;aACJ;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBACzB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aACnC;YAED,iCAAiC;YACjC,MAAM,UAAU,GAAe,EAAE,CAAC;YAClC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE;gBAC9B,IAAI,QAA4B,CAAC;gBACjC,IAAI;oBACA,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;iBAC7D;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,gCAAiC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC9E;gBAED,kBAAkB;gBAClB,IAAI,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE;oBAC9B,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;iBACvE;gBAED,2BAA2B;gBAC3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAO,EAAE;oBACtC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBAC9B;aACJ;YAED,+BAA+B;YAC/B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,sBAAsB;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;oBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,aAAa,GAA0B,MAAM,CAAC,MAAM,CAAC;wBACvD,UAAU;wBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACvB,EAAE,QAAQ,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,UAAU,CAAC;wBAClB,EAAE,EAAE,IAAA,SAAE,GAAE;wBACR,QAAQ,EAAE,aAAa;wBACvB,MAAM,EAAE,SAAS;qBACpB,CAAC,CAAC;iBACN;gBAED,6BAA6B;gBAC7B,IAAI,QAAQ,IAAI,SAAS,EAAE;oBACvB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClG;gBAED,yBAAyB;gBACzB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;gBAE1E,uBAAuB;gBACvB,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACjF;YAED,kBAAkB;YAClB,OAAO,IAAI,6BAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC;KAAA;IAGY,cAAc,CAAC,KAAa,EAAE,OAA8B;;YACrE,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,6BAA6B;YAC7B,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpB,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aAChB,EAAE,OAAO,CAAC,CAAC;YAEZ,gCAAgC;YAChC,IAAI,UAA8B,CAAC;YACnC,IAAI;gBACA,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;aAC/D;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA2C,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACxF;YAED,kBAAkB;YAClB,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE;gBAChC,MAAM,IAAI,KAAK,CAAC,0CAA0C,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;aACnF;YAED,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAwB,UAAU,CAAC,MAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAExH,2BAA2B;YAC3B,MAAM,cAAc,GAAoE,EAAE,CAAC;YAC3F,KAAK,MAAM,MAAM,IAAK,OAAO,EAAE;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE;oBAClD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;iBAC5C;gBACD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACpD;YAED,6CAA6C;YAC7C,MAAM,eAAe,GAA0B,EAAE,CAAC;YAClD,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE;gBACrC,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1G,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aACxC;YAED,wDAAwD;YACxD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,YAAa,CAAC,CAAC;QAC7F,CAAC;KAAA;IAED,YAAY;IAEC,WAAW;;;;;YACpB,MAAM,OAAM,WAAW,WAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;KAAA;IAEM,YAAY;QACf,KAAK,CAAC,YAAY,EAAE,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;IACjC,CAAC;IAEY,SAAS;;;;;YAClB,MAAM,OAAM,SAAS,WAAE,CAAC;YAExB,IAAI;gBACA,eAAe;gBACf,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;gBACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAChC;YAAC,OAAM,GAAY,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,kCAAmC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAChF;QACL,CAAC;KAAA;IAEe,aAAa;;;;;YACzB,MAAM,OAAM,aAAa,WAAE,CAAC;YAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,OAAO;aACV;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC/D,IAAI,MAAM,IAAI,CAAC,gBAAgB,EAAE,EAAE;gBAC/B,eAAe;gBACf,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjD;iBAAM;gBACH,IAAI;oBACA,qBAAqB;oBACrB,IAAI,CAAC,QAAQ,GAAG;wBACZ,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,EAAE;wBACX,OAAO,EAAE,EAAE;qBACd,CAAC;oBACF,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClE;gBAAC,OAAM,GAAY,EAAE;oBAClB,MAAM,IAAI,KAAK,CAAC,oCAAqC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAClF;aACJ;QACL,CAAC;KAAA;CACJ;AAlUD,gDAkUC"}