vectra 0.1.2 → 0.2.1

package/lib/vectra-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectra-cli.js","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,wDAAgC;AAChC,2CAAwC;AACxC,6DAA0D;AAC1D,6CAA0C;AAC1C,yDAAsD;AACtD,2CAAuC;AAEvC,SAAsB,GAAG;;QACrB,kBAAkB;QAClB,MAAM,IAAI,GAAG,MAAM,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;aAC1C,UAAU,CAAC,QAAQ,CAAC;aACpB,OAAO,CAAC,gBAAgB,EAAE,0BAA0B,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YACtE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,gCAAgC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,iBAAiB,EAAE,wCAAwC,EAAE,CAAC,KAAK,EAAE,EAAE;YAC5E,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;gBAC1F,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,sCAAsC;gBAChD,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,sDAAsD;gBAChE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,0DAA0D;gBACpE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,GAAG;aACf,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChD,OAAO,IAAI,CAAC;iBACf;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpE,OAAO,IAAI,CAAC;iBACf;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;iBACxK;YACL,CAAC,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAEzD,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAElG,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;gBACV,cAAc,EAAE;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;iBAC5B;aACJ,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,kBAAkB;YAClB,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;YACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACpB,IAAI;oBACA,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC;oBAClD,MAAM,OAAO,GAAI,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;oBACxE,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;iBACvE;gBAAC,OAAO,GAAY,EAAE;oBACnB,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,KAAK,CAAC,iBAAiB,GAAG,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;iBACxG;aACJ;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,6CAA6C,EAAE,CAAC,KAAK,EAAE,EAAE;YAChF,OAAO,KAAK;iBACP,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,6BAA6B;gBACvC,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,yDAAyD;gBACnE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChD,OAAO,IAAI,CAAC;iBACf;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpE,OAAO,IAAI,CAAC;iBACf;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;iBACxK;YACL,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YAErD,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,mBAAmB;YACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACpB,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;gBAC/B,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aACnC;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,eAAe,EAAE,oCAAoC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,uBAAuB,EAAE,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE;YACjE,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;aAC7F,CAAC;iBACD,MAAM,CAAC,gBAAgB,EAAE;gBACtB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,oDAAoD;gBAC9D,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,iDAAiD;gBAC3D,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,eAAe,EAAE;gBACrB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,2DAA2D;gBACrE,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,CAAC;aACb,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,6EAA6E;gBACvF,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,IAAI;aAChB,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,wDAAwD;gBAClE,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;gBACxC,OAAO,EAAE,UAAU;aACtB,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE9C,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAElG,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;aACb,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE;gBAC9C,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,SAAS,EAAE,IAAI,CAAC,UAAU;aAC7B,CAAC,CAAC;YAEH,iBAAiB;YACjB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC1B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE;oBAC3B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBACtC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChF,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;wBACpD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;wBAC1D,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;qBAC9C;iBACJ;qBAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE;oBAChC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;qBACtE;iBACJ;aACJ;QACL,CAAC,CAAA,CAAC;aACD,IAAI,EAAE;aACN,aAAa,EAAE;aACf,UAAU,EAAE,CAAC;IACtB,CAAC;CAAA;AAzND,kBAyNC;AAGD,SAAe,WAAW,CAAC,KAAe,EAAE,QAAgB,EAAE,OAAe;;QACzE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,OAAO,KAAK,CAAC;SAChB;aAAM,IAAI,OAAO,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;YAClE,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;SACxF;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,6EAA6E,CAAC,CAAA;SACvJ;IACL,CAAC;CAAA"}

package/package.json

@@ -2,7 +2,7 @@
     "name": "vectra",
     "author": "Steven Ickman",
     "description": "A vector database that uses the local file system for storage.",
-    "version": "0.1.2",
+    "version": "0.2.1",
     "license": "MIT",
     "keywords": [
         "gpt"
@@ -16,6 +16,9 @@
     },
     "main": "./lib/index.js",
     "types": "./lib/index.d.ts",
+    "bin": {
+        "vectra": "./bin/vectra.js"
+    },
     "typesVersions": {
         "<3.9": {
             "*": [
@@ -24,13 +27,28 @@
         }
     },
     "dependencies": {
-        "uuid": "^8.3.2"
+        "axios": "^1.3.4",
+        "cheerio": "^1.0.0-rc.12",
+        "dotenv": "^8.2.0",
+        "gpt-3-encoder": "1.1.4",
+        "json-colorizer": "^2.2.2",
+        "openai": "^3.2.1",
+        "uuid": "^9.0.0",
+        "yargs": "^17.7.2"
     },
     "resolutions": {
     },
     "devDependencies": {
         "@types/node": "^14.14.31",
-        "@types/uuid": "^8.3.0"
+        "@types/mocha": "^8.2.0",
+        "@types/assert": "^1.5.3",
+        "@types/uuid": "9.0.1",
+        "@types/yargs": "17.0.24",
+        "mocha": "10.2.0",
+        "nyc": "^15.1.0",
+        "shx": "^0.3.2",
+        "ts-mocha": "10.0.0",
+        "typescript": "^4.2.3"
     },
     "scripts": {
         "build": "tsc -b",

package/src/GPT3Tokenizer.ts

@@ -0,0 +1,15 @@
+import { Tokenizer } from "./types";
+import { encode, decode } from "gpt-3-encoder";
+
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+export class GPT3Tokenizer implements Tokenizer {
+    public decode(tokens: number[]): string {
+        return decode(tokens);
+    }
+
+    public encode(text: string): number[] {
+        return encode(text);
+    }
+}

package/src/ItemSelector.ts

@@ -1,4 +1,4 @@
-import { MetadataFilter, MetadataTypes } from './LocalIndex';
+import { MetadataFilter, MetadataTypes } from './types';
 
 export class ItemSelector {
     /**
@@ -62,12 +62,12 @@ export class ItemSelector {
         for (const key in filter) {
             switch (key) {
                 case '$and':
-                    if (!filter[key].every((f) => this.select(metadata, f))) {
+                    if (!filter[key]!.every((f: MetadataFilter) => this.select(metadata, f))) {
                         return false;
                     }
                     break;
                 case '$or':
-                    if (!filter[key].some((f) => this.select(metadata, f))) {
+                    if (!filter[key]!.some((f: MetadataFilter) => this.select(metadata, f))) {
                         return false;
                     }
                     break;
@@ -120,32 +120,32 @@ export class ItemSelector {
                     }
                     break;
                 case '$gt':
-                    if (typeof value != 'number' || value <= filter[key]) {
+                    if (typeof value != 'number' || value <= filter[key]!) {
                         return false;
                     }
                     break;
                 case '$gte':
-                    if (typeof value != 'number' || value < filter[key]) {
+                    if (typeof value != 'number' || value < filter[key]!) {
                         return false;
                     }
                     break;
                 case '$lt':
-                    if (typeof value != 'number' || value >= filter[key]) {
+                    if (typeof value != 'number' || value >= filter[key]!) {
                         return false;
                     }
                     break;
                 case '$lte':
-                    if (typeof value != 'number' || value > filter[key]) {
+                    if (typeof value != 'number' || value > filter[key]!) {
                         return false;
                     }
                     break;
                 case '$in':
-                    if (typeof value == 'boolean' || !filter[key].includes(value)) {
+                    if (typeof value == 'boolean' || !filter[key]!.includes(value)) {
                         return false;
                     }
                     break;
                 case '$nin':
-                    if (typeof value == 'boolean' || filter[key].includes(value)) {
+                    if (typeof value == 'boolean' || filter[key]!.includes(value)) {
                         return false;
                     }
                     break;

package/src/LocalDocument.ts

@@ -0,0 +1,70 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { MetadataTypes } from './types';
+
+export class LocalDocument {
+    private readonly _folderPath: string;
+    private readonly _id: string;
+    private readonly _uri: string;
+    private _metadata: Record<string,MetadataTypes>|undefined;
+    private _text: string|undefined;
+
+    public constructor(folderPath: string, id: string, uri: string) {
+        this._folderPath = folderPath;
+        this._id = id;
+        this._uri = uri;
+    }
+
+    public get folderPath(): string {
+        return this._folderPath;
+    }
+
+    public get id(): string {
+        return this._id;
+    }
+
+    public get uri(): string {
+        return this._uri;
+    }
+
+    public async hasMetadata(): Promise<boolean> {
+        try {
+            await fs.access(path.join(this.folderPath, `${this.id}.json`));
+            return true;
+        } catch (err: unknown) {
+            return false;
+        }
+    }
+
+    public async loadMetadata(): Promise<Record<string,MetadataTypes>> {
+        if (this._metadata == undefined) {
+            let json: string;
+            try {
+                json = (await fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
+            } catch (err: unknown) {
+                throw new Error(`Error reading metadata for document "${this.uri}": ${(err as any).toString()}`);
+            }
+
+            try {
+                this._metadata = JSON.parse(json);
+            } catch (err: unknown) {
+                throw new Error(`Error parsing metadata for document "${this.uri}": ${(err as any).toString()}`);
+            }
+        }
+
+        return this._metadata!;
+    }
+
+    public async loadText(): Promise<string> {
+        if (this._text == undefined) {
+            try {
+                this._text = (await fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
+            } catch (err: unknown) {
+                throw new Error(`Error reading text file for document "${this.uri}": ${(err as any).toString()}`);
+            }
+        }
+
+        return this._text;
+    }
+
+}

package/src/LocalDocumentIndex.ts

@@ -0,0 +1,355 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { v4 } from 'uuid';
+import { GPT3Tokenizer } from "./GPT3Tokenizer";
+import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
+import { TextSplitter, TextSplitterConfig } from "./TextSplitter";
+import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, EmbeddingsResponse, QueryResult, DocumentChunkMetadata, DocumentCatalogStats } from "./types";
+import { LocalDocumentResult } from './LocalDocumentResult';
+import { LocalDocument } from './LocalDocument';
+
+const EMBEDDINGS_BATCH_SIZE = 500;
+
+export interface DocumentQueryOptions {
+    maxDocuments?: number;
+    maxChunks?: number;
+    filter?: MetadataFilter;
+}
+
+export interface LocalDocumentIndexConfig {
+    folderPath: string;
+    embeddings?: EmbeddingsModel;
+    tokenizer?: Tokenizer;
+    chunkingConfig?: Partial<TextSplitterConfig>;
+}
+
+export class LocalDocumentIndex extends LocalIndex {
+    private readonly _embeddings?: EmbeddingsModel;
+    private readonly _tokenizer: Tokenizer;
+    private readonly _chunkingConfig?: TextSplitterConfig;
+    private _catalog?: DocumentCatalog;
+    private _newCatalog?: DocumentCatalog;
+
+
+    public constructor(config: LocalDocumentIndexConfig) {
+        super(config.folderPath);
+        this._embeddings = config.embeddings;
+        this._chunkingConfig = Object.assign({
+            keepSeparators: true,
+            chunkSize: 512,
+            chunkOverlap: 0,
+        } as TextSplitterConfig, config.chunkingConfig);
+        this._tokenizer = config.tokenizer ?? this._chunkingConfig.tokenizer ?? new GPT3Tokenizer();
+        this._chunkingConfig.tokenizer = this._tokenizer;
+    }
+
+    /**
+     * Returns true if the document catalog exists.
+     */
+    public async isCatalogCreated(): Promise<boolean> {
+        try {
+            await fs.access(path.join(this.folderPath, 'catalog.json'));
+            return true;
+        } catch (err: unknown) {
+            return false;
+        }
+    }
+
+    public async getDocumentId(uri: string): Promise<string | undefined> {
+        await this.loadIndexData();
+        return this._catalog?.uriToId[uri];
+    }
+
+    public async getDocumentUri(documentId: string): Promise<string | undefined> {
+        await this.loadIndexData();
+        return this._catalog?.idToUri[documentId];
+    }
+
+    public async createIndex(config?: CreateIndexConfig): Promise<void> {
+        await super.createIndex(config);
+        await this.loadIndexData();
+    }
+
+    public async deleteDocument(uri: string): Promise<void> {
+        // Lookup document ID
+        const documentId = await this.getDocumentId(uri);
+        if (documentId == undefined) {
+            return;
+        }
+
+        // Delete document chunks from index and remove from catalog
+        await this.beginUpdate();
+        try {
+            // Get list of chunks for document
+            const chunks = await this.listItemsByMetadata<DocumentChunkMetadata>({ documentId });
+
+            // Delete chunks
+            for (const chunk of chunks) {
+                await this.deleteItem(chunk.id);
+            }
+
+            // Remove entry from catalog
+            delete this._newCatalog!.uriToId[uri];
+            delete this._newCatalog!.idToUri[documentId];
+            this._newCatalog!.count--;
+
+            // Commit changes
+            await this.endUpdate();
+        } catch (err: unknown) {
+            // Cancel update and raise error
+            this.cancelUpdate();
+            throw new Error(`Error deleting document "${uri}": ${(err as any).toString()}`);
+        }
+
+        // Delete text file from disk
+        try {
+            await fs.unlink(path.join(this.folderPath, `${documentId}.txt`));
+        } catch (err: unknown) {
+            throw new Error(`Error removing text file for document "${uri}" from disk: ${(err as any).toString()}`);
+        }
+
+        // Delete metadata file from disk
+        try {
+            await fs.unlink(path.join(this.folderPath, `${documentId}.json`));
+        } catch (err: unknown) {
+            // Ignore error
+        }
+    }
+
+    public async getCatalogStats(): Promise<DocumentCatalogStats> {
+        const stats = await this.getIndexStats()
+        return {
+            version: this._catalog!.version,
+            documents: this._catalog!.count,
+            chunks: stats.items,
+            metadata_config: stats.metadata_config
+        };
+    }
+
+    /**
+     * Adds a document to the catalog.
+     * @remarks
+     * A new update is started if one is not already in progress. If an document with the same uri
+     * already exists, it will be replaced.
+     * @param item Item to insert
+     * @returns Inserted document
+     */
+    public async upsertDocument(uri: string, text: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument> {
+        // Ensure embeddings configured
+        if (!this._embeddings) {
+            throw new Error(`Embeddings model not configured.`);
+        }
+
+        // Check for existing document ID
+        let documentId = await this.getDocumentId(uri);
+        if (documentId != undefined) {
+            // Delete existing document
+            await this.deleteDocument(uri);
+        } else {
+            // Generate new document ID
+            documentId = v4();
+        }
+
+        // Populate docType based on extension
+        const config = Object.assign({}, this._chunkingConfig);
+        const pos = uri.lastIndexOf('.');
+        if (pos >= 0) {
+            const ext = uri.substring(pos + 1).toLowerCase();
+            config.docType = ext;
+        }
+
+        // Split text into chunks
+        const splitter = new TextSplitter(config);
+        const chunks = splitter.split(text);
+
+        // Break chunks into batches for embedding generation
+        const chunkBatches: string[][] = [];
+        let currentBatch: string[] = [];
+        for (const chunk of chunks) {
+            currentBatch.push(chunk.text);
+            if (currentBatch.length >= EMBEDDINGS_BATCH_SIZE) {
+                chunkBatches.push(currentBatch);
+                currentBatch = [];
+            }
+        }
+        if (currentBatch.length > 0) {
+            chunkBatches.push(currentBatch);
+        }
+
+        // Generate embeddings for chunks
+        const embeddings: number[][] = [];
+        for (const batch of chunkBatches) {
+            let response: EmbeddingsResponse;
+            try {
+                response = await this._embeddings.createEmbeddings(batch);
+            } catch (err: unknown) {
+                throw new Error(`Error generating embeddings: ${(err as any).toString()}`);
+            }
+
+            // Check for error
+            if (response.status != 'success') {
+                throw new Error(`Error generating embeddings: ${response.message}`);
+            }
+
+            // Add embeddings to output
+            for (const embedding of response.output!) {
+                embeddings.push(embedding);
+            }
+        }
+
+        // Add document chunks to index
+        await this.beginUpdate();
+        try {
+            // Add chunks to index
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                const embedding = embeddings[i];
+                const chunkMetadata: DocumentChunkMetadata = Object.assign({
+                    documentId,
+                    startPos: chunk.startPos,
+                    endPos: chunk.endPos,
+                }, metadata);
+                await this.insertItem({
+                    id: v4(),
+                    metadata: chunkMetadata,
+                    vector: embedding,
+                });
+            }
+
+            // Save metadata file to disk
+            if (metadata != undefined) {
+                await fs.writeFile(path.join(this.folderPath, `${documentId}.json`), JSON.stringify(metadata));
+            }
+
+            // Save text file to disk
+            await fs.writeFile(path.join(this.folderPath, `${documentId}.txt`), text);
+
+            // Add entry to catalog
+            this._newCatalog!.uriToId[uri] = documentId;
+            this._newCatalog!.idToUri[documentId] = uri;
+            this._newCatalog!.count++;
+
+            // Commit changes
+            await this.endUpdate();
+        } catch (err: unknown) {
+            // Cancel update and raise error
+            this.cancelUpdate();
+            throw new Error(`Error adding document "${uri}": ${(err as any).toString()}`);
+        }
+
+        // Return document
+        return new LocalDocument(this.folderPath, documentId, uri);
+    }
+
+
+    public async queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]> {
+        // Ensure embeddings configured
+        if (!this._embeddings) {
+            throw new Error(`Embeddings model not configured.`);
+        }
+
+        // Ensure options are defined
+        options = Object.assign({
+            maxDocuments: 10,
+            maxChunks: 50,
+        }, options);
+
+        // Generate embeddings for query
+        let embeddings: EmbeddingsResponse;
+        try {
+            embeddings = await this._embeddings.createEmbeddings(query);
+        } catch (err: unknown) {
+            throw new Error(`Error generating embeddings for query: ${(err as any).toString()}`);
+        }
+
+        // Check for error
+        if (embeddings.status != 'success') {
+            throw new Error(`Error generating embeddings for query: ${embeddings.message}`);
+        }
+
+        // Query index for chunks
+        const results = await this.queryItems<DocumentChunkMetadata>(embeddings.output![0], options.maxChunks!, options.filter);
+
+        // Group chunks by document
+        const documentChunks: { [documentId: string]: QueryResult<DocumentChunkMetadata>[]; } = {};
+        for (const result  of results) {
+            const metadata = result.item.metadata;
+            if (documentChunks[metadata.documentId] == undefined) {
+                documentChunks[metadata.documentId] = [];
+            }
+            documentChunks[metadata.documentId].push(result);
+        }
+
+        // Create a document result for each document
+        const documentResults: LocalDocumentResult[] = [];
+        for (const documentId in documentChunks) {
+            const chunks = documentChunks[documentId];
+            const uri = await this.getDocumentUri(documentId) as string;
+            const documentResult = new LocalDocumentResult(this.folderPath, documentId, uri, chunks, this._tokenizer);
+            documentResults.push(documentResult);
+        }
+
+        // Sort document results by score and return top results
+        return documentResults.sort((a, b) => b.score - a.score).slice(0, options.maxDocuments!);
+    }
+
+    // Overrides
+
+    public async beginUpdate(): Promise<void> {
+        await super.beginUpdate();
+        this._newCatalog = Object.assign({}, this._catalog);
+    }
+
+    public cancelUpdate(): void {
+        super.cancelUpdate();
+        this._newCatalog = undefined;
+    }
+
+    public async endUpdate(): Promise<void> {
+        await super.endUpdate();
+
+        try {
+            // Save catalog
+            await fs.writeFile(path.join(this.folderPath, 'catalog.json'), JSON.stringify(this._newCatalog));
+            this._catalog = this._newCatalog;
+            this._newCatalog = undefined;
+        } catch(err: unknown) {
+            throw new Error(`Error saving document catalog: ${(err as any).toString()}`);
+        }
+    }
+
+    protected async loadIndexData(): Promise<void> {
+        await super.loadIndexData();
+
+        if (this._catalog) {
+            return;
+        }
+
+        const catalogPath = path.join(this.folderPath, 'catalog.json');
+        if (await this.isCatalogCreated()) {
+            // Load catalog
+            const buffer = await fs.readFile(catalogPath);
+            this._catalog = JSON.parse(buffer.toString());
+        } else {
+            try {
+                // Initialize catalog
+                this._catalog = {
+                    version: 1,
+                    count: 0,
+                    uriToId: {},
+                    idToUri: {},
+                };
+                await fs.writeFile(catalogPath, JSON.stringify(this._catalog));
+            } catch(err: unknown) {
+                throw new Error(`Error creating document catalog: ${(err as any).toString()}`);
+            }
+        }
+    }
+}
+
+interface DocumentCatalog {
+    version: number;
+    count: number;
+    uriToId: { [uri: string]: string; };
+    idToUri: { [id: string]: string; };
+}