vectra 0.12.1 → 0.12.2

package/lib/GPT3Tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}

package/lib/GPT3Tokenizer.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GPT3Tokenizer = void 0;
+const gpt_tokenizer_1 = require("gpt-tokenizer");
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+class GPT3Tokenizer {
+    decode(tokens) {
+        return (0, gpt_tokenizer_1.decode)(tokens);
+    }
+    encode(text) {
+        return (0, gpt_tokenizer_1.encode)(text);
+    }
+}
+exports.GPT3Tokenizer = GPT3Tokenizer;
+//# sourceMappingURL=GPT3Tokenizer.js.map

package/lib/GPT3Tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}

package/lib/LocalDocument.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.d.ts","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D;;GAEG;AACH,qBAAa,aAAa;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,KAAK,CAAmB;IAEhC;;;;;OAKG;gBACgB,KAAK,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAMrE;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,IAAW,GAAG,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACU,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IASzC;;;OAGG;IACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAS5C;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,CAAC;IAmBlE;;;OAGG;IACU,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAW3C"}

package/lib/LocalDocumentIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAAmC,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;IAExB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAEpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE,eAAe,CAAC;IAE7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,UAAU,CAAC,qBAAqB,CAAC;IACrE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;IAEtC;;;OAGG;gBACgB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACH,IAAW,UAAU,IAAI,eAAe,GAAG,SAAS,CAEnD;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,SAAS,CAEhC;IAED;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IASjD;;;;OAIG;IACU,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKpE;;;;OAIG;IACU,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK5E;;;OAGG;IACU,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;OAGG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CvD;;;;;;;;;;OAUG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAiH1I;;;;;OAKG;IACU,aAAa,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAuB5D;;;;;OAKG;IACU,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAqD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAavB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BjD"}

package/lib/LocalDocumentResult.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocumentResult.d.ts","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE7F;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAEhC;;;OAGG;gBACgB,KAAK,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS;IAWzI;;OAEG;IACH,IAAW,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAExD;IAED;;OAEG;IACH,IAAW,KAAK,IAAI,MAAM,CAEzB;IAED;;;;;;OAMG;IACU,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAgEjF;;;;;;;;OAQG;IACU,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,iBAAiB,UAAO,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAgN7H,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;CAM1B"}

package/lib/LocalIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM5F,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED;;;;;GAKG;AACH,qBAAa,UAAU,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B,OAAO,CAAC,WAAW,CAAM;IAEzB;;;;OAIG;gBACgB,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAKzD;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,MAAM,CAE7B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,SAAS,CAAC;IAK5H;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;;;;;;;OAQG;IACU,gBAAgB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAiB5I;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1G;;;;;;OAMG;IACU,mBAAmB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1I;;;;;;;;;OASG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;IAuErM;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;OAEG;cACa,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAahC,eAAe;YA+Df,SAAS;YAsBT,UAAU;CAQ3B"}

package/lib/TextSplitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAI/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA2BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,aAAa;IA8BrB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,aAAa;CA4WxB"}

package/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC"}

package/lib/vectra-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectra-cli.d.ts","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":"AASA,wBAAsB,GAAG,kBA+PxB"}

package/package.json

@@ -2,7 +2,7 @@
     "name": "vectra",
     "author": "Steven Ickman",
     "description": "A vector database that uses the local file system for storage.",
-    "version": "0.12.1",
+    "version": "0.12.2",
     "license": "MIT",
     "keywords": [
         "gpt"
@@ -30,10 +30,10 @@
         }
     },
     "dependencies": {
-        "@paztis/gpt-3-encoder": "1.1.4",
         "axios": "^1.9.0",
         "cheerio": "^1.0.0",
         "dotenv": "^16.5.0",
+        "gpt-tokenizer": "^3.4.0",
         "json-colorizer": "^3.0.1",
         "openai": "^4.97.0",
         "turndown": "^7.2.0",
@@ -52,8 +52,8 @@
         "@types/yargs": "17.0.33",
         "mocha": "11.2.2",
         "npm-run-all": "^4.1.5",
-        "rimraf": "^5.0.1",
         "nyc": "^17.1.0",
+        "rimraf": "^5.0.1",
         "shx": "^0.4.0",
         "ts-mocha": "11.1.0",
         "ts-node": "^10.9.1",

package/src/GPT3Tokenizer.ts

@@ -1,5 +1,5 @@
 import { Tokenizer } from "./types";
-import { encode, decode } from "gpt-3-encoder";
+import { encode, decode } from "gpt-tokenizer";
 
 /**
  * Tokenizer that uses GPT-3's encoder.

package/bin/vectra.js

@@ -1,3 +0,0 @@
-#!/usr/bin/env node
-var app = require('../lib/vectra-cli.js');
-app.run();