vectra 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/bin/vectra.js +3 -0
  2. package/lib/GPT3Tokenizer.d.ts +9 -0
  3. package/lib/GPT3Tokenizer.d.ts.map +1 -0
  4. package/lib/GPT3Tokenizer.js +17 -0
  5. package/lib/GPT3Tokenizer.js.map +1 -0
  6. package/lib/ItemSelector.d.ts +1 -1
  7. package/lib/ItemSelector.d.ts.map +1 -1
  8. package/lib/ItemSelector.js.map +1 -1
  9. package/lib/LocalDocument.d.ts +16 -0
  10. package/lib/LocalDocument.d.ts.map +1 -0
  11. package/lib/LocalDocument.js +99 -0
  12. package/lib/LocalDocument.js.map +1 -0
  13. package/lib/LocalDocumentIndex.d.ts +48 -0
  14. package/lib/LocalDocumentIndex.d.ts.map +1 -0
  15. package/lib/LocalDocumentIndex.js +367 -0
  16. package/lib/LocalDocumentIndex.js.map +1 -0
  17. package/lib/LocalDocumentResult.d.ts +12 -0
  18. package/lib/LocalDocumentResult.d.ts.map +1 -0
  19. package/lib/LocalDocumentResult.js +186 -0
  20. package/lib/LocalDocumentResult.js.map +1 -0
  21. package/lib/LocalIndex.d.ts +9 -63
  22. package/lib/LocalIndex.d.ts.map +1 -1
  23. package/lib/LocalIndex.js +14 -1
  24. package/lib/LocalIndex.js.map +1 -1
  25. package/lib/OpenAIEmbeddings.d.ts +98 -0
  26. package/lib/OpenAIEmbeddings.d.ts.map +1 -0
  27. package/lib/OpenAIEmbeddings.js +139 -0
  28. package/lib/OpenAIEmbeddings.js.map +1 -0
  29. package/lib/TextSplitter.d.ts +17 -0
  30. package/lib/TextSplitter.d.ts.map +1 -0
  31. package/lib/TextSplitter.js +460 -0
  32. package/lib/TextSplitter.js.map +1 -0
  33. package/lib/WebFetcher.d.ts +16 -0
  34. package/lib/WebFetcher.d.ts.map +1 -0
  35. package/lib/WebFetcher.js +144 -0
  36. package/lib/WebFetcher.js.map +1 -0
  37. package/lib/index.d.ts +8 -0
  38. package/lib/index.d.ts.map +1 -1
  39. package/lib/index.js +13 -1
  40. package/lib/index.js.map +1 -1
  41. package/lib/internals/Colorize.d.ts +14 -0
  42. package/lib/internals/Colorize.d.ts.map +1 -0
  43. package/lib/internals/Colorize.js +64 -0
  44. package/lib/internals/Colorize.js.map +1 -0
  45. package/lib/internals/index.d.ts +3 -0
  46. package/lib/internals/index.d.ts.map +1 -0
  47. package/lib/internals/index.js +19 -0
  48. package/lib/internals/index.js.map +1 -0
  49. package/lib/internals/types.d.ts +42 -0
  50. package/lib/internals/types.d.ts.map +1 -0
  51. package/lib/internals/types.js +3 -0
  52. package/lib/internals/types.js.map +1 -0
  53. package/lib/types.d.ts +133 -0
  54. package/lib/types.d.ts.map +1 -0
  55. package/lib/types.js +3 -0
  56. package/lib/types.js.map +1 -0
  57. package/lib/vectra-cli.d.ts +2 -0
  58. package/lib/vectra-cli.d.ts.map +1 -0
  59. package/lib/vectra-cli.js +276 -0
  60. package/lib/vectra-cli.js.map +1 -0
  61. package/package.json +21 -3
  62. package/src/GPT3Tokenizer.ts +15 -0
  63. package/src/ItemSelector.ts +9 -9
  64. package/src/LocalDocument.ts +70 -0
  65. package/src/LocalDocumentIndex.ts +355 -0
  66. package/src/LocalDocumentResult.ts +206 -0
  67. package/src/LocalIndex.ts +12 -78
  68. package/src/OpenAIEmbeddings.ts +205 -0
  69. package/src/TextSplitter.ts +480 -0
  70. package/src/WebFetcher.ts +128 -0
  71. package/src/index.ts +8 -0
  72. package/src/internals/Colorize.ts +64 -0
  73. package/src/internals/index.ts +2 -0
  74. package/src/internals/types.ts +46 -0
  75. package/src/types.ts +160 -0
  76. package/src/vectra-cli.ts +238 -0
package/bin/vectra.js ADDED
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env node
2
+ var app = require('../lib/vectra-cli.js');
3
+ app.run();
@@ -0,0 +1,9 @@
1
+ import { Tokenizer } from "./types";
2
+ /**
3
+ * Tokenizer that uses GPT-3's encoder.
4
+ */
5
+ export declare class GPT3Tokenizer implements Tokenizer {
6
+ decode(tokens: number[]): string;
7
+ encode(text: string): number[];
8
+ }
9
+ //# sourceMappingURL=GPT3Tokenizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.GPT3Tokenizer = void 0;
4
+ const gpt_3_encoder_1 = require("gpt-3-encoder");
5
+ /**
6
+ * Tokenizer that uses GPT-3's encoder.
7
+ */
8
+ class GPT3Tokenizer {
9
+ decode(tokens) {
10
+ return (0, gpt_3_encoder_1.decode)(tokens);
11
+ }
12
+ encode(text) {
13
+ return (0, gpt_3_encoder_1.encode)(text);
14
+ }
15
+ }
16
+ exports.GPT3Tokenizer = GPT3Tokenizer;
17
+ //# sourceMappingURL=GPT3Tokenizer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}
@@ -1,4 +1,4 @@
1
- import { MetadataFilter, MetadataTypes } from './LocalIndex';
1
+ import { MetadataFilter, MetadataTypes } from './types';
2
2
  export declare class ItemSelector {
3
3
  /**
4
4
  * Returns the similarity between two vectors using the cosine similarity.
@@ -1 +1 @@
1
- {"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7D,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CAqDhC"}
1
+ {"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExD,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CAqDhC"}
@@ -1 +1 @@
1
- {"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACpD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE;wBACjD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE;wBACjD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC3D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC1D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA3JD,oCA2JC"}
1
+ {"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,KAAK,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACtE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC5D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC3D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA3JD,oCA2JC"}
@@ -0,0 +1,16 @@
1
+ import { MetadataTypes } from './types';
2
+ export declare class LocalDocument {
3
+ private readonly _folderPath;
4
+ private readonly _id;
5
+ private readonly _uri;
6
+ private _metadata;
7
+ private _text;
8
+ constructor(folderPath: string, id: string, uri: string);
9
+ get folderPath(): string;
10
+ get id(): string;
11
+ get uri(): string;
12
+ hasMetadata(): Promise<boolean>;
13
+ loadMetadata(): Promise<Record<string, MetadataTypes>>;
14
+ loadText(): Promise<string>;
15
+ }
16
+ //# sourceMappingURL=LocalDocument.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalDocument.d.ts","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,qBAAa,aAAa;IACtB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,KAAK,CAAmB;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAM9D,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED,IAAW,GAAG,IAAI,MAAM,CAEvB;IAEY,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/B,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,CAAC;IAmBrD,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAY3C"}
@@ -0,0 +1,99 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
+ return new (P || (P = Promise))(function (resolve, reject) {
28
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
29
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
30
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
31
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
32
+ });
33
+ };
34
+ Object.defineProperty(exports, "__esModule", { value: true });
35
+ exports.LocalDocument = void 0;
36
+ const fs = __importStar(require("fs/promises"));
37
+ const path = __importStar(require("path"));
38
+ class LocalDocument {
39
+ constructor(folderPath, id, uri) {
40
+ this._folderPath = folderPath;
41
+ this._id = id;
42
+ this._uri = uri;
43
+ }
44
+ get folderPath() {
45
+ return this._folderPath;
46
+ }
47
+ get id() {
48
+ return this._id;
49
+ }
50
+ get uri() {
51
+ return this._uri;
52
+ }
53
+ hasMetadata() {
54
+ return __awaiter(this, void 0, void 0, function* () {
55
+ try {
56
+ yield fs.access(path.join(this.folderPath, `${this.id}.json`));
57
+ return true;
58
+ }
59
+ catch (err) {
60
+ return false;
61
+ }
62
+ });
63
+ }
64
+ loadMetadata() {
65
+ return __awaiter(this, void 0, void 0, function* () {
66
+ if (this._metadata == undefined) {
67
+ let json;
68
+ try {
69
+ json = (yield fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
70
+ }
71
+ catch (err) {
72
+ throw new Error(`Error reading metadata for document "${this.uri}": ${err.toString()}`);
73
+ }
74
+ try {
75
+ this._metadata = JSON.parse(json);
76
+ }
77
+ catch (err) {
78
+ throw new Error(`Error parsing metadata for document "${this.uri}": ${err.toString()}`);
79
+ }
80
+ }
81
+ return this._metadata;
82
+ });
83
+ }
84
+ loadText() {
85
+ return __awaiter(this, void 0, void 0, function* () {
86
+ if (this._text == undefined) {
87
+ try {
88
+ this._text = (yield fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
89
+ }
90
+ catch (err) {
91
+ throw new Error(`Error reading text file for document "${this.uri}": ${err.toString()}`);
92
+ }
93
+ }
94
+ return this._text;
95
+ });
96
+ }
97
+ }
98
+ exports.LocalDocument = LocalDocument;
99
+ //# sourceMappingURL=LocalDocument.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalDocument.js","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAG7B,MAAa,aAAa;IAOtB,YAAmB,UAAkB,EAAE,EAAU,EAAE,GAAW;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAW,EAAE;QACT,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,IAAW,GAAG;QACV,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEY,WAAW;;YACpB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAEY,YAAY;;YACrB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE;gBAC7B,IAAI,IAAY,CAAC;gBACjB,IAAI;oBACA,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACxF;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;gBAED,IAAI;oBACA,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;iBACrC;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;aACJ;YAED,OAAO,IAAI,CAAC,SAAU,CAAC;QAC3B,CAAC;KAAA;IAEY,QAAQ;;YACjB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE;gBACzB,IAAI;oBACA,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC7F;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACrG;aACJ;YAED,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;KAAA;CAEJ;AAjED,sCAiEC"}
@@ -0,0 +1,48 @@
1
+ import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
2
+ import { TextSplitterConfig } from "./TextSplitter";
3
+ import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, DocumentCatalogStats } from "./types";
4
+ import { LocalDocumentResult } from './LocalDocumentResult';
5
+ import { LocalDocument } from './LocalDocument';
6
+ export interface DocumentQueryOptions {
7
+ maxDocuments?: number;
8
+ maxChunks?: number;
9
+ filter?: MetadataFilter;
10
+ }
11
+ export interface LocalDocumentIndexConfig {
12
+ folderPath: string;
13
+ embeddings?: EmbeddingsModel;
14
+ tokenizer?: Tokenizer;
15
+ chunkingConfig?: Partial<TextSplitterConfig>;
16
+ }
17
+ export declare class LocalDocumentIndex extends LocalIndex {
18
+ private readonly _embeddings?;
19
+ private readonly _tokenizer;
20
+ private readonly _chunkingConfig?;
21
+ private _catalog?;
22
+ private _newCatalog?;
23
+ constructor(config: LocalDocumentIndexConfig);
24
+ /**
25
+ * Returns true if the document catalog exists.
26
+ */
27
+ isCatalogCreated(): Promise<boolean>;
28
+ getDocumentId(uri: string): Promise<string | undefined>;
29
+ getDocumentUri(documentId: string): Promise<string | undefined>;
30
+ createIndex(config?: CreateIndexConfig): Promise<void>;
31
+ deleteDocument(uri: string): Promise<void>;
32
+ getCatalogStats(): Promise<DocumentCatalogStats>;
33
+ /**
34
+ * Adds a document to the catalog.
35
+ * @remarks
36
+ * A new update is started if one is not already in progress. If an document with the same uri
37
+ * already exists, it will be replaced.
38
+ * @param item Item to insert
39
+ * @returns Inserted document
40
+ */
41
+ upsertDocument(uri: string, text: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument>;
42
+ queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]>;
43
+ beginUpdate(): Promise<void>;
44
+ cancelUpdate(): void;
45
+ endUpdate(): Promise<void>;
46
+ protected loadIndexData(): Promise<void>;
47
+ }
48
+ //# sourceMappingURL=LocalDocumentIndex.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAA0D,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD,MAAM,WAAW,oBAAoB;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAwB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAChD;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;gBAGnB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IASpC,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKvD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK/D,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C1C,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;;;;;OAOG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IA4G3G,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAqD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAavB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BjD"}
@@ -0,0 +1,367 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
+ return new (P || (P = Promise))(function (resolve, reject) {
28
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
29
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
30
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
31
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
32
+ });
33
+ };
34
+ Object.defineProperty(exports, "__esModule", { value: true });
35
+ exports.LocalDocumentIndex = void 0;
36
+ const fs = __importStar(require("fs/promises"));
37
+ const path = __importStar(require("path"));
38
+ const uuid_1 = require("uuid");
39
+ const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
40
+ const LocalIndex_1 = require("./LocalIndex");
41
+ const TextSplitter_1 = require("./TextSplitter");
42
+ const LocalDocumentResult_1 = require("./LocalDocumentResult");
43
+ const LocalDocument_1 = require("./LocalDocument");
44
+ const EMBEDDINGS_BATCH_SIZE = 500;
45
+ class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
46
+ constructor(config) {
47
+ var _a, _b;
48
+ super(config.folderPath);
49
+ this._embeddings = config.embeddings;
50
+ this._chunkingConfig = Object.assign({
51
+ keepSeparators: true,
52
+ chunkSize: 512,
53
+ chunkOverlap: 0,
54
+ }, config.chunkingConfig);
55
+ this._tokenizer = (_b = (_a = config.tokenizer) !== null && _a !== void 0 ? _a : this._chunkingConfig.tokenizer) !== null && _b !== void 0 ? _b : new GPT3Tokenizer_1.GPT3Tokenizer();
56
+ this._chunkingConfig.tokenizer = this._tokenizer;
57
+ }
58
+ /**
59
+ * Returns true if the document catalog exists.
60
+ */
61
+ isCatalogCreated() {
62
+ return __awaiter(this, void 0, void 0, function* () {
63
+ try {
64
+ yield fs.access(path.join(this.folderPath, 'catalog.json'));
65
+ return true;
66
+ }
67
+ catch (err) {
68
+ return false;
69
+ }
70
+ });
71
+ }
72
+ getDocumentId(uri) {
73
+ var _a;
74
+ return __awaiter(this, void 0, void 0, function* () {
75
+ yield this.loadIndexData();
76
+ return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.uriToId[uri];
77
+ });
78
+ }
79
+ getDocumentUri(documentId) {
80
+ var _a;
81
+ return __awaiter(this, void 0, void 0, function* () {
82
+ yield this.loadIndexData();
83
+ return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.idToUri[documentId];
84
+ });
85
+ }
86
+ createIndex(config) {
87
+ const _super = Object.create(null, {
88
+ createIndex: { get: () => super.createIndex }
89
+ });
90
+ return __awaiter(this, void 0, void 0, function* () {
91
+ yield _super.createIndex.call(this, config);
92
+ yield this.loadIndexData();
93
+ });
94
+ }
95
+ deleteDocument(uri) {
96
+ return __awaiter(this, void 0, void 0, function* () {
97
+ // Lookup document ID
98
+ const documentId = yield this.getDocumentId(uri);
99
+ if (documentId == undefined) {
100
+ return;
101
+ }
102
+ // Delete document chunks from index and remove from catalog
103
+ yield this.beginUpdate();
104
+ try {
105
+ // Get list of chunks for document
106
+ const chunks = yield this.listItemsByMetadata({ documentId });
107
+ // Delete chunks
108
+ for (const chunk of chunks) {
109
+ yield this.deleteItem(chunk.id);
110
+ }
111
+ // Remove entry from catalog
112
+ delete this._newCatalog.uriToId[uri];
113
+ delete this._newCatalog.idToUri[documentId];
114
+ this._newCatalog.count--;
115
+ // Commit changes
116
+ yield this.endUpdate();
117
+ }
118
+ catch (err) {
119
+ // Cancel update and raise error
120
+ this.cancelUpdate();
121
+ throw new Error(`Error deleting document "${uri}": ${err.toString()}`);
122
+ }
123
+ // Delete text file from disk
124
+ try {
125
+ yield fs.unlink(path.join(this.folderPath, `${documentId}.txt`));
126
+ }
127
+ catch (err) {
128
+ throw new Error(`Error removing text file for document "${uri}" from disk: ${err.toString()}`);
129
+ }
130
+ // Delete metadata file from disk
131
+ try {
132
+ yield fs.unlink(path.join(this.folderPath, `${documentId}.json`));
133
+ }
134
+ catch (err) {
135
+ // Ignore error
136
+ }
137
+ });
138
+ }
139
+ getCatalogStats() {
140
+ return __awaiter(this, void 0, void 0, function* () {
141
+ const stats = yield this.getIndexStats();
142
+ return {
143
+ version: this._catalog.version,
144
+ documents: this._catalog.count,
145
+ chunks: stats.items,
146
+ metadata_config: stats.metadata_config
147
+ };
148
+ });
149
+ }
150
+ /**
151
+ * Adds a document to the catalog.
152
+ * @remarks
153
+ * A new update is started if one is not already in progress. If an document with the same uri
154
+ * already exists, it will be replaced.
155
+ * @param item Item to insert
156
+ * @returns Inserted document
157
+ */
158
+ upsertDocument(uri, text, metadata) {
159
+ return __awaiter(this, void 0, void 0, function* () {
160
+ // Ensure embeddings configured
161
+ if (!this._embeddings) {
162
+ throw new Error(`Embeddings model not configured.`);
163
+ }
164
+ // Check for existing document ID
165
+ let documentId = yield this.getDocumentId(uri);
166
+ if (documentId != undefined) {
167
+ // Delete existing document
168
+ yield this.deleteDocument(uri);
169
+ }
170
+ else {
171
+ // Generate new document ID
172
+ documentId = (0, uuid_1.v4)();
173
+ }
174
+ // Populate docType based on extension
175
+ const config = Object.assign({}, this._chunkingConfig);
176
+ const pos = uri.lastIndexOf('.');
177
+ if (pos >= 0) {
178
+ const ext = uri.substring(pos + 1).toLowerCase();
179
+ config.docType = ext;
180
+ }
181
+ // Split text into chunks
182
+ const splitter = new TextSplitter_1.TextSplitter(config);
183
+ const chunks = splitter.split(text);
184
+ // Break chunks into batches for embedding generation
185
+ const chunkBatches = [];
186
+ let currentBatch = [];
187
+ for (const chunk of chunks) {
188
+ currentBatch.push(chunk.text);
189
+ if (currentBatch.length >= EMBEDDINGS_BATCH_SIZE) {
190
+ chunkBatches.push(currentBatch);
191
+ currentBatch = [];
192
+ }
193
+ }
194
+ if (currentBatch.length > 0) {
195
+ chunkBatches.push(currentBatch);
196
+ }
197
+ // Generate embeddings for chunks
198
+ const embeddings = [];
199
+ for (const batch of chunkBatches) {
200
+ let response;
201
+ try {
202
+ response = yield this._embeddings.createEmbeddings(batch);
203
+ }
204
+ catch (err) {
205
+ throw new Error(`Error generating embeddings: ${err.toString()}`);
206
+ }
207
+ // Check for error
208
+ if (response.status != 'success') {
209
+ throw new Error(`Error generating embeddings: ${response.message}`);
210
+ }
211
+ // Add embeddings to output
212
+ for (const embedding of response.output) {
213
+ embeddings.push(embedding);
214
+ }
215
+ }
216
+ // Add document chunks to index
217
+ yield this.beginUpdate();
218
+ try {
219
+ // Add chunks to index
220
+ for (let i = 0; i < chunks.length; i++) {
221
+ const chunk = chunks[i];
222
+ const embedding = embeddings[i];
223
+ const chunkMetadata = Object.assign({
224
+ documentId,
225
+ startPos: chunk.startPos,
226
+ endPos: chunk.endPos,
227
+ }, metadata);
228
+ yield this.insertItem({
229
+ id: (0, uuid_1.v4)(),
230
+ metadata: chunkMetadata,
231
+ vector: embedding,
232
+ });
233
+ }
234
+ // Save metadata file to disk
235
+ if (metadata != undefined) {
236
+ yield fs.writeFile(path.join(this.folderPath, `${documentId}.json`), JSON.stringify(metadata));
237
+ }
238
+ // Save text file to disk
239
+ yield fs.writeFile(path.join(this.folderPath, `${documentId}.txt`), text);
240
+ // Add entry to catalog
241
+ this._newCatalog.uriToId[uri] = documentId;
242
+ this._newCatalog.idToUri[documentId] = uri;
243
+ this._newCatalog.count++;
244
+ // Commit changes
245
+ yield this.endUpdate();
246
+ }
247
+ catch (err) {
248
+ // Cancel update and raise error
249
+ this.cancelUpdate();
250
+ throw new Error(`Error adding document "${uri}": ${err.toString()}`);
251
+ }
252
+ // Return document
253
+ return new LocalDocument_1.LocalDocument(this.folderPath, documentId, uri);
254
+ });
255
+ }
256
+ queryDocuments(query, options) {
257
+ return __awaiter(this, void 0, void 0, function* () {
258
+ // Ensure embeddings configured
259
+ if (!this._embeddings) {
260
+ throw new Error(`Embeddings model not configured.`);
261
+ }
262
+ // Ensure options are defined
263
+ options = Object.assign({
264
+ maxDocuments: 10,
265
+ maxChunks: 50,
266
+ }, options);
267
+ // Generate embeddings for query
268
+ let embeddings;
269
+ try {
270
+ embeddings = yield this._embeddings.createEmbeddings(query);
271
+ }
272
+ catch (err) {
273
+ throw new Error(`Error generating embeddings for query: ${err.toString()}`);
274
+ }
275
+ // Check for error
276
+ if (embeddings.status != 'success') {
277
+ throw new Error(`Error generating embeddings for query: ${embeddings.message}`);
278
+ }
279
+ // Query index for chunks
280
+ const results = yield this.queryItems(embeddings.output[0], options.maxChunks, options.filter);
281
+ // Group chunks by document
282
+ const documentChunks = {};
283
+ for (const result of results) {
284
+ const metadata = result.item.metadata;
285
+ if (documentChunks[metadata.documentId] == undefined) {
286
+ documentChunks[metadata.documentId] = [];
287
+ }
288
+ documentChunks[metadata.documentId].push(result);
289
+ }
290
+ // Create a document result for each document
291
+ const documentResults = [];
292
+ for (const documentId in documentChunks) {
293
+ const chunks = documentChunks[documentId];
294
+ const uri = yield this.getDocumentUri(documentId);
295
+ const documentResult = new LocalDocumentResult_1.LocalDocumentResult(this.folderPath, documentId, uri, chunks, this._tokenizer);
296
+ documentResults.push(documentResult);
297
+ }
298
+ // Sort document results by score and return top results
299
+ return documentResults.sort((a, b) => b.score - a.score).slice(0, options.maxDocuments);
300
+ });
301
+ }
302
+ // Overrides
303
+ beginUpdate() {
304
+ const _super = Object.create(null, {
305
+ beginUpdate: { get: () => super.beginUpdate }
306
+ });
307
+ return __awaiter(this, void 0, void 0, function* () {
308
+ yield _super.beginUpdate.call(this);
309
+ this._newCatalog = Object.assign({}, this._catalog);
310
+ });
311
+ }
312
+ cancelUpdate() {
313
+ super.cancelUpdate();
314
+ this._newCatalog = undefined;
315
+ }
316
+ endUpdate() {
317
+ const _super = Object.create(null, {
318
+ endUpdate: { get: () => super.endUpdate }
319
+ });
320
+ return __awaiter(this, void 0, void 0, function* () {
321
+ yield _super.endUpdate.call(this);
322
+ try {
323
+ // Save catalog
324
+ yield fs.writeFile(path.join(this.folderPath, 'catalog.json'), JSON.stringify(this._newCatalog));
325
+ this._catalog = this._newCatalog;
326
+ this._newCatalog = undefined;
327
+ }
328
+ catch (err) {
329
+ throw new Error(`Error saving document catalog: ${err.toString()}`);
330
+ }
331
+ });
332
+ }
333
+ loadIndexData() {
334
+ const _super = Object.create(null, {
335
+ loadIndexData: { get: () => super.loadIndexData }
336
+ });
337
+ return __awaiter(this, void 0, void 0, function* () {
338
+ yield _super.loadIndexData.call(this);
339
+ if (this._catalog) {
340
+ return;
341
+ }
342
+ const catalogPath = path.join(this.folderPath, 'catalog.json');
343
+ if (yield this.isCatalogCreated()) {
344
+ // Load catalog
345
+ const buffer = yield fs.readFile(catalogPath);
346
+ this._catalog = JSON.parse(buffer.toString());
347
+ }
348
+ else {
349
+ try {
350
+ // Initialize catalog
351
+ this._catalog = {
352
+ version: 1,
353
+ count: 0,
354
+ uriToId: {},
355
+ idToUri: {},
356
+ };
357
+ yield fs.writeFile(catalogPath, JSON.stringify(this._catalog));
358
+ }
359
+ catch (err) {
360
+ throw new Error(`Error creating document catalog: ${err.toString()}`);
361
+ }
362
+ }
363
+ });
364
+ }
365
+ }
366
+ exports.LocalDocumentIndex = LocalDocumentIndex;
367
+ //# sourceMappingURL=LocalDocumentIndex.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalDocumentIndex.js","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,mDAAgD;AAChD,6CAA6D;AAC7D,iDAAkE;AAElE,+DAA4D;AAC5D,mDAAgD;AAEhD,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAelC,MAAa,kBAAmB,SAAQ,uBAAU;IAQ9C,YAAmB,MAAgC;;QAC/C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,CAAC;SACI,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAA,MAAA,MAAM,CAAC,SAAS,mCAAI,IAAI,CAAC,eAAe,CAAC,SAAS,mCAAI,IAAI,6BAAa,EAAE,CAAC;QAC5F,IAAI,CAAC,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;IACrD,CAAC;IAED;;OAEG;IACU,gBAAgB;;YACzB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;gBAC5D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAEY,aAAa,CAAC,GAAW;;;YAClC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,GAAG,CAAC,CAAC;;KACtC;IAEY,cAAc,CAAC,UAAkB;;;YAC1C,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,UAAU,CAAC,CAAC;;KAC7C;IAEY,WAAW,CAAC,MAA0B;;;;;YAC/C,MAAM,OAAM,WAAW,YAAC,MAAM,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,CAAC;KAAA;IAEY,cAAc,CAAC,GAAW;;YACnC,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,OAAO;aACV;YAED,4DAA4D;YAC5D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,kCAAkC;gBAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAwB,EAAE,UAAU,EAAE,CAAC,CAAC;gBAErF,gBAAgB;gBAChB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBACxB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACnC;gBAED,4BAA4B;gBAC5B,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACnF;YAED,6BAA6B;YAC7B,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,CAAC,CAAC;aACpE;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,gBAAiB,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAC3G;YAED,iCAAiC;YACjC,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC,CAAC;aACrE;YAAC,OAAO,GAAY,EAAE;gBACnB,eAAe;aAClB;QACL,CAAC;KAAA;IAEY,eAAe;;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;YACxC,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,QAAS,CAAC,OAAO;gBAC/B,SAAS,EAAE,IAAI,CAAC,QAAS,CAAC,KAAK;gBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK;gBACnB,eAAe,EAAE,KAAK,CAAC,eAAe;aACzC,CAAC;QACN,CAAC;KAAA;IAED;;;;;;;OAOG;IACU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,QAAwC;;YAC3F,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,iCAAiC;YACjC,IAAI,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aAClC;iBAAM;gBACH,2BAA2B;gBAC3B,UAAU,GAAG,IAAA,SAAE,GAAE,CAAC;aACrB;YAED,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,EAAE;gBACV,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjD,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;aACxB;YAED,yBAAyB;YACzB,MAAM,QAAQ,GAAG,IAAI,2BAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,qDAAqD;YACrD,MAAM,YAAY,GAAe,EAAE,CAAC;YACpC,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBACxB,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9B,IAAI,YAAY,CAAC,MAAM,IAAI,qBAAqB,EAAE;oBAC9C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAChC,YAAY,GAAG,EAAE,CAAC;iBACrB;aACJ;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBACzB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aACnC;YAED,iCAAiC;YACjC,MAAM,UAAU,GAAe,EAAE,CAAC;YAClC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE;gBAC9B,IAAI,QAA4B,CAAC;gBACjC,IAAI;oBACA,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;iBAC7D;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,gCAAiC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC9E;gBAED,kBAAkB;gBAClB,IAAI,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE;oBAC9B,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;iBACvE;gBAED,2BAA2B;gBAC3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAO,EAAE;oBACtC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBAC9B;aACJ;YAED,+BAA+B;YAC/B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,sBAAsB;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;oBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,aAAa,GAA0B,MAAM,CAAC,MAAM,CAAC;wBACvD,UAAU;wBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACvB,EAAE,QAAQ,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,UAAU,CAAC;wBAClB,EAAE,EAAE,IAAA,SAAE,GAAE;wBACR,QAAQ,EAAE,aAAa;wBACvB,MAAM,EAAE,SAAS;qBACpB,CAAC,CAAC;iBACN;gBAED,6BAA6B;gBAC7B,IAAI,QAAQ,IAAI,SAAS,EAAE;oBACvB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClG;gBAED,yBAAyB;gBACzB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;gBAE1E,uBAAuB;gBACvB,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACjF;YAED,kBAAkB;YAClB,OAAO,IAAI,6BAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC;KAAA;IAGY,cAAc,CAAC,KAAa,EAAE,OAA8B;;YACrE,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,6BAA6B;YAC7B,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpB,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aAChB,EAAE,OAAO,CAAC,CAAC;YAEZ,gCAAgC;YAChC,IAAI,UAA8B,CAAC;YACnC,IAAI;gBACA,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;aAC/D;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA2C,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACxF;YAED,kBAAkB;YAClB,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE;gBAChC,MAAM,IAAI,KAAK,CAAC,0CAA0C,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;aACnF;YAED,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAwB,UAAU,CAAC,MAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAExH,2BAA2B;YAC3B,MAAM,cAAc,GAAoE,EAAE,CAAC;YAC3F,KAAK,MAAM,MAAM,IAAK,OAAO,EAAE;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE;oBAClD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;iBAC5C;gBACD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACpD;YAED,6CAA6C;YAC7C,MAAM,eAAe,GAA0B,EAAE,CAAC;YAClD,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE;gBACrC,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1G,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aACxC;YAED,wDAAwD;YACxD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,YAAa,CAAC,CAAC;QAC7F,CAAC;KAAA;IAED,YAAY;IAEC,WAAW;;;;;YACpB,MAAM,OAAM,WAAW,WAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;KAAA;IAEM,YAAY;QACf,KAAK,CAAC,YAAY,EAAE,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;IACjC,CAAC;IAEY,SAAS;;;;;YAClB,MAAM,OAAM,SAAS,WAAE,CAAC;YAExB,IAAI;gBACA,eAAe;gBACf,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;gBACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAChC;YAAC,OAAM,GAAY,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,kCAAmC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAChF;QACL,CAAC;KAAA;IAEe,aAAa;;;;;YACzB,MAAM,OAAM,aAAa,WAAE,CAAC;YAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,OAAO;aACV;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC/D,IAAI,MAAM,IAAI,CAAC,gBAAgB,EAAE,EAAE;gBAC/B,eAAe;gBACf,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjD;iBAAM;gBACH,IAAI;oBACA,qBAAqB;oBACrB,IAAI,CAAC,QAAQ,GAAG;wBACZ,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,EAAE;wBACX,OAAO,EAAE,EAAE;qBACd,CAAC;oBACF,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClE;gBAAC,OAAM,GAAY,EAAE;oBAClB,MAAM,IAAI,KAAK,CAAC,oCAAqC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAClF;aACJ;QACL,CAAC;KAAA;CACJ;AAlUD,gDAkUC"}
@@ -0,0 +1,12 @@
1
+ import { LocalDocument } from "./LocalDocument";
2
+ import { QueryResult, DocumentChunkMetadata, Tokenizer, DocumentTextSection } from "./types";
3
+ export declare class LocalDocumentResult extends LocalDocument {
4
+ private readonly _chunks;
5
+ private readonly _tokenizer;
6
+ private readonly _score;
7
+ constructor(folderPath: string, id: string, uri: string, chunks: QueryResult<DocumentChunkMetadata>[], tokenizer: Tokenizer);
8
+ get chunks(): QueryResult<DocumentChunkMetadata>[];
9
+ get score(): number;
10
+ renderSections(maxTokens: number, maxSections: number): Promise<DocumentTextSection[]>;
11
+ }
12
+ //# sourceMappingURL=LocalDocumentResult.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalDocumentResult.d.ts","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE7F,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS;IAWlI,IAAW,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAExD;IAED,IAAW,KAAK,IAAI,MAAM,CAEzB;IAEY,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;CAmKtG"}