vectra 0.1.2 → 0.2.1

package/lib/index.d.ts

@@ -1,3 +1,11 @@
+export * from './GPT3Tokenizer';
 export * from './ItemSelector';
 export * from './LocalIndex';
+export * from './LocalDocument';
+export * from './LocalDocumentIndex';
+export * from './LocalDocumentResult';
+export * from './OpenAIEmbeddings';
+export * from './TextSplitter';
+export * from './types';
+export * from './WebFetcher';
 //# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC"}

package/lib/index.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
@@ -10,6 +14,14 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./GPT3Tokenizer"), exports);
 __exportStar(require("./ItemSelector"), exports);
 __exportStar(require("./LocalIndex"), exports);
+__exportStar(require("./LocalDocument"), exports);
+__exportStar(require("./LocalDocumentIndex"), exports);
+__exportStar(require("./LocalDocumentResult"), exports);
+__exportStar(require("./OpenAIEmbeddings"), exports);
+__exportStar(require("./TextSplitter"), exports);
+__exportStar(require("./types"), exports);
+__exportStar(require("./WebFetcher"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAA+B;AAC/B,+CAA6B"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC;AAChC,iDAA+B;AAC/B,+CAA6B;AAC7B,kDAAgC;AAChC,uDAAqC;AACrC,wDAAsC;AACtC,qDAAmC;AACnC,iDAA+B;AAC/B,0CAAwB;AACxB,+CAA6B"}

package/lib/internals/Colorize.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @private
+ */
+export declare class Colorize {
+    static replaceLine(text: string): string;
+    static error(error: Error | string): string;
+    static output(output: object | string, quote?: string, units?: string): string;
+    static progress(message: string): string;
+    static success(message: string): string;
+    static title(title: string): string;
+    static value(field: string, value: any, units?: string): string;
+    static warning(warning: string): string;
+}
+//# sourceMappingURL=Colorize.d.ts.map

package/lib/internals/Colorize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Colorize.d.ts","sourceRoot":"","sources":["../../src/internals/Colorize.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,qBAAa,QAAQ;WACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;WAIjC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAC,MAAM,GAAG,MAAM;WAQlC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,KAAK,GAAE,MAAW,EAAE,KAAK,GAAE,MAAW,GAAG,MAAM;WAyB/E,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;WAIjC,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;WAIhC,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;WAI5B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAE,MAAW,GAAG,MAAM;WAI5D,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;CAIjD"}

package/lib/internals/Colorize.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Colorize = void 0;
+const colorizer = require('json-colorizer');
+/**
+ * @private
+ */
+class Colorize {
+    static replaceLine(text) {
+        return '\x1b[A\x1b[2K' + text;
+    }
+    static error(error) {
+        if (typeof error === 'string') {
+            return `\x1b[31;1m${error}\x1b[0m`;
+        }
+        else {
+            return `\x1b[31;1m${error.message}\x1b[0m`;
+        }
+    }
+    static output(output, quote = '', units = '') {
+        if (typeof output === 'string') {
+            return `\x1b[32m${quote}${output}${quote}\x1b[0m`;
+        }
+        else if (typeof output === 'object' && output !== null) {
+            return colorizer(output, {
+                pretty: true,
+                colors: {
+                    BRACE: 'white',
+                    BRACKET: 'white',
+                    COLON: 'white',
+                    COMMA: 'white',
+                    STRING_KEY: 'white',
+                    STRING_LITERAL: 'green',
+                    NUMBER_LITERAL: 'blue',
+                    BOOLEAN_LITERAL: 'blue',
+                    NULL_LITERAL: 'blue'
+                }
+            });
+        }
+        else if (typeof output == 'number') {
+            return `\x1b[34m${output}${units}\x1b[0m`;
+        }
+        else {
+            return `\x1b[34m${output}\x1b[0m`;
+        }
+    }
+    static progress(message) {
+        return message;
+    }
+    static success(message) {
+        return `\x1b[32;1m${message}\x1b[0m`;
+    }
+    static title(title) {
+        return `\x1b[35;1m${title}\x1b[0m`;
+    }
+    static value(field, value, units = '') {
+        return `${field}: ${Colorize.output(value, '"', units)}`;
+    }
+    static warning(warning) {
+        return `\x1b[33m${warning}\x1b[0m`;
+    }
+}
+exports.Colorize = Colorize;
+//# sourceMappingURL=Colorize.js.map

package/lib/internals/Colorize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Colorize.js","sourceRoot":"","sources":["../../src/internals/Colorize.ts"],"names":[],"mappings":";;;AAAA,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE5C;;GAEG;AACH,MAAa,QAAQ;IACV,MAAM,CAAC,WAAW,CAAC,IAAY;QAClC,OAAO,eAAe,GAAG,IAAI,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,KAAmB;QACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC3B,OAAO,aAAa,KAAK,SAAS,CAAC;SACtC;aAAM;YACH,OAAO,aAAa,KAAK,CAAC,OAAO,SAAS,CAAC;SAC9C;IACL,CAAC;IAEM,MAAM,CAAC,MAAM,CAAC,MAAuB,EAAE,QAAgB,EAAE,EAAE,QAAgB,EAAE;QAChF,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;YAC5B,OAAO,WAAW,KAAK,GAAG,MAAM,GAAG,KAAK,SAAS,CAAC;SACrD;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE;YACtD,OAAO,SAAS,CAAC,MAAM,EAAE;gBACrB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE;oBACJ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,OAAO;oBAChB,KAAK,EAAE,OAAO;oBACd,KAAK,EAAE,OAAO;oBACd,UAAU,EAAE,OAAO;oBACnB,cAAc,EAAE,OAAO;oBACvB,cAAc,EAAE,MAAM;oBACtB,eAAe,EAAE,MAAM;oBACvB,YAAY,EAAE,MAAM;iBACvB;aACJ,CAAC,CAAC;SACN;aAAM,IAAI,OAAO,MAAM,IAAI,QAAQ,EAAE;YAClC,OAAO,WAAW,MAAM,GAAG,KAAK,SAAS,CAAC;SAC7C;aAAM;YACH,OAAO,WAAW,MAAM,SAAS,CAAC;SACrC;IACL,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,OAAe;QAClC,OAAO,OAAO,CAAC;IACnB,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,OAAe;QACjC,OAAO,aAAa,OAAO,SAAS,CAAC;IACzC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,KAAa;QAC7B,OAAO,aAAa,KAAK,SAAS,CAAC;IACvC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,KAAa,EAAE,KAAU,EAAE,QAAgB,EAAE;QAC7D,OAAO,GAAG,KAAK,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;IAC7D,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,OAAe;QACjC,OAAO,WAAW,OAAO,SAAS,CAAC;IAEvC,CAAC;CACJ;AA1DD,4BA0DC"}

package/lib/internals/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./Colorize";
+export * from "./types";
+//# sourceMappingURL=index.d.ts.map

package/lib/internals/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/internals/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC"}

package/lib/internals/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./Colorize"), exports);
+__exportStar(require("./types"), exports);
+//# sourceMappingURL=index.js.map

package/lib/internals/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/internals/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA2B;AAC3B,0CAAwB"}

package/lib/internals/types.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @private
+ */
+export interface CreateEmbeddingRequest {
+    input: CreateEmbeddingRequestInput;
+    user?: string;
+}
+/**
+ * @private
+ */
+export interface OpenAICreateEmbeddingRequest extends CreateEmbeddingRequest {
+    model: string;
+}
+/**
+ * @private
+ */
+export interface CreateEmbeddingResponse {
+    object: string;
+    model: string;
+    data: Array<CreateEmbeddingResponseDataInner>;
+    usage: CreateEmbeddingResponseUsage;
+}
+/**
+ * @private
+ */
+export interface CreateEmbeddingResponseDataInner {
+    index: number;
+    object: string;
+    embedding: Array<number>;
+}
+/**
+ * @private
+ */
+export interface CreateEmbeddingResponseUsage {
+    prompt_tokens: number;
+    total_tokens: number;
+}
+/**
+ * @private
+ */
+export type CreateEmbeddingRequestInput = Array<any> | Array<number> | Array<string> | string;
+//# sourceMappingURL=types.d.ts.map

package/lib/internals/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/internals/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACnC,KAAK,EAAE,2BAA2B,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,sBAAsB;IACxE,KAAK,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC9C,KAAK,EAAE,4BAA4B,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,2BAA2B,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC"}

package/lib/internals/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/lib/internals/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/internals/types.ts"],"names":[],"mappings":""}

package/lib/types.d.ts

@@ -0,0 +1,133 @@
+/**
+ * An AI model that can be used to create embeddings.
+ */
+export interface EmbeddingsModel {
+    /**
+     * Creates embeddings for the given inputs.
+     * @param inputs Text inputs to create embeddings for.
+     * @returns A `EmbeddingsResponse` with a status and the generated embeddings or a message when an error occurs.
+     */
+    createEmbeddings(inputs: string | string[]): Promise<EmbeddingsResponse>;
+}
+/**
+ * Status of the embeddings response.
+ * @remarks
+ * `success` - The embeddings were successfully created.
+ * `error` - An error occurred while creating the embeddings.
+ * `rate_limited` - The request was rate limited.
+ */
+export type EmbeddingsResponseStatus = 'success' | 'error' | 'rate_limited';
+/**
+ * Response returned by a `EmbeddingsClient`.
+ */
+export interface EmbeddingsResponse {
+    /**
+     * Status of the embeddings response.
+     */
+    status: EmbeddingsResponseStatus;
+    /**
+     * Optional. Embeddings for the given inputs.
+     */
+    output?: number[][];
+    /**
+     * Optional. Message when status is not equal to `success`.
+     */
+    message?: string;
+}
+export interface TextChunk {
+    text: string;
+    tokens: number[];
+    startPos: number;
+    endPos: number;
+    startOverlap: number[];
+    endOverlap: number[];
+}
+export interface TextFetcher {
+    fetch(uri: string): Promise<string>;
+}
+export interface IndexStats {
+    version: number;
+    metadata_config: {
+        indexed?: string[];
+    };
+    items: number;
+}
+export interface IndexItem<TMetadata = Record<string, MetadataTypes>> {
+    id: string;
+    metadata: TMetadata;
+    vector: number[];
+    norm: number;
+    metadataFile?: string;
+}
+export interface MetadataFilter {
+    /**
+     * Equal to (number, string, boolean)
+     */
+    '$eq'?: number | string | boolean;
+    /**
+     * Not equal to (number, string, boolean)
+     */
+    '$ne'?: number | string | boolean;
+    /**
+     * Greater than (number)
+     */
+    '$gt'?: number;
+    /**
+     * Greater than or equal to (number)
+     */
+    '$gte'?: number;
+    /**
+     * Less than (number)
+     */
+    '$lt'?: number;
+    /**
+     * Less than or equal to (number)
+     */
+    '$lte'?: number;
+    /**
+     * In array (string or number)
+     */
+    '$in'?: (number | string)[];
+    /**
+     * Not in array (string or number)
+     */
+    '$nin'?: (number | string)[];
+    /**
+     * AND (MetadataFilter[])
+     */
+    '$and'?: MetadataFilter[];
+    /**
+     * OR (MetadataFilter[])
+     */
+    '$or'?: MetadataFilter[];
+    [key: string]: unknown;
+}
+export type MetadataTypes = number | string | boolean;
+export interface QueryResult<TMetadata = Record<string, MetadataTypes>> {
+    item: IndexItem<TMetadata>;
+    score: number;
+}
+export interface Tokenizer {
+    decode(tokens: number[]): string;
+    encode(text: string): number[];
+}
+export interface DocumentChunkMetadata {
+    documentId: string;
+    startPos: number;
+    endPos: number;
+    [key: string]: MetadataTypes;
+}
+export interface DocumentCatalogStats {
+    version: number;
+    documents: number;
+    chunks: number;
+    metadata_config: {
+        indexed?: string[];
+    };
+}
+export interface DocumentTextSection {
+    text: string;
+    tokenCount: number;
+    score: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAC,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC1E;AAED;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,SAAS,GAAG,OAAO,GAAG,cAAc,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,wBAAwB,CAAC;IAEjC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAEpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,SAAS,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAE3B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE3B;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAE1B;;OAEG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IAEzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;AAElD,MAAM,WAAW,WAAW,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjE,IAAI,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACjB"}

package/lib/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/lib/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/lib/vectra-cli.d.ts

@@ -0,0 +1,2 @@
+export declare function run(): Promise<void>;
+//# sourceMappingURL=vectra-cli.d.ts.map

package/lib/vectra-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectra-cli.d.ts","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":"AAQA,wBAAsB,GAAG,kBAyNxB"}

package/lib/vectra-cli.js

@@ -0,0 +1,277 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = void 0;
+const fs = __importStar(require("fs/promises"));
+const yargs_1 = __importDefault(require("yargs/yargs"));
+const helpers_1 = require("yargs/helpers");
+const LocalDocumentIndex_1 = require("./LocalDocumentIndex");
+const WebFetcher_1 = require("./WebFetcher");
+const OpenAIEmbeddings_1 = require("./OpenAIEmbeddings");
+const internals_1 = require("./internals");
+function run() {
+    return __awaiter(this, void 0, void 0, function* () {
+        // prettier-ignore
+        const args = yield (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
+            .scriptName('vectra')
+            .command('create <index>', `create a new local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            console.log(internals_1.Colorize.output(`creating index at ${folderPath}`));
+            yield index.createIndex({ version: 1, deleteIfExists: true });
+        }))
+            .command('delete <index>', `delete an existing local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            console.log(internals_1.Colorize.output(`deleting index at ${folderPath}`));
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            yield index.deleteIndex();
+        }))
+            .command('add-web <index>', `adds one or more web pages to an index`, (yargs) => {
+            return yargs
+                .option('keys', {
+                alias: 'k',
+                describe: 'path of a JSON file containing the model keys to use for generating embeddings',
+                type: 'string'
+            })
+                .option('uri', {
+                alias: 'u',
+                array: true,
+                describe: 'http/https link to a web page to add',
+                type: 'string'
+            })
+                .option('list', {
+                alias: 'l',
+                describe: 'path to a file containing a list of web pages to add',
+                type: 'string'
+            })
+                .option('chunk-size', {
+                alias: 'cs',
+                describe: 'size of the generated chunks in tokens (defaults to 512)',
+                type: 'number',
+                default: 512
+            })
+                .check((argv) => {
+                if (Array.isArray(argv.uri) && argv.uri.length > 0) {
+                    return true;
+                }
+                else if (typeof argv.list == 'string' && argv.list.trim().length > 0) {
+                    return true;
+                }
+                else {
+                    throw new Error(`you must specify either one or more "--uri <link>" for the pages to add or a "--list <file path>" for a file containing the list of pages to add.`);
+                }
+            })
+                .demandOption(['keys']);
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            console.log(internals_1.Colorize.title('Adding Web Pages to Index'));
+            // Create embeddings
+            const keys = JSON.parse(yield fs.readFile(args.keys, 'utf-8'));
+            const embeddings = new OpenAIEmbeddings_1.OpenAIEmbeddings(Object.assign({ model: 'text-embedding-ada-002' }, keys));
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({
+                folderPath,
+                embeddings,
+                chunkingConfig: {
+                    chunkSize: args.chunkSize
+                }
+            });
+            // Get list of url's
+            const uris = yield getItemList(args.uri, args.list, 'web page');
+            // Fetch web pages
+            const fetcher = new WebFetcher_1.WebFetcher();
+            for (const uri of uris) {
+                try {
+                    console.log(internals_1.Colorize.progress(`fetching ${uri}`));
+                    const content = yield fetcher.fetch(uri);
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.progress(`indexing ${uri}`)));
+                    yield index.upsertDocument(uri, content);
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.success(`added ${uri}`)));
+                }
+                catch (err) {
+                    console.log(internals_1.Colorize.replaceLine(internals_1.Colorize.error(`Error adding: ${uri}\n${err.message}`)));
+                }
+            }
+        }))
+            .command('remove <index>', `removes one or more documents from an index`, (yargs) => {
+            return yargs
+                .option('uri', {
+                alias: 'u',
+                array: true,
+                describe: 'uri of a document to remove',
+                type: 'string'
+            })
+                .option('list', {
+                alias: 'l',
+                describe: 'path to a file containing a list of documents to remove',
+                type: 'string'
+            })
+                .check((argv) => {
+                if (Array.isArray(argv.uri) && argv.uri.length > 0) {
+                    return true;
+                }
+                else if (typeof argv.list == 'string' && argv.list.trim().length > 0) {
+                    return true;
+                }
+                else {
+                    throw new Error(`you must specify either one or more "--uri <link>" for the pages to add or a "--list <file path>" for a file containing the list of pages to add.`);
+                }
+            });
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            // Get list of uri's
+            const uris = yield getItemList(args.uri, args.list, 'document');
+            // Remove documents
+            for (const uri of uris) {
+                console.log(`removing ${uri}`);
+                yield index.deleteDocument(uri);
+            }
+        }))
+            .command('stats <index>', `prints the stats for a local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
+            const stats = yield index.getCatalogStats();
+            console.log(internals_1.Colorize.title('Index Stats'));
+            console.log(internals_1.Colorize.output(stats));
+        }))
+            .command('query <index> <query>', `queries a local index`, (yargs) => {
+            return yargs
+                .option('keys', {
+                alias: 'k',
+                describe: 'path of a JSON file containing the model keys to use for generating embeddings'
+            })
+                .option('document-count', {
+                alias: 'dc',
+                describe: 'max number of documents to return (defaults to 10)',
+                type: 'count',
+                default: 10
+            })
+                .option('chunk-count', {
+                alias: 'cc',
+                describe: 'max number of chunks to return (defaults to 50)',
+                type: 'count',
+                default: 50
+            })
+                .option('section-count', {
+                alias: 'sc',
+                describe: 'max number of document sections to render (defaults to 1)',
+                type: 'count',
+                default: 1
+            })
+                .option('tokens', {
+                alias: 't',
+                describe: 'max number of tokens to render for each document section (defaults to 2000)',
+                type: 'count',
+                default: 2000
+            })
+                .option('format', {
+                alias: 'f',
+                describe: `format of the rendered results. Defaults to 'sections'`,
+                choices: ['sections', 'stats', 'chunks'],
+                default: 'sections'
+            })
+                .demandOption(['keys']);
+        }, (args) => __awaiter(this, void 0, void 0, function* () {
+            console.log(internals_1.Colorize.title('Querying Index'));
+            // Create embeddings
+            const keys = JSON.parse(yield fs.readFile(args.keys, 'utf-8'));
+            const embeddings = new OpenAIEmbeddings_1.OpenAIEmbeddings(Object.assign({ model: 'text-embedding-ada-002' }, keys));
+            // Initialize index
+            const folderPath = args.index;
+            const index = new LocalDocumentIndex_1.LocalDocumentIndex({
+                folderPath,
+                embeddings
+            });
+            // Query index
+            const query = args.query;
+            const results = yield index.queryDocuments(query, {
+                maxDocuments: args.documentCount,
+                maxChunks: args.chunkCount,
+            });
+            // Render results
+            for (const result of results) {
+                console.log(internals_1.Colorize.output(result.uri));
+                console.log(internals_1.Colorize.value('score', result.score));
+                console.log(internals_1.Colorize.value('chunks', result.chunks.length));
+                if (args.format == 'sections') {
+                    const sections = yield result.renderSections(args.tokens, args.sectionCount);
+                    for (let i = 0; i < sections.length; i++) {
+                        const section = sections[i];
+                        console.log(internals_1.Colorize.title(args.sectionCount > 1 ? 'Section' : `Section ${1}`));
+                        console.log(internals_1.Colorize.value('score', section.score));
+                        console.log(internals_1.Colorize.value('tokens', section.tokenCount));
+                        console.log(internals_1.Colorize.output(section.text));
+                    }
+                }
+                else if (args.format == 'chunks') {
+                    const text = yield result.loadText();
+                    for (let i = 0; i < result.chunks.length; i++) {
+                        const chunk = result.chunks[i];
+                        const startPos = chunk.item.metadata.startPos;
+                        const endPos = chunk.item.metadata.endPos;
+                        console.log(internals_1.Colorize.title(`Chunk ${i + 1}`));
+                        console.log(internals_1.Colorize.value('score', chunk.score));
+                        console.log(internals_1.Colorize.value('startPos', startPos));
+                        console.log(internals_1.Colorize.value('endPos', endPos));
+                        console.log(internals_1.Colorize.output(text.substring(startPos, endPos + 1)));
+                    }
+                }
+            }
+        }))
+            .help()
+            .demandCommand()
+            .parseAsync();
+    });
+}
+exports.run = run;
+function getItemList(items, listFile, uriType) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (Array.isArray(items) && items.length > 0) {
+            return items;
+        }
+        else if (typeof listFile == 'string' && listFile.trim().length > 0) {
+            const list = yield fs.readFile(listFile, 'utf-8');
+            return list.split('\n').map((item) => item.trim()).filter((item) => item.length > 0);
+        }
+        else {
+            throw new Error(`you must specify either one or more "--uri <${uriType}>" for the items or a "--list <file path>" for a file containing the items.`);
+        }
+    });
+}
+//# sourceMappingURL=vectra-cli.js.map