vectra 0.11.0 → 0.12.0

package/lib/LocalDocument.js

@@ -1,146 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDocument = void 0;
-const fs = __importStar(require("fs/promises"));
-const path = __importStar(require("path"));
-/**
- * Represents an indexed document stored on disk.
- */
-class LocalDocument {
-    /**
-     * Creates a new `LocalDocument` instance.
-     * @param index Parent index that contains the document.
-     * @param id ID of the document.
-     * @param uri URI of the document.
-     */
-    constructor(index, id, uri) {
-        this._index = index;
-        this._id = id;
-        this._uri = uri;
-    }
-    /**
-     * Returns the folder path where the document is stored.
-     */
-    get folderPath() {
-        return this._index.folderPath;
-    }
-    /**
-     * Returns the ID of the document.
-     */
-    get id() {
-        return this._id;
-    }
-    /**
-     * Returns the URI of the document.
-     */
-    get uri() {
-        return this._uri;
-    }
-    /**
-     * Returns the length of the document in tokens.
-     * @remarks
-     * This value will be estimated for documents longer then 40k bytes.
-     * @returns Length of the document in tokens.
-     */
-    getLength() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const text = yield this.loadText();
-            if (text.length <= 40000) {
-                return this._index.tokenizer.encode(text).length;
-            }
-            else {
-                return Math.ceil(text.length / 4);
-            }
-        });
-    }
-    /**
-     * Determines if the document has additional metadata storred on disk.
-     * @returns True if the document has metadata; otherwise, false.
-     */
-    hasMetadata() {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield fs.access(path.join(this.folderPath, `${this.id}.json`));
-                return true;
-            }
-            catch (err) {
-                return false;
-            }
-        });
-    }
-    /**
-     * Loads the metadata for the document from disk.
-     * @returns Metadata for the document.
-     */
-    loadMetadata() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (this._metadata == undefined) {
-                let json;
-                try {
-                    json = (yield fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
-                }
-                catch (err) {
-                    throw new Error(`Error reading metadata for document "${this.uri}": ${err.toString()}`);
-                }
-                try {
-                    this._metadata = JSON.parse(json);
-                }
-                catch (err) {
-                    throw new Error(`Error parsing metadata for document "${this.uri}": ${err.toString()}`);
-                }
-            }
-            return this._metadata;
-        });
-    }
-    /**
-     * Loads the text for the document from disk.
-     * @returns Text for the document.
-     */
-    loadText() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (this._text == undefined) {
-                try {
-                    this._text = (yield fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
-                }
-                catch (err) {
-                    throw new Error(`Error reading text file for document "${this.uri}": ${err.toString()}`);
-                }
-            }
-            return this._text;
-        });
-    }
-}
-exports.LocalDocument = LocalDocument;
-//# sourceMappingURL=LocalDocument.js.map

package/lib/LocalDocument.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalDocument.js","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAI7B;;GAEG;AACH,MAAa,aAAa;IAOtB;;;;;OAKG;IACH,YAAmB,KAAyB,EAAE,EAAU,EAAE,GAAW;QACjE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAW,EAAE;QACT,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAW,GAAG;QACV,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACU,SAAS;;YAClB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE;gBACtB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;aACpD;iBAAM;gBACH,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACrC;QACL,CAAC;KAAA;IAED;;;OAGG;IACU,WAAW;;YACpB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAED;;;OAGG;IACU,YAAY;;YACrB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE;gBAC7B,IAAI,IAAY,CAAC;gBACjB,IAAI;oBACA,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACxF;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;gBAED,IAAI;oBACA,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;iBACrC;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;aACJ;YAED,OAAO,IAAI,CAAC,SAAU,CAAC;QAC3B,CAAC;KAAA;IAED;;;OAGG;IACU,QAAQ;;YACjB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE;gBACzB,IAAI;oBACA,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC7F;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACrG;aACJ;YAED,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;KAAA;CACJ;AA1GD,sCA0GC"}

package/lib/LocalDocumentIndex.d.ts

@@ -1,132 +0,0 @@
-import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
-import { TextSplitterConfig } from "./TextSplitter";
-import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, DocumentChunkMetadata, DocumentCatalogStats } from "./types";
-import { LocalDocumentResult } from './LocalDocumentResult';
-import { LocalDocument } from './LocalDocument';
-/**
- * Options for querying documents in the index.
- */
-export interface DocumentQueryOptions {
-    /**
-     * Optional. Maximum number of documents to return.
-     * @remarks
-     * Default is 10.
-     */
-    maxDocuments?: number;
-    /**
-     * Maximum number of chunks to return per document.
-     * @remarks
-     * Default is 50.
-     */
-    maxChunks?: number;
-    /**
-     * Optional. Filter to apply to the document metadata.
-     */
-    filter?: MetadataFilter;
-    /**
-     * Optional. Turn on bm25 keyword search to perform hybrid search - semantic + keyword
-     */
-    isBm25?: boolean;
-}
-/**
- * Configuration settings for a local document index.
- */
-export interface LocalDocumentIndexConfig {
-    /**
-     * Folder path where the index is stored.
-     */
-    folderPath: string;
-    /**
-     * Optional. Embeddings model to use for generating document embeddings.
-     */
-    embeddings?: EmbeddingsModel;
-    /**
-     * Optional. Tokenizer to use for splitting text into tokens.
-     */
-    tokenizer?: Tokenizer;
-    /**
-     * Optional. Configuration settings for splitting text into chunks.
-     */
-    chunkingConfig?: Partial<TextSplitterConfig>;
-}
-/**
- * Represents a local index of documents stored on disk.
- */
-export declare class LocalDocumentIndex extends LocalIndex<DocumentChunkMetadata> {
-    private readonly _embeddings?;
-    private readonly _tokenizer;
-    private readonly _chunkingConfig?;
-    private _catalog?;
-    private _newCatalog?;
-    /**
-     * Creates a new `LocalDocumentIndex` instance.
-     * @param config Configuration settings for the document index.
-     */
-    constructor(config: LocalDocumentIndexConfig);
-    /**
-     * Returns the embeddings model used by the index (if configured.)
-     */
-    get embeddings(): EmbeddingsModel | undefined;
-    /**
-     * Returns the tokenizer used by the index.
-     */
-    get tokenizer(): Tokenizer;
-    /**
-     * Returns true if the document catalog exists.
-     */
-    isCatalogCreated(): Promise<boolean>;
-    /**
-     * Returns the document ID for the given URI.
-     * @param uri URI of the document to lookup.
-     * @returns Document ID or undefined if not found.
-     */
-    getDocumentId(uri: string): Promise<string | undefined>;
-    /**
-     * Returns the document URI for the given ID.
-     * @param documentId ID of the document to lookup.
-     * @returns Document URI or undefined if not found.
-     */
-    getDocumentUri(documentId: string): Promise<string | undefined>;
-    /**
-     * Loads the document catalog from disk and returns its stats.
-     * @returns Catalog stats.
-     */
-    getCatalogStats(): Promise<DocumentCatalogStats>;
-    /**
-     * Deletes a document from the index.
-     * @param uri URI of the document to delete.
-     */
-    deleteDocument(uri: string): Promise<void>;
-    /**
-     * Adds a document to the catalog.
-     * @remarks
-     * A new update is started if one is not already in progress. If an document with the same uri
-     * already exists, it will be replaced.
-     * @param uri - Document URI
-     * @param text - Document text
-     * @param docType - Optional. Document type
-     * @param metadata - Optional. Document metadata to index
-     * @returns Inserted document
-     */
-    upsertDocument(uri: string, text: string, docType?: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument>;
-    /**
-     * Returns all documents in the index.
-     * @remarks
-     * Each document will contain all of the documents indexed chunks.
-     * @returns Array of documents.
-     */
-    listDocuments(): Promise<LocalDocumentResult[]>;
-    /**
-     * Queries the index for documents similar to the given query.
-     * @param query Text to query for.
-     * @param options Optional. Query options.
-     * @returns Array of document results.
-     */
-    queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]>;
-    beginUpdate(): Promise<void>;
-    cancelUpdate(): void;
-    createIndex(config?: CreateIndexConfig): Promise<void>;
-    endUpdate(): Promise<void>;
-    protected loadIndexData(): Promise<void>;
-}
-//# sourceMappingURL=LocalDocumentIndex.d.ts.map

package/lib/LocalDocumentIndex.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAAmC,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;IAExB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAEpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE,eAAe,CAAC;IAE7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,UAAU,CAAC,qBAAqB,CAAC;IACrE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;IAEtC;;;OAGG;gBACgB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACH,IAAW,UAAU,IAAI,eAAe,GAAG,SAAS,CAEnD;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,SAAS,CAEhC;IAED;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IASjD;;;;OAIG;IACU,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKpE;;;;OAIG;IACU,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK5E;;;OAGG;IACU,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;OAGG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CvD;;;;;;;;;;OAUG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAiH1I;;;;;OAKG;IACU,aAAa,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAuB5D;;;;;OAKG;IACU,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAqD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAavB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BjD"}