vectra 0.12.2 → 0.14.0

package/src/LocalDocument.spec.ts

@@ -0,0 +1,211 @@
+import { strict as assert } from "node:assert";
+import path from "./utils/pathUtils";
+import { describe, it, beforeEach } from "mocha";
+import * as LocalDocumentModule from "../src/LocalDocument";
+
+// Support both default and named export styles
+type LocalDocumentCtor = new (index: any, id: string, uri: string) => any;
+const LocalDocument: LocalDocumentCtor =
+  (LocalDocumentModule as any).LocalDocument ??
+  (LocalDocumentModule as any).default;
+
+if (!LocalDocument) {
+  throw new Error(
+    'Unable to import LocalDocument (neither named "LocalDocument" nor default export found).'
+  );
+}
+
+describe("LocalDocument", () => {
+  let calls: { method: string; args: any[] }[];
+  let indexStub: any;
+  let storageStub: any;
+  let tokenizerStub: any;
+  let doc: InstanceType<LocalDocumentCtor>;
+  const folderPath = "/folder";
+  const id = "doc1";
+  const uri = "file:///doc1";
+
+  beforeEach(() => {
+    calls = [];
+    storageStub = {
+      readFile: async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        if (filePath.endsWith(".txt")) {
+          return Buffer.from("hello world", "utf8");
+        }
+        if (filePath.endsWith(".json")) {
+          return Buffer.from(JSON.stringify({ key: "value" }), "utf8");
+        }
+        throw new Error("File not found");
+      },
+      pathExists: async (filePath: string) => {
+        calls.push({ method: "pathExists", args: [filePath] });
+        return filePath.endsWith(".json");
+      },
+    };
+    tokenizerStub = {
+      encode: (text: string) => {
+        calls.push({ method: "encode", args: [text] });
+        return Array.from(text);
+      },
+    };
+    indexStub = {
+      folderPath,
+      storage: storageStub,
+      tokenizer: tokenizerStub,
+    };
+    doc = new LocalDocument(indexStub, id, uri);
+  });
+
+  it("constructor and getters", () => {
+    assert.equal(doc.id, id);
+    assert.equal(doc.uri, uri);
+    assert.equal(doc.folderPath, folderPath);
+  });
+
+  describe("loadText", () => {
+    it("loads and caches text", async () => {
+      const text1 = await doc.loadText();
+      const text2 = await doc.loadText();
+      assert.equal(text1, "hello world");
+      assert.equal(text2, "hello world");
+
+      const readFileCalls = calls.filter((c) => c.method === "readFile");
+      assert.equal(readFileCalls.length, 1);
+      assert.equal(
+        readFileCalls[0].args[0],
+        path.join(folderPath, `${id}.txt`)
+      );
+    });
+
+    it("throws on read error with uri in message", async () => {
+      storageStub.readFile = async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        throw new Error("read error");
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+      await assert.rejects(
+        () => doc2.loadText(),
+        (err: any) =>
+          String(err.message).includes(
+            `Error reading text file for document "${uri}":`
+          )
+      );
+    });
+  });
+
+  describe("hasMetadata", () => {
+    it("returns true if metadata exists, using correct path", async () => {
+      const result = await doc.hasMetadata();
+      assert.equal(result, true);
+      const pe = calls.find((c) => c.method === "pathExists");
+      assert.ok(pe);
+      assert.equal(pe!.args[0], path.join(folderPath, `${id}.json`));
+    });
+
+    it("returns false if metadata does not exist", async () => {
+      storageStub.pathExists = async (filePath: string) => {
+        calls.push({ method: "pathExists", args: [filePath] });
+        return false;
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+      const result = await doc2.hasMetadata();
+      assert.equal(result, false);
+    });
+  });
+
+  describe("loadMetadata", () => {
+    it("loads and caches metadata, using correct path", async () => {
+      const meta1 = await doc.loadMetadata();
+      const meta2 = await doc.loadMetadata();
+      assert.deepEqual(meta1, { key: "value" });
+      assert.equal(meta1, meta2);
+
+      const jsonCalls = calls.filter(
+        (c) => c.method === "readFile" && c.args[0].endsWith(".json")
+      );
+      assert.equal(jsonCalls.length, 1);
+      assert.equal(jsonCalls[0].args[0], path.join(folderPath, `${id}.json`));
+    });
+
+    it("throws on read error with uri in message", async () => {
+      storageStub.readFile = async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        if (filePath.endsWith(".json")) throw new Error("read error");
+        return Buffer.from("ok", "utf8");
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+      await assert.rejects(
+        () => doc2.loadMetadata(),
+        (err: any) =>
+          String(err.message).includes(
+            `Error reading metadata for document "${uri}":`
+          )
+      );
+    });
+
+    it("throws on parse error with uri in message", async () => {
+      storageStub.readFile = async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        if (filePath.endsWith(".json")) return Buffer.from("{", "utf8");
+        return Buffer.from("ok", "utf8");
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+      await assert.rejects(
+        () => doc2.loadMetadata(),
+        (err: any) =>
+          String(err.message).includes(
+            `Error parsing metadata for document "${uri}":`
+          )
+      );
+    });
+  });
+
+  describe("getLength", () => {
+    it("uses tokenizer.encode for small text (≤ 40,000 chars)", async () => {
+      tokenizerStub.encode = (text: string) => {
+        calls.push({ method: "encode", args: [text] });
+        return new Array(5);
+      };
+      storageStub.readFile = async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        return Buffer.from("small text", "utf8");
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+      const length = await doc2.getLength();
+      assert.equal(length, 5);
+      assert.ok(calls.some((c) => c.method === "encode"));
+    });
+
+    it("estimates length for large text (> 40,000 chars) without encoding", async () => {
+      const largeText = "a".repeat(40001);
+      storageStub.readFile = async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        return Buffer.from(largeText, "utf8");
+      };
+      tokenizerStub.encode = () => {
+        throw new Error("encode should not be called for large text");
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+      const length = await doc2.getLength();
+      assert.equal(length, Math.ceil(40001 / 4));
+    });
+
+    it("reuses cached text across calls (no extra reads)", async () => {
+      tokenizerStub.encode = (text: string) => new Array(text.length);
+      storageStub.readFile = async (filePath: string) => {
+        calls.push({ method: "readFile", args: [filePath] });
+        return Buffer.from("small text", "utf8");
+      };
+      const doc2 = new LocalDocument(indexStub, id, uri);
+
+      const length1 = await doc2.getLength();
+      const length2 = await doc2.getLength();
+      assert.equal(length1, length2);
+
+      const readFileCalls = calls.filter((c) => c.method === "readFile");
+      assert.equal(readFileCalls.length, 1);
+      assert.equal(readFileCalls[0].args[0], path.join(folderPath, `${id}.txt`));
+    });
+  });
+});

package/src/LocalDocument.ts

@@ -1,5 +1,4 @@
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import { pathUtils as path } from './utils/pathUtils';
 import { MetadataTypes } from './types';
 import { LocalDocumentIndex } from './LocalDocumentIndex';
 
@@ -7,109 +6,104 @@ import { LocalDocumentIndex } from './LocalDocumentIndex';
  * Represents an indexed document stored on disk.
  */
 export class LocalDocument {
-    private readonly _index: LocalDocumentIndex;
-    private readonly _id: string;
-    private readonly _uri: string;
-    private _metadata: Record<string,MetadataTypes>|undefined;
-    private _text: string|undefined;
+  private readonly _index: LocalDocumentIndex;
+  private readonly _id: string;
+  private readonly _uri: string;
+  private _metadata: Record<string, MetadataTypes> | undefined;
+  private _text: string | undefined;
 
-    /**
-     * Creates a new `LocalDocument` instance.
-     * @param index Parent index that contains the document.
-     * @param id ID of the document.
-     * @param uri URI of the document.
-     */
-    public constructor(index: LocalDocumentIndex, id: string, uri: string) {
-        this._index = index;
-        this._id = id;
-        this._uri = uri;
-    }
+  /**
+   * Creates a new `LocalDocument` instance.
+   * @param index Parent index that contains the document.
+   * @param id ID of the document.
+   * @param uri URI of the document.
+   */
+  public constructor(index: LocalDocumentIndex, id: string, uri: string) {
+    this._index = index;
+    this._id = id;
+    this._uri = uri;
+  }
 
-    /**
-     * Returns the folder path where the document is stored.
-     */
-    public get folderPath(): string {
-        return this._index.folderPath;
-    }
+  /**
+   * Returns the folder path where the document is stored.
+   */
+  public get folderPath(): string {
+    return this._index.folderPath;
+  }
 
-    /**
-     * Returns the ID of the document.
-     */
-    public get id(): string {
-        return this._id;
-    }
+  /**
+   * Returns the ID of the document.
+   */
+  public get id(): string {
+    return this._id;
+  }
 
-    /**
-     * Returns the URI of the document.
-     */
-    public get uri(): string {
-        return this._uri;
-    }
+  /**
+   * Returns the URI of the document.
+   */
+  public get uri(): string {
+    return this._uri;
+  }
 
-    /**
-     * Returns the length of the document in tokens.
-     * @remarks
-     * This value will be estimated for documents longer then 40k bytes.
-     * @returns Length of the document in tokens.
-     */
-    public async getLength(): Promise<number> {
-        const text = await this.loadText();
-        if (text.length <= 40000) {
-            return this._index.tokenizer.encode(text).length;
-        } else {
-            return Math.ceil(text.length / 4);
-        }
+  /**
+   * Returns the length of the document in tokens.
+   * @remarks
+   * This value will be estimated for documents longer then 40k bytes.
+   * @returns Length of the document in tokens.
+   */
+  public async getLength(): Promise<number> {
+    const text = await this.loadText();
+    if (text.length <= 40000) {
+      return this._index.tokenizer.encode(text).length;
+    } else {
+      return Math.ceil(text.length / 4);
     }
+  }
 
-    /**
-     * Determines if the document has additional metadata storred on disk.
-     * @returns True if the document has metadata; otherwise, false.
-     */
-    public async hasMetadata(): Promise<boolean> {
-        try {
-            await fs.access(path.join(this.folderPath, `${this.id}.json`));
-            return true;
-        } catch (err: unknown) {
-            return false;
-        }
-    }
-
-    /**
-     * Loads the metadata for the document from disk.
-     * @returns Metadata for the document.
-     */
-    public async loadMetadata(): Promise<Record<string,MetadataTypes>> {
-        if (this._metadata == undefined) {
-            let json: string;
-            try {
-                json = (await fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
-            } catch (err: unknown) {
-                throw new Error(`Error reading metadata for document "${this.uri}": ${(err as any).toString()}`);
-            }
+  /**
+   * Determines if the document has additional metadata stored on disk.
+   * @returns True if the document has metadata; otherwise, false.
+   */
+  public async hasMetadata(): Promise<boolean> {
+    return this._index.storage.pathExists(path.join(this.folderPath, `${this.id}.json`));
+  }
 
-            try {
-                this._metadata = JSON.parse(json);
-            } catch (err: unknown) {
-                throw new Error(`Error parsing metadata for document "${this.uri}": ${(err as any).toString()}`);
-            }
-        }
+  /**
+   * Loads the metadata for the document from disk.
+   * @returns Metadata for the document.
+   */
+  public async loadMetadata(): Promise<Record<string, MetadataTypes>> {
+    if (this._metadata == undefined) {
+      let json: string;
+      try {
+        json = (await this._index.storage.readFile(path.join(this.folderPath, `${this.id}.json`))).toString('utf-8');
+      } catch (err: unknown) {
+        throw new Error(`Error reading metadata for document "${this.uri}": ${(err as any).toString()}`);
+      }
 
-        return this._metadata!;
+      try {
+        this._metadata = JSON.parse(json);
+      } catch (err: unknown) {
+        throw new Error(`Error parsing metadata for document "${this.uri}": ${(err as any).toString()}`);
+      }
     }
 
-    /**
-     * Loads the text for the document from disk.
-     * @returns Text for the document.
-     */
-    public async loadText(): Promise<string> {
-        if (this._text == undefined) {
-            try {
-                this._text = (await fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
-            } catch (err: unknown) {
-                throw new Error(`Error reading text file for document "${this.uri}": ${(err as any).toString()}`);
-            }
-        }
+    return this._metadata!;
+  }
 
-        return this._text;
+  /**
+   * Loads the text for the document from disk.
+   * @returns Text for the document.
+   */
+  public async loadText(): Promise<string> {
+    if (this._text == undefined) {
+      try {
+        this._text = (await this._index.storage.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString('utf-8');
+      } catch (err: unknown) {
+        throw new Error(`Error reading text file for document "${this.uri}": ${(err as any).toString()}`);
+      }
     }
+
+    return this._text;
+  }
 }