vectra 0.12.2 → 0.12.3

package/src/LocalIndex.spec.ts

@@ -1,4 +1,5 @@
 import assert from 'node:assert'
+import sinon from 'sinon'
 import { LocalIndex } from './LocalIndex'
 import { IndexItem } from './types'
 import fs from 'fs/promises'
@@ -7,12 +8,20 @@ import path from 'path'
 describe('LocalIndex', () => {
   const testIndexDir = path.join(__dirname, 'test_index');
 
+  const basicIndexItems: Partial<IndexItem>[] = [
+    { id: '1', vector: [1, 2, 3] },
+    { id: '2', vector: [2, 3, 4] },
+    { id: '3', vector: [3, 4, 5] }
+  ];
+
+
   beforeEach(async () => {
     await fs.rm(testIndexDir, { recursive: true, force: true });
   });
 
   afterEach(async () => {
     await fs.rm(testIndexDir, { recursive: true, force: true });
+    sinon.restore();
   });
 
   it('should create a new index', async () => {
@@ -20,20 +29,171 @@ describe('LocalIndex', () => {
     await index.createIndex();
     const created = await index.isIndexCreated();
     assert.equal(created, true);
+    assert.equal(index.folderPath, testIndexDir);
   });
 
-  describe('batchInsertItems', () => {
-    const indexItems: Partial<IndexItem>[] = [
-      { id: '1', vector: [1, 2, 3] },
-      { id: '2', vector: [2, 3, 4] },
-      { id: '3', vector: [3, 4, 5] }
-    ];
+  it('blocks concurrent operations when lock is held', async () => {
+    const index = new LocalIndex(testIndexDir);
+    await index.createIndex();
+    await index.beginUpdate(); // grab lock for a big update!
+    await assert.rejects(async () => {
+      await index.beginUpdate(); // try to grab lock again. should fail!
+    }, new Error('Update already in progress'))
+  })
+
+  describe('createIndex', () => {
+    it('checks for existing index on creation', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex(); // create first index.json
+
+      // create without deleteIfExists. Will reject
+      await assert.rejects(async () => {
+        await index.createIndex()
+      }, new Error('Index already exists'))
+
+      // create with deleteIfExists. Should remove old data
+      await index.insertItem({id:'1', vector: [1,2,3]})
+      const lengthBefore = (await index.listItems()).length
+      assert.equal(lengthBefore, 1)
+      await index.createIndex({deleteIfExists: true, version: 2, metadata_config: {}})
+      const lengthAfter = (await index.listItems()).length
+      assert.equal(lengthAfter, 0)
+    })
+
+    it('delete index if file creation fails', async () => {
+      const index = new LocalIndex(testIndexDir);
+      sinon.stub(fs, 'writeFile').rejects(new Error('fs error'))
+
+      await assert.rejects(async () => {
+        await index.createIndex();
+      }, new Error('Error creating index'))
+
+      await assert.rejects(async () => {
+        await index.listItems();
+      })
+    })
+  })
+
+  describe('deleteItem', () => {
+    it('does nothing when id not found', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.beginUpdate();
+      await index.insertItem(basicIndexItems[0])
+      await index.insertItem(basicIndexItems[1])
+      await index.insertItem(basicIndexItems[2])
+      await index.endUpdate();
+
+      await assert.doesNotReject(async () => {
+        await index.deleteItem('dne');
+      })
+      assert.equal((await index.listItems()).length, 3)
+    })
+
+    it('leaves existing empty index when last el deleted', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.insertItem(basicIndexItems[0]);
+
+      await index.deleteItem(basicIndexItems[0].id ?? '');
+      assert.equal(await index.isIndexCreated(), true);
+      assert.equal((await index.listItems()).length, 0);
+    })
+
+    it('removes elements from any position', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.batchInsertItems([
+        {id: '1', vector: []},
+        {id: '2', vector: []},
+        {id: '3', vector: []},
+        {id: '4', vector: []},
+        {id: '5', vector: []},
+      ]);
+
+      await index.beginUpdate();
+      await index.deleteItem('1');
+      await index.deleteItem('3');
+      await index.deleteItem('5');
+      await index.endUpdate();
+
+      assert.deepStrictEqual(await index.listItems(), [{id: '2', vector: [], metadata: {}, norm: 0}, {id: '4', vector: [], metadata: {}, norm: 0}])
+    })
+  })
+
+  describe('endUpdate', () => {
+    it('throws an error if no update has begun', async () => {
+      const index = new LocalIndex(testIndexDir);
+
+      await assert.rejects(async () => {
+        await index.endUpdate();
+      }, new Error('No update in progress'));
+    })
+
+    it('throws an error if the index could not be saved', async () => {
+      const index = new LocalIndex(testIndexDir, 'index.json');
+      await index.createIndex();
+      await index.beginUpdate();
+
+      sinon.stub(fs, 'writeFile').rejects(new Error('fs error'))
+
+      await assert.rejects(async () => {
+        await index.endUpdate();
+      }, new Error('Error saving index: Error: fs error'))
+    })
+  })
+
+  describe('getIndexStats', () => {
+    it('reports empty index correctly', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+
+      assert.deepStrictEqual(await index.getIndexStats(), {
+        version: 1,
+        metadata_config: {},
+        items: 0
+      })
+    })
+
+    it('correctly reports non-empty index stats', async () => {
+      const index = new LocalIndex(testIndexDir)
+      await index.createIndex({version: 1, metadata_config: {indexed: []}})
+      await index.batchInsertItems(basicIndexItems);
+
+      assert.deepStrictEqual(await index.getIndexStats(), {
+        version: 1,
+        metadata_config: {indexed: []},
+        items: 3
+      })
+    })
+  })
+
+  describe('getItem', () => {
+    it('returns undefined when item not found', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+
+      assert.equal(await index.getItem('1'), undefined)
+    })
 
+    it('returns requested item', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.batchInsertItems(basicIndexItems);
+
+      const item2 = await index.getItem('2');
+      assert.equal(item2?.id, basicIndexItems[1].id)
+      assert.equal(item2?.vector, basicIndexItems[1].vector)
+      assert.equal((await index.listItems()).length, 3)
+    })
+  })
+
+  describe('batchInsertItems', () => {
     it('should insert provided items', async () => {
       const index = new LocalIndex(testIndexDir);
       await index.createIndex();
 
-      const newItems = await index.batchInsertItems(indexItems);
+      const newItems = await index.batchInsertItems(basicIndexItems);
 
       assert.equal(newItems.length, 3);
 
@@ -50,7 +210,7 @@ describe('LocalIndex', () => {
       // ensures insert error is bubbled up to batchIndexItems caller
       await assert.rejects(
         async () => {
-          await index.batchInsertItems(indexItems);
+          await index.batchInsertItems(basicIndexItems);
         },
         {
           name: 'Error',
@@ -63,4 +223,101 @@ describe('LocalIndex', () => {
       assert.equal(storedItems.length, 1);
     });
   });
+
+  describe('listItemsByMetadata', () => {
+    it('returns items matching metadata filter', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.batchInsertItems([
+        {id: '1', vector: [], metadata: {category: 'food'}},
+        {id: '2', vector: [], metadata: {category: 'food'}},
+        {id: '3', vector: [], metadata: {category: 'electronics'}},
+        {id: '4', vector: [], metadata: {category: 'drink'}},
+        {id: '5', vector: [], metadata: {category: 'food'}},
+      ]);
+
+      const foodItems = await index.listItemsByMetadata({category: {'$eq': 'food'}})
+      assert.deepStrictEqual(foodItems.map((item) => item.id), ["1", "2", "5"])
+      const drinkItems = await index.listItemsByMetadata({category: {'$eq': 'drink'}})
+      assert.deepStrictEqual(drinkItems.map((item) => item.id), ["4"])
+      const clothingItems = await index.listItemsByMetadata({category: {'$eq': 'clothes'}})
+      assert.deepStrictEqual(clothingItems, [])
+    })
+
+    it('returns nothing when no items in index', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+
+      const items = await index.listItemsByMetadata({});
+      assert.deepStrictEqual(items, []);
+    })
+  });
+
+  describe("queryItems", () => {
+    it("returns empty array on empty index search", async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+
+      const result = await index.queryItems([1, 2, 3], "", 10);
+      assert.deepStrictEqual(result, []);
+    });
+
+    it("returns bad match when no better match exists", async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.insertItem({ id: "1", vector: [0.9, 0, 0, 0, 0] });
+
+      const result = await index.queryItems([0, 0, 0, 0, 0.1], "", 1);
+      assert.equal(result[0]?.score, 0);
+      assert.equal(result[0]?.item.id, "1");
+    });
+
+    it("returns all vectors when fewer than topK exist", async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.batchInsertItems(basicIndexItems);
+
+      const result = await index.queryItems([0, 0, 1], "", 10);
+      assert.equal(result.length, 3);
+      assert.deepStrictEqual(
+        result.map(({ item }) => item.id),
+        basicIndexItems.map((item) => item.id),
+      );
+    });
+
+    it("filters by metadata when filter provided", async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+      await index.batchInsertItems([
+        { id: "1", vector: [1, 0, 0], metadata: { category: "food" } },
+        { id: "2", vector: [0, 0, 1], metadata: { category: "drink" } },
+      ]);
+
+      const bestGeneralMatch = await index.queryItems([1, 0, 0], "", 1);
+      const bestDrinkMatch = await index.queryItems([1, 0, 0], "", 1, {
+        category: { $eq: "drink" },
+      });
+
+      assert.equal(bestGeneralMatch[0].item.id, "1");
+      assert.equal(bestDrinkMatch[0].item.id, "2");
+    });
+
+    it("reads item metadata file when provided", async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex({version: 1, metadata_config: {indexed: ['category']}});
+      await index.batchInsertItems([
+        { id: "1", vector: [1, 0, 0] },
+        { id: "2", vector: [0, 0, 1], metadata: {category: 'drink'} },
+      ]);
+
+      sinon
+        .stub(fs, "readFile")
+        .resolves(JSON.stringify({ category: "drink" }));
+
+      const bestDrinkMatch = await index.queryItems([1, 0, 0], "", 2, {category: {'$eq': 'drink'}});
+
+      assert.notEqual(bestDrinkMatch[0].item.metadataFile, undefined);
+      assert.equal(bestDrinkMatch[0].item.id, "2");
+    });
+  });
 });

package/src/LocalIndex.ts

@@ -275,6 +275,7 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
      * This method loads the index into memory and returns the top k items that are most similar.
      * An optional filter can be applied to the metadata of the items.
      * @param vector Vector to query against.
+     * @param query Query text used for BM25 keyword search.
      * @param topK Number of items to return.
      * @param filter Optional. Filter to apply.
      * @returns Similar items to the vector that matche the supplied filter.

package/src/TextSplitter.spec.ts

@@ -0,0 +1,87 @@
+import { describe, it } from 'mocha';
+import * as assert from 'node:assert';
+import { TextSplitter } from './TextSplitter';
+
+describe('TextSplitter', () => {
+  const makeSplitter = (opts?: Partial<ConstructorParameters<typeof TextSplitter>[0]>) =>
+    new TextSplitter({ chunkSize: 16, chunkOverlap: 0, ...opts });
+
+  it('keeps a leading punctuation-only chunk ("---")', () => {
+    const splitter = makeSplitter({ chunkSize: 3, chunkOverlap: 0 });
+    const chunks = splitter.split('---');
+    assert.deepStrictEqual(chunks.map(c => c.text), ['---']);
+  });
+
+  it('keeps punctuation-only separators (---, ***, ====) at start, middle, and end', () => {
+    const splitter = makeSplitter({ chunkSize: 4, chunkOverlap: 0 });
+    const text = ['---', 'Hello world', '***', 'Middle', '===='].join('\n');
+    const chunks = splitter.split(text);
+
+    assert.ok(chunks.some(c => c.text.includes('---')));
+    assert.ok(chunks.some(c => c.text.includes('***')));
+    assert.ok(chunks.some(c => c.text.includes('====')));
+  });
+
+  it('preserves frontmatter delimiters when chunk size is small and overlap is zero', () => {
+    const splitter = makeSplitter({ chunkSize: 12, chunkOverlap: 0 });
+    const md = [
+      '---',
+      'title: Test',
+      'tags: [a, b]',
+      '---',
+      '# Heading',
+      'Body text goes here.'
+    ].join('\n');
+
+    const chunks = splitter.split(md);
+    const joined = chunks.map(c => c.text).join('\n');
+
+    const delimiterCount = (joined.match(/^---$/gm) ?? []).length;
+    assert.strictEqual(delimiterCount, 2);
+  });
+
+  it('keeps trailing punctuation-only chunk', () => {
+    const splitter = makeSplitter({ chunkSize: 4, chunkOverlap: 0 });
+    const chunks = splitter.split('Content\n---');
+    assert.ok(chunks.some(c => c.text.includes('---')));
+  });
+
+  it('drops pure whitespace-only chunks', () => {
+    const splitter = makeSplitter({ chunkSize: 10, chunkOverlap: 0 });
+    const chunks1 = splitter.split('   \t  ');
+    const chunks2 = splitter.split('\n\n');
+    const chunks3 = splitter.split(' \n \n  ');
+    assert.strictEqual(chunks1.length, 0);
+    assert.strictEqual(chunks2.length, 0);
+    assert.strictEqual(chunks3.length, 0);
+  });
+
+  it('still returns alphanumeric chunks normally', () => {
+    const splitter = makeSplitter({ chunkSize: 5, chunkOverlap: 0 });
+    const chunks = splitter.split('abcde fghij');
+    assert.ok(chunks.length > 0);
+    assert.ok(chunks.map(c => c.text).join(' ').includes('abcde'));
+    assert.ok(chunks.map(c => c.text).join(' ').includes('fghij'));
+  });
+
+  it('does not regress with non-zero overlap', () => {
+    const splitter = makeSplitter({ chunkSize: 5, chunkOverlap: 2 });
+    const chunks = splitter.split('---\nabcdef');
+    assert.ok(chunks.some(c => c.text.includes('---')));
+  });
+
+  it('handles multiple punctuation-only separators interleaved with content', () => {
+    const splitter = makeSplitter({ chunkSize: 8, chunkOverlap: 0 });
+    const text = ['***', 'A', '---', 'B', '====', 'C'].join('\n');
+    const chunks = splitter.split(text);
+
+    assert.ok(chunks.some(c => c.text.includes('***')));
+    assert.ok(chunks.some(c => c.text.includes('---')));
+    assert.ok(chunks.some(c => c.text.includes('====')));
+
+    const joined = chunks.map(c => c.text).join('\n');
+    assert.ok(joined.includes('\nA\n'));
+    assert.ok(joined.includes('\nB\n'));
+    assert.ok(joined.includes('\nC'));
+  });
+});