xindex 1.0.0

package/rnd/keywords.ts

@@ -0,0 +1,38 @@
+import { createRequire } from "module";
+import { readFile } from "fs/promises";
+
+const require = createRequire(import.meta.url);
+const keyword_extractor = require("keyword-extractor");
+
+const filePath = process.argv[2];
+if (!filePath) {
+  console.error("Usage: npx tsx keywords.ts <file>");
+  process.exit(1);
+}
+
+const text = await readFile(filePath, "utf8");
+
+const keywords: string[] = keyword_extractor.extract(text, {
+  language: "english",
+  remove_digits: false,
+  return_changed_case: true,
+  remove_duplicates: true,
+  return_max_ngrams: 3,
+});
+
+// Count frequency of each keyword in original text (case-insensitive)
+const lower = text.toLowerCase();
+const scored = keywords
+  .filter((kw) => kw.length > 2 && !/^[^a-z]*$/.test(kw))
+  .map((kw) => {
+    const re = new RegExp(`\\b${kw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "gi");
+    const count = (lower.match(re) || []).length;
+    return { keyword: kw, count };
+  });
+
+scored.sort((a, b) => b.count - a.count);
+
+console.log(`Keywords from: ${filePath}\n`);
+for (const { keyword, count } of scored) {
+  console.log(`  ${keyword.padEnd(35)} (${count}x)`);
+}

package/rnd/test-vectra-memory.ts

@@ -0,0 +1,63 @@
+import { LocalIndex, VirtualFileStorage } from 'vectra';
+
+async function main() {
+    // 1. Create in-memory index
+    const storage = new VirtualFileStorage();
+    const index = new LocalIndex('mem://test', undefined, storage);
+    await index.createIndex({ version: 1 });
+    console.log('created index');
+
+    // 2. Insert items with fake 8-dim vectors
+    const dim = 8;
+    await index.insertItem({
+        id: 'doc-a',
+        vector: Array.from({ length: dim }, (_, i) => (i + 1) / dim),
+        metadata: { block: 'A', label: 'first' },
+    });
+    await index.insertItem({
+        id: 'doc-b',
+        vector: Array.from({ length: dim }, (_, i) => (dim - i) / dim),
+        metadata: { block: 'B', label: 'second' },
+    });
+    await index.insertItem({
+        id: 'doc-c',
+        vector: Array.from({ length: dim }, () => 0.5),
+        metadata: { block: 'C', label: 'third' },
+    });
+    console.log('inserted 3 items');
+
+    // 3. Query — vector close to doc-a
+    const queryVec = Array.from({ length: dim }, (_, i) => (i + 1) / dim);
+    const results = await index.queryItems(queryVec, '', 3);
+    console.log('query results:');
+    for (const r of results) {
+        console.log(`  ${r.item.id}  score=${r.score.toFixed(4)}  block=${r.item.metadata.block}`);
+    }
+
+    // 4. Upsert — update doc-a's vector
+    await index.upsertItem({
+        id: 'doc-a',
+        vector: Array.from({ length: dim }, () => 0.5),
+        metadata: { block: 'A', label: 'updated' },
+    });
+    const updated = await index.getItem('doc-a');
+    console.log(`upserted doc-a → label=${updated?.metadata.label}`);
+
+    // 5. List all items
+    const all = await index.listItems();
+    console.log(`total items: ${all.length}`);
+
+    // 6. Delete
+    await index.deleteItem('doc-b');
+    const afterDelete = await index.listItems();
+    console.log(`after delete: ${afterDelete.length} items`);
+
+    // 7. Verify no disk artifacts
+    const { default: fs } = await import('fs');
+    const exists = fs.existsSync('mem://test');
+    console.log(`disk folder exists: ${exists}`);
+
+    console.log('\ndone — all in-memory, nothing on disk');
+}
+
+main().catch(console.error);

package/rnd/vectra-keywords.ts

@@ -0,0 +1,95 @@
+import { LocalIndex } from "vectra";
+import { pipeline } from "@huggingface/transformers";
+import nlp from "compromise";
+import { createRequire } from "module";
+import { readFile } from "fs/promises";
+import { readdirSync } from "fs";
+
+const require = createRequire(import.meta.url);
+const keyword_extractor = require("keyword-extractor");
+
+// Init embedder + index
+const embedder = await pipeline(
+  "feature-extraction",
+  "sentence-transformers/all-MiniLM-L6-v2"
+);
+const index = new LocalIndex("./vectra-keyword-index");
+if (!(await index.isIndexCreated())) {
+  await index.createIndex();
+}
+
+async function embed(text: string): Promise<number[]> {
+  const result = await embedder(text, { pooling: "mean", normalize: true });
+  return Array.from(result.data as Float32Array);
+}
+
+function extractKeywords(text: string): string[] {
+  // Compromise: pull nouns, verbs, topics
+  const doc = nlp(text);
+  const parts = [
+    ...doc.topics().out("array"),
+    ...doc.nouns().out("array"),
+    ...doc.verbs().out("array"),
+  ] as string[];
+  const cleaned = parts.join(" ").replace(/\W+/g, " ").trim();
+
+  // keyword-extractor
+  const keywords: string[] = keyword_extractor.extract(cleaned, {
+    language: "english",
+    remove_digits: false,
+    return_changed_case: true,
+    remove_duplicates: true,
+    return_max_ngrams: 2,
+  });
+  return keywords.filter((kw: string) => kw.length > 2);
+}
+
+// --- Index files ---
+const files = process.argv.slice(2);
+if (!files.length) {
+  console.error("Usage: npx tsx vectra-keywords.ts <file1> [file2] ...");
+  process.exit(1);
+}
+
+console.log("=== Indexing ===");
+for (const filePath of files) {
+  const text = await readFile(filePath, "utf8");
+  const keywords = extractKeywords(text);
+  const keywordStr = keywords.join(", ");
+  const vector = await embed(keywordStr);
+
+  await index.upsertItem({
+    id: filePath,
+    vector,
+    metadata: { keywords: keywordStr, file: filePath },
+  });
+  console.log(`  ${filePath} → ${keywords.slice(0, 8).join(", ")}...`);
+}
+
+// --- Search by keyword ---
+async function search(query: string, topK = 5) {
+  const queryVector = await embed(query);
+  return index.queryItems(queryVector, query, topK);
+}
+
+console.log("\n=== Search: keyword 'fruit' ===");
+const r1 = await search("fruit");
+for (const r of r1) {
+  console.log(`  ${r.score.toFixed(4)} | ${r.item.id}`);
+  console.log(`    keywords: ${(r.item.metadata as any).keywords.slice(0, 100)}`);
+}
+
+// --- Search by synonym ---
+console.log("\n=== Search: synonym 'automobile' (for 'cars/vehicles') ===");
+const r2 = await search("automobile vehicle transportation");
+for (const r of r2) {
+  console.log(`  ${r.score.toFixed(4)} | ${r.item.id}`);
+  console.log(`    keywords: ${(r.item.metadata as any).keywords.slice(0, 100)}`);
+}
+
+console.log("\n=== Search: synonym 'embedding model neural network' ===");
+const r3 = await search("embedding model neural network");
+for (const r of r3) {
+  console.log(`  ${r.score.toFixed(4)} | ${r.item.id}`);
+  console.log(`    keywords: ${(r.item.metadata as any).keywords.slice(0, 100)}`);
+}

package/rnd/vectra.ts

@@ -0,0 +1,50 @@
+import { LocalIndex } from "vectra";
+import { pipeline } from "@huggingface/transformers";
+
+// Create/load index (folder-based)
+const index = new LocalIndex("./vectra-index");
+
+if (!(await index.isIndexCreated())) {
+  await index.createIndex();
+}
+
+// Embedding pipeline (fast local model)
+const embedder = await pipeline(
+  "feature-extraction",
+  "sentence-transformers/all-MiniLM-L6-v2"
+);
+
+async function embed(text: string): Promise<number[]> {
+  const result = await embedder(text, { pooling: "mean", normalize: true });
+  return Array.from(result.data as Float32Array);
+}
+
+// Add items with vectors + metadata
+const texts = [
+  { text: "Apples are red fruit", metadata: { category: "fruit" } },
+  { text: "Bananas are yellow", metadata: { category: "fruit" } },
+  { text: "Cars are vehicles", metadata: { category: "transport" } },
+];
+
+for (const item of texts) {
+  const vector = await embed(item.text);
+  await index.upsertItem({
+    id: item.text.slice(0, 20),
+    vector,
+    metadata: item.metadata,
+  });
+}
+console.log("Items indexed");
+
+// Query example
+const queryText = "red fruit";
+const queryVector = await embed(queryText);
+const results = await index.queryItems(queryVector, queryText, 3, {
+  category: { $eq: "fruit" },
+});
+
+for (const result of results) {
+  console.log(
+    `Score: ${result.score.toFixed(4)} | ID: ${result.item.id} | Metadata: ${JSON.stringify(result.item.metadata)}`
+  );
+}

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "esModuleInterop": true,
+    "strict": true,
+    "outDir": "dist",
+    "rootDir": ".",
+    "skipLibCheck": true
+  },
+  "include": ["*.ts"],
+  "exclude": ["node_modules", "dist"]
+}