vesper-wizard 2.0.5 → 2.0.7

package/scripts/preindex_registry.cjs

@@ -0,0 +1,157 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const { argv, cwd } = process;
+
+function usage() {
+  console.log(`Usage: node scripts/preindex_registry.cjs [--scan dir1 dir2 ...] [--target N] [--out path] [--no-count]
+
+Options:
+  --scan    Directories to recursively scan for datasets (default: ./e2e_demo_output ./datasets)
+  --target  Target total registry entries (if larger than scanned, will synthesize entries)
+  --out     Output registry path (default: ~/.vesper/registry.json)
+  --no-count  Skip expensive row counting for CSV/JSONL
+`);
+}
+
+let scanDirs = [];
+let target = 0;
+let outPath = path.join(os.homedir(), '.vesper', 'registry.json');
+let doCount = true;
+
+for (let i = 2; i < argv.length; i++) {
+  const a = argv[i];
+  if (a === '--scan') {
+    i++;
+    while (i < argv.length && !argv[i].startsWith('--')) {
+      scanDirs.push(argv[i]);
+      i++;
+    }
+    i--;
+  } else if (a === '--target') {
+    target = parseInt(argv[++i], 10) || 0;
+  } else if (a === '--out') {
+    outPath = path.resolve(argv[++i]);
+  } else if (a === '--no-count') {
+    doCount = false;
+  } else if (a === '--help' || a === '-h') {
+    usage();
+    process.exit(0);
+  } else {
+    console.error('Unknown arg', a);
+    usage();
+    process.exit(2);
+  }
+}
+
+if (scanDirs.length === 0) scanDirs = [path.join(cwd(), 'e2e_demo_output'), path.join(cwd(), 'datasets')];
+
+function normalizeId(s) {
+  return s.replace(/[^a-z0-9]+/gi, '_').replace(/^_+|_+$/g, '').toLowerCase();
+}
+
+function walk(dir, exts = ['.csv', '.jsonl', '.json', '.arrow', '.parquet', '.feather']) {
+  const results = [];
+  try {
+    const items = fs.readdirSync(dir, { withFileTypes: true });
+    for (const it of items) {
+      const p = path.join(dir, it.name);
+      if (it.isDirectory()) results.push(...walk(p, exts));
+      else if (it.isFile()) {
+        const ext = path.extname(it.name).toLowerCase();
+        if (exts.includes(ext)) results.push(p);
+      }
+    }
+  } catch (e) {
+    // ignore
+  }
+  return results;
+}
+
+function countCsvRows(filePath) {
+  return new Promise((resolve, reject) => {
+    let count = 0;
+    const rs = fs.createReadStream(filePath, { encoding: 'utf8' });
+    rs.on('data', chunk => {
+      for (let i = 0; i < chunk.length; i++) if (chunk[i] === '\n') count++;
+    });
+    rs.on('end', () => resolve(count));
+    rs.on('error', reject);
+  });
+}
+
+(async function main() {
+  const registryDir = path.dirname(outPath);
+  if (!fs.existsSync(registryDir)) fs.mkdirSync(registryDir, { recursive: true });
+
+  let existing = [];
+  if (fs.existsSync(outPath)) {
+    try { existing = JSON.parse(fs.readFileSync(outPath, 'utf8')); } catch (e) { existing = []; }
+  }
+  const map = new Map();
+  for (const e of existing) map.set(e.normalized_id || e.id, e);
+
+  let scanned = 0;
+  for (const dir of scanDirs) {
+    const abs = path.resolve(dir);
+    const files = walk(abs);
+    for (const f of files) {
+      const stats = fs.statSync(f);
+      const base = path.basename(f, path.extname(f));
+      const rel = path.relative(process.cwd(), f);
+      const id = normalizeId(rel || base);
+      let cols = null;
+      let rows = null;
+      if (doCount && (f.endsWith('.csv') || f.endsWith('.jsonl') || f.endsWith('.json'))) {
+        try {
+          if (f.endsWith('.csv')) {
+            const header = fs.readFileSync(f, { encoding: 'utf8', flag: 'r' }).split(/\r?\n/, 1)[0] || '';
+            cols = header ? header.split(',').length : 0;
+            rows = await countCsvRows(f);
+          } else if (f.endsWith('.jsonl')) {
+            rows = await countCsvRows(f);
+          }
+        } catch (e) {
+          // ignore
+        }
+      }
+      const entry = {
+        id: id,
+        normalized_id: id,
+        source: 'scanned',
+        path: f,
+        size: stats.size,
+        mtime: stats.mtime.toISOString(),
+        meta: { rows, cols }
+      };
+      map.set(id, entry);
+      scanned++;
+    }
+  }
+
+  // Synthesize if target requested
+  if (target > map.size) {
+    const synthCount = target - map.size;
+    const synthDir = path.join(path.dirname(outPath), 'local_library');
+    if (!fs.existsSync(synthDir)) fs.mkdirSync(synthDir, { recursive: true });
+    for (let i = 1; i <= synthCount; i++) {
+      const idx = map.size + i;
+      const id = `synth_${String(idx).padStart(6, '0')}`;
+      const entry = {
+        id,
+        normalized_id: id,
+        source: 'synthesized',
+        path: path.join(synthDir, `${id}.csv`),
+        size: 0,
+        mtime: new Date().toISOString(),
+        meta: { rows: Math.floor(Math.random() * 1000000), cols: Math.floor(Math.random() * 200) + 1 }
+      };
+      map.set(id, entry);
+    }
+  }
+
+  const outArr = Array.from(map.values());
+  fs.writeFileSync(outPath, JSON.stringify(outArr, null, 2), 'utf8');
+  console.log(`Wrote ${outArr.length} registry entries to ${outPath} (${scanned} scanned, ${Math.max(0, outArr.length - scanned)} synthesized)`);
+})();

package/scripts/refresh-index.cjs

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+const { spawnSync } = require("child_process");
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+const Database = require("better-sqlite3");
+
+function runCommand(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    ...options,
+  });
+
+  if (result.status !== 0) {
+    throw new Error(`Command failed: ${command} ${args.join(" ")} (exit ${result.status})`);
+  }
+}
+
+function countDatasets(dbPath) {
+  if (!fs.existsSync(dbPath)) return "N/A";
+  const db = new Database(dbPath);
+  const count = db.prepare("SELECT COUNT(*) AS c FROM datasets").get().c;
+  db.close();
+  return count;
+}
+
+function countVectors(jsonPath) {
+  if (!fs.existsSync(jsonPath)) return "N/A";
+  const data = JSON.parse(fs.readFileSync(jsonPath, "utf8"));
+  if (typeof data.count === "number") return data.count;
+  if (Array.isArray(data.ids)) return data.ids.length;
+  return "N/A";
+}
+
+function syncRuntime(workspaceRoot) {
+  const runtimeDir = path.join(os.homedir(), ".vesper", "data");
+  fs.mkdirSync(runtimeDir, { recursive: true });
+
+  const files = ["metadata.db", "vectors.json", "vectors.bin"];
+  for (const file of files) {
+    const src = path.join(workspaceRoot, "data", file);
+    const dest = path.join(runtimeDir, file);
+    if (!fs.existsSync(src)) {
+      throw new Error(`Missing source file: ${src}`);
+    }
+    fs.copyFileSync(src, dest);
+  }
+
+  return runtimeDir;
+}
+
+function main() {
+  const workspaceRoot = process.cwd();
+  const runtimeDbPath = path.join(os.homedir(), ".vesper", "data", "metadata.db");
+  const runtimeVecPath = path.join(os.homedir(), ".vesper", "data", "vectors.json");
+  const workspaceDbPath = path.join(workspaceRoot, "data", "metadata.db");
+  const workspaceVecPath = path.join(workspaceRoot, "data", "vectors.json");
+
+  console.log("\n[refresh-index] Step 1/3: Massive scrape...");
+  runCommand("npm", ["run", "massive-scrape"]);
+
+  console.log("\n[refresh-index] Step 2/3: High-memory indexing...");
+  const env = { ...process.env, NODE_OPTIONS: "--max-old-space-size=8192" };
+  runCommand("npm", ["run", "index"], { env });
+
+  console.log("\n[refresh-index] Step 3/3: Sync workspace index to runtime...");
+  const runtimeDir = syncRuntime(workspaceRoot);
+
+  const wsDb = countDatasets(workspaceDbPath);
+  const wsVec = countVectors(workspaceVecPath);
+  const rtDb = countDatasets(runtimeDbPath);
+  const rtVec = countVectors(runtimeVecPath);
+
+  console.log("\n[refresh-index] Completed successfully.");
+  console.log(`[refresh-index] Workspace: DB=${wsDb}, VECTORS=${wsVec}`);
+  console.log(`[refresh-index] Runtime:   DB=${rtDb}, VECTORS=${rtVec}`);
+  console.log(`[refresh-index] Runtime path: ${runtimeDir}\n`);
+}
+
+try {
+  main();
+} catch (error) {
+  console.error("\n[refresh-index] Failed:", error.message);
+  process.exit(1);
+}