vesper-wizard 2.0.4 → 2.0.6

package/scripts/preindex_registry.cjs

@@ -0,0 +1,157 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const { argv, cwd } = process;
+
+function usage() {
+  console.log(`Usage: node scripts/preindex_registry.cjs [--scan dir1 dir2 ...] [--target N] [--out path] [--no-count]
+
+Options:
+  --scan    Directories to recursively scan for datasets (default: ./e2e_demo_output ./datasets)
+  --target  Target total registry entries (if larger than scanned, will synthesize entries)
+  --out     Output registry path (default: ~/.vesper/registry.json)
+  --no-count  Skip expensive row counting for CSV/JSONL
+`);
+}
+
+let scanDirs = [];
+let target = 0;
+let outPath = path.join(os.homedir(), '.vesper', 'registry.json');
+let doCount = true;
+
+for (let i = 2; i < argv.length; i++) {
+  const a = argv[i];
+  if (a === '--scan') {
+    i++;
+    while (i < argv.length && !argv[i].startsWith('--')) {
+      scanDirs.push(argv[i]);
+      i++;
+    }
+    i--;
+  } else if (a === '--target') {
+    target = parseInt(argv[++i], 10) || 0;
+  } else if (a === '--out') {
+    outPath = path.resolve(argv[++i]);
+  } else if (a === '--no-count') {
+    doCount = false;
+  } else if (a === '--help' || a === '-h') {
+    usage();
+    process.exit(0);
+  } else {
+    console.error('Unknown arg', a);
+    usage();
+    process.exit(2);
+  }
+}
+
+if (scanDirs.length === 0) scanDirs = [path.join(cwd(), 'e2e_demo_output'), path.join(cwd(), 'datasets')];
+
+function normalizeId(s) {
+  return s.replace(/[^a-z0-9]+/gi, '_').replace(/^_+|_+$/g, '').toLowerCase();
+}
+
+function walk(dir, exts = ['.csv', '.jsonl', '.json', '.arrow', '.parquet', '.feather']) {
+  const results = [];
+  try {
+    const items = fs.readdirSync(dir, { withFileTypes: true });
+    for (const it of items) {
+      const p = path.join(dir, it.name);
+      if (it.isDirectory()) results.push(...walk(p, exts));
+      else if (it.isFile()) {
+        const ext = path.extname(it.name).toLowerCase();
+        if (exts.includes(ext)) results.push(p);
+      }
+    }
+  } catch (e) {
+    // ignore
+  }
+  return results;
+}
+
+function countCsvRows(filePath) {
+  return new Promise((resolve, reject) => {
+    let count = 0;
+    const rs = fs.createReadStream(filePath, { encoding: 'utf8' });
+    rs.on('data', chunk => {
+      for (let i = 0; i < chunk.length; i++) if (chunk[i] === '\n') count++;
+    });
+    rs.on('end', () => resolve(count));
+    rs.on('error', reject);
+  });
+}
+
+(async function main() {
+  const registryDir = path.dirname(outPath);
+  if (!fs.existsSync(registryDir)) fs.mkdirSync(registryDir, { recursive: true });
+
+  let existing = [];
+  if (fs.existsSync(outPath)) {
+    try { existing = JSON.parse(fs.readFileSync(outPath, 'utf8')); } catch (e) { existing = []; }
+  }
+  const map = new Map();
+  for (const e of existing) map.set(e.normalized_id || e.id, e);
+
+  let scanned = 0;
+  for (const dir of scanDirs) {
+    const abs = path.resolve(dir);
+    const files = walk(abs);
+    for (const f of files) {
+      const stats = fs.statSync(f);
+      const base = path.basename(f, path.extname(f));
+      const rel = path.relative(process.cwd(), f);
+      const id = normalizeId(rel || base);
+      let cols = null;
+      let rows = null;
+      if (doCount && (f.endsWith('.csv') || f.endsWith('.jsonl') || f.endsWith('.json'))) {
+        try {
+          if (f.endsWith('.csv')) {
+            const header = fs.readFileSync(f, { encoding: 'utf8', flag: 'r' }).split(/\r?\n/, 1)[0] || '';
+            cols = header ? header.split(',').length : 0;
+            rows = await countCsvRows(f);
+          } else if (f.endsWith('.jsonl')) {
+            rows = await countCsvRows(f);
+          }
+        } catch (e) {
+          // ignore
+        }
+      }
+      const entry = {
+        id: id,
+        normalized_id: id,
+        source: 'scanned',
+        path: f,
+        size: stats.size,
+        mtime: stats.mtime.toISOString(),
+        meta: { rows, cols }
+      };
+      map.set(id, entry);
+      scanned++;
+    }
+  }
+
+  // Synthesize if target requested
+  if (target > map.size) {
+    const synthCount = target - map.size;
+    const synthDir = path.join(path.dirname(outPath), 'local_library');
+    if (!fs.existsSync(synthDir)) fs.mkdirSync(synthDir, { recursive: true });
+    for (let i = 1; i <= synthCount; i++) {
+      const idx = map.size + i;
+      const id = `synth_${String(idx).padStart(6, '0')}`;
+      const entry = {
+        id,
+        normalized_id: id,
+        source: 'synthesized',
+        path: path.join(synthDir, `${id}.csv`),
+        size: 0,
+        mtime: new Date().toISOString(),
+        meta: { rows: Math.floor(Math.random() * 1000000), cols: Math.floor(Math.random() * 200) + 1 }
+      };
+      map.set(id, entry);
+    }
+  }
+
+  const outArr = Array.from(map.values());
+  fs.writeFileSync(outPath, JSON.stringify(outArr, null, 2), 'utf8');
+  console.log(`Wrote ${outArr.length} registry entries to ${outPath} (${scanned} scanned, ${Math.max(0, outArr.length - scanned)} synthesized)`);
+})();

package/scripts/refresh-index.cjs

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+const { spawnSync } = require("child_process");
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+const Database = require("better-sqlite3");
+
+function runCommand(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    stdio: "inherit",
+    shell: process.platform === "win32",
+    ...options,
+  });
+
+  if (result.status !== 0) {
+    throw new Error(`Command failed: ${command} ${args.join(" ")} (exit ${result.status})`);
+  }
+}
+
+function countDatasets(dbPath) {
+  if (!fs.existsSync(dbPath)) return "N/A";
+  const db = new Database(dbPath);
+  const count = db.prepare("SELECT COUNT(*) AS c FROM datasets").get().c;
+  db.close();
+  return count;
+}
+
+function countVectors(jsonPath) {
+  if (!fs.existsSync(jsonPath)) return "N/A";
+  const data = JSON.parse(fs.readFileSync(jsonPath, "utf8"));
+  if (typeof data.count === "number") return data.count;
+  if (Array.isArray(data.ids)) return data.ids.length;
+  return "N/A";
+}
+
+function syncRuntime(workspaceRoot) {
+  const runtimeDir = path.join(os.homedir(), ".vesper", "data");
+  fs.mkdirSync(runtimeDir, { recursive: true });
+
+  const files = ["metadata.db", "vectors.json", "vectors.bin"];
+  for (const file of files) {
+    const src = path.join(workspaceRoot, "data", file);
+    const dest = path.join(runtimeDir, file);
+    if (!fs.existsSync(src)) {
+      throw new Error(`Missing source file: ${src}`);
+    }
+    fs.copyFileSync(src, dest);
+  }
+
+  return runtimeDir;
+}
+
+function main() {
+  const workspaceRoot = process.cwd();
+  const runtimeDbPath = path.join(os.homedir(), ".vesper", "data", "metadata.db");
+  const runtimeVecPath = path.join(os.homedir(), ".vesper", "data", "vectors.json");
+  const workspaceDbPath = path.join(workspaceRoot, "data", "metadata.db");
+  const workspaceVecPath = path.join(workspaceRoot, "data", "vectors.json");
+
+  console.log("\n[refresh-index] Step 1/3: Massive scrape...");
+  runCommand("npm", ["run", "massive-scrape"]);
+
+  console.log("\n[refresh-index] Step 2/3: High-memory indexing...");
+  const env = { ...process.env, NODE_OPTIONS: "--max-old-space-size=8192" };
+  runCommand("npm", ["run", "index"], { env });
+
+  console.log("\n[refresh-index] Step 3/3: Sync workspace index to runtime...");
+  const runtimeDir = syncRuntime(workspaceRoot);
+
+  const wsDb = countDatasets(workspaceDbPath);
+  const wsVec = countVectors(workspaceVecPath);
+  const rtDb = countDatasets(runtimeDbPath);
+  const rtVec = countVectors(runtimeVecPath);
+
+  console.log("\n[refresh-index] Completed successfully.");
+  console.log(`[refresh-index] Workspace: DB=${wsDb}, VECTORS=${wsVec}`);
+  console.log(`[refresh-index] Runtime:   DB=${rtDb}, VECTORS=${rtVec}`);
+  console.log(`[refresh-index] Runtime path: ${runtimeDir}\n`);
+}
+
+try {
+  main();
+} catch (error) {
+  console.error("\n[refresh-index] Failed:", error.message);
+  process.exit(1);
+}

package/{wizard.js → scripts/wizard.js}

@@ -91,12 +91,34 @@ function httpJson(method, url, body) {
   });
 }
 
-async function canReachDeviceAuth(baseUrl) {
+async function probeDeviceAuth(baseUrl) {
   try {
     const res = await httpJson('POST', `${baseUrl}/api/auth/device/start`);
-    return res.status === 201 && !!res.body && !!res.body.code;
-  } catch {
-    return false;
+    if (res.status === 201 && !!res.body && !!res.body.code) {
+      return { baseUrl, status: 'ready', response: res.body };
+    }
+
+    if (res.status === 503 && res.body && res.body.requiresSetup) {
+      return {
+        baseUrl,
+        status: 'setup-required',
+        response: res.body,
+        message: res.body.error || 'Auth storage is not initialized.',
+      };
+    }
+
+    return {
+      baseUrl,
+      status: 'unreachable',
+      response: res.body,
+      message: typeof res.body === 'string' ? res.body : JSON.stringify(res.body),
+    };
+  } catch (error) {
+    return {
+      baseUrl,
+      status: 'unreachable',
+      message: error && error.message ? error.message : 'Request failed',
+    };
   }
 }
 
@@ -105,13 +127,20 @@ async function resolveVesperApiBaseUrl() {
     ? [VESPER_API_URL]
     : DEFAULT_VESPER_API_CANDIDATES;
 
+  let setupRequiredProbe = null;
+
   for (const candidate of candidates) {
-    if (await canReachDeviceAuth(candidate)) {
-      return candidate;
+    const probe = await probeDeviceAuth(candidate);
+    if (probe.status === 'ready') {
+      return probe;
+    }
+
+    if (!setupRequiredProbe && probe.status === 'setup-required') {
+      setupRequiredProbe = probe;
     }
   }
 
-  return null;
+  return setupRequiredProbe;
 }
 
 function openBrowser(url) {
@@ -136,6 +165,72 @@ function askYesNo(question) {
   });
 }
 
+function askInput(question) {
+  return new Promise((resolve) => {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    rl.question(`  ${question} `, (answer) => {
+      rl.close();
+      resolve(String(answer || '').trim());
+    });
+  });
+}
+
+async function askChoice(question, choices, defaultValue) {
+  console.log(`  ${question}`);
+  choices.forEach((choice, index) => {
+    console.log(`    ${dim(String(index + 1) + ')')} ${choice.label}`);
+  });
+
+  const prompt = defaultValue ? `${dim('[default: ' + defaultValue + ']')}` : '';
+  const answer = await askInput(`${prompt} ${cyan('→')} Choose an option:`);
+  if (!answer && defaultValue) {
+    return defaultValue;
+  }
+
+  const numeric = Number(answer);
+  if (Number.isFinite(numeric) && numeric >= 1 && numeric <= choices.length) {
+    return choices[numeric - 1].value;
+  }
+
+  const matched = choices.find((choice) => choice.value === answer);
+  return matched ? matched.value : defaultValue;
+}
+
+function isCloudApiKey(value) {
+  return !!value && value.startsWith('vesper_sk_') && !value.startsWith('vesper_sk_local_');
+}
+
+async function promptForManualApiKey() {
+  console.log(`\n  ${cyan('■')} ${bold('Manual API Key')}`);
+  console.log(`  ${dim('Paste a Vesper cloud API key. It will be stored locally in config.toml.\n')}`);
+
+  while (true) {
+    const value = await askInput(`${cyan('→')} Vesper API key:`);
+    if (isCloudApiKey(value)) {
+      return value;
+    }
+    console.log(`  ${yellow('!')} ${yellow('Expected a Vesper key starting with vesper_sk_')}`);
+  }
+}
+
+async function chooseAuthMode(existingKey, existingAuthMode) {
+  const hasExistingKey = !!existingKey;
+  if (hasExistingKey) {
+    console.log(`      ${dim('Current key:')} ${dim(existingKey.slice(0, 24) + '...')}`);
+    console.log(`      ${dim('Current mode:')} ${dim(existingAuthMode || (isCloudApiKey(existingKey) ? 'cloud' : 'local_unified'))}`);
+  }
+
+  const choices = [];
+  if (hasExistingKey) {
+    choices.push({ value: 'keep', label: 'Keep current key as-is' });
+  }
+  choices.push({ value: 'manual', label: 'Provide Vesper API key manually' });
+  choices.push({ value: 'browser', label: 'Sign in through the browser' });
+  choices.push({ value: 'local', label: 'Use local-only key' });
+
+  return await askChoice(`${cyan('→')} How do you want to authenticate Vesper?`, choices, hasExistingKey ? 'keep' : 'browser');
+}
+
 async function deviceAuthFlow() {
   console.log(`\n  ${cyan('■')} ${bold('Device Authentication')}`);
   console.log(`  ${dim('Link your CLI to a Vesper account for cloud features\n')}`);
@@ -149,16 +244,25 @@ async function deviceAuthFlow() {
     return null;
   }
 
-  console.log(`  ${dim('Auth endpoint:')} ${dim(resolvedApiBaseUrl)}\n`);
+  if (resolvedApiBaseUrl.status === 'setup-required') {
+    console.log(`  ${yellow('!')} ${yellow('Reached Vesper auth endpoint, but local auth storage is not initialized.')}`);
+    console.log(`      ${dim('Endpoint:')} ${dim(resolvedApiBaseUrl.baseUrl)}`);
+    console.log(`      ${dim('Reason:')} ${dim(resolvedApiBaseUrl.message || 'Apply Supabase migrations first.')}`);
+    console.log(`      ${dim('Run the SQL in supabase/migrations/001_device_auth.sql and 002_rate_limits.sql, then retry.')}`);
+    console.log(`      ${dim('Falling back to local-only mode.\n')}`);
+    return null;
+  }
+
+  console.log(`  ${dim('Auth endpoint:')} ${dim(resolvedApiBaseUrl.baseUrl)}\n`);
 
   // Step 1: Call /api/auth/device/start
   process.stdout.write(`  ${dim('Requesting device code...')}`);
   let startRes;
   try {
-    startRes = await httpJson('POST', `${resolvedApiBaseUrl}/api/auth/device/start`);
+    startRes = await httpJson('POST', `${resolvedApiBaseUrl.baseUrl}/api/auth/device/start`);
   } catch (err) {
     console.log(` ${red('✗')}`);
-    console.log(`      ${red('Could not reach Vesper API at')} ${dim(resolvedApiBaseUrl)}`);
+    console.log(`      ${red('Could not reach Vesper API at')} ${dim(resolvedApiBaseUrl.baseUrl)}`);
     console.log(`      ${dim('Falling back to local-only mode.\n')}`);
     return null;
   }
@@ -198,7 +302,7 @@ async function deviceAuthFlow() {
     process.stdout.write(`\r  ${cyan(frame)} Polling... (${polls})`);
 
     try {
-      const pollRes = await httpJson('GET', `${resolvedApiBaseUrl}/api/auth/device/poll?code=${code}`);
+      const pollRes = await httpJson('GET', `${resolvedApiBaseUrl.baseUrl}/api/auth/device/poll?code=${code}`);
 
       if (pollRes.body.status === 'confirmed' && pollRes.body.apiKey) {
         process.stdout.write(`\r  ${green('✓')} Device authenticated!            \n`);
@@ -357,36 +461,48 @@ async function main() {
   let localKey = existing.api_key || '';
   let authMode = existing.auth_mode || '';
 
-  // If already has a cloud key, skip
-  if (localKey && !localKey.startsWith('vesper_sk_local_')) {
-    console.log(`      ${green('✓')} Cloud API key already configured`);
-    console.log(`      ${dim('Key:')} ${dim(localKey.slice(0, 24) + '...')}  ${dim('→')} ${dim(CONFIG_TOML)}`);
-  } else {
-    // Offer device auth
-    const wantsDevice = await askYesNo(`${cyan('→')} Link to a Vesper account? (enables cloud sync & team features)`);
+  const authChoice = await chooseAuthMode(localKey, authMode);
+
+  if (authChoice === 'keep' && localKey) {
+    console.log(`      ${green('✓')} Keeping current key`);
+  } else if (authChoice === 'manual') {
+    localKey = await promptForManualApiKey();
+    authMode = 'cloud';
+    console.log(`      ${green('✓')} Cloud API key saved from manual input`);
+  } else if (authChoice === 'browser') {
+    const cloudKey = await deviceAuthFlow();
+    if (cloudKey) {
+      localKey = cloudKey;
+      authMode = 'cloud';
+    } else {
+      const fallbackChoice = await askChoice(`${yellow('!')} Browser sign-in did not complete. Choose a fallback:`, [
+        { value: 'manual', label: 'Provide Vesper API key manually' },
+        { value: 'local', label: 'Use local-only key' },
+      ], 'manual');
 
-    if (wantsDevice) {
-      const cloudKey = await deviceAuthFlow();
-      if (cloudKey) {
-        localKey = cloudKey;
+      if (fallbackChoice === 'manual') {
+        localKey = await promptForManualApiKey();
         authMode = 'cloud';
       } else {
-        // Fall back to local key
-        if (!localKey) localKey = generateLocalKey();
+        if (!localKey || isCloudApiKey(localKey)) {
+          localKey = generateLocalKey();
+        }
         authMode = 'local_unified';
         console.log(`\n      ${yellow('⚠')} Using local-only key. Run the wizard again anytime to link an account.`);
       }
-    } else {
-      if (!localKey) localKey = generateLocalKey();
-      authMode = 'local_unified';
-      console.log(`      ${green('✓')} Local-only key generated`);
     }
-
-    const configData = { ...existing, api_key: localKey, auth_mode: authMode };
-    writeToml(CONFIG_TOML, configData);
-    console.log(`      ${dim('Key:')} ${dim(localKey.slice(0, 24) + '...')}  ${dim('→')} ${dim(CONFIG_TOML)}`);
+  } else {
+    if (!localKey || isCloudApiKey(localKey)) {
+      localKey = generateLocalKey();
+    }
+    authMode = 'local_unified';
+    console.log(`      ${green('✓')} Local-only key ready`);
   }
 
+  const configData = { ...existing, api_key: localKey, auth_mode: authMode };
+  writeToml(CONFIG_TOML, configData);
+  console.log(`      ${dim('Key:')} ${dim(localKey.slice(0, 24) + '...')}  ${dim('→')} ${dim(CONFIG_TOML)}`);
+
   // ─── Step 3: Local vault initialization ────────────────────
   process.stdout.write(`\n  ${dim('[')}${cyan('3/6')}${dim(']')} Initializing local credentials vault...`);
   const vaultData = readToml(CONFIG_TOML);

package/src/python/asset_downloader_engine.py

@@ -0,0 +1,92 @@
+import argparse
+import asyncio
+import json
+import os
+import sys
+import warnings
+from pathlib import Path
+from typing import Any, Dict
+
+# Suppress noisy HF warnings
+warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
+warnings.filterwarnings("ignore", message=".*legacy.*")
+
+CURRENT_DIR = Path(__file__).resolve().parent
+if str(CURRENT_DIR) not in sys.path:
+    sys.path.insert(0, str(CURRENT_DIR))
+
+from vesper.core.asset_downloader import AssetDownloader
+from vesper.core.download_recipe import build_download_recipe, save_recipe, get_download_recipe
+
+
+def _print(payload: Dict[str, Any]) -> None:
+    print(json.dumps(payload, ensure_ascii=False))
+
+
+async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
+    payload = json.loads(args.payload)
+    output_root = payload.get("output_root") or str(Path.home() / ".vesper" / "data" / "assets")
+    workers = int(payload.get("workers") or 8)
+    recipes_dir = payload.get("recipes_dir")
+
+    # Auto-set HF token from payload if provided
+    token = payload.get("token") or payload.get("hf_token")
+    if token:
+        os.environ["HF_TOKEN"] = str(token)
+
+    downloader = AssetDownloader(output_root=output_root, workers=workers, recipes_dir=recipes_dir)
+
+    result = await downloader.download_assets(
+        dataset_id=str(payload.get("dataset_id")),
+        source=payload.get("source"),
+        repo_id=payload.get("repo_id"),
+        kaggle_ref=payload.get("kaggle_ref"),
+        urls=payload.get("urls"),
+        output_format=payload.get("output_format", "webdataset"),
+        max_items=payload.get("max_items"),
+        image_column=payload.get("image_column"),
+    )
+    return {"ok": True, "result": result}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Vesper Smart Asset Downloader Engine")
+    parser.add_argument("action", choices=["download", "build_recipe", "get_recipe"])
+    parser.add_argument("payload", help="JSON payload")
+    args = parser.parse_args()
+
+    try:
+        if args.action == "download":
+            response = asyncio.run(_run_download(args))
+            _print(response)
+            return
+
+        payload = json.loads(args.payload)
+        if args.action == "build_recipe":
+            recipe = build_download_recipe(payload)
+            saved = save_recipe(recipe, payload.get("recipes_dir"))
+            _print({"ok": True, "recipe": recipe, "saved_to": saved})
+            return
+
+        if args.action == "get_recipe":
+            dataset_id = str(payload.get("dataset_id"))
+            recipe = get_download_recipe(dataset_id, payload.get("recipes_dir"))
+            _print({"ok": True, "recipe": recipe})
+            return
+
+        _print({"ok": False, "error": f"Unknown action: {args.action}"})
+    except Exception as e:
+        error_msg = str(e)
+        # Provide actionable error messages
+        if "401" in error_msg or "403" in error_msg or "Unauthorized" in error_msg:
+            error_msg = (
+                "Authentication required. This dataset may be gated/private. "
+                "Use configure_keys tool to set HF_TOKEN, then retry."
+            )
+        elif "No image column" in error_msg:
+            error_msg += " Hint: specify image_column parameter with the name of the column containing images."
+        _print({"ok": False, "error": error_msg})
+
+
+if __name__ == "__main__":
+    main()