vesper-wizard 2.0.8 → 2.1.0

package/build/index.js

@@ -266,6 +266,34 @@ function logError(err, context) {
     fs.appendFileSync(errorLogPath, msg);
     console.error(`[Vesper] Critical error logged to ${errorLogPath}`);
 }
+// --- Request Queue: serialize all MCP tool calls to prevent crashes ---
+class RequestQueue {
+    queue = [];
+    running = false;
+    enqueue(task) {
+        return new Promise((resolve, reject) => {
+            this.queue.push({ resolve, reject, task });
+            this.drain();
+        });
+    }
+    async drain() {
+        if (this.running)
+            return;
+        this.running = true;
+        while (this.queue.length > 0) {
+            const item = this.queue.shift();
+            try {
+                const result = await item.task();
+                item.resolve(result);
+            }
+            catch (err) {
+                item.reject(err);
+            }
+        }
+        this.running = false;
+    }
+}
+const requestQueue = new RequestQueue();
 const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
 function printLaunchScreen() {
     const screen = `
@@ -502,7 +530,45 @@ function syncPythonScripts(appRoot, dataRoot) {
 }
 // Sync scripts immediately
 syncPythonScripts(appRoot, dataRoot);
-const metadataStore = new MetadataStore(dbPath);
+// Auto-rebuild better-sqlite3 if native binary doesn't match current Node version
+function tryRebuildSqlite() {
+    try {
+        const { execSync } = require("child_process");
+        const pkgRoot = path.resolve(__dirname, "..");
+        console.error("[Vesper] Rebuilding better-sqlite3 for Node " + process.version + "...");
+        execSync("npm rebuild better-sqlite3", {
+            stdio: "pipe",
+            timeout: 60000,
+            cwd: pkgRoot,
+        });
+        console.error("[Vesper] Rebuild succeeded. Retrying...");
+        // Clear require cache so the rebuilt module is loaded
+        for (const key of Object.keys(require.cache)) {
+            if (key.includes("better-sqlite3") || key.includes("better_sqlite3")) {
+                delete require.cache[key];
+            }
+        }
+        return true;
+    }
+    catch (e) {
+        console.error("[Vesper] Auto-rebuild failed: " + (e?.message || e));
+        return false;
+    }
+}
+let metadataStore;
+try {
+    metadataStore = new MetadataStore(dbPath);
+}
+catch (e) {
+    if (e?.code === "ERR_DLOPEN_FAILED" && tryRebuildSqlite()) {
+        metadataStore = new MetadataStore(dbPath);
+    }
+    else {
+        console.error("[Vesper] FATAL: Cannot load better-sqlite3.");
+        console.error("[Vesper] Run: npm rebuild better-sqlite3");
+        throw e;
+    }
+}
 const vectorStore = new VectorStore(vectorPath);
 const embedder = Embedder.getInstance();
 const searchEngine = new SearchEngine(metadataStore, vectorStore, embedder);
@@ -599,6 +665,18 @@ jobManager.on("processJob", async (job, execute) => {
 async function handlePrepareJob(jobId, query, requirements) {
     hydrateExternalKeys();
     const update = (updates) => jobManager.updateJob(jobId, updates);
+    const pipelineSteps = ["search", "validate", "download", "normalize", "quality", "register"];
+    const stepStatus = {};
+    for (const s of pipelineSteps)
+        stepStatus[s] = "pending";
+    const markPipelineStep = (step, status) => {
+        stepStatus[step] = status;
+        const summary = pipelineSteps.map(s => {
+            const st = stepStatus[s];
+            return st === "done" ? `[${s}]` : st === "running" ? `>${s}<` : st === "failed" ? `!${s}!` : st === "skipped" ? `~${s}~` : ` ${s} `;
+        }).join(" → ");
+        console.error(`[Pipeline] ${summary}`);
+    };
     // Ensure core Python packages are available for dataset operations
     try {
         await ensurePythonModules([
@@ -646,11 +724,14 @@ async function handlePrepareJob(jobId, query, requirements) {
             progress: 20,
             status_text: `Using explicit dataset id: ${datasetIdForDownload} (${source})`
         });
+        markPipelineStep("search", "skipped");
     }
     else {
+        markPipelineStep("search", "running");
         update({ progress: 10, status_text: "Searching for best dataset matching query..." });
         const results = await searchEngine.search(query, { limit: 10 });
         if (results.length === 0) {
+            markPipelineStep("search", "failed");
             throw new Error("No datasets found matching the query. Try refining your search terms.");
         }
         // Pick the best result that we can actually download (skip sources requiring missing credentials)
@@ -670,8 +751,10 @@ async function handlePrepareJob(jobId, query, requirements) {
             progress: 20,
             status_text: `Matched: ${selectedDataset.name} (${selectedDataset.source})`
         });
+        markPipelineStep("search", "done");
     }
     // Pre-check credentials for sources that require them
+    markPipelineStep("validate", "running");
     if (source === "kaggle") {
         const hasKaggleCreds = !!(process.env.KAGGLE_USERNAME && process.env.KAGGLE_KEY && process.env.KAGGLE_USERNAME !== "YOUR_KAGGLE_USERNAME");
         if (!hasKaggleCreds) {
@@ -679,8 +762,11 @@ async function handlePrepareJob(jobId, query, requirements) {
         }
     }
     if (source === "dataworld" && !hasDataWorldToken()) {
+        markPipelineStep("validate", "failed");
         throw new Error("data.world token not set. Use the configure_keys tool or set DW_AUTH_TOKEN environment variable.");
     }
+    markPipelineStep("validate", "done");
+    markPipelineStep("download", "running");
     update({ progress: 30, status_text: `Starting download from ${source}...` });
     // ensureData handles download and returns path to the raw file
     let rawFilePath = await dataIngestor.ensureData(datasetIdForDownload, source, (msg, prog) => {
@@ -743,15 +829,50 @@ async function handlePrepareJob(jobId, query, requirements) {
             update({ progress: 69, status_text: `Sample target met: ${currentRows.toLocaleString()} rows` });
         }
     }
+    markPipelineStep("download", "done");
+    // ── Normalize step: convert any raw format → parquet ──
+    markPipelineStep("normalize", "running");
+    const rawExt = path.extname(rawFilePath).toLowerCase();
+    if (rawExt !== ".parquet" && rawExt !== ".pq") {
+        update({ progress: 70, status_text: "Normalizing to parquet..." });
+        const normalizedDir = path.join(dataRoot, "data", "normalized");
+        if (!fs.existsSync(normalizedDir))
+            fs.mkdirSync(normalizedDir, { recursive: true });
+        const safeId = toSafeDatasetPathFragment(datasetIdForDownload);
+        const normalizedPath = path.join(normalizedDir, `${safeId}.parquet`);
+        try {
+            const normScript = path.join(dataRoot, "python", "normalize_engine.py");
+            const normResult = await runPythonJson(normScript, [rawFilePath, normalizedPath]);
+            if (normResult.ok && fs.existsSync(normalizedPath)) {
+                console.error(`[Prepare] Normalized ${rawExt} → parquet (${normResult.rows} rows)`);
+                rawFilePath = normalizedPath;
+                markPipelineStep("normalize", "done");
+            }
+            else {
+                console.error(`[Prepare] Normalize failed: ${normResult.error}, continuing with raw file`);
+                markPipelineStep("normalize", "skipped");
+            }
+        }
+        catch (e) {
+            console.error(`[Prepare] Normalize step failed: ${e?.message || e}, continuing with raw file`);
+            markPipelineStep("normalize", "skipped");
+        }
+    }
+    else {
+        markPipelineStep("normalize", "done");
+    }
     let qualityScore = selectedDataset?.quality_score ?? 70;
-    update({ progress: 70, status_text: "Analyzing dataset quality..." });
+    markPipelineStep("quality", "running");
+    update({ progress: 75, status_text: "Analyzing dataset quality..." });
     try {
         const report = await qualityAnalyzer.analyze(rawFilePath);
         qualityScore = report.overall_score;
+        markPipelineStep("quality", "done");
     }
     catch (error) {
         console.error(`[Prepare] Quality analysis failed for ${datasetIdForDownload}: ${error?.message || error}`);
         update({ progress: 78, status_text: "Quality analysis skipped (unsupported schema). Continuing installation..." });
+        markPipelineStep("quality", "skipped");
     }
     if (selectedDataset) {
         metadataStore.saveDataset({
@@ -759,15 +880,19 @@ async function handlePrepareJob(jobId, query, requirements) {
             quality_score: qualityScore
         });
     }
+    markPipelineStep("register", "running");
     update({ progress: 85, status_text: "Installing dataset into project..." });
     const installPath = await installService.install(datasetIdForDownload, rawFilePath);
     update({ progress: 100, status_text: "Preparation complete!" });
     // Register prepared dataset in local registry for lookup by export/list tools
     try {
         upsertRegistry(datasetIdForDownload, installPath, "completed");
+        markPipelineStep("register", "done");
+        markStepComplete(datasetIdForDownload, "prepare");
     }
     catch (e) {
         console.error(`[Registry] Failed to write registry for ${datasetIdForDownload}: ${e?.message || e}`);
+        markPipelineStep("register", "failed");
     }
     return installPath;
 }
@@ -1261,110 +1386,237 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
         ],
     };
 });
-// Call Tool
+// Call Tool — all requests are serialized through a queue to prevent crashes from parallel calls
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
-    // --- Pipeline Enforcement ---
-    // Map tool names to pipeline steps
-    const toolToStep = {
-        vesper_search: "search",
-        vesper_download: "download",
-        vesper_analyze: "analyze",
-        vesper_clean: "clean",
-        vesper_split: "split",
-        vesper_export: "export",
-        prepare_dataset: "prepare",
-    };
-    // Extract dataset_id if present and normalize
-    let datasetId = request.params.arguments?.dataset_id || request.params.arguments?.query || "";
-    if (datasetId)
-        datasetId = parseDatasetId(String(datasetId));
-    // Pipeline rules
-    const stepOrder = ["search", "download", "analyze", "clean", "split", "export"];
-    const prereqs = {
-        vesper_download: ["search"],
-        vesper_analyze: ["download"],
-        vesper_clean: ["analyze"],
-        vesper_split: ["clean"],
-        vesper_export: ["split"],
-    };
-    const tool = String(request.params.name);
-    const step = toolToStep[tool];
-    if (step && datasetId) {
-        // Check prerequisites
-        const required = prereqs[tool] || [];
-        for (const req of required) {
-            if (!hasStep(String(datasetId), req)) {
-                // Auto-run missing step if possible, else error
-                // For export, auto-run prepare_dataset if split missing
-                if (tool === "vesper_export" && req === "split") {
-                    // Auto-trigger prepare_dataset (start a background prepare job)
-                    try {
-                        jobManager.createJob("prepare", 0, { query: String(datasetId), requirements: undefined, downloadImages: false });
-                        // Mark split as complete so export can proceed; export handler will also wait for data if needed.
-                        markStepComplete(String(datasetId), "split");
+    return requestQueue.enqueue(async () => {
+        // --- Pipeline Enforcement ---
+        // Map tool names to pipeline steps
+        const toolToStep = {
+            vesper_search: "search",
+            vesper_download: "download",
+            vesper_analyze: "analyze",
+            vesper_clean: "clean",
+            vesper_split: "split",
+            vesper_export: "export",
+            prepare_dataset: "prepare",
+        };
+        // Extract dataset_id if present and normalize
+        let datasetId = request.params.arguments?.dataset_id || request.params.arguments?.query || "";
+        if (datasetId)
+            datasetId = parseDatasetId(String(datasetId));
+        // Pipeline rules
+        const stepOrder = ["search", "download", "analyze", "clean", "split", "export"];
+        const prereqs = {
+            vesper_download: ["search"],
+            vesper_analyze: ["download"],
+            vesper_clean: ["analyze"],
+            vesper_split: ["clean"],
+            vesper_export: ["split"],
+        };
+        const tool = String(request.params.name);
+        const step = toolToStep[tool];
+        if (step && datasetId) {
+            // Check prerequisites
+            const required = prereqs[tool] || [];
+            for (const req of required) {
+                if (!hasStep(String(datasetId), req)) {
+                    // Auto-run missing step if possible, else error
+                    // For export, auto-run prepare_dataset if split missing
+                    if (tool === "vesper_export" && req === "split") {
+                        // Auto-trigger prepare_dataset (start a background prepare job)
+                        try {
+                            jobManager.createJob("prepare", 0, { query: String(datasetId), requirements: undefined, downloadImages: false });
+                            // Mark split as complete so export can proceed; export handler will also wait for data if needed.
+                            markStepComplete(String(datasetId), "split");
+                        }
+                        catch (e) {
+                            console.error(`[Pipeline] Failed to auto-trigger prepare for ${datasetId}: ${e?.message || e}`);
+                            return {
+                                content: [{ type: "text", text: `ERROR: Failed to auto-run prepare for ${datasetId}. Please run prepare_dataset first.` }],
+                                isError: true,
+                            };
+                        }
                     }
-                    catch (e) {
-                        console.error(`[Pipeline] Failed to auto-trigger prepare for ${datasetId}: ${e?.message || e}`);
+                    else {
                         return {
-                            content: [{ type: "text", text: `ERROR: Failed to auto-run prepare for ${datasetId}. Please run prepare_dataset first.` }],
+                            content: [{ type: "text", text: `ERROR: Cannot run ${tool} before ${req}. Please run ${req} first.` }],
                             isError: true,
                         };
                     }
                 }
-                else {
+            }
+            // Mark this step as complete
+            markStepComplete(String(datasetId), String(step));
+        }
+        switch (request.params.name) {
+            case "unified_dataset_api": {
+                hydrateExternalKeys();
+                const operation = String(request.params.arguments?.operation || "").trim().toLowerCase();
+                const source = String(request.params.arguments?.source || "auto").trim().toLowerCase();
+                const includeUnavailable = request.params.arguments?.include_unavailable === true;
+                const publicOnly = request.params.arguments?.public_only !== false;
+                try {
+                    if (operation === "providers") {
+                        return {
+                            content: [{ type: "text", text: JSON.stringify({ providers: unifiedDatasetGateway.getProviderStatuses(includeUnavailable) }, null, 2) }],
+                        };
+                    }
+                    if (operation === "discover") {
+                        const query = String(request.params.arguments?.query || "").trim();
+                        if (!query) {
+                            throw new McpError(ErrorCode.InvalidParams, "query is required for operation='discover'");
+                        }
+                        const result = await unifiedDatasetGateway.discover({
+                            query,
+                            source,
+                            limit: Number(request.params.arguments?.limit || 10),
+                            publicOnly,
+                        });
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                        };
+                    }
+                    if (operation === "download") {
+                        const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                        if (!datasetId) {
+                            throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='download'");
+                        }
+                        try {
+                            await ensurePythonModules([{ module: "datasets", packageName: "datasets" }]);
+                        }
+                        catch {
+                            // best effort; non-HF providers do not require this
+                        }
+                        const result = await unifiedDatasetGateway.download({
+                            datasetId,
+                            source,
+                            targetDir: request.params.arguments?.target_dir ? String(request.params.arguments.target_dir) : undefined,
+                        });
+                        try {
+                            upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
+                        }
+                        catch (e) {
+                            console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
+                        }
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                        };
+                    }
+                    if (operation === "info") {
+                        const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                        if (!datasetId) {
+                            throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='info'");
+                        }
+                        const result = await unifiedDatasetGateway.info({
+                            datasetId,
+                            source,
+                            publicOnly,
+                        });
+                        return {
+                            content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                        };
+                    }
+                    throw new McpError(ErrorCode.InvalidParams, `Unsupported operation: ${operation}`);
+                }
+                catch (error) {
                     return {
-                        content: [{ type: "text", text: `ERROR: Cannot run ${tool} before ${req}. Please run ${req} first.` }],
+                        content: [{ type: "text", text: `ERROR: unified gateway failed: ${error.message}` }],
                         isError: true,
                     };
                 }
             }
-        }
-        // Mark this step as complete
-        markStepComplete(String(datasetId), String(step));
-    }
-    switch (request.params.name) {
-        case "unified_dataset_api": {
-            hydrateExternalKeys();
-            const operation = String(request.params.arguments?.operation || "").trim().toLowerCase();
-            const source = String(request.params.arguments?.source || "auto").trim().toLowerCase();
-            const includeUnavailable = request.params.arguments?.include_unavailable === true;
-            const publicOnly = request.params.arguments?.public_only !== false;
-            try {
-                if (operation === "providers") {
-                    return {
-                        content: [{ type: "text", text: JSON.stringify({ providers: unifiedDatasetGateway.getProviderStatuses(includeUnavailable) }, null, 2) }],
-                    };
+            case "vesper_search": {
+                const query = String(request.params.arguments?.query);
+                const limit = 5;
+                const safeOnly = true; // Enable safe filter by default
+                const enableJIT = request.params.arguments?.enable_jit === true;
+                if (!query) {
+                    throw new McpError(ErrorCode.InvalidParams, "Query is required");
                 }
-                if (operation === "discover") {
-                    const query = String(request.params.arguments?.query || "").trim();
-                    if (!query) {
-                        throw new McpError(ErrorCode.InvalidParams, "query is required for operation='discover'");
-                    }
-                    const result = await unifiedDatasetGateway.discover({
+                const results = await searchEngine.search(query, { limit, safeOnly, enableJIT });
+                const formattedOutput = formatSearchResults(results);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: formattedOutput,
+                        },
+                    ],
+                };
+            }
+            case "discover_datasets": {
+                hydrateExternalKeys();
+                const query = String(request.params.arguments?.query || "").trim();
+                const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
+                const limit = Number(request.params.arguments?.limit || 10);
+                if (!query) {
+                    throw new McpError(ErrorCode.InvalidParams, "query is required");
+                }
+                try {
+                    const gatewayResult = await unifiedDatasetGateway.discover({
                         query,
                         source,
-                        limit: Number(request.params.arguments?.limit || 10),
-                        publicOnly,
+                        limit,
+                        publicOnly: false,
                     });
+                    const results = gatewayResult.results;
+                    const recipeScript = path.join(dataRoot, "python", "asset_downloader_engine.py");
+                    for (const ds of results.slice(0, limit)) {
+                        const info = {
+                            dataset_id: ds.id,
+                            id: ds.id,
+                            source: ds.source,
+                            repo_id: ds.id,
+                            total_images: ds.total_examples || 0,
+                            image_column: undefined,
+                            recipes_dir: path.join(dataRoot, "recipes"),
+                        };
+                        try {
+                            await runPythonJson(recipeScript, ["build_recipe", JSON.stringify(info)]);
+                        }
+                        catch {
+                            // best-effort recipe generation; ignore discovery-time recipe failures
+                        }
+                    }
+                    const formattedOutput = formatSearchResults(results.slice(0, limit));
+                    const noteBlock = gatewayResult.notes.length > 0
+                        ? `\nGateway notes:\n- ${gatewayResult.notes.join("\n- ")}`
+                        : "";
                     return {
-                        content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                        content: [{ type: "text", text: `${formattedOutput}${noteBlock}` }]
                     };
                 }
-                if (operation === "download") {
-                    const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                    if (!datasetId) {
-                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='download'");
-                    }
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: discover failed: ${error.message}` }],
+                        isError: true,
+                    };
+                }
+            }
+            case "download_dataset": {
+                hydrateExternalKeys();
+                const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
+                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                const targetDir = request.params.arguments?.target_dir ? String(request.params.arguments.target_dir) : undefined;
+                if (!datasetId) {
+                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                }
+                // Pre-install Python datasets library for HuggingFace fallback
+                if (source === "huggingface") {
                     try {
-                        await ensurePythonModules([{ module: "datasets", packageName: "datasets" }]);
+                        await ensurePythonModules([
+                            { module: "datasets", packageName: "datasets" },
+                        ]);
                     }
                     catch {
-                        // best effort; non-HF providers do not require this
+                        // Continue - direct download may still work
                     }
+                }
+                try {
                     const result = await unifiedDatasetGateway.download({
                         datasetId,
                         source,
-                        targetDir: request.params.arguments?.target_dir ? String(request.params.arguments.target_dir) : undefined,
+                        targetDir,
                     });
                     try {
                         upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
@@ -1372,857 +1624,761 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     catch (e) {
                         console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
                     }
+                    const noteBlock = result.notes.length > 0 ? `\nNotes:\n- ${result.notes.join("\n- ")}` : "";
                     return {
-                        content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                        content: [{ type: "text", text: `Download complete: ${result.copied_to || result.local_path}${noteBlock}` }]
                     };
                 }
-                if (operation === "info") {
-                    const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-                    if (!datasetId) {
-                        throw new McpError(ErrorCode.InvalidParams, "dataset_id is required for operation='info'");
-                    }
-                    const result = await unifiedDatasetGateway.info({
-                        datasetId,
-                        source,
-                        publicOnly,
-                    });
+                catch (error) {
                     return {
-                        content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+                        content: [{ type: "text", text: `ERROR: download failed: ${error.message}` }],
+                        isError: true,
                     };
                 }
-                throw new McpError(ErrorCode.InvalidParams, `Unsupported operation: ${operation}`);
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: unified gateway failed: ${error.message}` }],
-                    isError: true,
-                };
-            }
-        }
-        case "vesper_search": {
-            const query = String(request.params.arguments?.query);
-            const limit = 5;
-            const safeOnly = true; // Enable safe filter by default
-            const enableJIT = request.params.arguments?.enable_jit === true;
-            if (!query) {
-                throw new McpError(ErrorCode.InvalidParams, "Query is required");
             }
-            const results = await searchEngine.search(query, { limit, safeOnly, enableJIT });
-            const formattedOutput = formatSearchResults(results);
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: formattedOutput,
-                    },
-                ],
-            };
-        }
-        case "discover_datasets": {
-            hydrateExternalKeys();
-            const query = String(request.params.arguments?.query || "").trim();
-            const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
-            const limit = Number(request.params.arguments?.limit || 10);
-            if (!query) {
-                throw new McpError(ErrorCode.InvalidParams, "query is required");
-            }
-            try {
-                const gatewayResult = await unifiedDatasetGateway.discover({
-                    query,
-                    source,
-                    limit,
-                    publicOnly: false,
-                });
-                const results = gatewayResult.results;
-                const recipeScript = path.join(dataRoot, "python", "asset_downloader_engine.py");
-                for (const ds of results.slice(0, limit)) {
-                    const info = {
-                        dataset_id: ds.id,
-                        id: ds.id,
-                        source: ds.source,
-                        repo_id: ds.id,
-                        total_images: ds.total_examples || 0,
-                        image_column: undefined,
-                        recipes_dir: path.join(dataRoot, "recipes"),
+            case "vesper_download_assets": {
+                hydrateExternalKeys();
+                const datasetId = String(request.params.arguments?.dataset_id || "").trim();
+                const source = String(request.params.arguments?.source || "").trim().toLowerCase();
+                // Auto-infer repo_id from dataset_id if not provided (common for HuggingFace)
+                const repoId = request.params.arguments?.repo_id
+                    ? String(request.params.arguments.repo_id)
+                    : (source === "huggingface" && datasetId.includes("/") ? datasetId : undefined);
+                const kaggleRef = request.params.arguments?.kaggle_ref ? String(request.params.arguments.kaggle_ref) : undefined;
+                const urls = Array.isArray(request.params.arguments?.urls)
+                    ? (request.params.arguments?.urls).map(v => String(v))
+                    : undefined;
+                const outputFormat = String(request.params.arguments?.output_format || "webdataset");
+                const requestedOutputDir = request.params.arguments?.target_dir
+                    ? String(request.params.arguments.target_dir).trim()
+                    : request.params.arguments?.output_dir
+                        ? String(request.params.arguments.output_dir).trim()
+                        : undefined;
+                const maxItems = request.params.arguments?.max_items ? Number(request.params.arguments.max_items) : undefined;
+                const workers = request.params.arguments?.workers ? Number(request.params.arguments.workers) : 8;
+                const imageColumn = request.params.arguments?.image_column ? String(request.params.arguments.image_column) : undefined;
+                if (!datasetId || !source) {
+                    throw new McpError(ErrorCode.InvalidParams, "dataset_id and source are required");
+                }
+                if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
+                    return {
+                        content: [{ type: "text", text: "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds)." }],
+                        isError: true,
                     };
-                    try {
-                        await runPythonJson(recipeScript, ["build_recipe", JSON.stringify(info)]);
-                    }
-                    catch {
-                        // best-effort recipe generation; ignore discovery-time recipe failures
-                    }
                 }
-                const formattedOutput = formatSearchResults(results.slice(0, limit));
-                const noteBlock = gatewayResult.notes.length > 0
-                    ? `\nGateway notes:\n- ${gatewayResult.notes.join("\n- ")}`
-                    : "";
-                return {
-                    content: [{ type: "text", text: `${formattedOutput}${noteBlock}` }]
-                };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: discover failed: ${error.message}` }],
-                    isError: true,
-                };
-            }
-        }
-        case "download_dataset": {
-            hydrateExternalKeys();
-            const source = String(request.params.arguments?.source || "huggingface").toLowerCase();
-            const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-            const targetDir = request.params.arguments?.target_dir ? String(request.params.arguments.target_dir) : undefined;
-            if (!datasetId) {
-                throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-            }
-            // Pre-install Python datasets library for HuggingFace fallback
-            if (source === "huggingface") {
+                const requiredModules = [
+                    { module: "aiohttp", packageName: "aiohttp" },
+                ];
+                if (source === "url") {
+                    requiredModules.push({ module: "aiofiles", packageName: "aiofiles" });
+                }
+                if (source === "huggingface") {
+                    requiredModules.push({ module: "datasets", packageName: "datasets" });
+                    requiredModules.push({ module: "PIL", packageName: "Pillow" });
+                }
+                if (source === "kaggle") {
+                    requiredModules.push({ module: "kaggle", packageName: "kaggle" });
+                }
                 try {
-                    await ensurePythonModules([
-                        { module: "datasets", packageName: "datasets" },
-                    ]);
+                    await ensurePythonModules(requiredModules);
                 }
-                catch {
-                    // Continue - direct download may still work
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Python dependency setup failed: ${error.message}` }],
+                        isError: true,
+                    };
                 }
-            }
-            try {
-                const result = await unifiedDatasetGateway.download({
-                    datasetId,
+                const scriptPath = path.join(dataRoot, "python", "asset_downloader_engine.py");
+                const payload = {
+                    dataset_id: datasetId,
                     source,
-                    targetDir,
-                });
+                    repo_id: repoId,
+                    kaggle_ref: kaggleRef,
+                    urls,
+                    output_format: outputFormat,
+                    output_dir: requestedOutputDir,
+                    max_items: maxItems,
+                    workers,
+                    image_column: imageColumn,
+                    output_root: path.join(dataRoot, "data", "assets"),
+                    recipes_dir: path.join(dataRoot, "recipes"),
+                };
                 try {
-                    upsertRegistry(result.dataset_id, result.copied_to || result.local_path, "completed");
+                    const result = await runPythonJson(scriptPath, ["download", JSON.stringify(payload)]);
+                    if (!result?.ok) {
+                        const errMsg = result?.error || "Unknown error";
+                        // Enhance error messages for common failures
+                        let hint = "";
+                        if (errMsg.includes("No image column")) {
+                            hint = "\n\nHint: Specify image_column parameter with the column name containing images/URLs.";
+                        }
+                        else if (errMsg.includes("Authentication") || errMsg.includes("401") || errMsg.includes("403")) {
+                            hint = "\n\nHint: Use configure_keys tool to set HF_TOKEN for gated/private datasets.";
+                        }
+                        return {
+                            content: [{ type: "text", text: `ERROR: asset download failed: ${errMsg}${hint}` }],
+                            isError: true,
+                        };
+                    }
+                    return {
+                        content: [{ type: "text", text: JSON.stringify(result.result, null, 2) }],
+                    };
                 }
-                catch (e) {
-                    console.error(`[Registry] Failed to write registry for ${result.dataset_id}: ${e?.message || e}`);
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: asset downloader execution failed: ${error.message}` }],
+                        isError: true,
+                    };
                 }
-                const noteBlock = result.notes.length > 0 ? `\nNotes:\n- ${result.notes.join("\n- ")}` : "";
-                return {
-                    content: [{ type: "text", text: `Download complete: ${result.copied_to || result.local_path}${noteBlock}` }]
-                };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: download failed: ${error.message}` }],
-                    isError: true,
-                };
             }
-        }
-        case "vesper_download_assets": {
-            hydrateExternalKeys();
-            const datasetId = String(request.params.arguments?.dataset_id || "").trim();
-            const source = String(request.params.arguments?.source || "").trim().toLowerCase();
-            // Auto-infer repo_id from dataset_id if not provided (common for HuggingFace)
-            const repoId = request.params.arguments?.repo_id
-                ? String(request.params.arguments.repo_id)
-                : (source === "huggingface" && datasetId.includes("/") ? datasetId : undefined);
-            const kaggleRef = request.params.arguments?.kaggle_ref ? String(request.params.arguments.kaggle_ref) : undefined;
-            const urls = Array.isArray(request.params.arguments?.urls)
-                ? (request.params.arguments?.urls).map(v => String(v))
-                : undefined;
-            const outputFormat = String(request.params.arguments?.output_format || "webdataset");
-            const requestedOutputDir = request.params.arguments?.target_dir
-                ? String(request.params.arguments.target_dir).trim()
-                : request.params.arguments?.output_dir
-                    ? String(request.params.arguments.output_dir).trim()
-                    : undefined;
-            const maxItems = request.params.arguments?.max_items ? Number(request.params.arguments.max_items) : undefined;
-            const workers = request.params.arguments?.workers ? Number(request.params.arguments.workers) : 8;
-            const imageColumn = request.params.arguments?.image_column ? String(request.params.arguments.image_column) : undefined;
-            if (!datasetId || !source) {
-                throw new McpError(ErrorCode.InvalidParams, "dataset_id and source are required");
-            }
-            if (source === "kaggle" && !dataIngestor.hasKaggleCredentials()) {
-                return {
-                    content: [{ type: "text", text: "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds)." }],
-                    isError: true,
-                };
-            }
-            const requiredModules = [
-                { module: "aiohttp", packageName: "aiohttp" },
-            ];
-            if (source === "url") {
-                requiredModules.push({ module: "aiofiles", packageName: "aiofiles" });
-            }
-            if (source === "huggingface") {
-                requiredModules.push({ module: "datasets", packageName: "datasets" });
-                requiredModules.push({ module: "PIL", packageName: "Pillow" });
-            }
-            if (source === "kaggle") {
-                requiredModules.push({ module: "kaggle", packageName: "kaggle" });
-            }
-            try {
-                await ensurePythonModules(requiredModules);
-            }
-            catch (error) {
+            case "configure_kaggle": {
+                const username = String(request.params.arguments?.username || "").trim();
+                const key = String(request.params.arguments?.key || "").trim();
+                if (!username || !key) {
+                    throw new McpError(ErrorCode.InvalidParams, "username and key are required");
+                }
+                const r1 = secureKeys.set("kaggle_username", username);
+                const r2 = secureKeys.set("kaggle_key", key);
+                process.env.KAGGLE_USERNAME = username;
+                process.env.KAGGLE_KEY = key;
                 return {
-                    content: [{ type: "text", text: `ERROR: Python dependency setup failed: ${error.message}` }],
-                    isError: true,
+                    content: [{ type: "text", text: `Kaggle credentials saved securely (${r1.method || "store"}/${r2.method || "store"}). You can now use Kaggle datasets.` }]
                 };
             }
-            const scriptPath = path.join(dataRoot, "python", "asset_downloader_engine.py");
-            const payload = {
-                dataset_id: datasetId,
-                source,
-                repo_id: repoId,
-                kaggle_ref: kaggleRef,
-                urls,
-                output_format: outputFormat,
-                output_dir: requestedOutputDir,
-                max_items: maxItems,
-                workers,
-                image_column: imageColumn,
-                output_root: path.join(dataRoot, "data", "assets"),
-                recipes_dir: path.join(dataRoot, "recipes"),
-            };
-            try {
-                const result = await runPythonJson(scriptPath, ["download", JSON.stringify(payload)]);
-                if (!result?.ok) {
-                    const errMsg = result?.error || "Unknown error";
-                    // Enhance error messages for common failures
-                    let hint = "";
-                    if (errMsg.includes("No image column")) {
-                        hint = "\n\nHint: Specify image_column parameter with the column name containing images/URLs.";
+            case "configure_keys": {
+                const hfToken = String(request.params.arguments?.hf_token || "").trim();
+                const kaggleUsername = String(request.params.arguments?.kaggle_username || "").trim();
+                const kaggleKey = String(request.params.arguments?.kaggle_key || "").trim();
+                const dataworldToken = String(request.params.arguments?.dataworld_token || "").trim();
+                const saved = [];
+                const methods = [];
+                if (hfToken) {
+                    const r = secureKeys.set("hf_token", hfToken);
+                    if (r.ok) {
+                        process.env.HF_TOKEN = hfToken;
+                        saved.push("HF token");
+                        if (r.method)
+                            methods.push(r.method);
                     }
-                    else if (errMsg.includes("Authentication") || errMsg.includes("401") || errMsg.includes("403")) {
-                        hint = "\n\nHint: Use configure_keys tool to set HF_TOKEN for gated/private datasets.";
+                }
+                if (kaggleUsername) {
+                    const r = secureKeys.set("kaggle_username", kaggleUsername);
+                    if (r.ok) {
+                        process.env.KAGGLE_USERNAME = kaggleUsername;
+                        saved.push("Kaggle username");
+                        if (r.method)
+                            methods.push(r.method);
+                    }
+                }
+                if (kaggleKey) {
+                    const r = secureKeys.set("kaggle_key", kaggleKey);
+                    if (r.ok) {
+                        process.env.KAGGLE_KEY = kaggleKey;
+                        saved.push("Kaggle key");
+                        if (r.method)
+                            methods.push(r.method);
                     }
+                }
+                if (dataworldToken) {
+                    const r = secureKeys.set("dataworld_token", dataworldToken);
+                    if (r.ok) {
+                        process.env.DW_AUTH_TOKEN = dataworldToken;
+                        saved.push("data.world token");
+                        if (r.method)
+                            methods.push(r.method);
+                    }
+                }
+                if (saved.length === 0) {
                     return {
-                        content: [{ type: "text", text: `ERROR: asset download failed: ${errMsg}${hint}` }],
-                        isError: true,
+                        content: [{ type: "text", text: "No keys provided. Core Vesper tools continue to work without API keys." }]
                     };
                 }
                 return {
-                    content: [{ type: "text", text: JSON.stringify(result.result, null, 2) }],
-                };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: asset downloader execution failed: ${error.message}` }],
-                    isError: true,
+                    content: [{ type: "text", text: `Key saved securely. Updated: ${saved.join(", ")}.` }]
                 };
             }
-        }
-        case "configure_kaggle": {
-            const username = String(request.params.arguments?.username || "").trim();
-            const key = String(request.params.arguments?.key || "").trim();
-            if (!username || !key) {
-                throw new McpError(ErrorCode.InvalidParams, "username and key are required");
-            }
-            const r1 = secureKeys.set("kaggle_username", username);
-            const r2 = secureKeys.set("kaggle_key", key);
-            process.env.KAGGLE_USERNAME = username;
-            process.env.KAGGLE_KEY = key;
-            return {
-                content: [{ type: "text", text: `Kaggle credentials saved securely (${r1.method || "store"}/${r2.method || "store"}). You can now use Kaggle datasets.` }]
-            };
-        }
-        case "configure_keys": {
-            const hfToken = String(request.params.arguments?.hf_token || "").trim();
-            const kaggleUsername = String(request.params.arguments?.kaggle_username || "").trim();
-            const kaggleKey = String(request.params.arguments?.kaggle_key || "").trim();
-            const dataworldToken = String(request.params.arguments?.dataworld_token || "").trim();
-            const saved = [];
-            const methods = [];
-            if (hfToken) {
-                const r = secureKeys.set("hf_token", hfToken);
-                if (r.ok) {
-                    process.env.HF_TOKEN = hfToken;
-                    saved.push("HF token");
-                    if (r.method)
-                        methods.push(r.method);
+            case "get_dataset_info": {
+                const datasetId = String(request.params.arguments?.dataset_id);
+                if (!datasetId) {
+                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
                 }
-            }
-            if (kaggleUsername) {
-                const r = secureKeys.set("kaggle_username", kaggleUsername);
-                if (r.ok) {
-                    process.env.KAGGLE_USERNAME = kaggleUsername;
-                    saved.push("Kaggle username");
-                    if (r.method)
-                        methods.push(r.method);
+                const dataset = metadataStore.getDataset(datasetId);
+                if (!dataset) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Dataset not found: ${datasetId}` }],
+                        isError: true,
+                    };
                 }
-            }
-            if (kaggleKey) {
-                const r = secureKeys.set("kaggle_key", kaggleKey);
-                if (r.ok) {
-                    process.env.KAGGLE_KEY = kaggleKey;
-                    saved.push("Kaggle key");
-                    if (r.method)
-                        methods.push(r.method);
+                // Enrich: if total_examples is 0-ish, try the HF datasets-server /size API
+                if ((!dataset.total_examples || dataset.total_examples === 0) && dataset.source === "huggingface") {
+                    try {
+                        const sizeResp = await fetch(`https://datasets-server.huggingface.co/size?dataset=${encodeURIComponent(dataset.id)}`);
+                        if (sizeResp.ok) {
+                            const sizeData = await sizeResp.json();
+                            const numRows = sizeData?.size?.dataset?.num_rows;
+                            if (numRows && numRows > 0) {
+                                dataset.total_examples = numRows;
+                                // Also backfill splits
+                                if (sizeData?.size?.splits && Array.isArray(sizeData.size.splits)) {
+                                    dataset.splits = sizeData.size.splits.map((s) => ({
+                                        name: s.split,
+                                        num_examples: s.num_rows || 0,
+                                        size_bytes: s.num_bytes_parquet_files || 0,
+                                    }));
+                                    dataset.has_train_split = dataset.splits.some((s) => s.name === "train");
+                                    dataset.has_test_split = dataset.splits.some((s) => s.name === "test");
+                                    dataset.has_validation_split = dataset.splits.some((s) => s.name === "validation" || s.name === "val");
+                                }
+                                // Persist enriched metadata
+                                metadataStore.saveDataset(dataset);
+                            }
+                        }
+                    }
+                    catch {
+                        // Enrichment is best-effort; continue with whatever we have
+                    }
                 }
-            }
-            if (dataworldToken) {
-                const r = secureKeys.set("dataworld_token", dataworldToken);
-                if (r.ok) {
-                    process.env.DW_AUTH_TOKEN = dataworldToken;
-                    saved.push("data.world token");
-                    if (r.method)
-                        methods.push(r.method);
+                const formattedOutput = formatDatasetInfo(dataset);
+                return { content: [{ type: "text", text: formattedOutput }] };
+            }
+            case "analyze_quality": {
+                const datasetId = String(request.params.arguments?.dataset_id);
+                const safeId = toSafeDatasetPathFragment(datasetId);
+                const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
+                const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
+                let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
+                // Demo Fallback for easy testing
+                if (datasetId === "demo" || !fs.existsSync(filePath)) {
+                    const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
+                    const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
+                    if (fs.existsSync(demoParquetPath)) {
+                        filePath = demoParquetPath;
+                    }
+                    else if (fs.existsSync(demoCsvPath)) {
+                        filePath = demoCsvPath;
+                    }
+                    else if (datasetId !== "demo") {
+                        return {
+                            content: [{ type: "text", text: `ERROR: Local data file for ${datasetId} not found. Try running prepare_dataset first, or use 'demo' as the dataset_id.` }],
+                            isError: true
+                        };
+                    }
                 }
-            }
-            if (saved.length === 0) {
+                const report = await qualityAnalyzer.analyze(filePath);
                 return {
-                    content: [{ type: "text", text: "No keys provided. Core Vesper tools continue to work without API keys." }]
+                    content: [{ type: "text", text: JSON.stringify(report, null, 2) }]
                 };
             }
-            return {
-                content: [{ type: "text", text: `Key saved securely. Updated: ${saved.join(", ")}.` }]
-            };
-        }
-        case "get_dataset_info": {
-            const datasetId = String(request.params.arguments?.dataset_id);
-            if (!datasetId) {
-                throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-            }
-            const dataset = metadataStore.getDataset(datasetId);
-            if (!dataset) {
+            case "preview_cleaning": {
+                const datasetId = String(request.params.arguments?.dataset_id);
+                const safeId = toSafeDatasetPathFragment(datasetId);
+                const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
+                const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
+                let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
+                if (datasetId === "demo" || !fs.existsSync(filePath)) {
+                    const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
+                    const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
+                    if (fs.existsSync(demoParquetPath)) {
+                        filePath = demoParquetPath;
+                    }
+                    else if (fs.existsSync(demoCsvPath)) {
+                        filePath = demoCsvPath;
+                    }
+                    else {
+                        throw new McpError(ErrorCode.InvalidParams, `Local data file not found for ${datasetId}. Please run prepare_dataset first.`);
+                    }
+                }
+                const report = await qualityAnalyzer.analyze(filePath);
+                // Phase 1: Target Detection
+                // We use the same TargetDetector instance inside CleaningPlanner now? 
+                // Actually, we instantiated it inside CleaningPlanner, so we just need to pass the file path 
+                // OR let the planner handle it if we update its signature to accept filePath.
+                // Let's check `CleaningPlanner.generatePlan` signature again.
+                // We updated it to accept `targetInfo`.
+                // So we need to run detection HERE and pass it.
+                // But `TargetDetector` is not exposed in `index.ts` scope yet.
+                // Let's create a global instance or use the one inside planner if exposed (it's private).
+                // Better approach: Instantiate TargetDetector here in index.ts for the tool content.
+                // Quick fix: Instantiate local detector or make global.
+                // I'll make a global `targetDetector` constant in index.ts
+                // But wait, I updated `CleaningPlanner` to instantiate its own detector.
+                // Does `CleaningPlanner` use it? No, I commented out the logic because it needed `filePath`.
+                // RETRY STRATEGY: 
+                // 1. Instantiate `targetDetector` in `index.ts`.
+                // 2. Run `detectTarget(filePath)`.
+                // 3. Pass result to `cleaningPlanner.generatePlan(..., targetInfo)`.
+                // I need to add `const targetDetector = new TargetDetector(__dirname);` to imports/init section first.
+                // But since I'm in this tool, I can't look back.
+                // I will assume I can add it, or just do it inside the case for now.
+                // To do it properly, I should have added `targetDetector` to the global scope in previous step.
+                // Let's do that in a separate step if needed. 
+                // For now, I'll instantiate it here.
+                const { TargetDetector } = await import("./preparation/target-detector.js");
+                const detector = new TargetDetector(__dirname);
+                const targetResult = await detector.detectTarget(filePath);
+                const targetInfo = targetResult.target_column ? {
+                    target: targetResult.target_column,
+                    confidence: targetResult.confidence
+                } : undefined;
+                const plan = await cleaningPlanner.generatePlan(datasetId, report, undefined, targetInfo);
+                let explanation = `### Cleaning Plan for ${datasetId}\n\n`;
+                if (targetInfo && targetInfo.target !== "target" && targetInfo.confidence > 0.7) {
+                    explanation += `**Target Detected**: '${targetInfo.target}' (Confidence: ${targetInfo.confidence.toFixed(2)})\n`;
+                    explanation += `   - **Action**: Renaming to 'target' for consistency.\n\n`;
+                }
+                explanation += `Estimated Quality Improvement: +${plan.estimated_impact.quality_score_improvement} points\n\n`;
+                if (plan.operations.length === 0) {
+                    explanation += "No cleaning operations required.";
+                }
+                else {
+                    plan.operations.forEach((op, i) => {
+                        explanation += `${i + 1}. **${op.type}**: ${op.reason}\n`;
+                    });
+                }
                 return {
-                    content: [{ type: "text", text: `ERROR: Dataset not found: ${datasetId}` }],
-                    isError: true,
+                    content: [{ type: "text", text: explanation }]
                 };
             }
-            const formattedOutput = formatDatasetInfo(dataset);
-            return { content: [{ type: "text", text: formattedOutput }] };
-        }
-        case "analyze_quality": {
-            const datasetId = String(request.params.arguments?.dataset_id);
-            const safeId = toSafeDatasetPathFragment(datasetId);
-            const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
-            const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
-            let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
-            // Demo Fallback for easy testing
-            if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
-                const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
-                if (fs.existsSync(demoParquetPath)) {
-                    filePath = demoParquetPath;
-                }
-                else if (fs.existsSync(demoCsvPath)) {
-                    filePath = demoCsvPath;
-                }
-                else if (datasetId !== "demo") {
+            case "custom_clean": {
+                const datasetId = String(request.params.arguments?.dataset_id);
+                const ops = request.params.arguments?.operations;
+                if (!datasetId || datasetId === "undefined") {
+                    throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
+                }
+                if (!ops || !Array.isArray(ops) || ops.length === 0) {
+                    throw new McpError(ErrorCode.InvalidParams, "operations array is required and must not be empty. Supported operations: RemoveDuplicates, DropColumns, FillMissing, FixTypes, RemoveOutliers, EncodeCategories");
+                }
+                // Pre-check: verify dataset file exists before starting the job
+                const cleanRegEntry = getRegistryEntry(datasetId);
+                const cleanRegPath = cleanRegEntry?.local_path || cleanRegEntry?.path;
+                const cleanDlStatus = metadataStore.getDownloadStatus(datasetId);
+                const cleanSafeId = toSafeDatasetPathFragment(datasetId);
+                const cleanDataExists = (cleanRegPath && fs.existsSync(cleanRegPath)) ||
+                    (cleanDlStatus?.local_path && fs.existsSync(cleanDlStatus.local_path)) ||
+                    fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.parquet`)) ||
+                    fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.csv`)) ||
+                    fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.feather`)) ||
+                    fs.existsSync(datasetId);
+                if (!cleanDataExists) {
                     return {
-                        content: [{ type: "text", text: `ERROR: Local data file for ${datasetId} not found. Try running prepare_dataset first, or use 'demo' as the dataset_id.` }],
-                        isError: true
+                        content: [{ type: "text", text: `Dataset '${datasetId}' not found locally. Download it first using download_dataset or prepare_dataset, then run custom_clean.` }],
+                        isError: true,
                     };
                 }
+                const job = jobManager.createJob("clean", 0, { datasetId, ops });
+                return {
+                    content: [{ type: "text", text: `Cleaning job started. ID: ${job.id}. Use check_job_status to monitor progress.` }]
+                };
             }
-            const report = await qualityAnalyzer.analyze(filePath);
-            return {
-                content: [{ type: "text", text: JSON.stringify(report, null, 2) }]
-            };
-        }
-        case "preview_cleaning": {
-            const datasetId = String(request.params.arguments?.dataset_id);
-            const safeId = toSafeDatasetPathFragment(datasetId);
-            const parquetPath = path.join(dataRoot, "data", "raw", `${safeId}.parquet`);
-            const csvPath = path.join(dataRoot, "data", "raw", `${safeId}.csv`);
-            let filePath = fs.existsSync(parquetPath) ? parquetPath : csvPath;
-            if (datasetId === "demo" || !fs.existsSync(filePath)) {
-                const demoParquetPath = path.join(dataRoot, "e2e_demo_output", "raw_data.parquet");
-                const demoCsvPath = path.join(dataRoot, "e2e_demo_output", "raw_data.csv");
-                if (fs.existsSync(demoParquetPath)) {
-                    filePath = demoParquetPath;
-                }
-                else if (fs.existsSync(demoCsvPath)) {
-                    filePath = demoCsvPath;
-                }
-                else {
-                    throw new McpError(ErrorCode.InvalidParams, `Local data file not found for ${datasetId}. Please run prepare_dataset first.`);
+            case "prepare_dataset": {
+                hydrateExternalKeys();
+                const query = String(request.params.arguments?.query);
+                const requirements = request.params.arguments?.requirements ? String(request.params.arguments?.requirements) : undefined;
+                const downloadImages = request.params.arguments?.download_images === true;
+                if (!query || query === "undefined") {
+                    throw new McpError(ErrorCode.InvalidParams, "query is required - describe the dataset you need or provide a dataset ID");
                 }
-            }
-            const report = await qualityAnalyzer.analyze(filePath);
-            // Phase 1: Target Detection
-            // We use the same TargetDetector instance inside CleaningPlanner now? 
-            // Actually, we instantiated it inside CleaningPlanner, so we just need to pass the file path 
-            // OR let the planner handle it if we update its signature to accept filePath.
-            // Let's check `CleaningPlanner.generatePlan` signature again.
-            // We updated it to accept `targetInfo`.
-            // So we need to run detection HERE and pass it.
-            // But `TargetDetector` is not exposed in `index.ts` scope yet.
-            // Let's create a global instance or use the one inside planner if exposed (it's private).
-            // Better approach: Instantiate TargetDetector here in index.ts for the tool content.
-            // Quick fix: Instantiate local detector or make global.
-            // I'll make a global `targetDetector` constant in index.ts
-            // But wait, I updated `CleaningPlanner` to instantiate its own detector.
-            // Does `CleaningPlanner` use it? No, I commented out the logic because it needed `filePath`.
-            // RETRY STRATEGY: 
-            // 1. Instantiate `targetDetector` in `index.ts`.
-            // 2. Run `detectTarget(filePath)`.
-            // 3. Pass result to `cleaningPlanner.generatePlan(..., targetInfo)`.
-            // I need to add `const targetDetector = new TargetDetector(__dirname);` to imports/init section first.
-            // But since I'm in this tool, I can't look back.
-            // I will assume I can add it, or just do it inside the case for now.
-            // To do it properly, I should have added `targetDetector` to the global scope in previous step.
-            // Let's do that in a separate step if needed. 
-            // For now, I'll instantiate it here.
-            const { TargetDetector } = await import("./preparation/target-detector.js");
-            const detector = new TargetDetector(__dirname);
-            const targetResult = await detector.detectTarget(filePath);
-            const targetInfo = targetResult.target_column ? {
-                target: targetResult.target_column,
-                confidence: targetResult.confidence
-            } : undefined;
-            const plan = await cleaningPlanner.generatePlan(datasetId, report, undefined, targetInfo);
-            let explanation = `### Cleaning Plan for ${datasetId}\n\n`;
-            if (targetInfo && targetInfo.target !== "target" && targetInfo.confidence > 0.7) {
-                explanation += `**Target Detected**: '${targetInfo.target}' (Confidence: ${targetInfo.confidence.toFixed(2)})\n`;
-                explanation += `   - **Action**: Renaming to 'target' for consistency.\n\n`;
-            }
-            explanation += `Estimated Quality Improvement: +${plan.estimated_impact.quality_score_improvement} points\n\n`;
-            if (plan.operations.length === 0) {
-                explanation += "No cleaning operations required.";
-            }
-            else {
-                plan.operations.forEach((op, i) => {
-                    explanation += `${i + 1}. **${op.type}**: ${op.reason}\n`;
-                });
-            }
-            return {
-                content: [{ type: "text", text: explanation }]
-            };
-        }
-        case "custom_clean": {
-            const datasetId = String(request.params.arguments?.dataset_id);
-            const ops = request.params.arguments?.operations;
-            if (!datasetId || datasetId === "undefined") {
-                throw new McpError(ErrorCode.InvalidParams, "dataset_id is required");
-            }
-            if (!ops || !Array.isArray(ops) || ops.length === 0) {
-                throw new McpError(ErrorCode.InvalidParams, "operations array is required and must not be empty. Supported operations: RemoveDuplicates, DropColumns, FillMissing, FixTypes, RemoveOutliers, EncodeCategories");
-            }
-            // Pre-check: verify dataset file exists before starting the job
-            const cleanRegEntry = getRegistryEntry(datasetId);
-            const cleanRegPath = cleanRegEntry?.local_path || cleanRegEntry?.path;
-            const cleanDlStatus = metadataStore.getDownloadStatus(datasetId);
-            const cleanSafeId = toSafeDatasetPathFragment(datasetId);
-            const cleanDataExists = (cleanRegPath && fs.existsSync(cleanRegPath)) ||
-                (cleanDlStatus?.local_path && fs.existsSync(cleanDlStatus.local_path)) ||
-                fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.parquet`)) ||
-                fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.csv`)) ||
-                fs.existsSync(path.join(dataRoot, "data", "raw", `${cleanSafeId}.feather`)) ||
-                fs.existsSync(datasetId);
-            if (!cleanDataExists) {
+                const job = jobManager.createJob("prepare", 0, { query, requirements, downloadImages });
                 return {
-                    content: [{ type: "text", text: `Dataset '${datasetId}' not found locally. Download it first using download_dataset or prepare_dataset, then run custom_clean.` }],
-                    isError: true,
+                    content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you. Use check_job_status to monitor progress.` }]
                 };
             }
-            const job = jobManager.createJob("clean", 0, { datasetId, ops });
-            return {
-                content: [{ type: "text", text: `Cleaning job started. ID: ${job.id}. Use check_job_status to monitor progress.` }]
-            };
-        }
-        case "prepare_dataset": {
-            hydrateExternalKeys();
-            const query = String(request.params.arguments?.query);
-            const requirements = request.params.arguments?.requirements ? String(request.params.arguments?.requirements) : undefined;
-            const downloadImages = request.params.arguments?.download_images === true;
-            if (!query || query === "undefined") {
-                throw new McpError(ErrorCode.InvalidParams, "query is required - describe the dataset you need or provide a dataset ID");
-            }
-            const job = jobManager.createJob("prepare", 0, { query, requirements, downloadImages });
-            return {
-                content: [{ type: "text", text: `Preparation job started. ID: ${job.id}. Vesper is finding and preparing the best dataset for you. Use check_job_status to monitor progress.` }]
-            };
-        }
-        case "compare_datasets": {
-            const datasetIds = request.params.arguments?.dataset_ids;
-            const datasets = datasetIds.map(id => metadataStore.getDataset(id)).filter(d => !!d);
-            let comparison = "| Metric | " + datasets.map(d => d.name).join(" | ") + " |\n";
-            comparison += "| :--- | " + datasets.map(() => " :---: ").join("|") + " |\n";
-            comparison += "| Quality Score | " + datasets.map(d => `${d.quality_score}/100`).join(" | ") + " |\n";
-            comparison += "| License | " + datasets.map(d => d.license.category).join(" | ") + " |\n";
-            comparison += "| Downloads | " + datasets.map(d => d.downloads).join(" | ") + " |\n";
-            comparison += "| Domain | " + datasets.map(d => d.domain).join(" | ") + " |\n";
-            return {
-                content: [{ type: "text", text: comparison }]
-            };
-        }
-        case "check_job_status": {
-            const jobId = String(request.params.arguments?.job_id);
-            const job = metadataStore.getJob(jobId);
-            if (!job) {
-                throw new McpError(ErrorCode.InvalidParams, `Job not found: ${jobId}`);
+            case "compare_datasets": {
+                const datasetIds = request.params.arguments?.dataset_ids;
+                const datasets = datasetIds.map(id => metadataStore.getDataset(id)).filter(d => !!d);
+                let comparison = "| Metric | " + datasets.map(d => d.name).join(" | ") + " |\n";
+                comparison += "| :--- | " + datasets.map(() => " :---: ").join("|") + " |\n";
+                comparison += "| Quality Score | " + datasets.map(d => `${d.quality_score}/100`).join(" | ") + " |\n";
+                comparison += "| License | " + datasets.map(d => d.license.category).join(" | ") + " |\n";
+                comparison += "| Downloads | " + datasets.map(d => d.downloads).join(" | ") + " |\n";
+                comparison += "| Domain | " + datasets.map(d => d.domain).join(" | ") + " |\n";
+                return {
+                    content: [{ type: "text", text: comparison }]
+                };
             }
-            const activeStatuses = new Set(["pending", "queued", "running", "retrying"]);
-            const now = Date.now();
-            const last = jobStatusLastPoll[jobId] || 0;
-            const minPollMs = 3000;
-            if (activeStatuses.has(job.status) && (now - last) < minPollMs) {
-                const waitMs = minPollMs - (now - last);
+            case "check_job_status": {
+                const jobId = String(request.params.arguments?.job_id);
+                const job = metadataStore.getJob(jobId);
+                if (!job) {
+                    throw new McpError(ErrorCode.InvalidParams, `Job not found: ${jobId}`);
+                }
+                const activeStatuses = new Set(["pending", "queued", "running", "retrying"]);
+                const now = Date.now();
+                const last = jobStatusLastPoll[jobId] || 0;
+                const minPollMs = 3000;
+                if (activeStatuses.has(job.status) && (now - last) < minPollMs) {
+                    const waitMs = minPollMs - (now - last);
+                    return {
+                        content: [{ type: "text", text: `No significant status change yet. Please poll again in ~${Math.ceil(waitMs / 1000)}s.` }]
+                    };
+                }
+                jobStatusLastPoll[jobId] = now;
                 return {
-                    content: [{ type: "text", text: `No significant status change yet. Please poll again in ~${Math.ceil(waitMs / 1000)}s.` }]
+                    content: [{ type: "text", text: formatJobStatus(job) }]
                 };
             }
-            jobStatusLastPoll[jobId] = now;
-            return {
-                content: [{ type: "text", text: formatJobStatus(job) }]
-            };
-        }
-        case "export_dataset": {
-            const datasetId = String(request.params.arguments?.dataset_id);
-            const targetDir = request.params.arguments?.target_dir ? String(request.params.arguments?.target_dir) : undefined;
-            const requestedFormat = String(request.params.arguments?.format || "feather");
-            const fastMode = request.params.arguments?.fast === true;
-            const preview = request.params.arguments?.preview === true;
-            const sampleRows = request.params.arguments?.sample_rows ? Number(request.params.arguments.sample_rows) : undefined;
-            const columns = request.params.arguments?.columns;
-            const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
-            // Use Metadata or Registry to find the actual local file
-            let sourcePath = resolveDatasetLocalPath(datasetId);
-            if (!sourcePath) {
-                console.error(`[Export] No local data found for ${datasetId}. Attempting to prepare automatically...`);
-                // Start a prepare job for this dataset id (acts like calling prepare_dataset)
+            case "export_dataset": {
+                const datasetId = String(request.params.arguments?.dataset_id);
+                const targetDir = request.params.arguments?.target_dir ? String(request.params.arguments?.target_dir) : undefined;
+                const requestedFormat = String(request.params.arguments?.format || "feather");
+                const fastMode = request.params.arguments?.fast === true;
+                const preview = request.params.arguments?.preview === true;
+                const sampleRows = request.params.arguments?.sample_rows ? Number(request.params.arguments.sample_rows) : undefined;
+                const columns = request.params.arguments?.columns;
+                const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
+                // Use Metadata or Registry to find the actual local file
+                let sourcePath = resolveDatasetLocalPath(datasetId);
+                if (!sourcePath) {
+                    console.error(`[Export] No local data found for ${datasetId}. Attempting to prepare automatically...`);
+                    // Start a prepare job for this dataset id (acts like calling prepare_dataset)
+                    try {
+                        jobManager.createJob("prepare", 0, { query: datasetId, requirements: undefined, downloadImages: false });
+                    }
+                    catch (e) {
+                        console.error(`[Export] Failed to start prepare job for ${datasetId}: ${e?.message || e}`);
+                    }
+                    // Poll for download status or registry entry until local_path appears or timeout
+                    const wait = (ms) => new Promise(res => setTimeout(res, ms));
+                    const maxWait = 120_000; // 120s
+                    const interval = 2000;
+                    let waited = 0;
+                    while (waited < maxWait) {
+                        const resolved = resolveDatasetLocalPath(datasetId);
+                        if (resolved) {
+                            sourcePath = resolved;
+                            console.error(`[Export] Local data is now available for ${datasetId}: ${sourcePath}`);
+                            break;
+                        }
+                        await wait(interval);
+                        waited += interval;
+                    }
+                    // If still no sourcePath, return helpful error listing prepared datasets
+                    if (!sourcePath) {
+                        const entries = readRegistry();
+                        const listText = entries.length === 0 ? "(no prepared datasets found)" : entries.map(e => `- ${e.dataset_id || e.id || "unknown"}: ${e.local_path || e.path || "unknown"}`).slice(0, 10).join("\n") + (entries.length > 10 ? "\n...and " + (entries.length - 10) + " more" : "");
+                        return {
+                            content: [{ type: "text", text: `ERROR: No local data found for ${datasetId} after attempting prepare. Check credentials and try running prepare_dataset manually. Prepared datasets:\n${listText}` }],
+                            isError: true
+                        };
+                    }
+                }
+                sourcePath = ensureExportableLocalPath(sourcePath);
                 try {
-                    jobManager.createJob("prepare", 0, { query: datasetId, requirements: undefined, downloadImages: false });
+                    upsertRegistry(datasetId, sourcePath, "completed");
                 }
                 catch (e) {
-                    console.error(`[Export] Failed to start prepare job for ${datasetId}: ${e?.message || e}`);
-                }
-                // Poll for download status or registry entry until local_path appears or timeout
-                const wait = (ms) => new Promise(res => setTimeout(res, ms));
-                const maxWait = 120_000; // 120s
-                const interval = 2000;
-                let waited = 0;
-                while (waited < maxWait) {
-                    const resolved = resolveDatasetLocalPath(datasetId);
-                    if (resolved) {
-                        sourcePath = resolved;
-                        console.error(`[Export] Local data is now available for ${datasetId}: ${sourcePath}`);
-                        break;
+                    console.error(`[Registry] Failed to normalize registry path for ${datasetId}: ${e?.message || e}`);
+                }
+                // If NOT fast mode, run quality/cleaning pipeline first (only for csv/parquet compat)
+                if (!fastMode) {
+                    const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
+                    const pipelineFmt = (requestedFormat === "csv" || requestedFormat === "parquet") ? requestedFormat : "parquet";
+                    const pipelineCompatibleInput = currentExt === "csv" || currentExt === "parquet";
+                    if (!pipelineCompatibleInput) {
+                        console.error(`[Export] Skipping pipeline for ${currentExt} input; using direct exporter conversion.`);
+                    }
+                    else if (currentExt !== pipelineFmt) {
+                        console.error(`[Export] Running quality/cleaning pipeline (use fast=true to skip)...`);
+                        try {
+                            const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
+                            if (pipelineResult.final_output_path) {
+                                sourcePath = pipelineResult.final_output_path;
+                                try {
+                                    // Update registry to point to pipeline's final output
+                                    upsertRegistry(datasetId, sourcePath, "completed");
+                                }
+                                catch (e) {
+                                    console.error(`[Registry] Failed to update registry for ${datasetId}: ${e?.message || e}`);
+                                }
+                            }
+                        }
+                        catch (err) {
+                            console.error(`[Export] Pipeline warning: ${err.message}. Continuing with raw file.`);
+                        }
                     }
-                    await wait(interval);
-                    waited += interval;
                 }
-                // If still no sourcePath, return helpful error listing prepared datasets
-                if (!sourcePath) {
-                    const entries = readRegistry();
-                    const listText = entries.length === 0 ? "(no prepared datasets found)" : entries.map(e => `- ${e.dataset_id || e.id || "unknown"}: ${e.local_path || e.path || "unknown"}`).slice(0, 10).join("\n") + (entries.length > 10 ? "\n...and " + (entries.length - 10) + " more" : "");
+                else {
+                    console.error(`[Export] Fast mode - skipping quality analysis and cleaning`);
+                }
+                // Build export options
+                const exportOpts = {};
+                if (compression)
+                    exportOpts.compression = compression;
+                if (preview)
+                    exportOpts.preview = true;
+                if (sampleRows)
+                    exportOpts.sample_rows = sampleRows;
+                if (columns)
+                    exportOpts.columns = columns;
+                try {
+                    // Determine output file name
+                    const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow", tfrecord: ".tfrecord" };
+                    const ext = extMap[requestedFormat] || ".feather";
+                    const safeName = toSafeDatasetPathFragment(datasetId);
+                    const outDir = targetDir || path.join(dataRoot, "exports");
+                    if (!fs.existsSync(outDir))
+                        fs.mkdirSync(outDir, { recursive: true });
+                    const outputFile = path.join(outDir, `${safeName}${ext}`);
+                    const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
+                    // Build rich response
+                    let msg = `**Export complete**\n`;
+                    msg += `- **File**: ${result.output_path}\n`;
+                    msg += `- **Format**: ${result.format}${result.compression ? ` (${result.compression})` : ""}\n`;
+                    msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
+                    if (result.file_size_mb !== undefined)
+                        msg += `- **Size**: ${result.file_size_mb} MB\n`;
+                    if (result.elapsed_seconds !== undefined)
+                        msg += `- **Time**: ${result.elapsed_seconds}s\n`;
+                    if (result.preview_path)
+                        msg += `- **Preview**: ${result.preview_path}\n`;
+                    msg += `\n`;
+                    if (requestedFormat === "feather") {
+                        msg += `**Inspect with:**\n`;
+                        msg += `  Python: \`pd.read_feather('${result.output_path}').head()\`\n`;
+                        msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                    }
+                    else if (requestedFormat === "parquet") {
+                        msg += `**Inspect with:**\n`;
+                        msg += `  Python: \`pd.read_parquet('${result.output_path}').head()\`\n`;
+                        msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
+                    }
+                    return { content: [{ type: "text", text: msg }] };
+                }
+                catch (error) {
                     return {
-                        content: [{ type: "text", text: `ERROR: No local data found for ${datasetId} after attempting prepare. Check credentials and try running prepare_dataset manually. Prepared datasets:\n${listText}` }],
+                        content: [{ type: "text", text: `ERROR: Export failed: ${error.message}` }],
                         isError: true
                     };
                 }
             }
-            sourcePath = ensureExportableLocalPath(sourcePath);
-            try {
-                upsertRegistry(datasetId, sourcePath, "completed");
-            }
-            catch (e) {
-                console.error(`[Registry] Failed to normalize registry path for ${datasetId}: ${e?.message || e}`);
-            }
-            // If NOT fast mode, run quality/cleaning pipeline first (only for csv/parquet compat)
-            if (!fastMode) {
-                const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
-                const pipelineFmt = (requestedFormat === "csv" || requestedFormat === "parquet") ? requestedFormat : "parquet";
-                const pipelineCompatibleInput = currentExt === "csv" || currentExt === "parquet";
-                if (!pipelineCompatibleInput) {
-                    console.error(`[Export] Skipping pipeline for ${currentExt} input; using direct exporter conversion.`);
-                }
-                else if (currentExt !== pipelineFmt) {
-                    console.error(`[Export] Running quality/cleaning pipeline (use fast=true to skip)...`);
-                    try {
-                        const pipelineResult = await pipelineExecutor.runPipeline(datasetId, sourcePath, pipelineFmt);
-                        if (pipelineResult.final_output_path) {
-                            sourcePath = pipelineResult.final_output_path;
-                            try {
-                                // Update registry to point to pipeline's final output
-                                upsertRegistry(datasetId, sourcePath, "completed");
-                            }
-                            catch (e) {
-                                console.error(`[Registry] Failed to update registry for ${datasetId}: ${e?.message || e}`);
-                            }
-                        }
+            case "fuse_datasets": {
+                const rawSources = request.params.arguments?.sources;
+                if (!rawSources || !Array.isArray(rawSources) || rawSources.length < 2) {
+                    throw new McpError(ErrorCode.InvalidParams, "sources must contain at least 2 dataset IDs/paths");
+                }
+                const strategy = request.params.arguments?.strategy || "concat";
+                const joinOn = request.params.arguments?.join_on;
+                const how = request.params.arguments?.how || "inner";
+                const dedup = request.params.arguments?.dedup !== false;
+                const runQualityAfter = request.params.arguments?.run_quality_after !== false;
+                const leakageCheck = request.params.arguments?.leakage_check !== false;
+                const outputFormat = request.params.arguments?.output_format || "feather";
+                const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
+                const preview = request.params.arguments?.preview !== false;
+                const resolvedPaths = [];
+                const unresolved = [];
+                for (const src of rawSources) {
+                    if (fs.existsSync(src)) {
+                        resolvedPaths.push(src);
+                        continue;
                     }
-                    catch (err) {
-                        console.error(`[Export] Pipeline warning: ${err.message}. Continuing with raw file.`);
+                    const status = metadataStore.getDownloadStatus(src);
+                    if (status?.local_path && fs.existsSync(status.local_path)) {
+                        resolvedPaths.push(status.local_path);
+                        continue;
                     }
+                    unresolved.push(src);
                 }
-            }
-            else {
-                console.error(`[Export] Fast mode - skipping quality analysis and cleaning`);
-            }
-            // Build export options
-            const exportOpts = {};
-            if (compression)
-                exportOpts.compression = compression;
-            if (preview)
-                exportOpts.preview = true;
-            if (sampleRows)
-                exportOpts.sample_rows = sampleRows;
-            if (columns)
-                exportOpts.columns = columns;
-            try {
-                // Determine output file name
-                const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow", tfrecord: ".tfrecord" };
-                const ext = extMap[requestedFormat] || ".feather";
-                const safeName = toSafeDatasetPathFragment(datasetId);
-                const outDir = targetDir || path.join(dataRoot, "exports");
-                if (!fs.existsSync(outDir))
-                    fs.mkdirSync(outDir, { recursive: true });
-                const outputFile = path.join(outDir, `${safeName}${ext}`);
-                const result = await dataExporter.export(sourcePath, outputFile, requestedFormat, exportOpts);
-                // Build rich response
-                let msg = `**Export complete**\n`;
-                msg += `- **File**: ${result.output_path}\n`;
-                msg += `- **Format**: ${result.format}${result.compression ? ` (${result.compression})` : ""}\n`;
-                msg += `- **Rows**: ${result.rows?.toLocaleString()}${result.columns ? " × " + result.columns + " cols" : ""}\n`;
-                if (result.file_size_mb !== undefined)
-                    msg += `- **Size**: ${result.file_size_mb} MB\n`;
-                if (result.elapsed_seconds !== undefined)
-                    msg += `- **Time**: ${result.elapsed_seconds}s\n`;
-                if (result.preview_path)
-                    msg += `- **Preview**: ${result.preview_path}\n`;
-                msg += `\n`;
-                if (requestedFormat === "feather") {
-                    msg += `**Inspect with:**\n`;
-                    msg += `  Python: \`pd.read_feather('${result.output_path}').head()\`\n`;
-                    msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
-                }
-                else if (requestedFormat === "parquet") {
-                    msg += `**Inspect with:**\n`;
-                    msg += `  Python: \`pd.read_parquet('${result.output_path}').head()\`\n`;
-                    msg += `  DuckDB: \`SELECT * FROM '${result.output_path}' LIMIT 50;\`\n`;
-                }
-                return { content: [{ type: "text", text: msg }] };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: Export failed: ${error.message}` }],
-                    isError: true
-                };
-            }
-        }
-        case "fuse_datasets": {
-            const rawSources = request.params.arguments?.sources;
-            if (!rawSources || !Array.isArray(rawSources) || rawSources.length < 2) {
-                throw new McpError(ErrorCode.InvalidParams, "sources must contain at least 2 dataset IDs/paths");
-            }
-            const strategy = request.params.arguments?.strategy || "concat";
-            const joinOn = request.params.arguments?.join_on;
-            const how = request.params.arguments?.how || "inner";
-            const dedup = request.params.arguments?.dedup !== false;
-            const runQualityAfter = request.params.arguments?.run_quality_after !== false;
-            const leakageCheck = request.params.arguments?.leakage_check !== false;
-            const outputFormat = request.params.arguments?.output_format || "feather";
-            const compression = request.params.arguments?.compression ? String(request.params.arguments.compression) : undefined;
-            const preview = request.params.arguments?.preview !== false;
-            const resolvedPaths = [];
-            const unresolved = [];
-            for (const src of rawSources) {
-                if (fs.existsSync(src)) {
-                    resolvedPaths.push(src);
-                    continue;
+                if (unresolved.length > 0) {
+                    return {
+                        content: [{
+                                type: "text",
+                                text: `ERROR: Could not resolve these sources to local files: ${unresolved.join(", ")}. Provide local paths or run prepare_dataset first.`
+                            }],
+                        isError: true
+                    };
                 }
-                const status = metadataStore.getDownloadStatus(src);
-                if (status?.local_path && fs.existsSync(status.local_path)) {
-                    resolvedPaths.push(status.local_path);
-                    continue;
+                try {
+                    const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow" };
+                    const ext = extMap[outputFormat] || ".feather";
+                    const outDir = path.join(dataRoot, "fusion");
+                    if (!fs.existsSync(outDir))
+                        fs.mkdirSync(outDir, { recursive: true });
+                    const outputPath = path.join(outDir, `fused_${Date.now()}${ext}`);
+                    const result = await fusionEngine.fuse(resolvedPaths, outputPath, {
+                        strategy,
+                        join_on: joinOn,
+                        how,
+                        dedup,
+                        run_quality_after: runQualityAfter,
+                        leakage_check: leakageCheck,
+                        output_format: outputFormat,
+                        compression: compression,
+                        preview,
+                    });
+                    const nullDelta = result.stats.null_delta;
+                    const nullText = nullDelta >= 0 ? `+${nullDelta}%` : `${nullDelta}%`;
+                    // Register fused dataset under a generated id so users can export it easily
+                    const fusedId = `fused_${Date.now()}`;
+                    try {
+                        upsertRegistry(fusedId, result.output_path, "completed");
+                    }
+                    catch (e) {
+                        console.error(`[Registry] Failed to register fused dataset ${fusedId}: ${e?.message || e}`);
+                    }
+                    let msg = `Fused ${result.stats.sources_count} sources -> ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).\n`;
+                    msg += `- Duplicates removed: ${result.stats.duplicates_removed.toLocaleString()}\n`;
+                    msg += `- Null change: ${nullText}\n`;
+                    msg += `- Output: ${result.output_path}\n`;
+                    if (result.preview_path)
+                        msg += `- Preview: ${result.preview_path}\n`;
+                    if (result.leakage_report) {
+                        msg += `- Leakage: ${result.leakage_report.leakage_detected ? "detected" : "none"}`;
+                        if (result.leakage_report.leakage_count) {
+                            msg += ` (${result.leakage_report.leakage_count})`;
+                        }
+                        msg += "\n";
+                    }
+                    msg += `\nNext: run split_dataset/export_dataset on fused output. Registered fused dataset id: ${fusedId}`;
+                    return { content: [{ type: "text", text: msg }] };
+                }
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Fusion failed: ${error.message}` }],
+                        isError: true
+                    };
                 }
-                unresolved.push(src);
-            }
-            if (unresolved.length > 0) {
-                return {
-                    content: [{
-                            type: "text",
-                            text: `ERROR: Could not resolve these sources to local files: ${unresolved.join(", ")}. Provide local paths or run prepare_dataset first.`
-                        }],
-                    isError: true
-                };
             }
-            try {
-                const extMap = { feather: ".feather", parquet: ".parquet", csv: ".csv", jsonl: ".jsonl", arrow: ".arrow" };
-                const ext = extMap[outputFormat] || ".feather";
-                const outDir = path.join(dataRoot, "fusion");
-                if (!fs.existsSync(outDir))
-                    fs.mkdirSync(outDir, { recursive: true });
-                const outputPath = path.join(outDir, `fused_${Date.now()}${ext}`);
-                const result = await fusionEngine.fuse(resolvedPaths, outputPath, {
-                    strategy,
-                    join_on: joinOn,
-                    how,
-                    dedup,
-                    run_quality_after: runQualityAfter,
-                    leakage_check: leakageCheck,
-                    output_format: outputFormat,
-                    compression: compression,
-                    preview,
-                });
-                const nullDelta = result.stats.null_delta;
-                const nullText = nullDelta >= 0 ? `+${nullDelta}%` : `${nullDelta}%`;
-                // Register fused dataset under a generated id so users can export it easily
-                const fusedId = `fused_${Date.now()}`;
-                try {
-                    upsertRegistry(fusedId, result.output_path, "completed");
+            case "analyze_image_quality": {
+                const inputPath = String(request.params.arguments?.path);
+                if (!fs.existsSync(inputPath)) {
+                    throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
                 }
-                catch (e) {
-                    console.error(`[Registry] Failed to register fused dataset ${fusedId}: ${e?.message || e}`);
-                }
-                let msg = `Fused ${result.stats.sources_count} sources -> ${result.stats.rows_after.toLocaleString()} rows (from ${result.stats.rows_before.toLocaleString()}).\n`;
-                msg += `- Duplicates removed: ${result.stats.duplicates_removed.toLocaleString()}\n`;
-                msg += `- Null change: ${nullText}\n`;
-                msg += `- Output: ${result.output_path}\n`;
-                if (result.preview_path)
-                    msg += `- Preview: ${result.preview_path}\n`;
-                if (result.leakage_report) {
-                    msg += `- Leakage: ${result.leakage_report.leakage_detected ? "detected" : "none"}`;
-                    if (result.leakage_report.leakage_count) {
-                        msg += ` (${result.leakage_report.leakage_count})`;
+                try {
+                    const report = await imageAnalyzer.analyze(inputPath);
+                    let output = `## Image Quality Report\n\n`;
+                    output += `- **Total Images**: ${report.total_images}\n`;
+                    output += `- **Corrupted**: ${report.corrupted_count}\n`;
+                    output += `- **Average Resolution**: ${Math.round(report.average_width)}x${Math.round(report.average_height)}\n`;
+                    output += `- **Blurry Images**: ${report.blurry_count}\n\n`;
+                    if (report.individual_results.length > 0) {
+                        output += `### Sample Detail (Top 5)\n`;
+                        report.individual_results.slice(0, 5).forEach(img => {
+                            const statusLabel = img.status === "ok" ? (img.is_blurry ? "[warn]" : "[ok]") : "[error]";
+                            output += `${statusLabel} **${img.filename}**: ${img.width}x${img.height} | Mode: ${img.mode} | Blur: ${img.blur_score?.toFixed(1)}\n`;
+                        });
                     }
-                    msg += "\n";
+                    return {
+                        content: [{ type: "text", text: output }]
+                    };
+                }
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Image analysis failed: ${error.message}` }],
+                        isError: true
+                    };
                 }
-                msg += `\nNext: run split_dataset/export_dataset on fused output. Registered fused dataset id: ${fusedId}`;
-                return { content: [{ type: "text", text: msg }] };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: Fusion failed: ${error.message}` }],
-                    isError: true
-                };
-            }
-        }
-        case "analyze_image_quality": {
-            const inputPath = String(request.params.arguments?.path);
-            if (!fs.existsSync(inputPath)) {
-                throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
             }
-            try {
-                const report = await imageAnalyzer.analyze(inputPath);
-                let output = `## Image Quality Report\n\n`;
-                output += `- **Total Images**: ${report.total_images}\n`;
-                output += `- **Corrupted**: ${report.corrupted_count}\n`;
-                output += `- **Average Resolution**: ${Math.round(report.average_width)}x${Math.round(report.average_height)}\n`;
-                output += `- **Blurry Images**: ${report.blurry_count}\n\n`;
-                if (report.individual_results.length > 0) {
-                    output += `### Sample Detail (Top 5)\n`;
-                    report.individual_results.slice(0, 5).forEach(img => {
-                        const statusLabel = img.status === "ok" ? (img.is_blurry ? "[warn]" : "[ok]") : "[error]";
-                        output += `${statusLabel} **${img.filename}**: ${img.width}x${img.height} | Mode: ${img.mode} | Blur: ${img.blur_score?.toFixed(1)}\n`;
+            case "analyze_media_quality": {
+                const inputPath = String(request.params.arguments?.path);
+                if (!fs.existsSync(inputPath)) {
+                    throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
+                }
+                try {
+                    const report = await mediaAnalyzer.analyze(inputPath);
+                    let output = `## Media Quality Report\n\n`;
+                    output += `- **Total Files**: ${report.total_files}\n`;
+                    output += `- **OK Files**: ${report.ok_files}\n`;
+                    output += `- **Failed Files**: ${report.failed_files}\n`;
+                    if ('avg_audio_duration' in report && report.avg_audio_duration) {
+                        output += `- **Average Audio Duration**: ${report.avg_audio_duration.toFixed(2)}s\n`;
+                    }
+                    if ('avg_video_duration' in report && report.avg_video_duration) {
+                        output += `- **Average Video Duration**: ${report.avg_video_duration.toFixed(2)}s\n`;
+                        output += `- **Average FPS**: ${report.avg_fps?.toFixed(1)}\n`;
+                    }
+                    output += `\n### Sample Detail (Top 5)\n`;
+                    report.details.slice(0, 5).forEach(item => {
+                        const statusLabel = item.status === "ok" ? "[ok]" : "[error]";
+                        if (item.type === "audio" && 'sample_rate' in item) {
+                            output += `${statusLabel} **${item.filename}**: ${item.duration?.toFixed(1)}s | SR: ${item.sample_rate}Hz | Silent: ${item.is_silent}\n`;
+                        }
+                        else if (item.type === "video" && 'width' in item) {
+                            output += `${statusLabel} **${item.filename}**: ${item.width}x${item.height} | ${item.fps?.toFixed(1)}fps | Risk: ${item.corruption_risk}\n`;
+                        }
+                        else {
+                            output += `${statusLabel} **${item.filename}**: ${item.error}\n`;
+                        }
                     });
+                    return {
+                        content: [{ type: "text", text: output }]
+                    };
+                }
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Media analysis failed: ${error.message}` }],
+                        isError: true
+                    };
                 }
-                return {
-                    content: [{ type: "text", text: output }]
-                };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: Image analysis failed: ${error.message}` }],
-                    isError: true
-                };
-            }
-        }
-        case "analyze_media_quality": {
-            const inputPath = String(request.params.arguments?.path);
-            if (!fs.existsSync(inputPath)) {
-                throw new McpError(ErrorCode.InvalidParams, `Path not found: ${inputPath}`);
             }
-            try {
-                const report = await mediaAnalyzer.analyze(inputPath);
-                let output = `## Media Quality Report\n\n`;
-                output += `- **Total Files**: ${report.total_files}\n`;
-                output += `- **OK Files**: ${report.ok_files}\n`;
-                output += `- **Failed Files**: ${report.failed_files}\n`;
-                if ('avg_audio_duration' in report && report.avg_audio_duration) {
-                    output += `- **Average Audio Duration**: ${report.avg_audio_duration.toFixed(2)}s\n`;
-                }
-                if ('avg_video_duration' in report && report.avg_video_duration) {
-                    output += `- **Average Video Duration**: ${report.avg_video_duration.toFixed(2)}s\n`;
-                    output += `- **Average FPS**: ${report.avg_fps?.toFixed(1)}\n`;
-                }
-                output += `\n### Sample Detail (Top 5)\n`;
-                report.details.slice(0, 5).forEach(item => {
-                    const statusLabel = item.status === "ok" ? "[ok]" : "[error]";
-                    if (item.type === "audio" && 'sample_rate' in item) {
-                        output += `${statusLabel} **${item.filename}**: ${item.duration?.toFixed(1)}s | SR: ${item.sample_rate}Hz | Silent: ${item.is_silent}\n`;
+            case "generate_quality_report": {
+                const datasetId = String(request.params.arguments?.dataset_id);
+                const datasetPath = String(request.params.arguments?.dataset_path);
+                if (!fs.existsSync(datasetPath)) {
+                    throw new McpError(ErrorCode.InvalidParams, `Dataset path not found: ${datasetPath}`);
+                }
+                try {
+                    // Optionally load text quality from metadata if available
+                    const metadata = await metadataStore.getDataset(datasetId);
+                    // TODO: Integrate text quality analysis when available
+                    const textQuality = null;
+                    const report = await qualityOrchestrator.generateReport(datasetId, datasetPath, textQuality);
+                    // Save report to metadata
+                    if (metadata) {
+                        metadata.unified_quality_report = report;
+                        await metadataStore.saveDataset(metadata);
                     }
-                    else if (item.type === "video" && 'width' in item) {
-                        output += `${statusLabel} **${item.filename}**: ${item.width}x${item.height} | ${item.fps?.toFixed(1)}fps | Risk: ${item.corruption_risk}\n`;
+                    let output = `# Unified Quality Report\n\n`;
+                    output += `**Dataset**: ${datasetId}\n`;
+                    output += `**Modalities**: ${report.modalities.join(", ")}\n`;
+                    output += `**Overall Quality Score**: ${report.overall_quality_score}/100\n\n`;
+                    if (report.text_quality) {
+                        output += `## Text Quality\n`;
+                        output += `- Rows: ${report.text_quality.row_count}\n`;
+                        output += `- Columns: ${report.text_quality.column_count}\n`;
+                        output += `- Missing: ${report.text_quality.missing_percentage.toFixed(1)}%\n`;
+                        output += `- Duplicates: ${report.text_quality.duplicate_percentage.toFixed(1)}%\n\n`;
                     }
-                    else {
-                        output += `${statusLabel} **${item.filename}**: ${item.error}\n`;
+                    if (report.image_quality) {
+                        output += `## Image Quality\n`;
+                        output += `- Total Images: ${report.image_quality.total_images}\n`;
+                        output += `- Corrupted: ${report.image_quality.corrupted_count}\n`;
+                        output += `- Avg Resolution: ${report.image_quality.avg_resolution}\n`;
+                        output += `- Blurry: ${report.image_quality.blurry_percentage.toFixed(1)}%\n\n`;
                     }
-                });
-                return {
-                    content: [{ type: "text", text: output }]
-                };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: Media analysis failed: ${error.message}` }],
-                    isError: true
-                };
-            }
-        }
-        case "generate_quality_report": {
-            const datasetId = String(request.params.arguments?.dataset_id);
-            const datasetPath = String(request.params.arguments?.dataset_path);
-            if (!fs.existsSync(datasetPath)) {
-                throw new McpError(ErrorCode.InvalidParams, `Dataset path not found: ${datasetPath}`);
-            }
-            try {
-                // Optionally load text quality from metadata if available
-                const metadata = await metadataStore.getDataset(datasetId);
-                // TODO: Integrate text quality analysis when available
-                const textQuality = null;
-                const report = await qualityOrchestrator.generateReport(datasetId, datasetPath, textQuality);
-                // Save report to metadata
-                if (metadata) {
-                    metadata.unified_quality_report = report;
-                    await metadataStore.saveDataset(metadata);
-                }
-                let output = `# Unified Quality Report\n\n`;
-                output += `**Dataset**: ${datasetId}\n`;
-                output += `**Modalities**: ${report.modalities.join(", ")}\n`;
-                output += `**Overall Quality Score**: ${report.overall_quality_score}/100\n\n`;
-                if (report.text_quality) {
-                    output += `## Text Quality\n`;
-                    output += `- Rows: ${report.text_quality.row_count}\n`;
-                    output += `- Columns: ${report.text_quality.column_count}\n`;
-                    output += `- Missing: ${report.text_quality.missing_percentage.toFixed(1)}%\n`;
-                    output += `- Duplicates: ${report.text_quality.duplicate_percentage.toFixed(1)}%\n\n`;
-                }
-                if (report.image_quality) {
-                    output += `## Image Quality\n`;
-                    output += `- Total Images: ${report.image_quality.total_images}\n`;
-                    output += `- Corrupted: ${report.image_quality.corrupted_count}\n`;
-                    output += `- Avg Resolution: ${report.image_quality.avg_resolution}\n`;
-                    output += `- Blurry: ${report.image_quality.blurry_percentage.toFixed(1)}%\n\n`;
-                }
-                if (report.audio_quality) {
-                    output += `## Audio Quality\n`;
-                    output += `- Total Files: ${report.audio_quality.total_files}\n`;
-                    output += `- Avg Duration: ${report.audio_quality.avg_duration.toFixed(1)}s\n`;
-                    output += `- Avg Sample Rate: ${report.audio_quality.avg_sample_rate.toFixed(0)}Hz\n`;
-                    output += `- Silent: ${report.audio_quality.silent_percentage.toFixed(1)}%\n\n`;
-                }
-                if (report.video_quality) {
-                    output += `## Video Quality\n`;
-                    output += `- Total Files: ${report.video_quality.total_files}\n`;
-                    output += `- Avg Duration: ${report.video_quality.avg_duration.toFixed(1)}s\n`;
-                    output += `- Avg FPS: ${report.video_quality.avg_fps.toFixed(1)}\n`;
-                    output += `- High Corruption Risk: ${report.video_quality.corruption_risk_high}\n\n`;
-                }
-                output += `## Recommendations\n`;
-                report.recommendations.forEach(rec => {
-                    output += `- ${rec}\n`;
-                });
-                return {
-                    content: [{ type: "text", text: output }]
-                };
-            }
-            catch (error) {
-                return {
-                    content: [{ type: "text", text: `ERROR: Quality report generation failed: ${error.message}` }],
-                    isError: true
-                };
+                    if (report.audio_quality) {
+                        output += `## Audio Quality\n`;
+                        output += `- Total Files: ${report.audio_quality.total_files}\n`;
+                        output += `- Avg Duration: ${report.audio_quality.avg_duration.toFixed(1)}s\n`;
+                        output += `- Avg Sample Rate: ${report.audio_quality.avg_sample_rate.toFixed(0)}Hz\n`;
+                        output += `- Silent: ${report.audio_quality.silent_percentage.toFixed(1)}%\n\n`;
+                    }
+                    if (report.video_quality) {
+                        output += `## Video Quality\n`;
+                        output += `- Total Files: ${report.video_quality.total_files}\n`;
+                        output += `- Avg Duration: ${report.video_quality.avg_duration.toFixed(1)}s\n`;
+                        output += `- Avg FPS: ${report.video_quality.avg_fps.toFixed(1)}\n`;
+                        output += `- High Corruption Risk: ${report.video_quality.corruption_risk_high}\n\n`;
+                    }
+                    output += `## Recommendations\n`;
+                    report.recommendations.forEach(rec => {
+                        output += `- ${rec}\n`;
+                    });
+                    return {
+                        content: [{ type: "text", text: output }]
+                    };
+                }
+                catch (error) {
+                    return {
+                        content: [{ type: "text", text: `ERROR: Quality report generation failed: ${error.message}` }],
+                        isError: true
+                    };
+                }
             }
+            default:
+                throw new McpError(ErrorCode.MethodNotFound, "Tool not found");
         }
-        default:
-            throw new McpError(ErrorCode.MethodNotFound, "Tool not found");
-    }
+    }); // end requestQueue.enqueue
 });
 async function main() {
     const args = process.argv.slice(2);