robot-resources 1.4.0 → 1.4.1

package/lib/tool-config.js

@@ -114,10 +114,6 @@ function configureOpenClaw() {
     // before_model_resolve hook actually fires for every request.
     const configActivated = activateRouterModel();
 
-    // Register scraper-mcp so the agent gets scraper_compress_url
-    // as a native tool, and the plugin can intercept web_fetch.
-    const scraperRegistered = registerScraperMcp();
-
     // Restart the gateway so it picks up the new plugin + config.
     let gatewayRestarted = false;
     try {
@@ -135,7 +131,6 @@ function configureOpenClaw() {
       action: 'installed',
       authMode,
       configActivated,
-      scraperRegistered,
       gatewayRestarted,
       note: authMode === 'subscription'
         ? 'Plugin required — subscription OAuth tokens are rejected by Anthropic when proxied via third-party clients.'
@@ -183,5 +178,5 @@ export function configureToolRouting() {
   return results;
 }
 
-// Exported for testing
-export { stripJson5, configureOpenClaw };
+// Exported for testing and direct use
+export { stripJson5, configureOpenClaw, registerScraperMcp };

package/lib/wizard.js

@@ -2,7 +2,7 @@ import { readConfig, writeConfig } from '@robot-resources/cli-core/config.mjs';
 import { findPython, isPortAvailable, isHeadless } from './detect.js';
 import { setupRouter, isRouterInstalled, getVenvPythonPath } from './python-bridge.js';
 import { installService, isServiceRunning, isServiceInstalled } from './service.js';
-import { configureToolRouting } from './tool-config.js';
+import { configureToolRouting, registerScraperMcp } from './tool-config.js';
 import { header, step, success, warn, error, info, blank, summary } from './ui.js';
 /**
  * Main setup wizard. Handles the full onboarding flow:
@@ -201,8 +201,104 @@ export async function runWizard({ nonInteractive = false } = {}) {
     }
   }
 
-  // ── Step 4.5: Healthcheck ─────────────────────────────────────────────
+  // ── Step 4: Scraper Installation ───────────────────────────────────────
+  //
+  // Independent of router. Scraper works even if router failed to install.
+  // 1. Register scraper-mcp in openclaw.json (if OC is present)
+  // 2. Pre-cache the scraper-mcp npm package
+  // 3. Restart gateway so OC picks up the new MCP server
+
+  blank();
+  step('Installing Scraper...');
+
+  results.scraper = false;
 
+  // Register MCP in openclaw.json
+  const scraperRegistered = registerScraperMcp();
+  if (scraperRegistered) {
+    success('Scraper MCP registered in OpenClaw — scraper_compress_url(url) available');
+  } else {
+    // Either already registered, or no openclaw.json
+    try {
+      const { readFileSync: readFs } = await import('node:fs');
+      const { join: joinPath } = await import('node:path');
+      const { homedir: home } = await import('node:os');
+      const ocConfig = JSON.parse(readFs(joinPath(home(), '.openclaw', 'openclaw.json'), 'utf-8'));
+      if (ocConfig?.mcp?.servers?.['robot-resources-scraper']) {
+        success('Scraper MCP already registered in OpenClaw');
+      }
+    } catch {
+      // No openclaw.json — not on OC, skip
+    }
+  }
+
+  // Pre-cache scraper-mcp package so OC doesn't need to download on first use
+  try {
+    const { execFileSync: execFs } = await import('node:child_process');
+    execFs('npx', ['-y', '@robot-resources/scraper-mcp', '--help'], {
+      stdio: 'ignore',
+      timeout: 60_000,
+    });
+    success('Scraper MCP package cached');
+    results.scraper = true;
+  } catch {
+    warn('Scraper MCP pre-cache failed — will download on first use');
+  }
+
+  // Restart gateway so OC picks up the scraper MCP server
+  if (scraperRegistered) {
+    try {
+      const { execFileSync: execFs2 } = await import('node:child_process');
+      execFs2('openclaw', ['gateway', 'restart'], {
+        stdio: 'ignore',
+        timeout: 15_000,
+      });
+      success('OpenClaw gateway restarted');
+      results.scraper = true;
+    } catch {
+      // Non-fatal — gateway may not be running
+    }
+  }
+
+  // ── Step 4.5: Healthchecks ────────────────────────────────────────────
+
+  // Scraper MCP: verify the server starts without crashing
+  if (results.scraper) {
+    step('Verifying Scraper MCP starts...');
+    try {
+      const { spawn } = await import('node:child_process');
+      const proc = spawn('npx', ['-y', '@robot-resources/scraper-mcp'], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        timeout: 10_000,
+      });
+
+      const healthy = await new Promise((resolve) => {
+        const timer = setTimeout(() => {
+          // Server stayed alive for 3s — it's working
+          proc.kill();
+          resolve(true);
+        }, 3000);
+
+        proc.on('error', () => { clearTimeout(timer); resolve(false); });
+        proc.on('exit', (code) => {
+          clearTimeout(timer);
+          // MCP servers don't exit on their own — if it exited, it crashed
+          resolve(code === 0);
+        });
+      });
+
+      if (healthy) {
+        success('Scraper MCP server healthy');
+      } else {
+        warn('Scraper MCP server exited unexpectedly');
+        results.scraper = false;
+      }
+    } catch {
+      warn('Could not verify Scraper MCP server');
+    }
+  }
+
+  // Router: verify it's responding on localhost:3838
   if (results.service) {
     blank();
     step('Verifying Router is responding...');
@@ -236,7 +332,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
 
   // ── Summary ─────────────────────────────────────────────────────────────
 
-  const somethingInstalled = results.router || results.service;
+  const somethingInstalled = results.router || results.service || results.scraper;
 
   const lines = [];
 
@@ -251,7 +347,11 @@ export async function runWizard({ nonInteractive = false } = {}) {
     lines.push('○ Router not installed (Python 3.10+ required)');
   }
 
-  lines.push('✓ Scraper installed (token compression for web content)');
+  if (results.scraper) {
+    lines.push('✓ Scraper MCP ready — use scraper_compress_url(url) to compress web content');
+  } else {
+    lines.push('○ Scraper MCP not configured');
+  }
 
   summary(lines);
 
@@ -276,7 +376,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     if (results.router)  info('  • Router installed in ~/.robot-resources/');
     if (results.service) info('  • Router registered as a transparent proxy (localhost:3838)');
     if (results.service) info('  • Reads API keys from requests — no keys stored by Router');
-    info('  • Scraper installed for token-efficient web compression');
+    if (results.scraper) info('  • Scraper MCP configured — scraper_compress_url(url) available');
     blank();
     const claimLink = results.claimUrl || 'https://robotresources.ai/dashboard';
     info('Suggested message to your human (COPY THIS EXACTLY):');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "robot-resources",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "Robot Resources — AI agent runtime tools. One command to install everything.",
   "type": "module",
   "bin": {