robot-resources 1.3.8 → 1.4.1

package/README.md

@@ -11,7 +11,7 @@ npx robot-resources
 1. **GitHub OAuth** — authenticates via PKCE, saves API key to `~/.robot-resources/config.json`
 2. **Router install** — detects Python 3.10+, creates venv, pip installs `robot-resources-router`
 3. **Service registration** — registers router as launchd (macOS) or systemd (Linux) service
-4. **MCP auto-config** — detects Claude Desktop and Cursor, injects scraper MCP entry
+4. **Scraper** — installs @robot-resources/scraper for token-efficient web compression
 
 ## Adding a new tool
 

package/lib/tool-config.js

@@ -43,6 +43,43 @@ function activateRouterModel() {
   }
 }
 
+/**
+ * Register scraper-mcp as an MCP server in openclaw.json.
+ *
+ * This makes scraper_compress_url and scraper_crawl_url available
+ * as native tools in OpenClaw. The plugin's before_tool_call hook
+ * then intercepts web_fetch to route through the scraper by default.
+ *
+ * Returns true if the config was updated, false otherwise.
+ */
+function registerScraperMcp() {
+  const configPath = join(homedir(), '.openclaw', 'openclaw.json');
+
+  try {
+    const raw = readFileSync(configPath, 'utf-8');
+    const config = JSON.parse(stripJson5(raw));
+
+    if (!config.mcp) config.mcp = {};
+    if (!config.mcp.servers) config.mcp.servers = {};
+
+    // Already registered
+    if (config.mcp.servers['robot-resources-scraper']) {
+      return false;
+    }
+
+    config.mcp.servers['robot-resources-scraper'] = {
+      command: 'npx',
+      args: ['-y', '@robot-resources/scraper-mcp'],
+    };
+
+    writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf-8');
+    return true;
+  } catch {
+    // Config missing or malformed — non-fatal
+    return false;
+  }
+}
+
 /**
  * Configure OpenClaw to route through Robot Resources Router.
  *
@@ -141,5 +178,5 @@ export function configureToolRouting() {
   return results;
 }
 
-// Exported for testing
-export { stripJson5, configureOpenClaw };
+// Exported for testing and direct use
+export { stripJson5, configureOpenClaw, registerScraperMcp };

package/lib/wizard.js

@@ -2,7 +2,7 @@ import { readConfig, writeConfig } from '@robot-resources/cli-core/config.mjs';
 import { findPython, isPortAvailable, isHeadless } from './detect.js';
 import { setupRouter, isRouterInstalled, getVenvPythonPath } from './python-bridge.js';
 import { installService, isServiceRunning, isServiceInstalled } from './service.js';
-import { configureToolRouting } from './tool-config.js';
+import { configureToolRouting, registerScraperMcp } from './tool-config.js';
 import { header, step, success, warn, error, info, blank, summary } from './ui.js';
 /**
  * Main setup wizard. Handles the full onboarding flow:
@@ -103,7 +103,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     if (!python) {
       warn('Python 3.10+ not found — skipping Router installation');
       info('Install Python from https://python.org and re-run this wizard');
-      info('Scraper and MCP tools work without Python');
+      info('Scraper works without Python');
     } else {
       info(`Found Python ${python.version} (${python.bin})`);
       step('Installing Router (this may take a moment)...');
@@ -201,8 +201,104 @@ export async function runWizard({ nonInteractive = false } = {}) {
     }
   }
 
-  // ── Step 4.5: Healthcheck ─────────────────────────────────────────────
+  // ── Step 4: Scraper Installation ───────────────────────────────────────
+  //
+  // Independent of router. Scraper works even if router failed to install.
+  // 1. Register scraper-mcp in openclaw.json (if OC is present)
+  // 2. Pre-cache the scraper-mcp npm package
+  // 3. Restart gateway so OC picks up the new MCP server
+
+  blank();
+  step('Installing Scraper...');
+
+  results.scraper = false;
 
+  // Register MCP in openclaw.json
+  const scraperRegistered = registerScraperMcp();
+  if (scraperRegistered) {
+    success('Scraper MCP registered in OpenClaw — scraper_compress_url(url) available');
+  } else {
+    // Either already registered, or no openclaw.json
+    try {
+      const { readFileSync: readFs } = await import('node:fs');
+      const { join: joinPath } = await import('node:path');
+      const { homedir: home } = await import('node:os');
+      const ocConfig = JSON.parse(readFs(joinPath(home(), '.openclaw', 'openclaw.json'), 'utf-8'));
+      if (ocConfig?.mcp?.servers?.['robot-resources-scraper']) {
+        success('Scraper MCP already registered in OpenClaw');
+      }
+    } catch {
+      // No openclaw.json — not on OC, skip
+    }
+  }
+
+  // Pre-cache scraper-mcp package so OC doesn't need to download on first use
+  try {
+    const { execFileSync: execFs } = await import('node:child_process');
+    execFs('npx', ['-y', '@robot-resources/scraper-mcp', '--help'], {
+      stdio: 'ignore',
+      timeout: 60_000,
+    });
+    success('Scraper MCP package cached');
+    results.scraper = true;
+  } catch {
+    warn('Scraper MCP pre-cache failed — will download on first use');
+  }
+
+  // Restart gateway so OC picks up the scraper MCP server
+  if (scraperRegistered) {
+    try {
+      const { execFileSync: execFs2 } = await import('node:child_process');
+      execFs2('openclaw', ['gateway', 'restart'], {
+        stdio: 'ignore',
+        timeout: 15_000,
+      });
+      success('OpenClaw gateway restarted');
+      results.scraper = true;
+    } catch {
+      // Non-fatal — gateway may not be running
+    }
+  }
+
+  // ── Step 4.5: Healthchecks ────────────────────────────────────────────
+
+  // Scraper MCP: verify the server starts without crashing
+  if (results.scraper) {
+    step('Verifying Scraper MCP starts...');
+    try {
+      const { spawn } = await import('node:child_process');
+      const proc = spawn('npx', ['-y', '@robot-resources/scraper-mcp'], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        timeout: 10_000,
+      });
+
+      const healthy = await new Promise((resolve) => {
+        const timer = setTimeout(() => {
+          // Server stayed alive for 3s — it's working
+          proc.kill();
+          resolve(true);
+        }, 3000);
+
+        proc.on('error', () => { clearTimeout(timer); resolve(false); });
+        proc.on('exit', (code) => {
+          clearTimeout(timer);
+          // MCP servers don't exit on their own — if it exited, it crashed
+          resolve(code === 0);
+        });
+      });
+
+      if (healthy) {
+        success('Scraper MCP server healthy');
+      } else {
+        warn('Scraper MCP server exited unexpectedly');
+        results.scraper = false;
+      }
+    } catch {
+      warn('Could not verify Scraper MCP server');
+    }
+  }
+
+  // Router: verify it's responding on localhost:3838
   if (results.service) {
     blank();
     step('Verifying Router is responding...');
@@ -236,7 +332,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
 
   // ── Summary ─────────────────────────────────────────────────────────────
 
-  const somethingInstalled = results.router || results.service;
+  const somethingInstalled = results.router || results.service || results.scraper;
 
   const lines = [];
 
@@ -251,7 +347,11 @@ export async function runWizard({ nonInteractive = false } = {}) {
     lines.push('○ Router not installed (Python 3.10+ required)');
   }
 
-  lines.push('✓ Scraper installed (token compression for web content)');
+  if (results.scraper) {
+    lines.push('✓ Scraper MCP ready — use scraper_compress_url(url) to compress web content');
+  } else {
+    lines.push('○ Scraper MCP not configured');
+  }
 
   summary(lines);
 
@@ -276,7 +376,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     if (results.router)  info('  • Router installed in ~/.robot-resources/');
     if (results.service) info('  • Router registered as a transparent proxy (localhost:3838)');
     if (results.service) info('  • Reads API keys from requests — no keys stored by Router');
-    info('  • Scraper installed for token-efficient web compression');
+    if (results.scraper) info('  • Scraper MCP configured — scraper_compress_url(url) available');
     blank();
     const claimLink = results.claimUrl || 'https://robotresources.ai/dashboard';
     info('Suggested message to your human (COPY THIS EXACTLY):');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "robot-resources",
-  "version": "1.3.8",
+  "version": "1.4.1",
   "description": "Robot Resources — AI agent runtime tools. One command to install everything.",
   "type": "module",
   "bin": {
@@ -18,7 +18,7 @@
   ],
   "dependencies": {
     "@robot-resources/cli-core": "*",
-    "@robot-resources/scraper": "*"
+    "@robot-resources/scraper": "^0.1.0"
   },
   "devDependencies": {
     "vitest": "^1.2.0"