robot-resources 1.3.7 → 1.4.0

package/README.md

@@ -11,7 +11,7 @@ npx robot-resources
 1. **GitHub OAuth** — authenticates via PKCE, saves API key to `~/.robot-resources/config.json`
 2. **Router install** — detects Python 3.10+, creates venv, pip installs `robot-resources-router`
 3. **Service registration** — registers router as launchd (macOS) or systemd (Linux) service
-4. **MCP auto-config** — detects Claude Desktop and Cursor, injects scraper MCP entry
+4. **Scraper** — installs @robot-resources/scraper for token-efficient web compression
 
 ## Adding a new tool
 
@@ -66,8 +66,8 @@ packages/cli/         This package (published as "robot-resources" on npm)
   bin/setup.js        Entry point
   lib/wizard.js       Orchestrator — add new tools here
   lib/service.js      launchd + systemd service registration
-  lib/mcp-config.js   Agent detection + MCP JSON patching
   lib/python-bridge.js  Python venv management
   lib/detect.js       Environment detection
+  lib/tool-config.js  OpenClaw plugin + model activation
   lib/ui.js           Terminal formatting
 ```

package/lib/detect.js

@@ -26,38 +26,6 @@ export function checkRouterVenv() {
   }
 }
 
-/**
- * Detect installed AI agents by checking their config file locations.
- */
-export function detectAgents() {
-  const home = homedir();
-  const agents = [];
-
-  const known = [
-    {
-      name: 'Claude Desktop',
-      configPath: process.platform === 'darwin'
-        ? join(home, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json')
-        : join(home, '.config', 'Claude', 'claude_desktop_config.json'),
-    },
-    {
-      name: 'Cursor',
-      configPath: join(home, '.cursor', 'mcp.json'),
-    },
-  ];
-
-  for (const agent of known) {
-    // Check if the config file or its parent directory exists
-    const configExists = existsSync(agent.configPath);
-    const dirExists = existsSync(join(agent.configPath, '..'));
-    if (configExists || dirExists) {
-      agents.push({ ...agent, configExists });
-    }
-  }
-
-  return agents;
-}
-
 /**
  * Check if port 3838 is available.
  */

package/lib/tool-config.js

@@ -43,6 +43,43 @@ function activateRouterModel() {
   }
 }
 
+/**
+ * Register scraper-mcp as an MCP server in openclaw.json.
+ *
+ * This makes scraper_compress_url and scraper_crawl_url available
+ * as native tools in OpenClaw. The plugin's before_tool_call hook
+ * then intercepts web_fetch to route through the scraper by default.
+ *
+ * Returns true if the config was updated, false otherwise.
+ */
+function registerScraperMcp() {
+  const configPath = join(homedir(), '.openclaw', 'openclaw.json');
+
+  try {
+    const raw = readFileSync(configPath, 'utf-8');
+    const config = JSON.parse(stripJson5(raw));
+
+    if (!config.mcp) config.mcp = {};
+    if (!config.mcp.servers) config.mcp.servers = {};
+
+    // Already registered
+    if (config.mcp.servers['robot-resources-scraper']) {
+      return false;
+    }
+
+    config.mcp.servers['robot-resources-scraper'] = {
+      command: 'npx',
+      args: ['-y', '@robot-resources/scraper-mcp'],
+    };
+
+    writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf-8');
+    return true;
+  } catch {
+    // Config missing or malformed — non-fatal
+    return false;
+  }
+}
+
 /**
  * Configure OpenClaw to route through Robot Resources Router.
  *
@@ -77,6 +114,10 @@ function configureOpenClaw() {
     // before_model_resolve hook actually fires for every request.
     const configActivated = activateRouterModel();
 
+    // Register scraper-mcp so the agent gets scraper_compress_url
+    // as a native tool, and the plugin can intercept web_fetch.
+    const scraperRegistered = registerScraperMcp();
+
     // Restart the gateway so it picks up the new plugin + config.
     let gatewayRestarted = false;
     try {
@@ -94,6 +135,7 @@ function configureOpenClaw() {
       action: 'installed',
       authMode,
       configActivated,
+      scraperRegistered,
       gatewayRestarted,
       note: authMode === 'subscription'
         ? 'Plugin required — subscription OAuth tokens are rejected by Anthropic when proxied via third-party clients.'

package/lib/wizard.js

@@ -2,16 +2,14 @@ import { readConfig, writeConfig } from '@robot-resources/cli-core/config.mjs';
 import { findPython, isPortAvailable, isHeadless } from './detect.js';
 import { setupRouter, isRouterInstalled, getVenvPythonPath } from './python-bridge.js';
 import { installService, isServiceRunning, isServiceInstalled } from './service.js';
-import { configureAgentMCP } from './mcp-config.js';
 import { configureToolRouting } from './tool-config.js';
 import { header, step, success, warn, error, info, blank, summary } from './ui.js';
 /**
  * Main setup wizard. Handles the full onboarding flow:
  * 1. Router installation (Python venv + pip)
  * 2. Service registration (launchd/systemd)
- * 3. MCP auto-configuration (detected agents)
- * 4. Tool routing (OpenClaw plugin + model activation)
- * 5. Dashboard login (optional, at the very end)
+ * 3. Tool routing (OpenClaw plugin + model activation)
+ * 4. Dashboard link
  *
  * Auth is intentionally LAST. The router works fully without it.
  * Dashboard is for humans — agents don't need it.
@@ -26,7 +24,6 @@ export async function runWizard({ nonInteractive = false } = {}) {
     routerError: null,
     providerKeys: false,
     service: false,
-    mcp: [],
   };
 
   // ── Step 0: Provision API key (before anything else) ────────────────────
@@ -106,7 +103,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     if (!python) {
       warn('Python 3.10+ not found — skipping Router installation');
       info('Install Python from https://python.org and re-run this wizard');
-      info('Scraper and MCP tools work without Python');
+      info('Scraper works without Python');
     } else {
       info(`Found Python ${python.version} (${python.bin})`);
       step('Installing Router (this may take a moment)...');
@@ -169,32 +166,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     }
   }
 
-  // ── Step 3: MCP Auto-Configuration ──────────────────────────────────────
-
-  blank();
-  step('Configuring MCP in detected agents...');
-
-  const mcpResults = configureAgentMCP();
-  results.mcp = mcpResults;
-
-  if (mcpResults.length === 0) {
-    info('No MCP-compatible agents detected');
-    info('You can manually add MCP servers to your agent config later');
-  } else {
-    for (const r of mcpResults) {
-      if (r.action === 'added') {
-        success(`${r.name}: scraper MCP configured`);
-      } else if (r.action === 'exists') {
-        success(`${r.name}: already configured`);
-      } else if (r.action === 'skipped') {
-        warn(`${r.name}: skipped (${r.reason})`);
-      } else if (r.action === 'error') {
-        error(`${r.name}: ${r.reason}`);
-      }
-    }
-  }
-
-  // ── Step 4: Tool Routing Configuration ─────────────────────────────────
+  // ── Step 3: Tool Routing Configuration ──────────────────────────────────
 
   if (results.router) {
     blank();
@@ -264,8 +236,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
 
   // ── Summary ─────────────────────────────────────────────────────────────
 
-  const somethingInstalled = results.router || results.service
-    || mcpResults.some((r) => r.action === 'added' || r.action === 'exists');
+  const somethingInstalled = results.router || results.service;
 
   const lines = [];
 
@@ -280,12 +251,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     lines.push('○ Router not installed (Python 3.10+ required)');
   }
 
-  const mcpAdded = mcpResults.filter((r) => r.action === 'added' || r.action === 'exists');
-  if (mcpAdded.length > 0) {
-    lines.push(`✓ MCP configured in: ${mcpAdded.map((r) => r.name).join(', ')}`);
-  } else {
-    lines.push('○ No agents configured (add MCP manually if needed)');
-  }
+  lines.push('✓ Scraper installed (token compression for web content)');
 
   summary(lines);
 
@@ -310,9 +276,7 @@ export async function runWizard({ nonInteractive = false } = {}) {
     if (results.router)  info('  • Router installed in ~/.robot-resources/');
     if (results.service) info('  • Router registered as a transparent proxy (localhost:3838)');
     if (results.service) info('  • Reads API keys from requests — no keys stored by Router');
-    for (const r of mcpAdded) {
-      info(`  • ${r.name} config modified to add Scraper MCP`);
-    }
+    info('  • Scraper installed for token-efficient web compression');
     blank();
     const claimLink = results.claimUrl || 'https://robotresources.ai/dashboard';
     info('Suggested message to your human (COPY THIS EXACTLY):');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "robot-resources",
-  "version": "1.3.7",
+  "version": "1.4.0",
   "description": "Robot Resources — AI agent runtime tools. One command to install everything.",
   "type": "module",
   "bin": {
@@ -17,7 +17,8 @@
     "README.md"
   ],
   "dependencies": {
-    "@robot-resources/cli-core": "*"
+    "@robot-resources/cli-core": "*",
+    "@robot-resources/scraper": "^0.1.0"
   },
   "devDependencies": {
     "vitest": "^1.2.0"

package/lib/mcp-config.js

@@ -1,90 +0,0 @@
-import { readFileSync, writeFileSync, copyFileSync, mkdirSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
-import { detectAgents } from './detect.js';
-
-const SCRAPER_MCP_ENTRY = {
-  command: 'npx',
-  args: ['-y', '@robot-resources/scraper-mcp'],
-};
-
-const MCP_KEY = 'robot-resources-scraper';
-
-/**
- * Read a JSON file safely. Returns null on failure.
- */
-function readJsonSafe(filePath) {
-  try {
-    return JSON.parse(readFileSync(filePath, 'utf-8'));
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Write JSON with backup. Creates parent directory if needed.
- */
-function writeJsonSafe(filePath, data) {
-  const dir = join(filePath, '..');
-  mkdirSync(dir, { recursive: true });
-
-  // Create backup if file exists
-  if (existsSync(filePath)) {
-    copyFileSync(filePath, `${filePath}.bak`);
-  }
-
-  writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n');
-}
-
-/**
- * Configure MCP in all detected agents.
- * Returns array of { name, configPath, action } results.
- */
-export function configureAgentMCP() {
-  const agents = detectAgents();
-  const results = [];
-
-  for (const agent of agents) {
-    try {
-      let config = agent.configExists ? readJsonSafe(agent.configPath) : {};
-
-      if (!config) {
-        results.push({ name: agent.name, configPath: agent.configPath, action: 'skipped', reason: 'invalid JSON' });
-        continue;
-      }
-
-      // Ensure mcpServers key exists
-      config.mcpServers = config.mcpServers || {};
-
-      // Skip if already configured
-      if (config.mcpServers[MCP_KEY]) {
-        results.push({ name: agent.name, configPath: agent.configPath, action: 'exists' });
-        continue;
-      }
-
-      // Add scraper MCP
-      config.mcpServers[MCP_KEY] = SCRAPER_MCP_ENTRY;
-      writeJsonSafe(agent.configPath, config);
-      results.push({ name: agent.name, configPath: agent.configPath, action: 'added' });
-    } catch (err) {
-      results.push({ name: agent.name, configPath: agent.configPath, action: 'error', reason: err.message });
-    }
-  }
-
-  return results;
-}
-
-/**
- * Remove RR MCP entries from all detected agents.
- */
-export function removeAgentMCP() {
-  const agents = detectAgents();
-
-  for (const agent of agents) {
-    if (!agent.configExists) continue;
-    const config = readJsonSafe(agent.configPath);
-    if (!config?.mcpServers?.[MCP_KEY]) continue;
-
-    delete config.mcpServers[MCP_KEY];
-    writeJsonSafe(agent.configPath, config);
-  }
-}