seo-intel 1.0.0

package/setup/models.js

@@ -0,0 +1,328 @@
+/**
+ * SEO Intel — Model Recommendations
+ *
+ * Two-tier model system:
+ *   Extraction tier: local Ollama model for structured JSON extraction during crawl
+ *   Analysis tier: powerful cloud model (or large local) for strategic gap analysis
+ *
+ * Includes hardware-based auto-recommendation using VRAM detection from checks.js
+ */
+
+// ── Extraction Models (local, runs during crawl) ────────────────────────────
+//
+// The extraction task is structured JSON extraction:
+//   Input:  ~1-4KB text (URL + title + headings + body excerpt)
+//   Output: ~900 tokens JSON (13 fields: enums, keyword arrays, entities)
+//   Complexity: low-medium — pattern matching + classification
+//   Minimum viable: 4B parameters for reliable JSON output
+
+export const EXTRACTION_MODELS = [
+  {
+    id: 'qwen3.5:4b',
+    name: 'Qwen 3.5 4B',
+    family: 'qwen3.5',
+    tier: 'budget',
+    vram: '~3 GB',
+    minVramMB: 2500,
+    speed: '~2s/page',
+    quality: 'good',
+    description: 'Minimum recommended. Reliable JSON extraction, decent keyword detection. Great for laptops and older GPUs.',
+    recommended: false,
+  },
+  {
+    id: 'qwen3.5:9b',
+    name: 'Qwen 3.5 9B',
+    family: 'qwen3.5',
+    tier: 'balanced',
+    vram: '~5 GB',
+    minVramMB: 4500,
+    speed: '~3s/page',
+    quality: 'better',
+    description: 'Default recommendation. Better entity detection and intent classification. Works on most modern GPUs.',
+    recommended: true,
+  },
+  {
+    id: 'qwen3.5:27b',
+    name: 'Qwen 3.5 27B',
+    family: 'qwen3.5',
+    tier: 'quality',
+    vram: '~17 GB',
+    minVramMB: 15000,
+    speed: '~6s/page',
+    quality: 'great',
+    description: 'Nuanced intent classification, better keyword quality. Needs RTX 3090+ or M-series with 24GB+.',
+    recommended: false,
+  },
+  {
+    id: 'qwen3.5:35b',
+    name: 'Qwen 3.5 35B',
+    family: 'qwen3.5',
+    tier: 'power',
+    vram: '~22 GB',
+    minVramMB: 18000,
+    speed: '~8s/page',
+    quality: 'excellent',
+    description: 'Near-cloud quality extraction. Needs RTX 3090/4090 or M2 Ultra. Overkill for most users.',
+    recommended: false,
+  },
+  // Alternative providers
+  {
+    id: 'nemotron-nano:4b',
+    name: 'Nemotron 3 Nano 4B',
+    family: 'nemotron',
+    tier: 'budget',
+    vram: '~3 GB',
+    minVramMB: 2500,
+    speed: '~2s/page',
+    quality: 'good',
+    description: 'NVIDIA agentic model. Efficient extraction with tool-use training. Good alternative to Qwen 3.5 4B.',
+    recommended: false,
+  },
+  // Legacy / fallback models (already installed by many users)
+  {
+    id: 'qwen3:4b',
+    name: 'Qwen 3 4B (legacy)',
+    family: 'qwen3',
+    tier: 'budget',
+    vram: '~3 GB',
+    minVramMB: 2500,
+    speed: '~2s/page',
+    quality: 'good',
+    description: 'Previous generation. Works well but Qwen 3.5 is better if you can upgrade.',
+    recommended: false,
+    legacy: true,
+  },
+  {
+    id: 'qwen3:8b',
+    name: 'Qwen 3 8B (legacy)',
+    family: 'qwen3',
+    tier: 'balanced',
+    vram: '~5 GB',
+    minVramMB: 4500,
+    speed: '~3s/page',
+    quality: 'better',
+    description: 'Previous generation. Solid extraction. Qwen 3.5 recommended for new installs.',
+    recommended: false,
+    legacy: true,
+  },
+];
+
+// ── Analysis Models (local Ollama, runs during analysis) ─────────────────────
+//
+// The analysis task is heavy strategic reasoning:
+//   Input:  10K-100K tokens (full crawl dataset, keyword matrices, competitor data)
+//   Output: structured JSON with strategic recommendations, positioning, gap analysis
+//   Complexity: high — comparative reasoning across multiple domains
+//   Minimum viable: 14B+ parameters for reliable strategic output
+//   Cloud models (Claude, GPT-4o, DeepSeek) available via OpenClaw agent setup
+
+export const ANALYSIS_MODELS = [
+  {
+    id: 'qwen3:14b',
+    name: 'Qwen 3 14B',
+    family: 'qwen3',
+    type: 'local',
+    vram: '~9 GB',
+    minVramMB: 8000,
+    context: '32K tokens',
+    costNote: 'Free (your GPU)',
+    quality: 'decent',
+    recommended: false,
+    description: 'Minimum viable for analysis. Handles small-medium projects. Needs RTX 3070+ or M1 Pro+.',
+  },
+  {
+    id: 'qwen3.5:27b',
+    name: 'Qwen 3.5 27B',
+    family: 'qwen3.5',
+    type: 'local',
+    vram: '~17 GB',
+    minVramMB: 15000,
+    context: '32K tokens',
+    costNote: 'Free (your GPU)',
+    quality: 'good',
+    recommended: true,
+    description: 'Sweet spot for local analysis. Strong reasoning with 27.8B params. Needs RTX 3090/4080+ or M-series with 24GB+.',
+  },
+  {
+    id: 'qwen3.5:35b',
+    name: 'Qwen 3.5 35B',
+    family: 'qwen3.5',
+    type: 'local',
+    vram: '~22 GB',
+    minVramMB: 18000,
+    context: '32K tokens',
+    costNote: 'Free (your GPU)',
+    quality: 'great',
+    recommended: false,
+    description: 'High quality analysis. Best Qwen 3.5 for strategic reasoning. Needs RTX 3090/4090 or M2 Ultra.',
+  },
+  {
+    id: 'nemotron-3-super:120b',
+    name: 'Nemotron 3 Super 120B',
+    family: 'nemotron-3-super',
+    type: 'local',
+    vram: '~87 GB',
+    minVramMB: 48000,
+    context: '32K tokens',
+    costNote: 'Free (your GPU)',
+    quality: 'excellent',
+    recommended: false,
+    description: 'MoE — 120B total but only 12B active params. Excellent reasoning at efficient compute. Needs 64GB+ unified memory or multi-GPU.',
+  },
+];
+
+// ── VRAM-Based Recommendations ──────────────────────────────────────────────
+
+const VRAM_TIERS = [
+  { maxMB: 2500,  extraction: null,          note: 'Not enough VRAM for local extraction. Use cloud or CPU mode (slow).' },
+  { maxMB: 4500,  extraction: 'qwen3.5:4b',  note: 'Budget tier — Qwen 3.5 4B fits your GPU.' },
+  { maxMB: 8000,  extraction: 'qwen3.5:9b',  note: 'Balanced tier — Qwen 3.5 9B recommended for best quality/speed.' },
+  { maxMB: 18000, extraction: 'qwen3.5:27b', note: 'Quality tier — Qwen 3.5 27B for nuanced extraction.' },
+  { maxMB: 48000, extraction: 'qwen3.5:35b', note: 'Power tier — Qwen 3.5 35B for near-cloud quality.' },
+  { maxMB: Infinity, extraction: 'qwen3.5:35b', note: 'Power tier — Qwen 3.5 35B recommended. Your GPU can handle anything.' },
+];
+
+/**
+ * Recommend an extraction model based on available models and VRAM.
+ * Priority: prefer already-installed models that match VRAM tier.
+ *
+ * @param {string[]} availableModels - models currently in Ollama
+ * @param {number} [vramMB] - detected VRAM in MB (0 if unknown)
+ * @returns {{ model: object, installed: boolean, autoRecommended: boolean, note: string } | null}
+ */
+export function recommendExtractionModel(availableModels = [], vramMB = 0) {
+  // Find VRAM tier
+  const tier = VRAM_TIERS.find(t => vramMB <= t.maxMB) || VRAM_TIERS[VRAM_TIERS.length - 1];
+
+  // Preferred model order (newest → legacy)
+  const preferenceOrder = [
+    'qwen3.5:9b', 'qwen3.5:27b', 'qwen3.5:4b', 'qwen3.5:35b',
+    'qwen3:8b', 'qwen3:4b', 'qwen3.5:0.6b',
+  ];
+
+  // Filter to models that fit VRAM
+  const fittingModels = EXTRACTION_MODELS.filter(m => !vramMB || vramMB >= m.minVramMB);
+
+  // 1. Best installed model that fits VRAM
+  for (const prefId of preferenceOrder) {
+    const isInstalled = availableModels.some(m => m.startsWith(prefId.split(':')[0]) && m.includes(prefId.split(':')[1]));
+    const modelDef = fittingModels.find(m => m.id === prefId);
+    if (isInstalled && modelDef) {
+      return {
+        model: modelDef,
+        installed: true,
+        autoRecommended: modelDef.id === tier.extraction,
+        note: `Already installed. ${tier.note}`,
+      };
+    }
+  }
+
+  // 2. VRAM-recommended model (not installed)
+  if (tier.extraction) {
+    const modelDef = EXTRACTION_MODELS.find(m => m.id === tier.extraction);
+    if (modelDef) {
+      return {
+        model: modelDef,
+        installed: false,
+        autoRecommended: true,
+        note: `Recommended for your hardware. ${tier.note}`,
+      };
+    }
+  }
+
+  // 3. Any installed model
+  for (const prefId of preferenceOrder) {
+    const isInstalled = availableModels.some(m => m.startsWith(prefId.split(':')[0]));
+    const modelDef = EXTRACTION_MODELS.find(m => m.id === prefId);
+    if (isInstalled && modelDef) {
+      return {
+        model: modelDef,
+        installed: true,
+        autoRecommended: false,
+        note: `Installed but may be slow for your VRAM. Consider upgrading.`,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Recommend an analysis model based on available Ollama models and VRAM.
+ *
+ * @param {string[]} availableModels - models currently in Ollama
+ * @param {number} [vramMB] - detected VRAM in MB
+ * @returns {{ model: object, installed: boolean, note: string }}
+ */
+export function recommendAnalysisModel(availableModels = [], vramMB = 0) {
+  const preferenceOrder = [
+    'qwen3.5:27b', 'qwen3.5:35b', 'qwen3:14b', 'nemotron-3-super:120b',
+  ];
+
+  // Filter to models that fit VRAM
+  const fittingModels = ANALYSIS_MODELS.filter(m => !vramMB || vramMB >= m.minVramMB);
+
+  // 1. Best installed model that fits VRAM
+  for (const prefId of preferenceOrder) {
+    const isInstalled = availableModels.some(m => m.startsWith(prefId.split(':')[0]) && m.includes(prefId.split(':')[1]));
+    const modelDef = fittingModels.find(m => m.id === prefId);
+    if (isInstalled && modelDef) {
+      return {
+        model: modelDef,
+        installed: true,
+        note: `Already installed. Ready for analysis.`,
+      };
+    }
+  }
+
+  // 2. VRAM-based recommendation
+  let recId = 'qwen3:14b'; // default minimum
+  if (vramMB >= 48000) recId = 'nemotron-3-super:120b';
+  else if (vramMB >= 18000) recId = 'qwen3.5:35b';
+  else if (vramMB >= 15000) recId = 'qwen3.5:27b';
+
+  const recModel = ANALYSIS_MODELS.find(m => m.id === recId);
+  if (recModel) {
+    return {
+      model: recModel,
+      installed: false,
+      note: `Recommended for your hardware. Use OpenClaw for cloud models.`,
+    };
+  }
+
+  return {
+    model: ANALYSIS_MODELS[0],
+    installed: false,
+    note: 'Minimum viable model for local analysis.',
+  };
+}
+
+/**
+ * Get all model recommendations for display.
+ *
+ * @param {string[]} availableModels - models in Ollama
+ * @param {object} envKeys - { GEMINI_API_KEY: true, ... }
+ * @param {number} vramMB - detected VRAM
+ * @returns {{ extraction: object, analysis: object, allExtraction: object[], allAnalysis: object[] }}
+ */
+export function getModelRecommendations(availableModels = [], envKeys = {}, vramMB = 0) {
+  return {
+    extraction: recommendExtractionModel(availableModels, vramMB),
+    analysis: recommendAnalysisModel(availableModels, vramMB),
+    allExtraction: EXTRACTION_MODELS.map(m => ({
+      ...m,
+      installed: availableModels.some(am =>
+        am.startsWith(m.family) && am.includes(m.id.split(':')[1])
+      ),
+      fitsVram: !vramMB || vramMB >= m.minVramMB,
+    })),
+    allAnalysis: ANALYSIS_MODELS.map(m => ({
+      ...m,
+      installed: availableModels.some(am =>
+        am.startsWith(m.family) && am.includes(m.id.split(':')[1])
+      ),
+      fitsVram: !vramMB || vramMB >= m.minVramMB,
+    })),
+    vramMB,
+  };
+}

package/setup/openclaw-bridge.js

@@ -0,0 +1,329 @@
+/**
+ * SEO Intel — OpenClaw Setup Bridge
+ *
+ * When OpenClaw gateway is running, this module lets the setup process
+ * delegate to the agent for a conversational, intelligent setup flow.
+ *
+ * Instead of a rigid wizard with fixed steps, the agent:
+ *   1. Reads the system check results
+ *   2. Knows what's missing / misconfigured
+ *   3. Guides the user conversationally through fixes
+ *   4. Can troubleshoot errors in real-time
+ *   5. Configures everything via the setup engine API
+ *
+ * Falls back to the standard wizard if OpenClaw is not available.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dirname, '..');
+
+const OPENCLAW_API = 'http://127.0.0.1:18789';
+
+// ── OpenClaw Gateway Communication ─────────────────────────────────────────
+
+/**
+ * Send a message to OpenClaw's agent via the OpenAI-compatible API.
+ * Returns the agent's text response.
+ */
+async function askAgent(messages, opts = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), opts.timeout || 60000);
+
+  try {
+    const res = await fetch(`${OPENCLAW_API}/v1/chat/completions`, {
+      method: 'POST',
+      signal: controller.signal,
+      headers: {
+        'Content-Type': 'application/json',
+        ...(opts.token ? { 'Authorization': `Bearer ${opts.token}` } : {}),
+      },
+      body: JSON.stringify({
+        model: opts.model || 'default',
+        messages,
+        temperature: 0.3,
+        max_tokens: 2000,
+      }),
+    });
+
+    if (!res.ok) {
+      const err = await res.text();
+      throw new Error(`OpenClaw API error: ${res.status} ${err}`);
+    }
+
+    const data = await res.json();
+    return data.choices?.[0]?.message?.content || '';
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+/**
+ * Check if the OpenClaw gateway is reachable and ready.
+ */
+export async function isGatewayReady() {
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 3000);
+    const res = await fetch(`${OPENCLAW_API}/v1/models`, {
+      signal: controller.signal,
+    });
+    clearTimeout(timeout);
+    return res.ok;
+  } catch {
+    return false;
+  }
+}
+
+// ── Setup Context Builder ──────────────────────────────────────────────────
+
+/**
+ * Build a context message for the agent with full system status.
+ * This gives the agent everything it needs to guide the setup.
+ */
+function buildSetupContext(systemCheck) {
+  const ctx = {
+    tool: 'SEO Intel',
+    version: systemCheck._version || '0.2.0',
+    installDir: ROOT,
+    systemStatus: {
+      node: {
+        version: systemCheck.node.version,
+        ok: systemCheck.node.meetsMinimum,
+      },
+      ollama: {
+        available: systemCheck.ollama.available,
+        host: systemCheck.ollama.host,
+        models: systemCheck.ollama.models?.map(m => m.name) || [],
+      },
+      playwright: {
+        installed: systemCheck.playwright.installed,
+      },
+      apiKeys: {
+        gemini: systemCheck.env.keys.GEMINI_API_KEY || false,
+        anthropic: systemCheck.env.keys.ANTHROPIC_API_KEY || false,
+        openai: systemCheck.env.keys.OPENAI_API_KEY || false,
+        deepseek: systemCheck.env.keys.DEEPSEEK_API_KEY || false,
+      },
+      gpu: {
+        vram: systemCheck.vram.vramMB,
+        name: systemCheck.vram.gpuName,
+      },
+      existingProjects: systemCheck.configs.projects?.map(p => p.project) || [],
+      gsc: {
+        hasData: systemCheck.gsc?.hasData || false,
+      },
+    },
+    capabilities: systemCheck.summary,
+    setupApiBase: 'http://localhost:3000/api/setup',
+  };
+
+  return JSON.stringify(ctx, null, 2);
+}
+
+// ── The System Prompt ──────────────────────────────────────────────────────
+
+const SETUP_SYSTEM_PROMPT = `You are the SEO Intel setup assistant. You help users configure SEO Intel — a competitive SEO intelligence tool that runs locally.
+
+IMPORTANT RULES:
+- Be concise and friendly. No walls of text.
+- Guide step by step. One thing at a time.
+- When something is already configured, acknowledge it and move on.
+- If something is broken, explain what's wrong and offer to fix it.
+- You have access to bash to run commands. Use it to install things, check status, and configure.
+
+THE SETUP FLOW:
+1. Check what's already working (I'll give you the system status)
+2. Fix any missing dependencies (Node, npm deps, Playwright, Ollama)
+3. Help choose and configure the extraction model (local Ollama recommended)
+4. Help configure an analysis API key (Gemini recommended for best value)
+5. Create a project config (target domain + competitors)
+6. Optionally set up Google Search Console data
+7. Run a quick pipeline test to verify everything works
+8. Show the user their first commands to run
+
+AVAILABLE COMMANDS (run these from ${ROOT}):
+- node cli.js setup-web → opens web wizard at http://localhost:3000/setup
+- node cli.js crawl <project> → crawl domains
+- node cli.js extract <project> → extract with local AI
+- node cli.js analyze <project> → run gap analysis
+- node cli.js html <project> → generate dashboard
+- node cli.js serve → start dashboard server
+- node cli.js status → show system status
+- node cli.js competitors <project> → manage domains
+- node cli.js auth → show auth connections
+- node cli.js export-actions <project> [--scope technical|competitive|suggestive|all] [--format json|brief] → agentic export (technical free, rest Solo)
+- node cli.js competitive-actions <project> [--vs domain] [--format json|brief] → competitive gap export (Solo)
+- node cli.js suggest-usecases <project> [--scope docs|product-pages|onboarding|all] [--format json|brief] → suggest what to build (Solo)
+
+TO INSTALL THINGS:
+- npm install (in ${ROOT}) → install Node dependencies
+- npx playwright install chromium → install browser
+- ollama pull qwen3.5:9b → install extraction model
+
+TO CONFIGURE:
+- Edit ${ROOT}/.env for API keys and settings
+- Project configs go in ${ROOT}/config/<project>.json
+
+ANALYSIS MODELS (user needs at least one API key):
+- Gemini: Best value, 1M context (~$0.01-0.05/analysis) → GEMINI_API_KEY
+- Claude: Best quality, nuanced reasoning (~$0.10-0.30) → ANTHROPIC_API_KEY
+- OpenAI: Solid all-around (~$0.05-0.15) → OPENAI_API_KEY
+- DeepSeek: Budget option (~$0.02-0.08) → DEEPSEEK_API_KEY
+
+EXTRACTION MODELS (local, free):
+- qwen3.5:9b (recommended, needs 6GB+ VRAM)
+- qwen3.5:4b (budget, needs 3GB+ VRAM)
+- qwen3.5:27b (quality, needs 16GB+ VRAM)`;
+
+// ── Agent-Driven Setup Flow ────────────────────────────────────────────────
+
+/**
+ * Start an agent-driven setup session.
+ * Uses OpenClaw's agent to guide the user conversationally.
+ *
+ * @param {object} systemCheck - Result from fullSystemCheck()
+ * @param {object} [opts]
+ * @param {function} [opts.onMessage] - Callback for agent messages
+ * @param {function} [opts.onInput] - Callback to get user input
+ * @returns {Promise<{ completed: boolean, project?: string }>}
+ */
+export async function runAgentSetup(systemCheck, opts = {}) {
+  const { onMessage, onInput } = opts;
+
+  const context = buildSetupContext(systemCheck);
+
+  const messages = [
+    {
+      role: 'system',
+      content: SETUP_SYSTEM_PROMPT,
+    },
+    {
+      role: 'user',
+      content: `Here's the current system status. Guide me through setting up SEO Intel. Be concise — one step at a time.\n\n${context}`,
+    },
+  ];
+
+  // Get initial response
+  const initialResponse = await askAgent(messages);
+  messages.push({ role: 'assistant', content: initialResponse });
+
+  if (onMessage) onMessage(initialResponse);
+
+  // Conversational loop
+  let completed = false;
+  let maxTurns = 20; // safety limit
+
+  while (!completed && maxTurns > 0) {
+    maxTurns--;
+
+    // Get user input
+    const userInput = onInput ? await onInput() : null;
+    if (!userInput || userInput.toLowerCase() === 'done' || userInput.toLowerCase() === 'exit') {
+      completed = true;
+      break;
+    }
+
+    messages.push({ role: 'user', content: userInput });
+
+    const response = await askAgent(messages);
+    messages.push({ role: 'assistant', content: response });
+
+    if (onMessage) onMessage(response);
+
+    // Check if setup seems complete
+    if (response.includes('setup is complete') || response.includes('all set') || response.includes('ready to go')) {
+      completed = true;
+    }
+  }
+
+  return { completed };
+}
+
+// ── CLI Integration ────────────────────────────────────────────────────────
+
+/**
+ * Run the OpenClaw-powered setup from the CLI.
+ * Falls back to web wizard if OpenClaw isn't available.
+ */
+export async function cliAgentSetup(systemCheck) {
+  const readline = await import('readline');
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  const ask = (prompt) => new Promise(resolve => rl.question(prompt, resolve));
+
+  console.log('\n  \x1b[36m\x1b[1m🐸 SEO Intel — Agent-Powered Setup\x1b[0m\n');
+  console.log('  \x1b[2mOpenClaw is guiding your setup. Type your answers, or "done" to finish.\x1b[0m\n');
+
+  try {
+    await runAgentSetup(systemCheck, {
+      onMessage: (msg) => {
+        // Format agent output with indentation
+        const lines = msg.split('\n');
+        for (const line of lines) {
+          console.log(`  ${line}`);
+        }
+        console.log();
+      },
+      onInput: async () => {
+        const input = await ask('  \x1b[36m>\x1b[0m ');
+        return input;
+      },
+    });
+  } finally {
+    rl.close();
+  }
+
+  console.log('\n  \x1b[32m✓ Setup session ended.\x1b[0m\n');
+}
+
+// ── Web API Integration ────────────────────────────────────────────────────
+
+/**
+ * Handle a setup chat message via the web API.
+ * Used by the web wizard's "Agent Mode" chat panel.
+ *
+ * @param {object} body - { message, history, systemCheck }
+ * @returns {Promise<{ response: string }>}
+ */
+export async function handleAgentChat(body) {
+  const { message, history = [], systemCheck } = body;
+
+  const context = systemCheck ? buildSetupContext(systemCheck) : '';
+
+  const messages = [
+    {
+      role: 'system',
+      content: SETUP_SYSTEM_PROMPT,
+    },
+  ];
+
+  // Add context as first user message if this is a new conversation
+  if (history.length === 0 && context) {
+    messages.push({
+      role: 'user',
+      content: `System status:\n${context}`,
+    });
+  }
+
+  // Add conversation history
+  for (const msg of history) {
+    messages.push({
+      role: msg.role,
+      content: msg.content,
+    });
+  }
+
+  // Add current message
+  messages.push({ role: 'user', content: message });
+
+  const response = await askAgent(messages);
+
+  return { response };
+}