seo-intel 1.0.0

package/setup/config-builder.js

@@ -0,0 +1,227 @@
+/**
+ * SEO Intel — Config Builder
+ *
+ * Generates project configuration files and manages .env updates.
+ * Extracted from config/setup-wizard.js for reuse by both CLI and web wizard.
+ */
+
+import { writeFileSync, existsSync, readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dirname, '..');
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+export function slugify(s) {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+}
+
+export function domainFromUrl(url) {
+  try {
+    return new URL(url.startsWith('http') ? url : `https://${url}`).hostname.replace(/^www\./, '');
+  } catch {
+    return url;
+  }
+}
+
+// ── Build Project Config ────────────────────────────────────────────────────
+
+/**
+ * Build a complete project config object.
+ *
+ * @param {object} params
+ * @param {string} params.projectName
+ * @param {string} params.targetUrl
+ * @param {string} params.siteName
+ * @param {string} params.industry
+ * @param {string} params.audience
+ * @param {string} params.goal
+ * @param {string} [params.maturity='early stage']
+ * @param {Array<{url: string}>} [params.competitors=[]]
+ * @param {Array<{url: string}>} [params.owned=[]]
+ * @param {string} [params.crawlMode='standard']
+ * @param {number} [params.pagesPerDomain=50]
+ * @param {string} [params.ollamaHost]
+ * @param {string} [params.extractionModel]
+ * @returns {object} Full config JSON
+ */
+export function buildProjectConfig({
+  projectName,
+  targetUrl,
+  siteName,
+  industry,
+  audience,
+  goal,
+  maturity = 'early stage',
+  competitors = [],
+  owned = [],
+  crawlMode = 'standard',
+  pagesPerDomain = 50,
+  ollamaHost,
+  extractionModel,
+}) {
+  const slug = slugify(projectName);
+  const targetDomain = domainFromUrl(targetUrl);
+  const normalizedUrl = targetUrl.startsWith('http') ? targetUrl : `https://${targetUrl}`;
+
+  const config = {
+    project: slug,
+    crawl: {
+      mode: crawlMode,
+      pagesPerDomain,
+      depth: 3,
+    },
+    context: {
+      siteName: siteName || slug,
+      url: normalizedUrl,
+      industry: industry || '',
+      audience: audience || '',
+      goal: goal || '',
+      maturity,
+    },
+    target: {
+      url: normalizedUrl,
+      domain: targetDomain,
+      role: 'target',
+    },
+    competitors: competitors.map(c => {
+      const url = c.url.startsWith('http') ? c.url : `https://${c.url}`;
+      return {
+        url,
+        domain: domainFromUrl(url),
+        role: 'competitor',
+      };
+    }),
+  };
+
+  // Optional: owned subdomains
+  if (owned.length > 0) {
+    config.owned = owned.map(o => {
+      const url = o.url.startsWith('http') ? o.url : `https://${o.url}`;
+      return {
+        url,
+        domain: domainFromUrl(url),
+        role: 'owned',
+      };
+    });
+  }
+
+  // Optional: Ollama settings
+  if (ollamaHost || extractionModel) {
+    config.crawl.ollamaHost = ollamaHost;
+    config.crawl.extractionModel = extractionModel;
+  }
+
+  return config;
+}
+
+// ── Write Project Config ────────────────────────────────────────────────────
+
+/**
+ * Write a project config to disk.
+ *
+ * @param {object} config - full config object from buildProjectConfig()
+ * @param {string} [rootDir] - override root directory
+ * @returns {{ path: string, overwritten: boolean }}
+ */
+export function writeProjectConfig(config, rootDir = ROOT) {
+  const configPath = join(rootDir, 'config', `${config.project}.json`);
+  const overwritten = existsSync(configPath);
+  writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
+  return { path: configPath, overwritten };
+}
+
+// ── .env Management ─────────────────────────────────────────────────────────
+
+/**
+ * Write a single key to .env (create or update).
+ */
+export function writeEnvKey(key, value, rootDir = ROOT) {
+  const envPath = join(rootDir, '.env');
+  let content = existsSync(envPath) ? readFileSync(envPath, 'utf8') : '';
+  const regex = new RegExp(`^${key}=.*$`, 'm');
+  if (regex.test(content)) {
+    content = content.replace(regex, `${key}=${value}`);
+  } else {
+    content += `\n${key}=${value}`;
+  }
+  writeFileSync(envPath, content.trim() + '\n');
+}
+
+/**
+ * Batch-update .env with setup choices.
+ * Creates .env from .env.example if it doesn't exist.
+ */
+export function updateEnvForSetup(values = {}, rootDir = ROOT) {
+  const envPath = join(rootDir, '.env');
+  const examplePath = join(rootDir, '.env.example');
+
+  // Create .env from template if missing
+  if (!existsSync(envPath)) {
+    if (existsSync(examplePath)) {
+      writeFileSync(envPath, readFileSync(examplePath, 'utf8'));
+    } else {
+      writeFileSync(envPath, '# SEO Intel Configuration\n');
+    }
+  }
+
+  // Apply each value
+  const keyMap = {
+    ollamaUrl: 'OLLAMA_URL',
+    ollamaModel: 'OLLAMA_MODEL',
+    ollamaCtx: 'OLLAMA_CTX',
+    ollamaTimeout: 'OLLAMA_TIMEOUT_MS',
+    geminiKey: 'GEMINI_API_KEY',
+    anthropicKey: 'ANTHROPIC_API_KEY',
+    openaiKey: 'OPENAI_API_KEY',
+    deepseekKey: 'DEEPSEEK_API_KEY',
+    crawlDelay: 'CRAWL_DELAY_MS',
+    crawlMaxPages: 'CRAWL_MAX_PAGES',
+    crawlTimeout: 'CRAWL_TIMEOUT_MS',
+  };
+
+  for (const [jsKey, envKey] of Object.entries(keyMap)) {
+    if (values[jsKey] !== undefined && values[jsKey] !== null && values[jsKey] !== '') {
+      writeEnvKey(envKey, String(values[jsKey]), rootDir);
+    }
+  }
+
+  return { path: envPath };
+}
+
+/**
+ * Validate a project config for completeness.
+ *
+ * @param {object} config
+ * @returns {{ valid: boolean, errors: string[] }}
+ */
+export function validateConfig(config) {
+  const errors = [];
+
+  if (!config.project) errors.push('Missing project name');
+  if (!config.target?.url) errors.push('Missing target URL');
+  if (!config.target?.domain) errors.push('Missing target domain');
+  if (!config.context?.siteName) errors.push('Missing site name');
+
+  // Validate URL format
+  if (config.target?.url) {
+    try {
+      new URL(config.target.url);
+    } catch {
+      errors.push(`Invalid target URL: ${config.target.url}`);
+    }
+  }
+
+  // Validate competitors
+  if (config.competitors) {
+    for (const c of config.competitors) {
+      if (!c.url || !c.domain) {
+        errors.push(`Competitor missing URL or domain: ${JSON.stringify(c)}`);
+      }
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}

package/setup/engine.js

@@ -0,0 +1,65 @@
+/**
+ * SEO Intel — Setup Engine
+ *
+ * Facade that re-exports all setup modules.
+ * Used by both CLI wizard (config/setup-wizard.js) and web wizard (setup/web-routes.js).
+ *
+ * Usage:
+ *   import { fullSystemCheck, getModelRecommendations, ... } from './setup/engine.js';
+ */
+
+// System detection
+export {
+  checkNodeVersion,
+  checkNpm,
+  checkOllamaLocal,
+  checkOllamaRemote,
+  checkOllamaAuto,
+  checkPlaywright,
+  checkNpmDeps,
+  checkEnvFile,
+  checkExistingConfigs,
+  checkGscData,
+  checkOpenClaw,
+  detectOS,
+  detectVRAM,
+  fullSystemCheck,
+  parseEnvFile,
+} from './checks.js';
+
+// Model recommendations
+export {
+  EXTRACTION_MODELS,
+  ANALYSIS_MODELS,
+  recommendExtractionModel,
+  recommendAnalysisModel,
+  getModelRecommendations,
+} from './models.js';
+
+// Auto-installers
+export {
+  installNpmDeps,
+  installPlaywright,
+  pullOllamaModel,
+  createEnvFile,
+} from './installers.js';
+
+// Pipeline validation
+export {
+  testOllamaConnectivity,
+  testApiKey,
+  testCrawl,
+  testExtraction,
+  runFullValidation,
+} from './validator.js';
+
+// Config generation
+export {
+  slugify,
+  domainFromUrl,
+  buildProjectConfig,
+  writeProjectConfig,
+  writeEnvKey,
+  updateEnvForSetup,
+  validateConfig,
+} from './config-builder.js';

package/setup/installers.js

@@ -0,0 +1,197 @@
+/**
+ * SEO Intel — Auto-Installers
+ *
+ * Async generator functions that install dependencies and yield progress events.
+ * Both CLI and web wizard consume the same generators — CLI prints, web streams via SSE.
+ *
+ * Usage:
+ *   for await (const ev of installNpmDeps()) console.log(ev.message);
+ */
+
+import { spawn } from 'child_process';
+import { existsSync, readFileSync, writeFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dirname, '..');
+
+// ── Event helpers ───────────────────────────────────────────────────────────
+
+function ev(phase, status, message, extra = {}) {
+  return { phase, status, message, ts: Date.now(), ...extra };
+}
+
+// ── npm install ─────────────────────────────────────────────────────────────
+
+export async function* installNpmDeps(rootDir = ROOT) {
+  yield ev('npm-install', 'start', 'Installing npm dependencies...');
+
+  try {
+    const result = await runCommand('npm', ['install', '--no-audit', '--no-fund'], rootDir);
+
+    if (result.exitCode === 0) {
+      yield ev('npm-install', 'done', 'npm dependencies installed successfully.');
+    } else {
+      yield ev('npm-install', 'error', `npm install failed (exit ${result.exitCode}): ${result.stderr.slice(0, 500)}`);
+    }
+  } catch (err) {
+    yield ev('npm-install', 'error', `npm install error: ${err.message}`);
+  }
+}
+
+// ── Playwright Chromium ─────────────────────────────────────────────────────
+
+export async function* installPlaywright(rootDir = ROOT) {
+  yield ev('playwright', 'start', 'Installing Playwright Chromium browser (~150MB)...');
+
+  try {
+    const result = await runCommand('npx', ['playwright', 'install', 'chromium'], rootDir);
+
+    if (result.exitCode === 0) {
+      yield ev('playwright', 'done', 'Playwright Chromium installed successfully.');
+    } else {
+      yield ev('playwright', 'error', `Playwright install failed (exit ${result.exitCode}): ${result.stderr.slice(0, 500)}`);
+    }
+  } catch (err) {
+    yield ev('playwright', 'error', `Playwright install error: ${err.message}`);
+  }
+}
+
+// ── Ollama model pull ───────────────────────────────────────────────────────
+
+export async function* pullOllamaModel(model, host = 'http://localhost:11434') {
+  yield ev('ollama-pull', 'start', `Pulling model ${model} from Ollama...`);
+
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 600000); // 10 min timeout for large models
+
+    const res = await fetch(`${host}/api/pull`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ name: model, stream: true }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      yield ev('ollama-pull', 'error', `Ollama pull failed: HTTP ${res.status} ${text.slice(0, 200)}`);
+      return;
+    }
+
+    const reader = res.body;
+    let lastPercent = -1;
+
+    // Stream Ollama's NDJSON progress
+    for await (const chunk of reader) {
+      const text = typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk);
+      const lines = text.split('\n').filter(Boolean);
+
+      for (const line of lines) {
+        try {
+          const data = JSON.parse(line);
+
+          if (data.error) {
+            yield ev('ollama-pull', 'error', `Ollama error: ${data.error}`);
+            return;
+          }
+
+          if (data.total && data.completed) {
+            const percent = Math.round((data.completed / data.total) * 100);
+            if (percent !== lastPercent && percent % 5 === 0) {
+              lastPercent = percent;
+              yield ev('ollama-pull', 'progress', `Downloading ${model}... ${percent}%`, { progress: percent });
+            }
+          } else if (data.status) {
+            // Status messages like "pulling manifest", "verifying sha256 digest"
+            yield ev('ollama-pull', 'progress', data.status);
+          }
+        } catch {
+          // Skip unparseable lines
+        }
+      }
+    }
+
+    yield ev('ollama-pull', 'done', `Model ${model} pulled successfully.`);
+  } catch (err) {
+    if (err.name === 'AbortError') {
+      yield ev('ollama-pull', 'error', `Ollama pull timed out after 10 minutes.`);
+    } else {
+      yield ev('ollama-pull', 'error', `Ollama pull error: ${err.message}`);
+    }
+  }
+}
+
+// ── Create .env from template ───────────────────────────────────────────────
+
+export function* createEnvFile(rootDir = ROOT) {
+  const envPath = join(rootDir, '.env');
+  const examplePath = join(rootDir, '.env.example');
+
+  if (existsSync(envPath)) {
+    yield ev('env-create', 'done', '.env file already exists — keeping it.');
+    return;
+  }
+
+  if (!existsSync(examplePath)) {
+    // Create a minimal .env
+    const minimal = [
+      '# SEO Intel Configuration',
+      '',
+      '# Cloud model for analysis (pick one)',
+      'GEMINI_API_KEY=',
+      '# ANTHROPIC_API_KEY=',
+      '# OPENAI_API_KEY=',
+      '',
+      '# Local Ollama for extraction',
+      'OLLAMA_URL=http://localhost:11434',
+      'OLLAMA_MODEL=qwen3.5:9b',
+      'OLLAMA_CTX=8192',
+      '',
+      '# Crawler settings',
+      'CRAWL_DELAY_MS=1500',
+      'CRAWL_MAX_PAGES=50',
+      'CRAWL_TIMEOUT_MS=15000',
+      '',
+    ].join('\n');
+
+    writeFileSync(envPath, minimal);
+    yield ev('env-create', 'done', 'Created .env with default values.');
+    return;
+  }
+
+  writeFileSync(envPath, readFileSync(examplePath, 'utf8'));
+  yield ev('env-create', 'done', 'Created .env from .env.example template.');
+}
+
+// ── Spawn helper ────────────────────────────────────────────────────────────
+
+function runCommand(cmd, args, cwd = ROOT) {
+  return new Promise((resolve, reject) => {
+    let stdout = '';
+    let stderr = '';
+
+    const proc = spawn(cmd, args, {
+      cwd,
+      shell: true,
+      env: { ...process.env, FORCE_COLOR: '0' },
+    });
+
+    proc.stdout?.on('data', (d) => { stdout += d.toString(); });
+    proc.stderr?.on('data', (d) => { stderr += d.toString(); });
+
+    proc.on('error', reject);
+    proc.on('close', (exitCode) => {
+      resolve({ exitCode, stdout, stderr });
+    });
+
+    // Timeout after 5 minutes for npm/playwright
+    setTimeout(() => {
+      proc.kill('SIGTERM');
+      reject(new Error('Command timed out after 5 minutes'));
+    }, 300000);
+  });
+}