seo-intel 1.5.45 → 1.5.50

package/cli.js

@@ -20,7 +20,6 @@ import { totalmem } from 'os';
 import { fileURLToPath } from 'url';
 import chalk from 'chalk';
 
-import { crawlDomain } from './crawler/index.js';
 // Paid modules — loaded lazily inside gated commands only.
 let _extractPage, _buildAnalysisPrompt;
 async function getExtractPage() {
@@ -57,6 +56,13 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 // Start background update check (non-blocking, never slows startup)
 checkForUpdates();
 
+// Long-running commands (serve, setup-web) intentionally keep the process alive.
+// Everything else is one-shot: once the action resolves we force-exit so a
+// hung background fetch (update check / license phone-home) can't hold the
+// shell — an aborted fetch's socket lingers until the OS connect timeout, so
+// AbortController alone is not enough to make the process exit promptly.
+let _keepProcessAlive = false;
+
 // Ensure reports/ and config/ directories exist
 try { mkdirSync(join(__dirname, 'reports'), { recursive: true }); } catch { /* ok */ }
 try { mkdirSync(join(__dirname, 'config'), { recursive: true }); } catch { /* ok */ }
@@ -575,6 +581,8 @@ program
         },
       };
 
+      // Lazy: crawler/index.js pulls Playwright + turndown — keep them off CLI startup
+      const { crawlDomain } = await import('./crawler/index.js');
       for await (const page of crawlDomain(site.url, crawlOpts)) {
         if (page._blocked) {
           totalBlocked++;
@@ -1323,6 +1331,7 @@ program
     let pageCount = 0;
     let skipped = 0;
     let blocked = false;
+    const { crawlDomain } = await import('./crawler/index.js');
     for await (const page of crawlDomain(next.url, {
       onSitemapDiscovered: (urls) => {
         try { upsertSitemapUrls(db, domainId, urls.map(u => u.url), `${next.url}/sitemap.xml`); }
@@ -1547,6 +1556,84 @@ program
     }
   });
 
+// ── MODELS ─────────────────────────────────────────────────────────────────
+program
+  .command('models')
+  .description('Suggest local extraction models for your hardware (Gemma / Qwen) — local is strongly recommended over cloud')
+  .option('--format <type>', 'Output format: brief or json', 'brief')
+  .action(async (opts) => {
+    const isJson = opts.format === 'json';
+    const { suggestExtractionModels, CLOUD_EXTRACTION_DISCLAIMER, detectVRAM } = await import('./setup/engine.js');
+
+    // Detect hardware (best-effort) + installed Ollama models (quick, short timeout).
+    let vram = { available: false, vramMB: 0, gpuName: null };
+    try { vram = detectVRAM(); } catch { /* keep default */ }
+    let installed = [];
+    try {
+      const controller = new AbortController();
+      const t = setTimeout(() => controller.abort(), 1500);
+      const res = await fetch('http://localhost:11434/api/tags', { signal: controller.signal });
+      clearTimeout(t);
+      if (res.ok) {
+        const data = await res.json();
+        installed = (data.models || []).map(m => m.name);
+      }
+    } catch { /* Ollama not reachable — fine */ }
+
+    const { suggestions, recommendedId } = suggestExtractionModels(vram.vramMB || 0, installed);
+
+    if (isJson) {
+      console.log(JSON.stringify({
+        command: 'models',
+        hardware: { gpu: vram.gpuName || null, vramMB: vram.vramMB || 0 },
+        recommended: recommendedId,
+        suggestions,
+        cloud_disclaimer: CLOUD_EXTRACTION_DISCLAIMER,
+      }, null, 2));
+      return;
+    }
+
+    console.log('');
+    console.log(chalk.bold('  🧠 Local extraction models'));
+    console.log('');
+    if (vram.available && vram.vramMB) {
+      console.log(chalk.gray(`  Detected: ${vram.gpuName || 'GPU'} · ~${(vram.vramMB / 1024).toFixed(1)} GB VRAM`));
+    } else {
+      console.log(chalk.gray('  GPU/VRAM not detected — showing the full range. Pick by your machine.'));
+    }
+    console.log('');
+
+    const qFmt = (q) => q === 'excellent' ? chalk.green(q) : q === 'great' || q === 'better' ? chalk.cyan(q) : chalk.gray(q);
+    for (const s of suggestions) {
+      const star = s.id === recommendedId ? chalk.bold.green(' ◀ recommended') : '';
+      const fit = vram.vramMB && !s.fitsVram ? chalk.red(' (needs more VRAM)') : '';
+      const got = s.installed ? chalk.green(' ✓ installed') : chalk.gray(` ollama pull ${s.id}`);
+      console.log(`  ${chalk.bold(s.name.padEnd(16))} ${chalk.gray(s.vram.padEnd(8))} ${s.speed.padEnd(11)} ${qFmt(s.quality)}${star}${fit}`);
+      console.log(`  ${' '.repeat(16)} ${got}`);
+    }
+    console.log('');
+
+    // The MUST: extraction-should-be-local disclaimer, every time.
+    console.log(chalk.bold.yellow('  ⚠  Extraction should be done with a LOCAL model'));
+    const wrap = (text, width) => {
+      const out = []; let line = '';
+      for (const word of text.split(/\s+/)) {
+        if ((line + ' ' + word).trim().length > width) { out.push(line.trim()); line = word; }
+        else line += ' ' + word;
+      }
+      if (line.trim()) out.push(line.trim());
+      return out;
+    };
+    for (const line of wrap(CLOUD_EXTRACTION_DISCLAIMER, 78)) {
+      console.log(chalk.yellow('     ' + line));
+    }
+    console.log('');
+    if (recommendedId && !suggestions.find(s => s.id === recommendedId)?.installed) {
+      console.log(chalk.gray(`  Get started:  `) + chalk.white(`ollama pull ${recommendedId}`) + chalk.gray(`  then  `) + chalk.white(`seo-intel setup`));
+      console.log('');
+    }
+  });
+
 // ── STATUS ─────────────────────────────────────────────────────────────────
 program
   .command('status')
@@ -2357,6 +2444,7 @@ program
   .option('--open', 'Open browser automatically', true)
   .option('--no-open', 'Do not open browser')
   .action(async (opts) => {
+    _keepProcessAlive = true;
     const port = parseInt(opts.port, 10);
     process.env.PORT = String(port);
     if (opts.open) process.env.SEO_INTEL_AUTO_OPEN = '1';
@@ -2369,6 +2457,7 @@ program
   .description('Open the web-based setup wizard in your browser')
   .option('--port <n>', 'Server port', '3000')
   .action(async (opts) => {
+    _keepProcessAlive = true;
     const port = parseInt(opts.port, 10);
     process.env.PORT = String(port);
     await import('./server.js');
@@ -4326,9 +4415,24 @@ program
     }
 
     const { runAeoAnalysis, persistAeoScores, upsertCitabilityInsights } = await import('./analyses/aeo/index.js');
+    const { fetchAiAccessForDomains } = await import('./analyses/aeo/ai-access.js');
+
+    // AI-crawler access (robots.txt) — domain-level signal. Network, but cheap
+    // (one robots.txt per target/owned domain) and best-effort.
+    const targetDomains = db
+      .prepare("SELECT DISTINCT domain FROM domains WHERE project = ? AND role IN ('target','owned')")
+      .all(project)
+      .map(r => r.domain);
+    let aiAccessByDomain = null;
+    if (targetDomains.length) {
+      if (isBrief) console.log(chalk.gray(`  Checking AI-crawler access (robots.txt) for ${targetDomains.length} domain(s)…`));
+      try { aiAccessByDomain = await fetchAiAccessForDomains(targetDomains); }
+      catch { aiAccessByDomain = null; }
+    }
 
     const results = runAeoAnalysis(db, project, {
       includeCompetitors: !opts.targetOnly,
+      aiAccessByDomain,
       log: (msg) => isBrief ? console.log(chalk.gray(msg)) : null,
     });
 
@@ -4340,7 +4444,7 @@ program
 
     // Persist scores
     persistAeoScores(db, results);
-    upsertCitabilityInsights(db, project, results.target);
+    upsertCitabilityInsights(db, project, results.target, results.summary.aiAccess);
 
     const { summary } = results;
     const { tierCounts } = summary;
@@ -4439,6 +4543,21 @@ program
       console.log(`    ${chalk.red('●')} Poor (<35):        ${tierCounts.poor}`);
       console.log('');
 
+      if (summary.aiAccess && summary.aiAccess.length) {
+        console.log(chalk.bold('  🤖 AI Crawler Access (robots.txt)'));
+        console.log('');
+        for (const a of summary.aiAccess) {
+          const icon = a.verdict === 'blocked' ? chalk.red('✗') : a.verdict === 'partial' ? chalk.yellow('⚠') : chalk.green('✓');
+          const label = a.verdict === 'blocked' ? chalk.red('BLOCKED') : a.verdict === 'partial' ? chalk.yellow('PARTIAL') : chalk.green('OPEN');
+          console.log(`    ${icon} ${chalk.bold(a.domain)}  ${label}  ${chalk.gray(a.score + '/100')}`);
+          if (a.verdict !== 'open') console.log(chalk.gray(`         ${a.detail}`));
+        }
+        if (summary.gatedPages > 0) {
+          console.log(chalk.red(`    ⛔ ${summary.gatedPages} page(s) capped at 30/100 — AI assistants can't read them, so on-page quality can't help.`));
+        }
+        console.log('');
+      }
+
       if (summary.weakestSignals.length) {
         console.log(chalk.bold('  🔍 Weakest Signals (target average)'));
         console.log('');
@@ -4488,12 +4607,16 @@ program
     }
 
     // ── Regenerate dashboard ──
-    try {
-      const configs = loadAllConfigs();
-      generateMultiDashboard(db, configs);
-      console.log(chalk.green('  ✅ Dashboard updated with AI Citability card\n'));
-    } catch (e) {
-      console.log(chalk.gray(`  (Dashboard not updated: ${e.message})\n`));
+    // Skip in JSON mode: generateMultiDashboard logs progress to stdout, which
+    // would corrupt the JSON object that machine/agent consumers parse.
+    if (isBrief) {
+      try {
+        const configs = loadAllConfigs();
+        generateMultiDashboard(db, configs);
+        console.log(chalk.green('  ✅ Dashboard updated with AI Citability card\n'));
+      } catch (e) {
+        console.log(chalk.gray(`  (Dashboard not updated: ${e.message})\n`));
+      }
     }
 
     // ── Save report ──
@@ -4674,11 +4797,14 @@ program
       console.log(chalk.green(`\n  ✅ Report saved: ${opts.out}\n`));
     }
 
-    // Regenerate dashboard
-    try {
-      const configs = loadAllConfigs();
-      generateMultiDashboard(db, configs);
-    } catch {}
+    // Regenerate dashboard — skip in JSON mode so generateMultiDashboard's
+    // stdout progress logs don't corrupt the JSON output.
+    if (opts.format !== 'json') {
+      try {
+        const configs = loadAllConfigs();
+        generateMultiDashboard(db, configs);
+      } catch {}
+    }
   });
 
 // ── AEO BLOG DRAFT GENERATOR ─────────────────────────────────────────────
@@ -5057,10 +5183,20 @@ program
   });
 
 // ── License activation hook — phone-home if cache is stale/missing ──────────
+// Hard cap so a slow/unreachable license server can never block the command.
+// If activation doesn't finish in time we proceed on cached/offline behavior;
+// the in-flight request is abandoned when the process exits (see bottom of file).
+const LICENSE_ACTIVATION_BUDGET_MS = 2500;
 program.hook('preAction', async () => {
   const license = loadLicense();
   if (license.needsActivation || license.stale) {
-    await activateLicense().catch(() => {});
+    await Promise.race([
+      activateLicense().catch(() => {}),
+      new Promise((resolve) => {
+        const t = setTimeout(resolve, LICENSE_ACTIVATION_BUDGET_MS);
+        if (typeof t.unref === 'function') t.unref();
+      }),
+    ]);
   }
 });
 
@@ -5164,6 +5300,7 @@ program
     const tag = chalk.cyan(`[${domain.split('.')[0]}]`);
 
     try {
+      const { crawlDomain } = await import('./crawler/index.js');
       for await (const page of crawlDomain(siteUrl, {
         maxPages, stealth: useStealth, tiered: true,
         onSitemapDiscovered: (urls) => {
@@ -5516,7 +5653,27 @@ program
   });
 
 // Global error handler — ensures uncaught errors in async actions exit non-zero (BUG-004)
-program.parseAsync().catch(err => {
-  console.error(chalk.red(`\n✗ ${err.message}\n`));
-  process.exit(1);
-});
+program.parseAsync()
+  .then(() => {
+    // One-shot command finished — exit now instead of waiting on the event loop
+    // to drain. A hung background fetch (update check / license phone-home) would
+    // otherwise keep the process alive until the OS connect timeout (~10s).
+    if (!_keepProcessAlive) flushThenExit(process.exitCode ?? 0);
+  })
+  .catch(err => {
+    console.error(chalk.red(`\n✗ ${err.message}\n`));
+    process.exit(1);
+  });
+
+// Drain any buffered stdout/stderr before exiting. process.exit() truncates
+// async pipe writes (large `--format json` output piped to another process),
+// so we wait for both streams to flush, with a short safety net so a stalled
+// pipe can never hang the process.
+function flushThenExit(code) {
+  let pending = 2;
+  const done = () => { if (--pending === 0) process.exit(code); };
+  process.stdout.write('', done);
+  process.stderr.write('', done);
+  const t = setTimeout(() => process.exit(code), 2000);
+  if (typeof t.unref === 'function') t.unref();
+}

package/lib/license.js

@@ -36,6 +36,23 @@ const CACHE_PATH = join(CACHE_DIR, 'license-cache.json');
 const LS_CACHE_TTL = 24 * 60 * 60 * 1000;        // 24h fresh
 const LS_STALE_LIMIT = 7 * 24 * 60 * 60 * 1000;   // 7 days stale max
 
+// Lemon Squeezy License API base — overridable for testing (e.g. point at an
+// unreachable host to verify graceful degradation).
+const LS_API_BASE = (process.env.SEO_INTEL_LICENSE_API || 'https://api.lemonsqueezy.com').replace(/\/+$/, '');
+
+// Per-request network budget. A reachable server answers well under this; a
+// slow/unreachable one aborts here instead of hanging. The CLI also caps the
+// total activation wait in its preAction hook, so this is a backstop.
+const NETWORK_TIMEOUT_MS = 6000;
+
+/** AbortController whose timeout won't keep the event loop alive on its own. */
+function abortAfter(ms) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), ms);
+  if (typeof timer.unref === 'function') timer.unref();
+  return { controller, timer };
+}
+
 // ── Tiers ──────────────────────────────────────────────────────────────────
 
 export const TIERS = {
@@ -132,23 +149,21 @@ function checkCache(key) {
  * Returns instance_id on success.
  */
 async function activateWithLS(key) {
+  const { controller, timer } = abortAfter(NETWORK_TIMEOUT_MS);
   try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 8000);
-
     const body = new URLSearchParams({
       license_key: key,
       instance_name: `seo-intel-${getMachineId()}`,
     });
 
-    const res = await fetch('https://api.lemonsqueezy.com/v1/licenses/activate', {
+    const res = await fetch(`${LS_API_BASE}/v1/licenses/activate`, {
       signal: controller.signal,
       method: 'POST',
       headers: { 'Accept': 'application/json' },
       body,
     });
 
-    clearTimeout(timeout);
+    clearTimeout(timer);
     const data = await res.json();
 
     if (data.activated) {
@@ -179,22 +194,20 @@ async function activateWithLS(key) {
  * Params: license_key, instance_id (optional)
  */
 async function validateWithLS(key, instanceId) {
+  const { controller, timer } = abortAfter(NETWORK_TIMEOUT_MS);
   try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 8000);
-
     const params = { license_key: key };
     if (instanceId) params.instance_id = instanceId;
     const body = new URLSearchParams(params);
 
-    const res = await fetch('https://api.lemonsqueezy.com/v1/licenses/validate', {
+    const res = await fetch(`${LS_API_BASE}/v1/licenses/validate`, {
       signal: controller.signal,
       method: 'POST',
       headers: { 'Accept': 'application/json' },
       body,
     });
 
-    clearTimeout(timeout);
+    clearTimeout(timer);
     const data = await res.json();
 
     if (data.valid) {
@@ -228,23 +241,21 @@ export async function deactivateLicense() {
   const instanceId = cache?.instanceId;
   if (!instanceId) return { deactivated: false, error: 'No active instance to deactivate' };
 
+  const { controller, timer } = abortAfter(NETWORK_TIMEOUT_MS);
   try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 8000);
-
     const body = new URLSearchParams({
       license_key: keyInfo.value,
       instance_id: instanceId,
     });
 
-    const res = await fetch('https://api.lemonsqueezy.com/v1/licenses/deactivate', {
+    const res = await fetch(`${LS_API_BASE}/v1/licenses/deactivate`, {
       signal: controller.signal,
       method: 'POST',
       headers: { 'Accept': 'application/json' },
       body,
     });
 
-    clearTimeout(timeout);
+    clearTimeout(timer);
     const data = await res.json();
 
     if (data.deactivated) {

package/lib/updater.js

@@ -45,6 +45,19 @@ const CACHE_DIR = join(homedir(), '.seo-intel');
 const CACHE_FILE = join(CACHE_DIR, 'update-cache.json');
 const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
 
+// Network budget for a single update-source fetch. Update checks are advisory
+// and fire-and-forget, so keep this tight — a slow source must never hold up
+// the CLI. The abort timer is unref'd so it can't keep the event loop alive.
+const FETCH_TIMEOUT_MS = 3000;
+
+/** Start an AbortController whose timeout won't keep the process alive. */
+function abortAfter(ms) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), ms);
+  if (typeof timer.unref === 'function') timer.unref();
+  return { controller, timer };
+}
+
 function readCache() {
   try {
     if (!existsSync(CACHE_FILE)) return null;
@@ -95,8 +108,7 @@ export function compareSemver(a, b) {
  * Uses the abbreviated metadata endpoint (fast, no auth needed).
  */
 async function checkNpm() {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), 5000);
+  const { controller, timer } = abortAfter(FETCH_TIMEOUT_MS);
 
   try {
     const res = await fetch('https://registry.npmjs.org/seo-intel/latest', {
@@ -109,7 +121,7 @@ async function checkNpm() {
   } catch {
     return null;
   } finally {
-    clearTimeout(timeout);
+    clearTimeout(timer);
   }
 }
 
@@ -118,8 +130,7 @@ async function checkNpm() {
  * Endpoint returns { version, changelog?, downloadUrl? }
  */
 async function checkUkkometa() {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), 5000);
+  const { controller, timer } = abortAfter(FETCH_TIMEOUT_MS);
 
   try {
     const res = await fetch('https://ukkometa.fi/api/seo-intel/version', {
@@ -139,7 +150,7 @@ async function checkUkkometa() {
   } catch {
     return null;
   } finally {
-    clearTimeout(timeout);
+    clearTimeout(timer);
   }
 }