seo-intel 1.5.46 → 1.5.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/agent-harness.js +851 -0
- package/analyses/aeo/ai-access.js +210 -0
- package/analyses/aeo/index.js +52 -9
- package/analyses/aeo/rescore.js +59 -0
- package/analyses/aeo/scorer.js +36 -13
- package/cli.js +227 -19
- package/config/example.json +1 -0
- package/extractor/qwen.js +23 -7
- package/lib/intel.js +101 -0
- package/lib/license.js +26 -15
- package/lib/updater.js +17 -6
- package/mcp/server.js +250 -6
- package/package.json +4 -2
- package/seo-intel.png +0 -0
- package/setup/engine.js +3 -0
- package/setup/models.js +90 -2
package/cli.js
CHANGED
|
@@ -16,11 +16,10 @@ import { program } from 'commander';
|
|
|
16
16
|
import { spawnSync } from 'child_process';
|
|
17
17
|
import { readFileSync, writeFileSync, readdirSync, unlinkSync, existsSync, mkdirSync } from 'fs';
|
|
18
18
|
import { dirname, join } from 'path';
|
|
19
|
-
import { totalmem } from 'os';
|
|
19
|
+
import { totalmem, homedir } from 'os';
|
|
20
20
|
import { fileURLToPath } from 'url';
|
|
21
21
|
import chalk from 'chalk';
|
|
22
22
|
|
|
23
|
-
import { crawlDomain } from './crawler/index.js';
|
|
24
23
|
// Paid modules — loaded lazily inside gated commands only.
|
|
25
24
|
let _extractPage, _buildAnalysisPrompt;
|
|
26
25
|
async function getExtractPage() {
|
|
@@ -54,9 +53,34 @@ import { getCurrentVersion, checkForUpdates, printUpdateNotice, forceUpdateCheck
|
|
|
54
53
|
|
|
55
54
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
56
55
|
|
|
56
|
+
// Self-register where this checkout/install lives, in ~/.seo-intel/install.json.
|
|
57
|
+
// Any local agent harness (Hermes, a future Claude Code plugin, anything) can
|
|
58
|
+
// read this one file to find SEO Intel — no env vars, no hardcoded paths, no
|
|
59
|
+
// per-harness wiring. Cheap to refresh on every run; last run to execute wins,
|
|
60
|
+
// which is the right default for the common one-checkout-per-machine case.
|
|
61
|
+
function registerInstallLocation() {
|
|
62
|
+
try {
|
|
63
|
+
const dir = join(homedir(), '.seo-intel');
|
|
64
|
+
mkdirSync(dir, { recursive: true });
|
|
65
|
+
writeFileSync(join(dir, 'install.json'), JSON.stringify({
|
|
66
|
+
root: __dirname,
|
|
67
|
+
version: getCurrentVersion(),
|
|
68
|
+
updatedAt: new Date().toISOString(),
|
|
69
|
+
}, null, 2));
|
|
70
|
+
} catch { /* non-fatal — harnesses fall back to PATH resolution */ }
|
|
71
|
+
}
|
|
72
|
+
registerInstallLocation();
|
|
73
|
+
|
|
57
74
|
// Start background update check (non-blocking, never slows startup)
|
|
58
75
|
checkForUpdates();
|
|
59
76
|
|
|
77
|
+
// Long-running commands (serve, setup-web) intentionally keep the process alive.
|
|
78
|
+
// Everything else is one-shot: once the action resolves we force-exit so a
|
|
79
|
+
// hung background fetch (update check / license phone-home) can't hold the
|
|
80
|
+
// shell — an aborted fetch's socket lingers until the OS connect timeout, so
|
|
81
|
+
// AbortController alone is not enough to make the process exit promptly.
|
|
82
|
+
let _keepProcessAlive = false;
|
|
83
|
+
|
|
60
84
|
// Ensure reports/ and config/ directories exist
|
|
61
85
|
try { mkdirSync(join(__dirname, 'reports'), { recursive: true }); } catch { /* ok */ }
|
|
62
86
|
try { mkdirSync(join(__dirname, 'config'), { recursive: true }); } catch { /* ok */ }
|
|
@@ -575,6 +599,8 @@ program
|
|
|
575
599
|
},
|
|
576
600
|
};
|
|
577
601
|
|
|
602
|
+
// Lazy: crawler/index.js pulls Playwright + turndown — keep them off CLI startup
|
|
603
|
+
const { crawlDomain } = await import('./crawler/index.js');
|
|
578
604
|
for await (const page of crawlDomain(site.url, crawlOpts)) {
|
|
579
605
|
if (page._blocked) {
|
|
580
606
|
totalBlocked++;
|
|
@@ -1323,6 +1349,7 @@ program
|
|
|
1323
1349
|
let pageCount = 0;
|
|
1324
1350
|
let skipped = 0;
|
|
1325
1351
|
let blocked = false;
|
|
1352
|
+
const { crawlDomain } = await import('./crawler/index.js');
|
|
1326
1353
|
for await (const page of crawlDomain(next.url, {
|
|
1327
1354
|
onSitemapDiscovered: (urls) => {
|
|
1328
1355
|
try { upsertSitemapUrls(db, domainId, urls.map(u => u.url), `${next.url}/sitemap.xml`); }
|
|
@@ -1547,6 +1574,84 @@ program
|
|
|
1547
1574
|
}
|
|
1548
1575
|
});
|
|
1549
1576
|
|
|
1577
|
+
// ── MODELS ─────────────────────────────────────────────────────────────────
|
|
1578
|
+
program
|
|
1579
|
+
.command('models')
|
|
1580
|
+
.description('Suggest local extraction models for your hardware (Gemma / Qwen) — local is strongly recommended over cloud')
|
|
1581
|
+
.option('--format <type>', 'Output format: brief or json', 'brief')
|
|
1582
|
+
.action(async (opts) => {
|
|
1583
|
+
const isJson = opts.format === 'json';
|
|
1584
|
+
const { suggestExtractionModels, CLOUD_EXTRACTION_DISCLAIMER, detectVRAM } = await import('./setup/engine.js');
|
|
1585
|
+
|
|
1586
|
+
// Detect hardware (best-effort) + installed Ollama models (quick, short timeout).
|
|
1587
|
+
let vram = { available: false, vramMB: 0, gpuName: null };
|
|
1588
|
+
try { vram = detectVRAM(); } catch { /* keep default */ }
|
|
1589
|
+
let installed = [];
|
|
1590
|
+
try {
|
|
1591
|
+
const controller = new AbortController();
|
|
1592
|
+
const t = setTimeout(() => controller.abort(), 1500);
|
|
1593
|
+
const res = await fetch('http://localhost:11434/api/tags', { signal: controller.signal });
|
|
1594
|
+
clearTimeout(t);
|
|
1595
|
+
if (res.ok) {
|
|
1596
|
+
const data = await res.json();
|
|
1597
|
+
installed = (data.models || []).map(m => m.name);
|
|
1598
|
+
}
|
|
1599
|
+
} catch { /* Ollama not reachable — fine */ }
|
|
1600
|
+
|
|
1601
|
+
const { suggestions, recommendedId } = suggestExtractionModels(vram.vramMB || 0, installed);
|
|
1602
|
+
|
|
1603
|
+
if (isJson) {
|
|
1604
|
+
console.log(JSON.stringify({
|
|
1605
|
+
command: 'models',
|
|
1606
|
+
hardware: { gpu: vram.gpuName || null, vramMB: vram.vramMB || 0 },
|
|
1607
|
+
recommended: recommendedId,
|
|
1608
|
+
suggestions,
|
|
1609
|
+
cloud_disclaimer: CLOUD_EXTRACTION_DISCLAIMER,
|
|
1610
|
+
}, null, 2));
|
|
1611
|
+
return;
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
console.log('');
|
|
1615
|
+
console.log(chalk.bold(' 🧠 Local extraction models'));
|
|
1616
|
+
console.log('');
|
|
1617
|
+
if (vram.available && vram.vramMB) {
|
|
1618
|
+
console.log(chalk.gray(` Detected: ${vram.gpuName || 'GPU'} · ~${(vram.vramMB / 1024).toFixed(1)} GB VRAM`));
|
|
1619
|
+
} else {
|
|
1620
|
+
console.log(chalk.gray(' GPU/VRAM not detected — showing the full range. Pick by your machine.'));
|
|
1621
|
+
}
|
|
1622
|
+
console.log('');
|
|
1623
|
+
|
|
1624
|
+
const qFmt = (q) => q === 'excellent' ? chalk.green(q) : q === 'great' || q === 'better' ? chalk.cyan(q) : chalk.gray(q);
|
|
1625
|
+
for (const s of suggestions) {
|
|
1626
|
+
const star = s.id === recommendedId ? chalk.bold.green(' ◀ recommended') : '';
|
|
1627
|
+
const fit = vram.vramMB && !s.fitsVram ? chalk.red(' (needs more VRAM)') : '';
|
|
1628
|
+
const got = s.installed ? chalk.green(' ✓ installed') : chalk.gray(` ollama pull ${s.id}`);
|
|
1629
|
+
console.log(` ${chalk.bold(s.name.padEnd(16))} ${chalk.gray(s.vram.padEnd(8))} ${s.speed.padEnd(11)} ${qFmt(s.quality)}${star}${fit}`);
|
|
1630
|
+
console.log(` ${' '.repeat(16)} ${got}`);
|
|
1631
|
+
}
|
|
1632
|
+
console.log('');
|
|
1633
|
+
|
|
1634
|
+
// The MUST: extraction-should-be-local disclaimer, every time.
|
|
1635
|
+
console.log(chalk.bold.yellow(' ⚠ Extraction should be done with a LOCAL model'));
|
|
1636
|
+
const wrap = (text, width) => {
|
|
1637
|
+
const out = []; let line = '';
|
|
1638
|
+
for (const word of text.split(/\s+/)) {
|
|
1639
|
+
if ((line + ' ' + word).trim().length > width) { out.push(line.trim()); line = word; }
|
|
1640
|
+
else line += ' ' + word;
|
|
1641
|
+
}
|
|
1642
|
+
if (line.trim()) out.push(line.trim());
|
|
1643
|
+
return out;
|
|
1644
|
+
};
|
|
1645
|
+
for (const line of wrap(CLOUD_EXTRACTION_DISCLAIMER, 78)) {
|
|
1646
|
+
console.log(chalk.yellow(' ' + line));
|
|
1647
|
+
}
|
|
1648
|
+
console.log('');
|
|
1649
|
+
if (recommendedId && !suggestions.find(s => s.id === recommendedId)?.installed) {
|
|
1650
|
+
console.log(chalk.gray(` Get started: `) + chalk.white(`ollama pull ${recommendedId}`) + chalk.gray(` then `) + chalk.white(`seo-intel setup`));
|
|
1651
|
+
console.log('');
|
|
1652
|
+
}
|
|
1653
|
+
});
|
|
1654
|
+
|
|
1550
1655
|
// ── STATUS ─────────────────────────────────────────────────────────────────
|
|
1551
1656
|
program
|
|
1552
1657
|
.command('status')
|
|
@@ -2357,6 +2462,7 @@ program
|
|
|
2357
2462
|
.option('--open', 'Open browser automatically', true)
|
|
2358
2463
|
.option('--no-open', 'Do not open browser')
|
|
2359
2464
|
.action(async (opts) => {
|
|
2465
|
+
_keepProcessAlive = true;
|
|
2360
2466
|
const port = parseInt(opts.port, 10);
|
|
2361
2467
|
process.env.PORT = String(port);
|
|
2362
2468
|
if (opts.open) process.env.SEO_INTEL_AUTO_OPEN = '1';
|
|
@@ -2369,6 +2475,7 @@ program
|
|
|
2369
2475
|
.description('Open the web-based setup wizard in your browser')
|
|
2370
2476
|
.option('--port <n>', 'Server port', '3000')
|
|
2371
2477
|
.action(async (opts) => {
|
|
2478
|
+
_keepProcessAlive = true;
|
|
2372
2479
|
const port = parseInt(opts.port, 10);
|
|
2373
2480
|
process.env.PORT = String(port);
|
|
2374
2481
|
await import('./server.js');
|
|
@@ -4326,9 +4433,24 @@ program
|
|
|
4326
4433
|
}
|
|
4327
4434
|
|
|
4328
4435
|
const { runAeoAnalysis, persistAeoScores, upsertCitabilityInsights } = await import('./analyses/aeo/index.js');
|
|
4436
|
+
const { fetchAiAccessForDomains } = await import('./analyses/aeo/ai-access.js');
|
|
4437
|
+
|
|
4438
|
+
// AI-crawler access (robots.txt) — domain-level signal. Network, but cheap
|
|
4439
|
+
// (one robots.txt per target/owned domain) and best-effort.
|
|
4440
|
+
const targetDomains = db
|
|
4441
|
+
.prepare("SELECT DISTINCT domain FROM domains WHERE project = ? AND role IN ('target','owned')")
|
|
4442
|
+
.all(project)
|
|
4443
|
+
.map(r => r.domain);
|
|
4444
|
+
let aiAccessByDomain = null;
|
|
4445
|
+
if (targetDomains.length) {
|
|
4446
|
+
if (isBrief) console.log(chalk.gray(` Checking AI-crawler access (robots.txt) for ${targetDomains.length} domain(s)…`));
|
|
4447
|
+
try { aiAccessByDomain = await fetchAiAccessForDomains(targetDomains); }
|
|
4448
|
+
catch { aiAccessByDomain = null; }
|
|
4449
|
+
}
|
|
4329
4450
|
|
|
4330
4451
|
const results = runAeoAnalysis(db, project, {
|
|
4331
4452
|
includeCompetitors: !opts.targetOnly,
|
|
4453
|
+
aiAccessByDomain,
|
|
4332
4454
|
log: (msg) => isBrief ? console.log(chalk.gray(msg)) : null,
|
|
4333
4455
|
});
|
|
4334
4456
|
|
|
@@ -4340,7 +4462,7 @@ program
|
|
|
4340
4462
|
|
|
4341
4463
|
// Persist scores
|
|
4342
4464
|
persistAeoScores(db, results);
|
|
4343
|
-
upsertCitabilityInsights(db, project, results.target);
|
|
4465
|
+
upsertCitabilityInsights(db, project, results.target, results.summary.aiAccess);
|
|
4344
4466
|
|
|
4345
4467
|
const { summary } = results;
|
|
4346
4468
|
const { tierCounts } = summary;
|
|
@@ -4439,6 +4561,21 @@ program
|
|
|
4439
4561
|
console.log(` ${chalk.red('●')} Poor (<35): ${tierCounts.poor}`);
|
|
4440
4562
|
console.log('');
|
|
4441
4563
|
|
|
4564
|
+
if (summary.aiAccess && summary.aiAccess.length) {
|
|
4565
|
+
console.log(chalk.bold(' 🤖 AI Crawler Access (robots.txt)'));
|
|
4566
|
+
console.log('');
|
|
4567
|
+
for (const a of summary.aiAccess) {
|
|
4568
|
+
const icon = a.verdict === 'blocked' ? chalk.red('✗') : a.verdict === 'partial' ? chalk.yellow('⚠') : chalk.green('✓');
|
|
4569
|
+
const label = a.verdict === 'blocked' ? chalk.red('BLOCKED') : a.verdict === 'partial' ? chalk.yellow('PARTIAL') : chalk.green('OPEN');
|
|
4570
|
+
console.log(` ${icon} ${chalk.bold(a.domain)} ${label} ${chalk.gray(a.score + '/100')}`);
|
|
4571
|
+
if (a.verdict !== 'open') console.log(chalk.gray(` ${a.detail}`));
|
|
4572
|
+
}
|
|
4573
|
+
if (summary.gatedPages > 0) {
|
|
4574
|
+
console.log(chalk.red(` ⛔ ${summary.gatedPages} page(s) capped at 30/100 — AI assistants can't read them, so on-page quality can't help.`));
|
|
4575
|
+
}
|
|
4576
|
+
console.log('');
|
|
4577
|
+
}
|
|
4578
|
+
|
|
4442
4579
|
if (summary.weakestSignals.length) {
|
|
4443
4580
|
console.log(chalk.bold(' 🔍 Weakest Signals (target average)'));
|
|
4444
4581
|
console.log('');
|
|
@@ -4488,12 +4625,16 @@ program
|
|
|
4488
4625
|
}
|
|
4489
4626
|
|
|
4490
4627
|
// ── Regenerate dashboard ──
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4628
|
+
// Skip in JSON mode: generateMultiDashboard logs progress to stdout, which
|
|
4629
|
+
// would corrupt the JSON object that machine/agent consumers parse.
|
|
4630
|
+
if (isBrief) {
|
|
4631
|
+
try {
|
|
4632
|
+
const configs = loadAllConfigs();
|
|
4633
|
+
generateMultiDashboard(db, configs);
|
|
4634
|
+
console.log(chalk.green(' ✅ Dashboard updated with AI Citability card\n'));
|
|
4635
|
+
} catch (e) {
|
|
4636
|
+
console.log(chalk.gray(` (Dashboard not updated: ${e.message})\n`));
|
|
4637
|
+
}
|
|
4497
4638
|
}
|
|
4498
4639
|
|
|
4499
4640
|
// ── Save report ──
|
|
@@ -4511,6 +4652,39 @@ program
|
|
|
4511
4652
|
}
|
|
4512
4653
|
});
|
|
4513
4654
|
|
|
4655
|
+
// ── RESCORE (single-URL re-check, closes the agent loop) ──────────────────
|
|
4656
|
+
program
|
|
4657
|
+
.command('rescore <project> <url>')
|
|
4658
|
+
.description("Re-check one URL's AI citability after a fix — read-only, raw-HTML (bot's-eye) score + delta vs baseline")
|
|
4659
|
+
.option('--format <type>', 'Output format: brief or json', 'brief')
|
|
4660
|
+
.action(async (project, url, opts) => {
|
|
4661
|
+
const isJson = opts.format === 'json';
|
|
4662
|
+
const { getDb } = await import('./db/db.js');
|
|
4663
|
+
const { rescorePage } = await import('./analyses/aeo/rescore.js');
|
|
4664
|
+
const db = getDb();
|
|
4665
|
+
const res = await rescorePage(db, project, url, {
|
|
4666
|
+
log: isJson ? undefined : (m) => console.log(chalk.gray(' ' + m)),
|
|
4667
|
+
});
|
|
4668
|
+
if (isJson) {
|
|
4669
|
+
console.log(JSON.stringify(res, null, 2));
|
|
4670
|
+
return;
|
|
4671
|
+
}
|
|
4672
|
+
const tone = (s) => (s == null ? chalk.gray : s >= 60 ? chalk.green : s >= 35 ? chalk.yellow : chalk.red);
|
|
4673
|
+
console.log('');
|
|
4674
|
+
console.log(chalk.bold(` 🔁 Re-score — ${url}`));
|
|
4675
|
+
console.log(chalk.gray(` lens: raw HTML (what bots see) · HTTP ${res.status_code ?? '—'}`));
|
|
4676
|
+
if (res.after == null) {
|
|
4677
|
+
console.log(chalk.red(` ✗ ${res.note || 'could not score'}`));
|
|
4678
|
+
} else {
|
|
4679
|
+
const b = res.before == null ? chalk.gray('—') : tone(res.before)(String(res.before));
|
|
4680
|
+
const a = tone(res.after)(String(res.after));
|
|
4681
|
+
const arrow =
|
|
4682
|
+
res.delta == null ? '' : res.delta > 0 ? chalk.green(` ▲ +${res.delta}`) : res.delta < 0 ? chalk.red(` ▼ ${res.delta}`) : chalk.gray(' ±0');
|
|
4683
|
+
console.log(` citability ${b} → ${a}${arrow} (${res.status_before ?? '—'} → ${res.status_after ?? '—'})`);
|
|
4684
|
+
}
|
|
4685
|
+
console.log('');
|
|
4686
|
+
});
|
|
4687
|
+
|
|
4514
4688
|
// ── SITE WATCH ──────────────────────────────────────────────────────────
|
|
4515
4689
|
program
|
|
4516
4690
|
.command('watch <project>')
|
|
@@ -4674,11 +4848,14 @@ program
|
|
|
4674
4848
|
console.log(chalk.green(`\n ✅ Report saved: ${opts.out}\n`));
|
|
4675
4849
|
}
|
|
4676
4850
|
|
|
4677
|
-
// Regenerate dashboard
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4851
|
+
// Regenerate dashboard — skip in JSON mode so generateMultiDashboard's
|
|
4852
|
+
// stdout progress logs don't corrupt the JSON output.
|
|
4853
|
+
if (opts.format !== 'json') {
|
|
4854
|
+
try {
|
|
4855
|
+
const configs = loadAllConfigs();
|
|
4856
|
+
generateMultiDashboard(db, configs);
|
|
4857
|
+
} catch {}
|
|
4858
|
+
}
|
|
4682
4859
|
});
|
|
4683
4860
|
|
|
4684
4861
|
// ── AEO BLOG DRAFT GENERATOR ─────────────────────────────────────────────
|
|
@@ -5057,10 +5234,20 @@ program
|
|
|
5057
5234
|
});
|
|
5058
5235
|
|
|
5059
5236
|
// ── License activation hook — phone-home if cache is stale/missing ──────────
|
|
5237
|
+
// Hard cap so a slow/unreachable license server can never block the command.
|
|
5238
|
+
// If activation doesn't finish in time we proceed on cached/offline behavior;
|
|
5239
|
+
// the in-flight request is abandoned when the process exits (see bottom of file).
|
|
5240
|
+
const LICENSE_ACTIVATION_BUDGET_MS = 2500;
|
|
5060
5241
|
program.hook('preAction', async () => {
|
|
5061
5242
|
const license = loadLicense();
|
|
5062
5243
|
if (license.needsActivation || license.stale) {
|
|
5063
|
-
await
|
|
5244
|
+
await Promise.race([
|
|
5245
|
+
activateLicense().catch(() => {}),
|
|
5246
|
+
new Promise((resolve) => {
|
|
5247
|
+
const t = setTimeout(resolve, LICENSE_ACTIVATION_BUDGET_MS);
|
|
5248
|
+
if (typeof t.unref === 'function') t.unref();
|
|
5249
|
+
}),
|
|
5250
|
+
]);
|
|
5064
5251
|
}
|
|
5065
5252
|
});
|
|
5066
5253
|
|
|
@@ -5164,6 +5351,7 @@ program
|
|
|
5164
5351
|
const tag = chalk.cyan(`[${domain.split('.')[0]}]`);
|
|
5165
5352
|
|
|
5166
5353
|
try {
|
|
5354
|
+
const { crawlDomain } = await import('./crawler/index.js');
|
|
5167
5355
|
for await (const page of crawlDomain(siteUrl, {
|
|
5168
5356
|
maxPages, stealth: useStealth, tiered: true,
|
|
5169
5357
|
onSitemapDiscovered: (urls) => {
|
|
@@ -5516,7 +5704,27 @@ program
|
|
|
5516
5704
|
});
|
|
5517
5705
|
|
|
5518
5706
|
// Global error handler — ensures uncaught errors in async actions exit non-zero (BUG-004)
|
|
5519
|
-
program.parseAsync()
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
|
|
5707
|
+
program.parseAsync()
|
|
5708
|
+
.then(() => {
|
|
5709
|
+
// One-shot command finished — exit now instead of waiting on the event loop
|
|
5710
|
+
// to drain. A hung background fetch (update check / license phone-home) would
|
|
5711
|
+
// otherwise keep the process alive until the OS connect timeout (~10s).
|
|
5712
|
+
if (!_keepProcessAlive) flushThenExit(process.exitCode ?? 0);
|
|
5713
|
+
})
|
|
5714
|
+
.catch(err => {
|
|
5715
|
+
console.error(chalk.red(`\n✗ ${err.message}\n`));
|
|
5716
|
+
process.exit(1);
|
|
5717
|
+
});
|
|
5718
|
+
|
|
5719
|
+
// Drain any buffered stdout/stderr before exiting. process.exit() truncates
|
|
5720
|
+
// async pipe writes (large `--format json` output piped to another process),
|
|
5721
|
+
// so we wait for both streams to flush, with a short safety net so a stalled
|
|
5722
|
+
// pipe can never hang the process.
|
|
5723
|
+
function flushThenExit(code) {
|
|
5724
|
+
let pending = 2;
|
|
5725
|
+
const done = () => { if (--pending === 0) process.exit(code); };
|
|
5726
|
+
process.stdout.write('', done);
|
|
5727
|
+
process.stderr.write('', done);
|
|
5728
|
+
const t = setTimeout(() => process.exit(code), 2000);
|
|
5729
|
+
if (typeof t.unref === 'function') t.unref();
|
|
5730
|
+
}
|
package/config/example.json
CHANGED
package/extractor/qwen.js
CHANGED
|
@@ -24,14 +24,17 @@ function modelMatches(available, target) {
|
|
|
24
24
|
// ── LM Studio support (OpenAI-compatible API) ──────────────────────────────
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
|
-
* Ping an LM Studio host. Uses GET /
|
|
27
|
+
* Ping an LM Studio host. Uses the OpenAI-compatible GET /v1/models (returns
|
|
28
|
+
* { data: [{ id }] } and lists models that are actually loaded/servable) —
|
|
29
|
+
* NOT the native /api/v1/models, which returns a different shape ({ models:
|
|
30
|
+
* [{ key }] }) and lists every downloaded model regardless of load state.
|
|
28
31
|
*/
|
|
29
32
|
export async function pingLmStudioHost(host, model, timeoutMs = OLLAMA_PREFLIGHT_TIMEOUT_MS) {
|
|
30
33
|
const controller = new AbortController();
|
|
31
34
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
32
35
|
|
|
33
36
|
try {
|
|
34
|
-
const res = await fetch(`${host}/
|
|
37
|
+
const res = await fetch(`${host}/v1/models`, { signal: controller.signal });
|
|
35
38
|
if (!res.ok) {
|
|
36
39
|
return { host, model, reachable: false, modelAvailable: false, type: 'lmstudio',
|
|
37
40
|
error: `HTTP ${res.status} ${res.statusText}`.trim() };
|
|
@@ -62,14 +65,17 @@ async function callLmStudio(route, prompt) {
|
|
|
62
65
|
const timeout = setTimeout(() => controller.abort(), OLLAMA_TIMEOUT_MS);
|
|
63
66
|
|
|
64
67
|
try {
|
|
65
|
-
const res = await fetch(`${route.host}/
|
|
68
|
+
const res = await fetch(`${route.host}/v1/chat/completions`, {
|
|
66
69
|
signal: controller.signal,
|
|
67
70
|
method: 'POST',
|
|
68
71
|
headers: { 'Content-Type': 'application/json' },
|
|
69
72
|
body: JSON.stringify({
|
|
70
73
|
model: route.model,
|
|
71
74
|
messages: [{ role: 'user', content: prompt }],
|
|
72
|
-
|
|
75
|
+
// LM Studio rejects OpenAI's 'json_object'; it accepts only 'json_schema'
|
|
76
|
+
// or 'text'. The prompt already demands JSON and the response is run
|
|
77
|
+
// through extractLastJsonObject/repairJson, so 'text' is the portable choice.
|
|
78
|
+
response_format: { type: 'text' },
|
|
73
79
|
temperature: 0,
|
|
74
80
|
max_tokens: 1200,
|
|
75
81
|
stream: false,
|
|
@@ -427,10 +433,20 @@ JSON output:`;
|
|
|
427
433
|
console.log(`[extractor] used ${route.label} for ${url}`);
|
|
428
434
|
break;
|
|
429
435
|
} catch (err) {
|
|
430
|
-
route.failures = (route.failures || 0) + 1;
|
|
431
436
|
console.warn(`[extractor] ${route.label} failed for ${url}: ${describeOllamaError(err, route)}`);
|
|
432
|
-
|
|
433
|
-
|
|
437
|
+
// Only TRANSPORT failures (host down/unreachable) should retire a host for
|
|
438
|
+
// the run. CONTENT failures (a small model returns unparseable JSON for one
|
|
439
|
+
// long page) must NOT disable the host — that page degrades on its own; the
|
|
440
|
+
// next page may extract fine. Conflating the two killed local extraction
|
|
441
|
+
// after a couple of hard pages.
|
|
442
|
+
const msg = String(err?.message || '');
|
|
443
|
+
const transportFail = err?.name === 'AbortError'
|
|
444
|
+
|| /fetch failed|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH|socket hang up|network|timeout|HTTP 5\d\d/i.test(msg);
|
|
445
|
+
if (transportFail) {
|
|
446
|
+
route.failures = (route.failures || 0) + 1;
|
|
447
|
+
if (route.failures >= OLLAMA_HOST_FAILURE_LIMIT) {
|
|
448
|
+
removeRouteFromActivePool(runtimeState, route);
|
|
449
|
+
}
|
|
434
450
|
}
|
|
435
451
|
}
|
|
436
452
|
}
|
package/lib/intel.js
CHANGED
|
@@ -108,11 +108,112 @@ function collectRaw(db, project) {
|
|
|
108
108
|
};
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
// Map a page's weakest citability signal to a concrete on-page fix.
|
|
112
|
+
const SIGNAL_FIX = {
|
|
113
|
+
'schema coverage': 'Add JSON-LD schema (FAQPage / Product / Organization)',
|
|
114
|
+
'qa proximity': 'Add a Q&A / FAQ section answering real questions',
|
|
115
|
+
'entity authority': 'Add author + credentials and sameAs entity grounding',
|
|
116
|
+
'structured claims': 'Add extractable, standalone factual claims',
|
|
117
|
+
'answer density': 'Lead each section with a direct one-line answer',
|
|
118
|
+
'freshness': 'Add visible published / updated dates',
|
|
119
|
+
};
|
|
120
|
+
const scoreBucket = (s) => (s >= 75 ? 'good' : s >= 55 ? 'needs-work' : s >= 35 ? 'weak' : 'poor');
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Ranked, role-attributed opportunities from citability scores. Each row is
|
|
124
|
+
* dashboard-ready: real value/status/confidence + a concrete suggested fix.
|
|
125
|
+
* side='fix' = your own pages to improve; side='attack' = thin competitor pages.
|
|
126
|
+
*/
|
|
127
|
+
function buildOpportunities(db, project) {
|
|
128
|
+
let rows = [];
|
|
129
|
+
try {
|
|
130
|
+
rows = db.prepare(`
|
|
131
|
+
SELECT p.url, p.title, d.domain, d.role,
|
|
132
|
+
c.score, c.entity_authority, c.structured_claims, c.answer_density,
|
|
133
|
+
c.qa_proximity, c.freshness, c.schema_coverage, c.ai_intents
|
|
134
|
+
FROM citability_scores c
|
|
135
|
+
JOIN pages p ON p.id = c.page_id
|
|
136
|
+
JOIN domains d ON d.id = p.domain_id
|
|
137
|
+
WHERE d.project = ?
|
|
138
|
+
ORDER BY c.score ASC
|
|
139
|
+
`).all(project);
|
|
140
|
+
} catch { return { opportunities: [], summary: null }; }
|
|
141
|
+
|
|
142
|
+
const opportunities = rows.map((r, i) => {
|
|
143
|
+
const own = r.role === 'target' || r.role === 'owned';
|
|
144
|
+
const signals = {
|
|
145
|
+
'entity authority': r.entity_authority, 'structured claims': r.structured_claims,
|
|
146
|
+
'answer density': r.answer_density, 'qa proximity': r.qa_proximity,
|
|
147
|
+
'freshness': r.freshness, 'schema coverage': r.schema_coverage,
|
|
148
|
+
};
|
|
149
|
+
const ranked = Object.entries(signals).filter(([, v]) => v != null).sort((a, b) => a[1] - b[1]);
|
|
150
|
+
const weak = ranked.slice(0, 2).map(([k]) => k);
|
|
151
|
+
let intent = null;
|
|
152
|
+
try { intent = (JSON.parse(r.ai_intents || '[]') || [])[0] || null; } catch { /* ignore */ }
|
|
153
|
+
let label = r.title || r.url;
|
|
154
|
+
try { const u = new URL(r.url); label = u.hostname + (u.pathname === '/' ? '' : u.pathname); } catch { /* keep label */ }
|
|
155
|
+
return {
|
|
156
|
+
id: `cit:${i}`,
|
|
157
|
+
side: own ? 'fix' : 'attack',
|
|
158
|
+
role: r.role,
|
|
159
|
+
domain: r.domain,
|
|
160
|
+
url: r.url,
|
|
161
|
+
title: r.title || null,
|
|
162
|
+
finding: label,
|
|
163
|
+
kind: 'citability',
|
|
164
|
+
score: r.score,
|
|
165
|
+
status: scoreBucket(r.score),
|
|
166
|
+
value: Math.round((100 - r.score) * (own ? 1 : 0.6)), // your low pages rank highest
|
|
167
|
+
confidence: +(ranked.length / 6).toFixed(2), // signal completeness
|
|
168
|
+
intent,
|
|
169
|
+
weak_signals: weak,
|
|
170
|
+
suggested_action: own ? (SIGNAL_FIX[weak[0]] || 'Improve on-page citability') : 'Outrank — thin competitor page',
|
|
171
|
+
proof: r.url,
|
|
172
|
+
};
|
|
173
|
+
}).sort((a, b) => b.value - a.value);
|
|
174
|
+
|
|
175
|
+
const owned = opportunities.filter((o) => o.side === 'fix');
|
|
176
|
+
const attack = opportunities.filter((o) => o.side === 'attack');
|
|
177
|
+
const avg = (arr) => (arr.length ? Math.round((arr.reduce((s, o) => s + o.score, 0) / arr.length) * 10) / 10 : null);
|
|
178
|
+
|
|
179
|
+
// Per-signal averages across YOUR pages — the AEO citability breakdown.
|
|
180
|
+
const ownedRows = rows.filter((r) => r.role === 'target' || r.role === 'owned');
|
|
181
|
+
const signalAvg = (key) => {
|
|
182
|
+
const vals = ownedRows.map((r) => r[key]).filter((v) => v != null);
|
|
183
|
+
return vals.length ? Math.round(vals.reduce((a, b) => a + b, 0) / vals.length) : null;
|
|
184
|
+
};
|
|
185
|
+
const signals = {
|
|
186
|
+
entity_authority: signalAvg('entity_authority'),
|
|
187
|
+
structured_claims: signalAvg('structured_claims'),
|
|
188
|
+
answer_density: signalAvg('answer_density'),
|
|
189
|
+
qa_proximity: signalAvg('qa_proximity'),
|
|
190
|
+
freshness: signalAvg('freshness'),
|
|
191
|
+
schema_coverage: signalAvg('schema_coverage'),
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
opportunities,
|
|
196
|
+
summary: {
|
|
197
|
+
pages_scored: rows.length,
|
|
198
|
+
owned_pages: owned.length,
|
|
199
|
+
competitor_pages: attack.length,
|
|
200
|
+
avg_citability_owned: avg(owned),
|
|
201
|
+
avg_citability_all: avg(opportunities),
|
|
202
|
+
fix_opportunities: owned.length,
|
|
203
|
+
attack_targets: attack.length,
|
|
204
|
+
signals,
|
|
205
|
+
},
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
111
209
|
function collectAudit(db, project) {
|
|
112
210
|
const insights = getActiveInsights(db, project);
|
|
113
211
|
let citability = null;
|
|
114
212
|
try { citability = getCitabilityScores(db, project); } catch { /* citability_scores table may not exist if AEO never run */ }
|
|
213
|
+
const { opportunities, summary } = buildOpportunities(db, project);
|
|
115
214
|
return {
|
|
215
|
+
summary,
|
|
216
|
+
opportunities,
|
|
116
217
|
citability,
|
|
117
218
|
insights: {
|
|
118
219
|
keyword_gaps: insights.keyword_gaps,
|
package/lib/license.js
CHANGED
|
@@ -36,6 +36,23 @@ const CACHE_PATH = join(CACHE_DIR, 'license-cache.json');
|
|
|
36
36
|
const LS_CACHE_TTL = 24 * 60 * 60 * 1000; // 24h fresh
|
|
37
37
|
const LS_STALE_LIMIT = 7 * 24 * 60 * 60 * 1000; // 7 days stale max
|
|
38
38
|
|
|
39
|
+
// Lemon Squeezy License API base — overridable for testing (e.g. point at an
|
|
40
|
+
// unreachable host to verify graceful degradation).
|
|
41
|
+
const LS_API_BASE = (process.env.SEO_INTEL_LICENSE_API || 'https://api.lemonsqueezy.com').replace(/\/+$/, '');
|
|
42
|
+
|
|
43
|
+
// Per-request network budget. A reachable server answers well under this; a
|
|
44
|
+
// slow/unreachable one aborts here instead of hanging. The CLI also caps the
|
|
45
|
+
// total activation wait in its preAction hook, so this is a backstop.
|
|
46
|
+
const NETWORK_TIMEOUT_MS = 6000;
|
|
47
|
+
|
|
48
|
+
/** AbortController whose timeout won't keep the event loop alive on its own. */
|
|
49
|
+
function abortAfter(ms) {
|
|
50
|
+
const controller = new AbortController();
|
|
51
|
+
const timer = setTimeout(() => controller.abort(), ms);
|
|
52
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
53
|
+
return { controller, timer };
|
|
54
|
+
}
|
|
55
|
+
|
|
39
56
|
// ── Tiers ──────────────────────────────────────────────────────────────────
|
|
40
57
|
|
|
41
58
|
export const TIERS = {
|
|
@@ -132,23 +149,21 @@ function checkCache(key) {
|
|
|
132
149
|
* Returns instance_id on success.
|
|
133
150
|
*/
|
|
134
151
|
async function activateWithLS(key) {
|
|
152
|
+
const { controller, timer } = abortAfter(NETWORK_TIMEOUT_MS);
|
|
135
153
|
try {
|
|
136
|
-
const controller = new AbortController();
|
|
137
|
-
const timeout = setTimeout(() => controller.abort(), 8000);
|
|
138
|
-
|
|
139
154
|
const body = new URLSearchParams({
|
|
140
155
|
license_key: key,
|
|
141
156
|
instance_name: `seo-intel-${getMachineId()}`,
|
|
142
157
|
});
|
|
143
158
|
|
|
144
|
-
const res = await fetch(
|
|
159
|
+
const res = await fetch(`${LS_API_BASE}/v1/licenses/activate`, {
|
|
145
160
|
signal: controller.signal,
|
|
146
161
|
method: 'POST',
|
|
147
162
|
headers: { 'Accept': 'application/json' },
|
|
148
163
|
body,
|
|
149
164
|
});
|
|
150
165
|
|
|
151
|
-
clearTimeout(
|
|
166
|
+
clearTimeout(timer);
|
|
152
167
|
const data = await res.json();
|
|
153
168
|
|
|
154
169
|
if (data.activated) {
|
|
@@ -179,22 +194,20 @@ async function activateWithLS(key) {
|
|
|
179
194
|
* Params: license_key, instance_id (optional)
|
|
180
195
|
*/
|
|
181
196
|
async function validateWithLS(key, instanceId) {
|
|
197
|
+
const { controller, timer } = abortAfter(NETWORK_TIMEOUT_MS);
|
|
182
198
|
try {
|
|
183
|
-
const controller = new AbortController();
|
|
184
|
-
const timeout = setTimeout(() => controller.abort(), 8000);
|
|
185
|
-
|
|
186
199
|
const params = { license_key: key };
|
|
187
200
|
if (instanceId) params.instance_id = instanceId;
|
|
188
201
|
const body = new URLSearchParams(params);
|
|
189
202
|
|
|
190
|
-
const res = await fetch(
|
|
203
|
+
const res = await fetch(`${LS_API_BASE}/v1/licenses/validate`, {
|
|
191
204
|
signal: controller.signal,
|
|
192
205
|
method: 'POST',
|
|
193
206
|
headers: { 'Accept': 'application/json' },
|
|
194
207
|
body,
|
|
195
208
|
});
|
|
196
209
|
|
|
197
|
-
clearTimeout(
|
|
210
|
+
clearTimeout(timer);
|
|
198
211
|
const data = await res.json();
|
|
199
212
|
|
|
200
213
|
if (data.valid) {
|
|
@@ -228,23 +241,21 @@ export async function deactivateLicense() {
|
|
|
228
241
|
const instanceId = cache?.instanceId;
|
|
229
242
|
if (!instanceId) return { deactivated: false, error: 'No active instance to deactivate' };
|
|
230
243
|
|
|
244
|
+
const { controller, timer } = abortAfter(NETWORK_TIMEOUT_MS);
|
|
231
245
|
try {
|
|
232
|
-
const controller = new AbortController();
|
|
233
|
-
const timeout = setTimeout(() => controller.abort(), 8000);
|
|
234
|
-
|
|
235
246
|
const body = new URLSearchParams({
|
|
236
247
|
license_key: keyInfo.value,
|
|
237
248
|
instance_id: instanceId,
|
|
238
249
|
});
|
|
239
250
|
|
|
240
|
-
const res = await fetch(
|
|
251
|
+
const res = await fetch(`${LS_API_BASE}/v1/licenses/deactivate`, {
|
|
241
252
|
signal: controller.signal,
|
|
242
253
|
method: 'POST',
|
|
243
254
|
headers: { 'Accept': 'application/json' },
|
|
244
255
|
body,
|
|
245
256
|
});
|
|
246
257
|
|
|
247
|
-
clearTimeout(
|
|
258
|
+
clearTimeout(timer);
|
|
248
259
|
const data = await res.json();
|
|
249
260
|
|
|
250
261
|
if (data.deactivated) {
|