seo-intel 1.0.1 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Setup SEO Intel.command +11 -0
- package/cli.js +11 -4
- package/crawler/index.js +5 -0
- package/db/db.js +13 -0
- package/package.json +2 -1
- package/reports/generate-html.js +147 -14
- package/server.js +40 -0
- package/setup/checks.js +24 -17
- package/setup/models.js +62 -5
- package/setup/validator.js +60 -16
- package/setup/web-routes.js +78 -2
- package/setup/wizard.html +287 -46
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
cd "$(dirname "$0")" || cd ~
|
|
3
|
+
clear
|
|
4
|
+
echo ""
|
|
5
|
+
echo " SEO Intel — Setup Wizard"
|
|
6
|
+
echo " ========================"
|
|
7
|
+
echo ""
|
|
8
|
+
npx seo-intel setup
|
|
9
|
+
echo ""
|
|
10
|
+
echo " Setup complete. You can close this window."
|
|
11
|
+
read -n 1 -s -r -p " Press any key to exit..."
|
package/cli.js
CHANGED
|
@@ -24,6 +24,7 @@ import { getNextCrawlTarget, needsAnalysis, getCrawlStatus, loadAllConfigs } fro
|
|
|
24
24
|
import {
|
|
25
25
|
getDb, upsertDomain, upsertPage, insertExtraction,
|
|
26
26
|
insertKeywords, insertHeadings, insertLinks, insertPageSchemas,
|
|
27
|
+
upsertTechnical,
|
|
27
28
|
getCompetitorSummary, getKeywordMatrix, getHeadingStructure,
|
|
28
29
|
getPageHash, getSchemasByProject
|
|
29
30
|
} from './db/db.js';
|
|
@@ -53,10 +54,13 @@ try { mkdirSync(join(__dirname, 'config'), { recursive: true }); } catch { /* ok
|
|
|
53
54
|
* Fast: 2s timeout per host, runs sequentially.
|
|
54
55
|
*/
|
|
55
56
|
async function checkOllamaAvailability() {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
57
|
+
// Build host chain: primary → fallback → always try localhost as last resort
|
|
58
|
+
const configured = [
|
|
59
|
+
process.env.OLLAMA_URL,
|
|
60
|
+
process.env.OLLAMA_FALLBACK_URL,
|
|
61
|
+
].filter(Boolean);
|
|
62
|
+
const localhost = 'http://localhost:11434';
|
|
63
|
+
const hosts = [...new Set([...configured, localhost])];
|
|
60
64
|
|
|
61
65
|
for (const host of hosts) {
|
|
62
66
|
try {
|
|
@@ -478,6 +482,7 @@ program
|
|
|
478
482
|
started_at: crawlStart,
|
|
479
483
|
failed: totalFailed,
|
|
480
484
|
});
|
|
485
|
+
upsertTechnical(db, { pageId, hasCanonical: page.hasCanonical, hasOgTags: page.hasOgTags, hasSchema: page.hasSchema, hasRobots: page.hasRobots });
|
|
481
486
|
try {
|
|
482
487
|
const extractFn = await getExtractPage();
|
|
483
488
|
const extraction = await extractFn(page);
|
|
@@ -494,6 +499,7 @@ program
|
|
|
494
499
|
totalFailed++;
|
|
495
500
|
}
|
|
496
501
|
} else {
|
|
502
|
+
upsertTechnical(db, { pageId, hasCanonical: page.hasCanonical, hasOgTags: page.hasOgTags, hasSchema: page.hasSchema, hasRobots: page.hasRobots });
|
|
497
503
|
insertHeadings(db, pageId, page.headings);
|
|
498
504
|
insertLinks(db, pageId, page.links);
|
|
499
505
|
if (page.parsedSchemas?.length) insertPageSchemas(db, pageId, page.parsedSchemas);
|
|
@@ -1051,6 +1057,7 @@ program
|
|
|
1051
1057
|
}
|
|
1052
1058
|
}
|
|
1053
1059
|
|
|
1060
|
+
upsertTechnical(db, { pageId, hasCanonical: page.hasCanonical, hasOgTags: page.hasOgTags, hasSchema: page.hasSchema, hasRobots: page.hasRobots });
|
|
1054
1061
|
process.stdout.write(chalk.gray(` [${pageCount + 1}] d${page.depth ?? 0} ${page.url.slice(0, 65)} → extracting...`));
|
|
1055
1062
|
writeProgress({
|
|
1056
1063
|
status: 'running', command: 'run', project: next.project,
|
package/crawler/index.js
CHANGED
|
@@ -495,6 +495,8 @@ async function processPage(page, url, base, depth, queue, maxDepth) {
|
|
|
495
495
|
|
|
496
496
|
const robotsMeta = await page.$eval('meta[name="robots"]', el => el.content).catch(() => '');
|
|
497
497
|
const isIndexable = !robotsMeta.toLowerCase().includes('noindex');
|
|
498
|
+
const hasCanonical = await page.$('link[rel="canonical"]').then(el => !!el).catch(() => false);
|
|
499
|
+
const hasOgTags = await page.$('meta[property^="og:"]').then(el => !!el).catch(() => false);
|
|
498
500
|
|
|
499
501
|
const publishedDate = await page.evaluate(() => {
|
|
500
502
|
for (const sel of ['meta[property="article:published_time"]','meta[name="date"]','meta[itemprop="datePublished"]']) {
|
|
@@ -553,6 +555,9 @@ async function processPage(page, url, base, depth, queue, maxDepth) {
|
|
|
553
555
|
schemaTypes, parsedSchemas, vitals, publishedDate, modifiedDate,
|
|
554
556
|
contentHash: hash,
|
|
555
557
|
quality: quality.ok, qualityReason: quality.reason,
|
|
558
|
+
hasCanonical, hasOgTags,
|
|
559
|
+
hasRobots: !!robotsMeta,
|
|
560
|
+
hasSchema: schemaTypes.length > 0,
|
|
556
561
|
};
|
|
557
562
|
}
|
|
558
563
|
|
package/db/db.js
CHANGED
|
@@ -61,6 +61,19 @@ export function upsertPage(db, { domainId, url, statusCode, wordCount, loadMs, i
|
|
|
61
61
|
return db.prepare('SELECT id FROM pages WHERE url = ?').get(url);
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
export function upsertTechnical(db, { pageId, hasCanonical, hasOgTags, hasSchema, hasRobots, isMobileOk = 0 }) {
|
|
65
|
+
db.prepare(`
|
|
66
|
+
INSERT INTO technical (page_id, has_canonical, has_og_tags, has_schema, has_robots, is_mobile_ok)
|
|
67
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
68
|
+
ON CONFLICT(page_id) DO UPDATE SET
|
|
69
|
+
has_canonical = excluded.has_canonical,
|
|
70
|
+
has_og_tags = excluded.has_og_tags,
|
|
71
|
+
has_schema = excluded.has_schema,
|
|
72
|
+
has_robots = excluded.has_robots,
|
|
73
|
+
is_mobile_ok = excluded.is_mobile_ok
|
|
74
|
+
`).run(pageId, hasCanonical ? 1 : 0, hasOgTags ? 1 : 0, hasSchema ? 1 : 0, hasRobots ? 1 : 0, isMobileOk ? 1 : 0);
|
|
75
|
+
}
|
|
76
|
+
|
|
64
77
|
export function getPageHash(db, url) {
|
|
65
78
|
return db.prepare('SELECT content_hash FROM pages WHERE url = ?').get(url)?.content_hash || null;
|
|
66
79
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "seo-intel",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "Local Ahrefs-style SEO competitor intelligence. Crawl → SQLite → cloud analysis.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "SEE LICENSE IN LICENSE",
|
|
@@ -46,6 +46,7 @@
|
|
|
46
46
|
"README.md",
|
|
47
47
|
"seo-intel.png",
|
|
48
48
|
"Start SEO Intel.command",
|
|
49
|
+
"Setup SEO Intel.command",
|
|
49
50
|
"Start SEO Intel.bat",
|
|
50
51
|
"start-seo-intel.sh"
|
|
51
52
|
],
|
package/reports/generate-html.js
CHANGED
|
@@ -42,7 +42,12 @@ function gatherProjectData(db, project, config) {
|
|
|
42
42
|
// Merge owned subdomains (blog.x, docs.x) into target at the SQL level.
|
|
43
43
|
// Uses a savepoint so changes are rolled back after report generation —
|
|
44
44
|
// the actual DB stays intact, but ALL downstream queries see unified data.
|
|
45
|
-
|
|
45
|
+
// Include BOTH config-owned domains AND DB role='owned' domains.
|
|
46
|
+
const configOwned = (config.owned || []).map(o => o.domain);
|
|
47
|
+
const dbOwned = db.prepare(
|
|
48
|
+
`SELECT domain FROM domains WHERE project = ? AND role = 'owned'`
|
|
49
|
+
).all(project).map(r => r.domain);
|
|
50
|
+
const ownedDomains = [...new Set([...configOwned, ...dbOwned])];
|
|
46
51
|
const hasOwned = ownedDomains.length > 0;
|
|
47
52
|
if (hasOwned) {
|
|
48
53
|
db.prepare('SAVEPOINT owned_merge').run();
|
|
@@ -84,7 +89,7 @@ function gatherProjectData(db, project, config) {
|
|
|
84
89
|
const schemaBreakdown = getSchemaBreakdown(db, project);
|
|
85
90
|
|
|
86
91
|
// Advanced visualization data
|
|
87
|
-
const gravityMap = getGravityMapData(db, project);
|
|
92
|
+
const gravityMap = getGravityMapData(db, project, config);
|
|
88
93
|
const contentTerrain = getContentTerrainData(db, project);
|
|
89
94
|
const keywordVenn = getKeywordVennData(db, project);
|
|
90
95
|
const performanceBubbles = getPerformanceBubbleData(db, project);
|
|
@@ -256,6 +261,44 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
256
261
|
}
|
|
257
262
|
|
|
258
263
|
/* ─── Header Bar ─────────────────────────────────────────────────────── */
|
|
264
|
+
.update-banner {
|
|
265
|
+
padding: 10px 20px;
|
|
266
|
+
border-radius: var(--radius);
|
|
267
|
+
display: flex;
|
|
268
|
+
align-items: center;
|
|
269
|
+
gap: 12px;
|
|
270
|
+
font-size: 0.78rem;
|
|
271
|
+
margin-bottom: 12px;
|
|
272
|
+
}
|
|
273
|
+
.update-banner.update-normal {
|
|
274
|
+
background: rgba(232,213,163,0.06);
|
|
275
|
+
border: 1px solid rgba(232,213,163,0.2);
|
|
276
|
+
color: var(--accent-gold);
|
|
277
|
+
}
|
|
278
|
+
.update-banner.update-security {
|
|
279
|
+
background: rgba(220,80,80,0.08);
|
|
280
|
+
border: 1px solid rgba(220,80,80,0.25);
|
|
281
|
+
color: #ff6b6b;
|
|
282
|
+
}
|
|
283
|
+
.update-banner .update-version { font-family: var(--font-mono); font-weight: 600; }
|
|
284
|
+
.update-banner .update-changelog { font-size: 0.68rem; color: var(--text-muted); flex:1; }
|
|
285
|
+
.update-banner .update-btn {
|
|
286
|
+
padding: 5px 14px;
|
|
287
|
+
border-radius: 6px;
|
|
288
|
+
font-size: 0.7rem;
|
|
289
|
+
font-weight: 600;
|
|
290
|
+
border: 1px solid currentColor;
|
|
291
|
+
background: transparent;
|
|
292
|
+
color: inherit;
|
|
293
|
+
cursor: pointer;
|
|
294
|
+
white-space: nowrap;
|
|
295
|
+
}
|
|
296
|
+
.update-banner .update-btn:hover { background: rgba(255,255,255,0.06); }
|
|
297
|
+
.update-banner .update-dismiss {
|
|
298
|
+
cursor: pointer; opacity: 0.5; font-size: 0.7rem;
|
|
299
|
+
}
|
|
300
|
+
.update-banner .update-dismiss:hover { opacity: 1; }
|
|
301
|
+
|
|
259
302
|
.header-bar {
|
|
260
303
|
background: var(--bg-card);
|
|
261
304
|
border: 1px solid var(--border-card);
|
|
@@ -468,6 +511,8 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
468
511
|
white-space: nowrap;
|
|
469
512
|
}
|
|
470
513
|
.es-btn:hover { border-color: var(--accent-gold); color: var(--accent-gold); }
|
|
514
|
+
.es-btn-stop { border-color: rgba(220,80,80,0.3); color: #dc5050; }
|
|
515
|
+
.es-btn-stop:hover { border-color: #dc5050; color: #ff6b6b; background: rgba(220,80,80,0.08); }
|
|
471
516
|
.es-btn:disabled {
|
|
472
517
|
opacity: 0.4; cursor: not-allowed;
|
|
473
518
|
border-color: var(--border-card);
|
|
@@ -1740,9 +1785,9 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
1740
1785
|
bar('Open Graph Tags', techCoverage.og) +
|
|
1741
1786
|
bar('Schema Markup', techCoverage.schema) +
|
|
1742
1787
|
bar('Robots Meta', techCoverage.robots) +
|
|
1743
|
-
`<div style="font-size:0.65rem;color:var(--text-muted);margin-top:8px;">
|
|
1788
|
+
(avgLoad > 0 ? `<div style="font-size:0.65rem;color:var(--text-muted);margin-top:8px;">
|
|
1744
1789
|
Avg load: <span style="color:${avgLoad > 3000 ? 'var(--color-danger)' : avgLoad > 1500 ? 'var(--color-warning)' : 'var(--color-success)'};">${avgLoad > 1000 ? (avgLoad/1000).toFixed(1) + 's' : avgLoad + 'ms'}</span>
|
|
1745
|
-
</div
|
|
1790
|
+
</div>` : '');
|
|
1746
1791
|
})() : '<div style="font-size:0.72rem;color:var(--text-muted);">No technical data yet. Run a crawl first.</div>'}
|
|
1747
1792
|
</div>
|
|
1748
1793
|
</div>
|
|
@@ -1788,6 +1833,8 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
1788
1833
|
// ── Panel HTML (project-specific body content) ──
|
|
1789
1834
|
const panelHtml = `
|
|
1790
1835
|
<div class="project-panel" data-project="${project}">
|
|
1836
|
+
<!-- UPDATE BANNER (populated by JS if update available) -->
|
|
1837
|
+
<div id="updateBanner${suffix}" style="display:none;"></div>
|
|
1791
1838
|
<!-- HEADER BAR -->
|
|
1792
1839
|
<div class="header-bar" id="header">
|
|
1793
1840
|
<div class="header-left">
|
|
@@ -1874,6 +1921,9 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
1874
1921
|
<button class="es-btn" id="btnExtract${suffix}" onclick="startJob('extract','${project}')">
|
|
1875
1922
|
<i class="fa-solid fa-brain"></i> Extract
|
|
1876
1923
|
</button>
|
|
1924
|
+
<button class="es-btn es-btn-stop" id="btnStop${suffix}" onclick="stopJob()" style="display:none;">
|
|
1925
|
+
<i class="fa-solid fa-stop"></i> Stop
|
|
1926
|
+
</button>
|
|
1877
1927
|
<label class="es-stealth-toggle">
|
|
1878
1928
|
<input type="checkbox" id="stealthToggle${suffix}">
|
|
1879
1929
|
<i class="fa-solid fa-user-ninja"></i> Stealth
|
|
@@ -1904,6 +1954,7 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
1904
1954
|
<button class="term-btn" data-cmd="keywords" data-project="${project}"><i class="fa-solid fa-key"></i> Keywords</button>` : ''}
|
|
1905
1955
|
<button class="term-btn" data-cmd="status" data-project=""><i class="fa-solid fa-circle-info"></i> Status</button>
|
|
1906
1956
|
<button class="term-btn" data-cmd="guide" data-project="${project}"><i class="fa-solid fa-map"></i> Guide</button>
|
|
1957
|
+
<button class="term-btn" data-cmd="setup" data-project="" style="margin-left:auto;border-color:rgba(232,213,163,0.25);"><i class="fa-solid fa-gear"></i> Setup</button>
|
|
1907
1958
|
${!pro ? `<span style="font-size:0.55rem;color:var(--text-muted);margin-left:auto;"><i class="fa-solid fa-lock" style="color:var(--accent-gold);margin-right:3px;"></i><a href="https://ukkometa.fi/en/seo-intel/" target="_blank" style="color:var(--accent-gold);text-decoration:none;">Solo</a> for extract, analyze, exports</span>` : ''}
|
|
1908
1959
|
</div>
|
|
1909
1960
|
<!-- Terminal output -->
|
|
@@ -1972,7 +2023,18 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
1972
2023
|
}
|
|
1973
2024
|
|
|
1974
2025
|
function runCommand(command, proj, extra) {
|
|
1975
|
-
|
|
2026
|
+
// Setup always works — even during a running crawl
|
|
2027
|
+
if (command === 'setup') {
|
|
2028
|
+
if (isServed) {
|
|
2029
|
+
window.open('/setup', '_blank');
|
|
2030
|
+
} else {
|
|
2031
|
+
window.open('http://localhost:3000/setup', '_blank');
|
|
2032
|
+
appendLine('Opening setup wizard at localhost:3000/setup', 'stdout');
|
|
2033
|
+
appendLine('If it does not open, run: seo-intel setup', 'cmd');
|
|
2034
|
+
}
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
2037
|
+
|
|
1976
2038
|
if (!isServed) {
|
|
1977
2039
|
appendLine('', 'cmd');
|
|
1978
2040
|
appendLine('Not connected to server. Run in your terminal:', 'error');
|
|
@@ -3383,6 +3445,27 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
3383
3445
|
bg: '#111111', grid: '#222222', text: '#b8b8b8', muted: '#555555'
|
|
3384
3446
|
};
|
|
3385
3447
|
|
|
3448
|
+
// ═══ UPDATE CHECK ═══
|
|
3449
|
+
(function() {
|
|
3450
|
+
if (!window.location.protocol.startsWith('http')) return;
|
|
3451
|
+
fetch('/api/update-check').then(r => r.json()).then(function(info) {
|
|
3452
|
+
if (!info.hasUpdate) return;
|
|
3453
|
+
var banner = document.getElementById('updateBanner${suffix}');
|
|
3454
|
+
if (!banner) return;
|
|
3455
|
+
var cls = info.security ? 'update-security' : 'update-normal';
|
|
3456
|
+
var icon = info.security ? 'fa-shield-halved' : 'fa-arrow-up';
|
|
3457
|
+
var changelogHtml = info.changelog ? '<span class="update-changelog">' + info.changelog.split('\\n')[0] + '</span>' : '';
|
|
3458
|
+
banner.style.display = 'block';
|
|
3459
|
+
banner.innerHTML = '<div class="update-banner ' + cls + '">' +
|
|
3460
|
+
'<i class="fa-solid ' + icon + '"></i>' +
|
|
3461
|
+
'<span class="update-version">' + info.current + ' → ' + info.latest + '</span>' +
|
|
3462
|
+
changelogHtml +
|
|
3463
|
+
'<button class="update-btn" onclick="navigator.clipboard.writeText(\\'npm update -g seo-intel\\');this.textContent=\\'Copied!\\';setTimeout(()=>this.textContent=\\'Update\\',2000)">Update</button>' +
|
|
3464
|
+
'<span class="update-dismiss" onclick="this.closest(\\'.update-banner\\').style.display=\\'none\\'"><i class="fa-solid fa-xmark"></i></span>' +
|
|
3465
|
+
'</div>';
|
|
3466
|
+
}).catch(function() {});
|
|
3467
|
+
})();
|
|
3468
|
+
|
|
3386
3469
|
// ═══ LIVE DASHBOARD CONTROLS ═══
|
|
3387
3470
|
(function() {
|
|
3388
3471
|
const isServed = window.location.protocol.startsWith('http');
|
|
@@ -3415,9 +3498,22 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
3415
3498
|
.catch(function(err) { alert('Server error: ' + err.message); });
|
|
3416
3499
|
};
|
|
3417
3500
|
|
|
3501
|
+
window.stopJob = function() {
|
|
3502
|
+
if (!confirm('Stop the running job?')) return;
|
|
3503
|
+
fetch('/api/stop', { method: 'POST' })
|
|
3504
|
+
.then(function(r) { return r.json(); })
|
|
3505
|
+
.then(function(data) {
|
|
3506
|
+
if (data.stopped) {
|
|
3507
|
+
setButtonsState(false, null);
|
|
3508
|
+
}
|
|
3509
|
+
})
|
|
3510
|
+
.catch(function(err) { alert('Stop failed: ' + err.message); });
|
|
3511
|
+
};
|
|
3512
|
+
|
|
3418
3513
|
function setButtonsState(disabled, activeCmd) {
|
|
3419
3514
|
var btnC = document.getElementById('btnCrawl' + sfx);
|
|
3420
3515
|
var btnE = document.getElementById('btnExtract' + sfx);
|
|
3516
|
+
var btnS = document.getElementById('btnStop' + sfx);
|
|
3421
3517
|
if (btnC) {
|
|
3422
3518
|
btnC.disabled = disabled;
|
|
3423
3519
|
if (disabled && activeCmd === 'crawl') {
|
|
@@ -3438,6 +3534,9 @@ function buildHtmlTemplate(data, opts = {}) {
|
|
|
3438
3534
|
btnE.innerHTML = '<i class="fa-solid fa-brain"></i> Extract';
|
|
3439
3535
|
}
|
|
3440
3536
|
}
|
|
3537
|
+
if (btnS) {
|
|
3538
|
+
btnS.style.display = disabled ? 'inline-flex' : 'none';
|
|
3539
|
+
}
|
|
3441
3540
|
}
|
|
3442
3541
|
|
|
3443
3542
|
function startPolling() {
|
|
@@ -4591,10 +4690,21 @@ function buildMultiHtmlTemplate(allProjectData) {
|
|
|
4591
4690
|
.catch(function(err) { alert('Server error: ' + err.message); });
|
|
4592
4691
|
};
|
|
4593
4692
|
|
|
4693
|
+
window.stopJob = function() {
|
|
4694
|
+
if (!confirm('Stop the running job?')) return;
|
|
4695
|
+
fetch('/api/stop', { method: 'POST' })
|
|
4696
|
+
.then(function(r) { return r.json(); })
|
|
4697
|
+
.then(function(data) {
|
|
4698
|
+
if (data.stopped) setButtonsState(false, null);
|
|
4699
|
+
})
|
|
4700
|
+
.catch(function(err) { alert('Stop failed: ' + err.message); });
|
|
4701
|
+
};
|
|
4702
|
+
|
|
4594
4703
|
function setButtonsState(disabled, activeCmd) {
|
|
4595
4704
|
var sfx = '-' + currentProject;
|
|
4596
4705
|
var btnC = document.getElementById('btnCrawl' + sfx);
|
|
4597
4706
|
var btnE = document.getElementById('btnExtract' + sfx);
|
|
4707
|
+
var btnS = document.getElementById('btnStop' + sfx);
|
|
4598
4708
|
if (btnC) {
|
|
4599
4709
|
btnC.disabled = disabled;
|
|
4600
4710
|
btnC.classList.toggle('running', disabled && activeCmd === 'crawl');
|
|
@@ -4609,6 +4719,9 @@ function buildMultiHtmlTemplate(allProjectData) {
|
|
|
4609
4719
|
? '<i class="fa-solid fa-spinner fa-spin"></i> Extracting\u2026'
|
|
4610
4720
|
: '<i class="fa-solid fa-brain"></i> Extract';
|
|
4611
4721
|
}
|
|
4722
|
+
if (btnS) {
|
|
4723
|
+
btnS.style.display = disabled ? 'inline-flex' : 'none';
|
|
4724
|
+
}
|
|
4612
4725
|
}
|
|
4613
4726
|
|
|
4614
4727
|
function startPolling() { if (!pollTimer) { pollTimer = setInterval(pollProgress, 2000); pollProgress(); } }
|
|
@@ -4679,6 +4792,23 @@ function getLatestKeywordsReport(project) {
|
|
|
4679
4792
|
* Works on any array of objects with { domain, role, ...numeric fields }.
|
|
4680
4793
|
* Numeric fields are summed; role is set to 'target'.
|
|
4681
4794
|
*/
|
|
4795
|
+
|
|
4796
|
+
/**
|
|
4797
|
+
* Build a domain resolver closure. Returns a function that maps
|
|
4798
|
+
* any (domain, role) pair to the resolved domain name.
|
|
4799
|
+
* Owned domains (by config OR by DB role) resolve to the target domain.
|
|
4800
|
+
*/
|
|
4801
|
+
function buildDomainResolver(config) {
|
|
4802
|
+
const targetDomain = config?.target?.domain;
|
|
4803
|
+
const ownedSet = new Set((config?.owned || []).map(o => o.domain));
|
|
4804
|
+
return function(domain, role) {
|
|
4805
|
+
if (!targetDomain) return domain;
|
|
4806
|
+
if (domain === targetDomain) return targetDomain;
|
|
4807
|
+
if (ownedSet.has(domain) || role === 'owned') return targetDomain;
|
|
4808
|
+
return domain;
|
|
4809
|
+
};
|
|
4810
|
+
}
|
|
4811
|
+
|
|
4682
4812
|
function mergeOwnedDomains(rows, config) {
|
|
4683
4813
|
if (!config?.target?.domain) return rows;
|
|
4684
4814
|
|
|
@@ -4940,7 +5070,7 @@ function getTechnicalScores(db, project, config) {
|
|
|
4940
5070
|
let targetRow = null;
|
|
4941
5071
|
|
|
4942
5072
|
for (const row of rawScores) {
|
|
4943
|
-
const isOwned = ownedDomains.has(row.domain);
|
|
5073
|
+
const isOwned = ownedDomains.has(row.domain) || row.role === 'owned';
|
|
4944
5074
|
const isTarget = row.domain === targetDomain;
|
|
4945
5075
|
|
|
4946
5076
|
if (isOwned || isTarget) {
|
|
@@ -5543,7 +5673,7 @@ function getSchemaBreakdown(db, project) {
|
|
|
5543
5673
|
|
|
5544
5674
|
// ─── Advanced Visualization Data Functions ───────────────────────────────────
|
|
5545
5675
|
|
|
5546
|
-
function getGravityMapData(db, project) {
|
|
5676
|
+
function getGravityMapData(db, project, config) {
|
|
5547
5677
|
// Get keyword sets per domain for overlap calculation
|
|
5548
5678
|
const rows = db.prepare(`
|
|
5549
5679
|
SELECT DISTINCT d.domain, d.role, k.keyword
|
|
@@ -5551,10 +5681,13 @@ function getGravityMapData(db, project) {
|
|
|
5551
5681
|
WHERE d.project = ? AND k.keyword LIKE '% %'
|
|
5552
5682
|
`).all(project);
|
|
5553
5683
|
|
|
5684
|
+
const resolve = buildDomainResolver(config);
|
|
5554
5685
|
const domainKws = {};
|
|
5555
5686
|
for (const r of rows) {
|
|
5556
|
-
|
|
5557
|
-
|
|
5687
|
+
const domain = resolve(r.domain, r.role);
|
|
5688
|
+
const role = domain === config?.target?.domain ? 'target' : r.role;
|
|
5689
|
+
if (!domainKws[domain]) domainKws[domain] = { role, keywords: new Set() };
|
|
5690
|
+
domainKws[domain].keywords.add(r.keyword.toLowerCase());
|
|
5558
5691
|
}
|
|
5559
5692
|
|
|
5560
5693
|
// Build nodes
|
|
@@ -5661,11 +5794,11 @@ function getHeadingFlowData(db, project, config) {
|
|
|
5661
5794
|
// Resolve owned → target
|
|
5662
5795
|
const targetDomain = config?.target?.domain;
|
|
5663
5796
|
const ownedDomains = new Set((config?.owned || []).map(o => o.domain));
|
|
5664
|
-
const resolve = (d) => ownedDomains.has(d) ? targetDomain : d;
|
|
5797
|
+
const resolve = (d, role) => (ownedDomains.has(d) || role === 'owned') ? targetDomain : d;
|
|
5665
5798
|
|
|
5666
5799
|
const domains = {};
|
|
5667
5800
|
for (const r of rows) {
|
|
5668
|
-
const domain = resolve(r.domain);
|
|
5801
|
+
const domain = resolve(r.domain, r.role);
|
|
5669
5802
|
const role = domain === targetDomain ? 'target' : r.role;
|
|
5670
5803
|
if (!domains[domain]) domains[domain] = { role, h1: 0, h2: 0, h3: 0 };
|
|
5671
5804
|
domains[domain][`h${r.level}`] += r.cnt;
|
|
@@ -5786,11 +5919,11 @@ function getLinkRadarPulseData(db, project, config) {
|
|
|
5786
5919
|
// Resolve owned domains → target
|
|
5787
5920
|
const targetDomain = config?.target?.domain;
|
|
5788
5921
|
const ownedDomains = new Set((config?.owned || []).map(o => o.domain));
|
|
5789
|
-
const resolveDomain = (d) => ownedDomains.has(d) ? targetDomain : d;
|
|
5922
|
+
const resolveDomain = (d, role) => (ownedDomains.has(d) || role === 'owned') ? targetDomain : d;
|
|
5790
5923
|
|
|
5791
5924
|
const domains = {};
|
|
5792
5925
|
for (const r of rows) {
|
|
5793
|
-
const domain = resolveDomain(r.domain);
|
|
5926
|
+
const domain = resolveDomain(r.domain, r.role);
|
|
5794
5927
|
const role = domain === targetDomain ? 'target' : r.role;
|
|
5795
5928
|
if (!domains[domain]) domains[domain] = { role, depths: [] };
|
|
5796
5929
|
|
|
@@ -5832,7 +5965,7 @@ function getExtractionStatus(db, project, config) {
|
|
|
5832
5965
|
let targetRow = null;
|
|
5833
5966
|
|
|
5834
5967
|
for (const row of rawCoverage) {
|
|
5835
|
-
if (ownedDomains.has(row.domain) && targetDomain) {
|
|
5968
|
+
if ((ownedDomains.has(row.domain) || row.role === 'owned') && targetDomain) {
|
|
5836
5969
|
// Merge into target
|
|
5837
5970
|
if (!targetRow) {
|
|
5838
5971
|
targetRow = { ...row, domain: targetDomain, role: 'target' };
|
package/server.js
CHANGED
|
@@ -3,6 +3,7 @@ import { readFileSync, writeFileSync, existsSync, readdirSync } from 'fs';
|
|
|
3
3
|
import { spawn } from 'child_process';
|
|
4
4
|
import { dirname, join, extname } from 'path';
|
|
5
5
|
import { fileURLToPath } from 'url';
|
|
6
|
+
import { checkForUpdates, getUpdateInfo } from './lib/updater.js';
|
|
6
7
|
|
|
7
8
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
8
9
|
const PORT = parseInt(process.env.PORT || '3000', 10);
|
|
@@ -324,6 +325,42 @@ async function handleRequest(req, res) {
|
|
|
324
325
|
}
|
|
325
326
|
|
|
326
327
|
|
|
328
|
+
// ─── API: Stop running job ───
|
|
329
|
+
// ─── API: Update check ───
|
|
330
|
+
if (req.method === 'GET' && path === '/api/update-check') {
|
|
331
|
+
try {
|
|
332
|
+
const info = await getUpdateInfo();
|
|
333
|
+
json(res, 200, info);
|
|
334
|
+
} catch (e) {
|
|
335
|
+
json(res, 200, { hasUpdate: false, error: e.message });
|
|
336
|
+
}
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (req.method === 'POST' && path === '/api/stop') {
|
|
341
|
+
try {
|
|
342
|
+
const progress = readProgress();
|
|
343
|
+
if (!progress || progress.status !== 'running' || !progress.pid) {
|
|
344
|
+
json(res, 404, { error: 'No running job to stop' });
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
try {
|
|
348
|
+
process.kill(progress.pid, 'SIGTERM');
|
|
349
|
+
// Give it a moment, then force kill if still alive
|
|
350
|
+
setTimeout(() => {
|
|
351
|
+
try { process.kill(progress.pid, 'SIGKILL'); } catch {}
|
|
352
|
+
}, 3000);
|
|
353
|
+
} catch (e) {
|
|
354
|
+
if (e.code !== 'ESRCH') throw e;
|
|
355
|
+
// Already dead
|
|
356
|
+
}
|
|
357
|
+
json(res, 200, { stopped: true, pid: progress.pid, command: progress.command });
|
|
358
|
+
} catch (e) {
|
|
359
|
+
json(res, 500, { error: e.message });
|
|
360
|
+
}
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
|
|
327
364
|
// ─── API: Export actions ───
|
|
328
365
|
if (req.method === 'POST' && path === '/api/export-actions') {
|
|
329
366
|
try {
|
|
@@ -579,6 +616,9 @@ const server = createServer((req, res) => {
|
|
|
579
616
|
});
|
|
580
617
|
});
|
|
581
618
|
|
|
619
|
+
// Start background update check
|
|
620
|
+
checkForUpdates();
|
|
621
|
+
|
|
582
622
|
server.listen(PORT, '127.0.0.1', () => {
|
|
583
623
|
console.log(`\n SEO Intel Dashboard Server`);
|
|
584
624
|
console.log(` http://localhost:${PORT}\n`);
|
package/setup/checks.js
CHANGED
|
@@ -79,28 +79,33 @@ export async function checkOllamaRemote(host) {
|
|
|
79
79
|
export async function checkOllamaAuto(customHosts = []) {
|
|
80
80
|
// 1. Try local
|
|
81
81
|
const local = checkOllamaLocal();
|
|
82
|
+
const allHosts = []; // Track all reachable hosts for UI
|
|
83
|
+
|
|
82
84
|
if (local.running && local.models.length > 0) {
|
|
83
|
-
|
|
84
|
-
available: true,
|
|
85
|
-
mode: 'local',
|
|
86
|
-
host: local.host,
|
|
87
|
-
models: local.models,
|
|
88
|
-
installed: local.installed,
|
|
89
|
-
};
|
|
85
|
+
allHosts.push({ host: local.host, mode: 'local', models: local.models, reachable: true });
|
|
90
86
|
}
|
|
91
87
|
|
|
92
|
-
// 2. Try custom/LAN hosts
|
|
88
|
+
// 2. Try custom/LAN hosts (check ALL, not just first)
|
|
93
89
|
for (const host of customHosts) {
|
|
90
|
+
if (host === 'http://localhost:11434') continue; // already checked
|
|
94
91
|
const remote = await checkOllamaRemote(host);
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
92
|
+
allHosts.push({ host: remote.host, mode: 'remote', models: remote.models, reachable: remote.reachable });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Pick best available host (first with models)
|
|
96
|
+
const best = allHosts.find(h => h.reachable && h.models.length > 0);
|
|
97
|
+
|
|
98
|
+
if (best) {
|
|
99
|
+
// Combine models from all reachable hosts
|
|
100
|
+
const allModels = [...new Set(allHosts.filter(h => h.reachable).flatMap(h => h.models))];
|
|
101
|
+
return {
|
|
102
|
+
available: true,
|
|
103
|
+
mode: best.mode,
|
|
104
|
+
host: best.host,
|
|
105
|
+
models: allModels,
|
|
106
|
+
installed: local.installed,
|
|
107
|
+
allHosts,
|
|
108
|
+
};
|
|
104
109
|
}
|
|
105
110
|
|
|
106
111
|
// 3. Local installed but not running or no models
|
|
@@ -111,6 +116,7 @@ export async function checkOllamaAuto(customHosts = []) {
|
|
|
111
116
|
host: local.host,
|
|
112
117
|
models: [],
|
|
113
118
|
installed: true,
|
|
119
|
+
allHosts,
|
|
114
120
|
};
|
|
115
121
|
}
|
|
116
122
|
|
|
@@ -120,6 +126,7 @@ export async function checkOllamaAuto(customHosts = []) {
|
|
|
120
126
|
host: null,
|
|
121
127
|
models: [],
|
|
122
128
|
installed: false,
|
|
129
|
+
allHosts,
|
|
123
130
|
};
|
|
124
131
|
}
|
|
125
132
|
|
package/setup/models.js
CHANGED
|
@@ -114,7 +114,7 @@ export const EXTRACTION_MODELS = [
|
|
|
114
114
|
// Output: structured JSON with strategic recommendations, positioning, gap analysis
|
|
115
115
|
// Complexity: high — comparative reasoning across multiple domains
|
|
116
116
|
// Minimum viable: 14B+ parameters for reliable strategic output
|
|
117
|
-
// Cloud models (Claude, GPT-
|
|
117
|
+
// Cloud models (Claude, GPT-5.4, Gemini) available via OpenClaw agent setup
|
|
118
118
|
|
|
119
119
|
export const ANALYSIS_MODELS = [
|
|
120
120
|
{
|
|
@@ -169,6 +169,60 @@ export const ANALYSIS_MODELS = [
|
|
|
169
169
|
recommended: false,
|
|
170
170
|
description: 'MoE — 120B total but only 12B active params. Excellent reasoning at efficient compute. Needs 64GB+ unified memory or multi-GPU.',
|
|
171
171
|
},
|
|
172
|
+
// ── Cloud frontier models (require API key in .env) ──
|
|
173
|
+
// ── Cloud frontier models (require API key in .env or via OpenClaw) ──
|
|
174
|
+
{
|
|
175
|
+
id: 'gemini-3.1-pro',
|
|
176
|
+
name: 'Gemini 3.1 Pro',
|
|
177
|
+
family: 'gemini',
|
|
178
|
+
type: 'cloud',
|
|
179
|
+
provider: 'gemini',
|
|
180
|
+
envKey: 'GEMINI_API_KEY',
|
|
181
|
+
context: '2M tokens',
|
|
182
|
+
costNote: '~$0.01–0.05/analysis',
|
|
183
|
+
quality: 'frontier',
|
|
184
|
+
recommended: false,
|
|
185
|
+
description: 'Google\'s latest frontier model. Massive 2M context handles the largest competitive datasets. Best value for cloud analysis.',
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
id: 'claude-opus-4.6',
|
|
189
|
+
name: 'Claude Opus 4.6',
|
|
190
|
+
family: 'claude',
|
|
191
|
+
type: 'cloud',
|
|
192
|
+
provider: 'anthropic',
|
|
193
|
+
envKey: 'ANTHROPIC_API_KEY',
|
|
194
|
+
context: '1M tokens',
|
|
195
|
+
costNote: '~$0.10–0.30/analysis',
|
|
196
|
+
quality: 'frontier',
|
|
197
|
+
recommended: false,
|
|
198
|
+
description: 'Anthropic\'s most capable model. Deepest reasoning for competitive gap analysis, strategic positioning, and implementation briefs.',
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
id: 'gpt-5.4',
|
|
202
|
+
name: 'GPT-5.4',
|
|
203
|
+
family: 'gpt',
|
|
204
|
+
type: 'cloud',
|
|
205
|
+
provider: 'openai',
|
|
206
|
+
envKey: 'OPENAI_API_KEY',
|
|
207
|
+
context: '256K tokens',
|
|
208
|
+
costNote: '~$0.05–0.15/analysis',
|
|
209
|
+
quality: 'frontier',
|
|
210
|
+
recommended: false,
|
|
211
|
+
description: 'OpenAI\'s flagship frontier model. Strong analytical reasoning for competitive intelligence and strategic recommendations.',
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
id: 'deepseek-r1',
|
|
215
|
+
name: 'DeepSeek R1',
|
|
216
|
+
family: 'deepseek',
|
|
217
|
+
type: 'cloud',
|
|
218
|
+
provider: 'deepseek',
|
|
219
|
+
envKey: 'DEEPSEEK_API_KEY',
|
|
220
|
+
context: '128K tokens',
|
|
221
|
+
costNote: '~$0.005–0.02/analysis',
|
|
222
|
+
quality: 'great',
|
|
223
|
+
recommended: false,
|
|
224
|
+
description: 'Reasoning-optimized model at a fraction of the cost. Excellent quality-to-price ratio for budget-conscious analysis.',
|
|
225
|
+
},
|
|
172
226
|
];
|
|
173
227
|
|
|
174
228
|
// ── VRAM-Based Recommendations ──────────────────────────────────────────────
|
|
@@ -318,10 +372,13 @@ export function getModelRecommendations(availableModels = [], envKeys = {}, vram
|
|
|
318
372
|
})),
|
|
319
373
|
allAnalysis: ANALYSIS_MODELS.map(m => ({
|
|
320
374
|
...m,
|
|
321
|
-
installed:
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
375
|
+
installed: m.type === 'cloud'
|
|
376
|
+
? !!(m.envKey && envKeys[m.envKey])
|
|
377
|
+
: availableModels.some(am =>
|
|
378
|
+
am.startsWith(m.family) && am.includes(m.id.split(':')[1])
|
|
379
|
+
),
|
|
380
|
+
configured: m.type === 'cloud' ? !!(m.envKey && envKeys[m.envKey]) : undefined,
|
|
381
|
+
fitsVram: m.type === 'cloud' ? true : (!vramMB || vramMB >= m.minVramMB),
|
|
325
382
|
})),
|
|
326
383
|
vramMB,
|
|
327
384
|
};
|