seo-intel 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.env.example +41 -0
  2. package/LICENSE +75 -0
  3. package/README.md +243 -0
  4. package/Start SEO Intel.bat +9 -0
  5. package/Start SEO Intel.command +8 -0
  6. package/cli.js +3727 -0
  7. package/config/example.json +29 -0
  8. package/config/setup-wizard.js +522 -0
  9. package/crawler/index.js +566 -0
  10. package/crawler/robots.js +103 -0
  11. package/crawler/sanitize.js +124 -0
  12. package/crawler/schema-parser.js +168 -0
  13. package/crawler/sitemap.js +103 -0
  14. package/crawler/stealth.js +393 -0
  15. package/crawler/subdomain-discovery.js +341 -0
  16. package/db/db.js +213 -0
  17. package/db/schema.sql +120 -0
  18. package/exports/competitive.js +186 -0
  19. package/exports/heuristics.js +67 -0
  20. package/exports/queries.js +197 -0
  21. package/exports/suggestive.js +230 -0
  22. package/exports/technical.js +180 -0
  23. package/exports/templates.js +77 -0
  24. package/lib/gate.js +204 -0
  25. package/lib/license.js +369 -0
  26. package/lib/oauth.js +432 -0
  27. package/lib/updater.js +324 -0
  28. package/package.json +68 -0
  29. package/reports/generate-html.js +6194 -0
  30. package/reports/generate-site-graph.js +949 -0
  31. package/reports/gsc-loader.js +190 -0
  32. package/scheduler.js +142 -0
  33. package/seo-audit.js +619 -0
  34. package/seo-intel.png +0 -0
  35. package/server.js +602 -0
  36. package/setup/ROADMAP.md +109 -0
  37. package/setup/checks.js +483 -0
  38. package/setup/config-builder.js +227 -0
  39. package/setup/engine.js +65 -0
  40. package/setup/installers.js +197 -0
  41. package/setup/models.js +328 -0
  42. package/setup/openclaw-bridge.js +329 -0
  43. package/setup/validator.js +395 -0
  44. package/setup/web-routes.js +688 -0
  45. package/setup/wizard.html +2920 -0
  46. package/start-seo-intel.sh +8 -0
package/lib/updater.js ADDED
@@ -0,0 +1,324 @@
1
+ /**
2
+ * SEO Intel — Update Checker
3
+ *
4
+ * Non-blocking version check that runs in the background.
5
+ * Never slows down CLI startup. Caches results for 24 hours.
6
+ *
7
+ * Two update channels:
8
+ * - npm registry (public installs via `npm install -g seo-intel`)
9
+ * - froggo.pro (direct downloads / pro users)
10
+ *
11
+ * Usage:
12
+ * import { checkForUpdates, printUpdateNotice } from './updater.js';
13
+ * // At CLI startup (non-blocking):
14
+ * checkForUpdates();
15
+ * // At end of command output:
16
+ * printUpdateNotice();
17
+ */
18
+
19
+ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
20
+ import { join, dirname } from 'path';
21
+ import { fileURLToPath } from 'url';
22
+
23
+ const __dirname = dirname(fileURLToPath(import.meta.url));
24
+ const ROOT = join(__dirname, '..');
25
+
26
+ // ── Version source of truth ────────────────────────────────────────────────
27
+
28
+ let _currentVersion = null;
29
+
30
+ export function getCurrentVersion() {
31
+ if (_currentVersion) return _currentVersion;
32
+ try {
33
+ const pkg = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8'));
34
+ _currentVersion = pkg.version;
35
+ } catch {
36
+ _currentVersion = '0.0.0';
37
+ }
38
+ return _currentVersion;
39
+ }
40
+
41
+ // ── Cache file ─────────────────────────────────────────────────────────────
42
+
43
+ const CACHE_DIR = join(ROOT, '.cache');
44
+ const CACHE_FILE = join(CACHE_DIR, 'update-check.json');
45
+ const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
46
+
47
+ function readCache() {
48
+ try {
49
+ if (!existsSync(CACHE_FILE)) return null;
50
+ const data = JSON.parse(readFileSync(CACHE_FILE, 'utf8'));
51
+ if (Date.now() - data.checkedAt > CACHE_TTL_MS) return null; // expired
52
+ return data;
53
+ } catch {
54
+ return null;
55
+ }
56
+ }
57
+
58
+ function writeCache(data) {
59
+ try {
60
+ if (!existsSync(CACHE_DIR)) mkdirSync(CACHE_DIR, { recursive: true });
61
+ writeFileSync(CACHE_FILE, JSON.stringify({
62
+ ...data,
63
+ checkedAt: Date.now(),
64
+ }, null, 2));
65
+ } catch {
66
+ // Best-effort — no crash if write fails
67
+ }
68
+ }
69
+
70
+ // ── Version comparison ─────────────────────────────────────────────────────
71
+
72
+ /**
73
+ * Compare semver strings. Returns:
74
+ * 1 if a > b
75
+ * 0 if a === b
76
+ * -1 if a < b
77
+ */
78
+ export function compareSemver(a, b) {
79
+ const pa = a.split('.').map(Number);
80
+ const pb = b.split('.').map(Number);
81
+ for (let i = 0; i < 3; i++) {
82
+ const va = pa[i] || 0;
83
+ const vb = pb[i] || 0;
84
+ if (va > vb) return 1;
85
+ if (va < vb) return -1;
86
+ }
87
+ return 0;
88
+ }
89
+
90
+ // ── Update sources ─────────────────────────────────────────────────────────
91
+
92
+ /**
93
+ * Check npm registry for latest version.
94
+ * Uses the abbreviated metadata endpoint (fast, no auth needed).
95
+ */
96
+ async function checkNpm() {
97
+ const controller = new AbortController();
98
+ const timeout = setTimeout(() => controller.abort(), 5000);
99
+
100
+ try {
101
+ const res = await fetch('https://registry.npmjs.org/seo-intel/latest', {
102
+ signal: controller.signal,
103
+ headers: { 'Accept': 'application/json' },
104
+ });
105
+ if (!res.ok) return null;
106
+ const data = await res.json();
107
+ return data.version || null;
108
+ } catch {
109
+ return null;
110
+ } finally {
111
+ clearTimeout(timeout);
112
+ }
113
+ }
114
+
115
+ /**
116
+ * Check froggo.pro for latest version.
117
+ * Endpoint returns { version, changelog?, downloadUrl? }
118
+ */
119
+ async function checkFroggo() {
120
+ const controller = new AbortController();
121
+ const timeout = setTimeout(() => controller.abort(), 5000);
122
+
123
+ try {
124
+ const res = await fetch('https://froggo.pro/api/seo-intel/version', {
125
+ signal: controller.signal,
126
+ headers: { 'Accept': 'application/json' },
127
+ });
128
+ if (!res.ok) return null;
129
+ const data = await res.json();
130
+ return {
131
+ version: data.version || null,
132
+ changelog: data.changelog || null,
133
+ downloadUrl: data.downloadUrl || null,
134
+ security: data.security || false,
135
+ securitySeverity: data.securitySeverity || null,
136
+ updatePolicy: data.updatePolicy || null,
137
+ };
138
+ } catch {
139
+ return null;
140
+ } finally {
141
+ clearTimeout(timeout);
142
+ }
143
+ }
144
+
145
+ // ── Main check ─────────────────────────────────────────────────────────────
146
+
147
+ let _updateResult = null;
148
+ let _checkPromise = null;
149
+
150
+ /**
151
+ * Start a background update check.
152
+ * Non-blocking — fires and forgets. Call printUpdateNotice() later to show results.
153
+ */
154
+ export function checkForUpdates() {
155
+ // Use cached result if fresh
156
+ const cached = readCache();
157
+ if (cached) {
158
+ _updateResult = cached;
159
+ return;
160
+ }
161
+
162
+ // Fire background check — never await this
163
+ _checkPromise = (async () => {
164
+ try {
165
+ const current = getCurrentVersion();
166
+
167
+ // Check both sources in parallel
168
+ const [npmVersion, froggoData] = await Promise.all([
169
+ checkNpm(),
170
+ checkFroggo(),
171
+ ]);
172
+
173
+ const froggoVersion = froggoData?.version || null;
174
+
175
+ // Determine the highest available version
176
+ let latestVersion = current;
177
+ let source = 'current';
178
+ let changelog = null;
179
+ let downloadUrl = null;
180
+
181
+ if (npmVersion && compareSemver(npmVersion, latestVersion) > 0) {
182
+ latestVersion = npmVersion;
183
+ source = 'npm';
184
+ }
185
+ if (froggoVersion && compareSemver(froggoVersion, latestVersion) > 0) {
186
+ latestVersion = froggoVersion;
187
+ source = 'froggo';
188
+ changelog = froggoData.changelog;
189
+ downloadUrl = froggoData.downloadUrl;
190
+ }
191
+
192
+ const hasUpdate = compareSemver(latestVersion, current) > 0;
193
+
194
+ _updateResult = {
195
+ current,
196
+ latest: latestVersion,
197
+ hasUpdate,
198
+ source,
199
+ changelog,
200
+ downloadUrl,
201
+ npmVersion,
202
+ froggoVersion,
203
+ security: froggoData?.security || false,
204
+ securitySeverity: froggoData?.securitySeverity || null,
205
+ updatePolicy: froggoData?.updatePolicy || null,
206
+ };
207
+
208
+ writeCache(_updateResult);
209
+ } catch {
210
+ // Silent fail — updates are non-critical
211
+ _updateResult = null;
212
+ }
213
+ })();
214
+ }
215
+
216
+ /**
217
+ * Print update notification if a newer version is available.
218
+ * Call at end of command output so it doesn't interfere with results.
219
+ *
220
+ * Returns true if an update notice was printed.
221
+ */
222
+ export async function printUpdateNotice() {
223
+ // Wait for background check to finish (with timeout)
224
+ if (_checkPromise) {
225
+ const timeout = new Promise(resolve => setTimeout(resolve, 2000));
226
+ await Promise.race([_checkPromise, timeout]);
227
+ }
228
+
229
+ if (!_updateResult || !_updateResult.hasUpdate) return false;
230
+
231
+ const { current, latest, source, changelog, downloadUrl, security, securitySeverity } = _updateResult;
232
+
233
+ const GOLD = '\x1b[38;5;214m';
234
+ const RED = '\x1b[31m';
235
+ const DIM = '\x1b[2m';
236
+ const BOLD = '\x1b[1m';
237
+ const RESET = '\x1b[0m';
238
+ const CYAN = '\x1b[36m';
239
+
240
+ // Security updates get a red urgent banner
241
+ const COLOR = security ? RED : GOLD;
242
+ const PREFIX = security ? '🔒 SECURITY UPDATE' : 'Update available';
243
+
244
+ console.log('');
245
+ console.log(`${COLOR}${BOLD} ╭─────────────────────────────────────────╮${RESET}`);
246
+ if (security) {
247
+ console.log(`${RED}${BOLD} │ 🔒 SECURITY UPDATE: ${DIM}${current}${RESET}${RED}${BOLD} → ${CYAN}${latest}${RESET}${RED}${BOLD}${' '.repeat(Math.max(0, 10 - current.length - latest.length))}│${RESET}`);
248
+ if (securitySeverity) {
249
+ console.log(`${RED}${BOLD} │ Severity: ${securitySeverity.toUpperCase()}${' '.repeat(Math.max(0, 28 - securitySeverity.length))}│${RESET}`);
250
+ }
251
+ } else {
252
+ console.log(`${GOLD}${BOLD} │ Update available: ${DIM}${current}${RESET}${GOLD}${BOLD} → ${CYAN}${latest}${RESET}${GOLD}${BOLD}${' '.repeat(Math.max(0, 16 - current.length - latest.length))}│${RESET}`);
253
+ }
254
+
255
+ if (changelog) {
256
+ // Show first line of changelog
257
+ const firstLine = changelog.split('\n')[0].slice(0, 35);
258
+ console.log(`${GOLD}${BOLD} │ ${DIM}${firstLine}${' '.repeat(Math.max(0, 38 - firstLine.length))}${RESET}${GOLD}${BOLD}│${RESET}`);
259
+ }
260
+
261
+ // Show appropriate update command
262
+ if (source === 'npm') {
263
+ console.log(`${COLOR}${BOLD} │ ${RESET}${DIM}npm update -g seo-intel${' '.repeat(16)}${COLOR}${BOLD}│${RESET}`);
264
+ } else if (downloadUrl) {
265
+ console.log(`${COLOR}${BOLD} │ ${RESET}${CYAN}${downloadUrl.slice(0, 37)}${' '.repeat(Math.max(0, 38 - downloadUrl.length))}${COLOR}${BOLD}│${RESET}`);
266
+ } else {
267
+ console.log(`${COLOR}${BOLD} │ ${RESET}${DIM}npm update -g seo-intel${' '.repeat(16)}${COLOR}${BOLD}│${RESET}`);
268
+ }
269
+
270
+ if (security) {
271
+ console.log(`${RED}${BOLD} │ ${RESET}${DIM}or: seo-intel update --apply${' '.repeat(11)}${RED}${BOLD}│${RESET}`);
272
+ }
273
+
274
+ console.log(`${COLOR}${BOLD} ╰─────────────────────────────────────────╯${RESET}`);
275
+ console.log('');
276
+
277
+ return true;
278
+ }
279
+
280
+ // ── For web/API ────────────────────────────────────────────────────────────
281
+
282
+ /**
283
+ * Get update info as JSON (for web wizard / dashboard API).
284
+ * Awaits the background check with a short timeout.
285
+ */
286
+ export async function getUpdateInfo() {
287
+ if (_checkPromise) {
288
+ const timeout = new Promise(resolve => setTimeout(resolve, 3000));
289
+ await Promise.race([_checkPromise, timeout]);
290
+ }
291
+
292
+ if (!_updateResult) {
293
+ return {
294
+ current: getCurrentVersion(),
295
+ hasUpdate: false,
296
+ };
297
+ }
298
+
299
+ return { ..._updateResult };
300
+ }
301
+
302
+ /**
303
+ * Force a fresh update check (ignores cache).
304
+ * Used by `seo-intel update` command.
305
+ */
306
+ export async function forceUpdateCheck() {
307
+ // Clear cache
308
+ try {
309
+ if (existsSync(CACHE_FILE)) {
310
+ const { unlinkSync } = await import('fs');
311
+ unlinkSync(CACHE_FILE);
312
+ }
313
+ } catch { /* ok */ }
314
+
315
+ _updateResult = null;
316
+ _checkPromise = null;
317
+
318
+ checkForUpdates();
319
+
320
+ // Actually wait for result
321
+ if (_checkPromise) await _checkPromise;
322
+
323
+ return _updateResult || { current: getCurrentVersion(), hasUpdate: false };
324
+ }
package/package.json ADDED
@@ -0,0 +1,68 @@
1
+ {
2
+ "name": "seo-intel",
3
+ "version": "1.0.0",
4
+ "description": "Local Ahrefs-style SEO competitor intelligence. Crawl → SQLite → cloud analysis.",
5
+ "type": "module",
6
+ "license": "SEE LICENSE IN LICENSE",
7
+ "author": "Ukkometa <anssi@ukkometa.fi> (https://ukkometa.fi)",
8
+ "homepage": "https://ukkometa.fi/en/seo-intel/",
9
+ "repository": {
10
+ "type": "git",
11
+ "url": "https://github.com/ukkometa/seo-intel"
12
+ },
13
+ "keywords": [
14
+ "seo",
15
+ "competitor-analysis",
16
+ "web-crawler",
17
+ "content-intelligence",
18
+ "search-console",
19
+ "ollama",
20
+ "local-first"
21
+ ],
22
+ "bin": {
23
+ "seo-intel": "./cli.js"
24
+ },
25
+ "engines": {
26
+ "node": ">=22.5.0"
27
+ },
28
+ "files": [
29
+ "cli.js",
30
+ "server.js",
31
+ "scheduler.js",
32
+ "seo-audit.js",
33
+ "lib/",
34
+ "setup/",
35
+ "config/setup-wizard.js",
36
+ "config/example.json",
37
+ "crawler/",
38
+ "db/db.js",
39
+ "db/schema.sql",
40
+ "exports/",
41
+ "reports/generate-html.js",
42
+ "reports/generate-site-graph.js",
43
+ "reports/gsc-loader.js",
44
+ ".env.example",
45
+ "LICENSE",
46
+ "README.md",
47
+ "seo-intel.png",
48
+ "Start SEO Intel.command",
49
+ "Start SEO Intel.bat",
50
+ "start-seo-intel.sh"
51
+ ],
52
+ "scripts": {
53
+ "crawl": "node cli.js crawl",
54
+ "analyze": "node cli.js analyze",
55
+ "report": "node cli.js report",
56
+ "setup": "node cli.js setup",
57
+ "serve": "node cli.js serve",
58
+ "postinstall": "echo '\\n SEO Intel installed.\\n Run: seo-intel setup\\n Or: seo-intel serve (opens dashboard)\\n'"
59
+ },
60
+ "dependencies": {
61
+ "chalk": "^5.3.0",
62
+ "commander": "^12.0.0",
63
+ "dotenv": "^16.4.5",
64
+ "node-fetch": "^3.3.2",
65
+ "playwright": "^1.43.0",
66
+ "turndown": "^7.2.2"
67
+ }
68
+ }