seo-intel 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.env.example +41 -0
  2. package/LICENSE +75 -0
  3. package/README.md +243 -0
  4. package/Start SEO Intel.bat +9 -0
  5. package/Start SEO Intel.command +8 -0
  6. package/cli.js +3727 -0
  7. package/config/example.json +29 -0
  8. package/config/setup-wizard.js +522 -0
  9. package/crawler/index.js +566 -0
  10. package/crawler/robots.js +103 -0
  11. package/crawler/sanitize.js +124 -0
  12. package/crawler/schema-parser.js +168 -0
  13. package/crawler/sitemap.js +103 -0
  14. package/crawler/stealth.js +393 -0
  15. package/crawler/subdomain-discovery.js +341 -0
  16. package/db/db.js +213 -0
  17. package/db/schema.sql +120 -0
  18. package/exports/competitive.js +186 -0
  19. package/exports/heuristics.js +67 -0
  20. package/exports/queries.js +197 -0
  21. package/exports/suggestive.js +230 -0
  22. package/exports/technical.js +180 -0
  23. package/exports/templates.js +77 -0
  24. package/lib/gate.js +204 -0
  25. package/lib/license.js +369 -0
  26. package/lib/oauth.js +432 -0
  27. package/lib/updater.js +324 -0
  28. package/package.json +68 -0
  29. package/reports/generate-html.js +6194 -0
  30. package/reports/generate-site-graph.js +949 -0
  31. package/reports/gsc-loader.js +190 -0
  32. package/scheduler.js +142 -0
  33. package/seo-audit.js +619 -0
  34. package/seo-intel.png +0 -0
  35. package/server.js +602 -0
  36. package/setup/ROADMAP.md +109 -0
  37. package/setup/checks.js +483 -0
  38. package/setup/config-builder.js +227 -0
  39. package/setup/engine.js +65 -0
  40. package/setup/installers.js +197 -0
  41. package/setup/models.js +328 -0
  42. package/setup/openclaw-bridge.js +329 -0
  43. package/setup/validator.js +395 -0
  44. package/setup/web-routes.js +688 -0
  45. package/setup/wizard.html +2920 -0
  46. package/start-seo-intel.sh +8 -0
package/lib/oauth.js ADDED
@@ -0,0 +1,432 @@
1
+ /**
2
+ * SEO Intel — OAuth Manager
3
+ *
4
+ * Handles OAuth 2.0 flows for services that require user authorization:
5
+ * - Google Search Console (GSC)
6
+ * - Google Analytics (future)
7
+ * - Slack notifications (future)
8
+ *
9
+ * Architecture:
10
+ * - Tokens stored in .tokens/ directory (gitignored)
11
+ * - Local callback server on configurable port for OAuth redirects
12
+ * - Refresh tokens auto-renewed before expiry
13
+ * - Works alongside API key auth (users can choose either)
14
+ *
15
+ * Flow:
16
+ * 1. User runs `seo-intel auth google` or clicks "Connect" in web wizard
17
+ * 2. Opens browser to Google consent screen
18
+ * 3. Google redirects to localhost:PORT/oauth/callback
19
+ * 4. We exchange code for tokens, store them
20
+ * 5. Subsequent API calls use the stored access token (auto-refresh)
21
+ */
22
+
23
+ import { existsSync, readFileSync, writeFileSync, mkdirSync, unlinkSync } from 'fs';
24
+ import { join, dirname } from 'path';
25
+ import { fileURLToPath } from 'url';
26
+ import { createServer } from 'http';
27
+
28
+ const __dirname = dirname(fileURLToPath(import.meta.url));
29
+ const ROOT = join(__dirname, '..');
30
+ const TOKENS_DIR = join(ROOT, '.tokens');
31
+
32
+ // ── Provider Configs ───────────────────────────────────────────────────────
33
+
34
+ const PROVIDERS = {
35
+ google: {
36
+ name: 'Google',
37
+ authUrl: 'https://accounts.google.com/o/oauth2/v2/auth',
38
+ tokenUrl: 'https://oauth2.googleapis.com/token',
39
+ scopes: [
40
+ 'https://www.googleapis.com/auth/webmasters.readonly', // Search Console
41
+ 'https://www.googleapis.com/auth/analytics.readonly', // Analytics (future)
42
+ ],
43
+ // Client ID/Secret come from .env or project config
44
+ envClientId: 'GOOGLE_CLIENT_ID',
45
+ envClientSecret: 'GOOGLE_CLIENT_SECRET',
46
+ },
47
+ // Future providers:
48
+ // slack: { ... },
49
+ // github: { ... },
50
+ };
51
+
52
+ // ── Token Storage ──────────────────────────────────────────────────────────
53
+
54
+ function ensureTokenDir() {
55
+ if (!existsSync(TOKENS_DIR)) mkdirSync(TOKENS_DIR, { recursive: true });
56
+ }
57
+
58
+ function tokenPath(provider) {
59
+ return join(TOKENS_DIR, `${provider}.json`);
60
+ }
61
+
62
+ /**
63
+ * Read stored tokens for a provider.
64
+ * @param {string} provider - e.g. 'google'
65
+ * @returns {{ accessToken, refreshToken, expiresAt, scopes } | null}
66
+ */
67
+ export function getTokens(provider) {
68
+ try {
69
+ const path = tokenPath(provider);
70
+ if (!existsSync(path)) return null;
71
+ const data = JSON.parse(readFileSync(path, 'utf8'));
72
+ return data;
73
+ } catch {
74
+ return null;
75
+ }
76
+ }
77
+
78
+ /**
79
+ * Save tokens for a provider.
80
+ */
81
+ function saveTokens(provider, tokens) {
82
+ ensureTokenDir();
83
+ writeFileSync(tokenPath(provider), JSON.stringify({
84
+ ...tokens,
85
+ savedAt: Date.now(),
86
+ }, null, 2));
87
+ }
88
+
89
+ /**
90
+ * Delete stored tokens (disconnect).
91
+ */
92
+ export function clearTokens(provider) {
93
+ try {
94
+ const p = tokenPath(provider);
95
+ if (existsSync(p)) unlinkSync(p);
96
+ } catch { /* ok */ }
97
+ }
98
+
99
+ /**
100
+ * Check if a provider is connected (has valid-looking tokens).
101
+ */
102
+ export function isConnected(provider) {
103
+ const tokens = getTokens(provider);
104
+ return !!(tokens?.accessToken && tokens?.refreshToken);
105
+ }
106
+
107
+ // ── Client Credentials ─────────────────────────────────────────────────────
108
+
109
+ /**
110
+ * Get OAuth client credentials from .env.
111
+ * For Google, users create a project at console.cloud.google.com
112
+ * and paste client_id + client_secret.
113
+ */
114
+ function getClientCredentials(provider) {
115
+ const config = PROVIDERS[provider];
116
+ if (!config) throw new Error(`Unknown OAuth provider: ${provider}`);
117
+
118
+ const clientId = process.env[config.envClientId];
119
+ const clientSecret = process.env[config.envClientSecret];
120
+
121
+ if (!clientId || !clientSecret) {
122
+ return null;
123
+ }
124
+
125
+ return { clientId, clientSecret };
126
+ }
127
+
128
+ // ── OAuth Flow ─────────────────────────────────────────────────────────────
129
+
130
+ const DEFAULT_CALLBACK_PORT = 9876;
131
+
132
+ /**
133
+ * Build the OAuth authorization URL.
134
+ * @param {string} provider
135
+ * @param {object} [opts]
136
+ * @param {number} [opts.port] - callback port (default 9876)
137
+ * @param {string[]} [opts.scopes] - override default scopes
138
+ * @returns {{ url: string, state: string }}
139
+ */
140
+ export function getAuthUrl(provider, opts = {}) {
141
+ const config = PROVIDERS[provider];
142
+ if (!config) throw new Error(`Unknown OAuth provider: ${provider}`);
143
+
144
+ const creds = getClientCredentials(provider);
145
+ if (!creds) {
146
+ throw new Error(
147
+ `Missing ${config.envClientId} and ${config.envClientSecret} in .env.\n` +
148
+ ` → Set up OAuth credentials at https://console.cloud.google.com/apis/credentials`
149
+ );
150
+ }
151
+
152
+ const port = opts.port || DEFAULT_CALLBACK_PORT;
153
+ const redirectUri = `http://localhost:${port}/oauth/callback`;
154
+ const scopes = opts.scopes || config.scopes;
155
+ const state = Math.random().toString(36).slice(2, 14);
156
+
157
+ const params = new URLSearchParams({
158
+ client_id: creds.clientId,
159
+ redirect_uri: redirectUri,
160
+ response_type: 'code',
161
+ scope: scopes.join(' '),
162
+ access_type: 'offline', // get refresh token
163
+ prompt: 'consent', // always show consent to ensure refresh token
164
+ state,
165
+ });
166
+
167
+ return {
168
+ url: `${config.authUrl}?${params.toString()}`,
169
+ state,
170
+ redirectUri,
171
+ port,
172
+ };
173
+ }
174
+
175
+ /**
176
+ * Exchange authorization code for tokens.
177
+ * @param {string} provider
178
+ * @param {string} code - authorization code from callback
179
+ * @param {string} redirectUri
180
+ * @returns {Promise<{ accessToken, refreshToken, expiresAt, scopes }>}
181
+ */
182
+ async function exchangeCode(provider, code, redirectUri) {
183
+ const config = PROVIDERS[provider];
184
+ const creds = getClientCredentials(provider);
185
+
186
+ const body = new URLSearchParams({
187
+ code,
188
+ client_id: creds.clientId,
189
+ client_secret: creds.clientSecret,
190
+ redirect_uri: redirectUri,
191
+ grant_type: 'authorization_code',
192
+ });
193
+
194
+ const res = await fetch(config.tokenUrl, {
195
+ method: 'POST',
196
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
197
+ body: body.toString(),
198
+ });
199
+
200
+ if (!res.ok) {
201
+ const err = await res.text();
202
+ throw new Error(`Token exchange failed: ${res.status} ${err}`);
203
+ }
204
+
205
+ const data = await res.json();
206
+
207
+ const tokens = {
208
+ accessToken: data.access_token,
209
+ refreshToken: data.refresh_token,
210
+ expiresAt: Date.now() + (data.expires_in * 1000) - 60000, // 1min buffer
211
+ scopes: data.scope?.split(' ') || [],
212
+ tokenType: data.token_type,
213
+ };
214
+
215
+ saveTokens(provider, tokens);
216
+ return tokens;
217
+ }
218
+
219
+ /**
220
+ * Refresh an expired access token.
221
+ * @param {string} provider
222
+ * @returns {Promise<string>} new access token
223
+ */
224
+ export async function refreshAccessToken(provider) {
225
+ const config = PROVIDERS[provider];
226
+ const creds = getClientCredentials(provider);
227
+ const stored = getTokens(provider);
228
+
229
+ if (!stored?.refreshToken) {
230
+ throw new Error(`No refresh token for ${provider}. Re-authenticate with: seo-intel auth ${provider}`);
231
+ }
232
+
233
+ const body = new URLSearchParams({
234
+ refresh_token: stored.refreshToken,
235
+ client_id: creds.clientId,
236
+ client_secret: creds.clientSecret,
237
+ grant_type: 'refresh_token',
238
+ });
239
+
240
+ const res = await fetch(config.tokenUrl, {
241
+ method: 'POST',
242
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
243
+ body: body.toString(),
244
+ });
245
+
246
+ if (!res.ok) {
247
+ const err = await res.text();
248
+ throw new Error(`Token refresh failed: ${res.status} ${err}`);
249
+ }
250
+
251
+ const data = await res.json();
252
+
253
+ const tokens = {
254
+ ...stored,
255
+ accessToken: data.access_token,
256
+ expiresAt: Date.now() + (data.expires_in * 1000) - 60000,
257
+ };
258
+
259
+ // Google doesn't always return a new refresh token
260
+ if (data.refresh_token) {
261
+ tokens.refreshToken = data.refresh_token;
262
+ }
263
+
264
+ saveTokens(provider, tokens);
265
+ return tokens.accessToken;
266
+ }
267
+
268
+ /**
269
+ * Get a valid access token (auto-refreshes if expired).
270
+ * @param {string} provider
271
+ * @returns {Promise<string>} access token ready to use
272
+ */
273
+ export async function getAccessToken(provider) {
274
+ const tokens = getTokens(provider);
275
+ if (!tokens) {
276
+ throw new Error(`Not connected to ${provider}. Run: seo-intel auth ${provider}`);
277
+ }
278
+
279
+ // Refresh if expired (or within 1min of expiry)
280
+ if (Date.now() >= tokens.expiresAt) {
281
+ return refreshAccessToken(provider);
282
+ }
283
+
284
+ return tokens.accessToken;
285
+ }
286
+
287
+ // ── Local Callback Server ──────────────────────────────────────────────────
288
+
289
+ /**
290
+ * Start a temporary local server to handle the OAuth callback.
291
+ * Returns a promise that resolves with the auth code.
292
+ *
293
+ * @param {string} provider
294
+ * @param {string} expectedState
295
+ * @param {number} port
296
+ * @returns {Promise<{ code: string }>}
297
+ */
298
+ function startCallbackServer(provider, expectedState, port) {
299
+ return new Promise((resolve, reject) => {
300
+ const server = createServer((req, res) => {
301
+ const url = new URL(req.url, `http://localhost:${port}`);
302
+
303
+ if (url.pathname === '/oauth/callback') {
304
+ const code = url.searchParams.get('code');
305
+ const state = url.searchParams.get('state');
306
+ const error = url.searchParams.get('error');
307
+
308
+ if (error) {
309
+ res.writeHead(200, { 'Content-Type': 'text/html' });
310
+ res.end(`
311
+ <html><body style="font-family:system-ui;text-align:center;padding:60px;">
312
+ <h2 style="color:#e74c3c;">❌ Authorization failed</h2>
313
+ <p>${error}</p>
314
+ <p style="color:#888;">You can close this tab.</p>
315
+ </body></html>
316
+ `);
317
+ server.close();
318
+ reject(new Error(`OAuth denied: ${error}`));
319
+ return;
320
+ }
321
+
322
+ if (state !== expectedState) {
323
+ res.writeHead(400, { 'Content-Type': 'text/html' });
324
+ res.end('<h2>State mismatch — possible CSRF. Try again.</h2>');
325
+ server.close();
326
+ reject(new Error('OAuth state mismatch'));
327
+ return;
328
+ }
329
+
330
+ res.writeHead(200, { 'Content-Type': 'text/html' });
331
+ res.end(`
332
+ <html><body style="font-family:system-ui;text-align:center;padding:60px;">
333
+ <h2 style="color:#2ecc71;">✅ Connected to ${PROVIDERS[provider]?.name || provider}!</h2>
334
+ <p style="color:#888;">You can close this tab and return to the terminal.</p>
335
+ </body></html>
336
+ `);
337
+
338
+ server.close();
339
+ resolve({ code });
340
+ } else {
341
+ res.writeHead(404);
342
+ res.end();
343
+ }
344
+ });
345
+
346
+ server.listen(port, () => {
347
+ // Server ready
348
+ });
349
+
350
+ // Auto-timeout after 3 minutes
351
+ setTimeout(() => {
352
+ server.close();
353
+ reject(new Error('OAuth callback timed out (3 minutes). Try again.'));
354
+ }, 180000);
355
+ });
356
+ }
357
+
358
+ // ── Public API: Full OAuth Flow ────────────────────────────────────────────
359
+
360
+ /**
361
+ * Run the full OAuth flow for a provider.
362
+ * Opens browser → waits for callback → exchanges code → stores tokens.
363
+ *
364
+ * @param {string} provider - e.g. 'google'
365
+ * @param {object} [opts]
366
+ * @param {number} [opts.port] - callback port
367
+ * @param {boolean} [opts.openBrowser] - auto-open browser (default true)
368
+ * @returns {Promise<{ success: boolean, provider: string, scopes: string[] }>}
369
+ */
370
+ export async function startOAuthFlow(provider, opts = {}) {
371
+ const { url, state, redirectUri, port } = getAuthUrl(provider, opts);
372
+
373
+ // Start callback server BEFORE opening browser
374
+ const callbackPromise = startCallbackServer(provider, state, port);
375
+
376
+ // Open browser
377
+ if (opts.openBrowser !== false) {
378
+ const { exec } = await import('child_process');
379
+ const cmd = process.platform === 'darwin' ? 'open' :
380
+ process.platform === 'win32' ? 'start' : 'xdg-open';
381
+ exec(`${cmd} "${url}"`);
382
+ }
383
+
384
+ // Wait for callback
385
+ const { code } = await callbackPromise;
386
+
387
+ // Exchange code for tokens
388
+ const tokens = await exchangeCode(provider, code, redirectUri);
389
+
390
+ return {
391
+ success: true,
392
+ provider,
393
+ scopes: tokens.scopes,
394
+ };
395
+ }
396
+
397
+ // ── Provider Status ────────────────────────────────────────────────────────
398
+
399
+ /**
400
+ * Get connection status for all providers.
401
+ * Useful for the web wizard and status command.
402
+ */
403
+ export function getAllConnectionStatus() {
404
+ const statuses = {};
405
+ for (const [key, config] of Object.entries(PROVIDERS)) {
406
+ const tokens = getTokens(key);
407
+ const creds = getClientCredentials(key);
408
+ statuses[key] = {
409
+ name: config.name,
410
+ connected: !!(tokens?.accessToken && tokens?.refreshToken),
411
+ hasCredentials: !!creds,
412
+ expiresAt: tokens?.expiresAt || null,
413
+ scopes: tokens?.scopes || [],
414
+ needsSetup: !creds,
415
+ };
416
+ }
417
+ return statuses;
418
+ }
419
+
420
+ /**
421
+ * Available providers and their required env vars.
422
+ * Shown in setup wizard to guide users.
423
+ */
424
+ export function getProviderRequirements() {
425
+ return Object.entries(PROVIDERS).map(([key, config]) => ({
426
+ id: key,
427
+ name: config.name,
428
+ envVars: [config.envClientId, config.envClientSecret],
429
+ scopes: config.scopes,
430
+ setupUrl: key === 'google' ? 'https://console.cloud.google.com/apis/credentials' : null,
431
+ }));
432
+ }