seo-intel 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +41 -0
- package/LICENSE +75 -0
- package/README.md +243 -0
- package/Start SEO Intel.bat +9 -0
- package/Start SEO Intel.command +8 -0
- package/cli.js +3727 -0
- package/config/example.json +29 -0
- package/config/setup-wizard.js +522 -0
- package/crawler/index.js +566 -0
- package/crawler/robots.js +103 -0
- package/crawler/sanitize.js +124 -0
- package/crawler/schema-parser.js +168 -0
- package/crawler/sitemap.js +103 -0
- package/crawler/stealth.js +393 -0
- package/crawler/subdomain-discovery.js +341 -0
- package/db/db.js +213 -0
- package/db/schema.sql +120 -0
- package/exports/competitive.js +186 -0
- package/exports/heuristics.js +67 -0
- package/exports/queries.js +197 -0
- package/exports/suggestive.js +230 -0
- package/exports/technical.js +180 -0
- package/exports/templates.js +77 -0
- package/lib/gate.js +204 -0
- package/lib/license.js +369 -0
- package/lib/oauth.js +432 -0
- package/lib/updater.js +324 -0
- package/package.json +68 -0
- package/reports/generate-html.js +6194 -0
- package/reports/generate-site-graph.js +949 -0
- package/reports/gsc-loader.js +190 -0
- package/scheduler.js +142 -0
- package/seo-audit.js +619 -0
- package/seo-intel.png +0 -0
- package/server.js +602 -0
- package/setup/ROADMAP.md +109 -0
- package/setup/checks.js +483 -0
- package/setup/config-builder.js +227 -0
- package/setup/engine.js +65 -0
- package/setup/installers.js +197 -0
- package/setup/models.js +328 -0
- package/setup/openclaw-bridge.js +329 -0
- package/setup/validator.js +395 -0
- package/setup/web-routes.js +688 -0
- package/setup/wizard.html +2920 -0
- package/start-seo-intel.sh +8 -0
package/lib/oauth.js
ADDED
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SEO Intel — OAuth Manager
|
|
3
|
+
*
|
|
4
|
+
* Handles OAuth 2.0 flows for services that require user authorization:
|
|
5
|
+
* - Google Search Console (GSC)
|
|
6
|
+
* - Google Analytics (future)
|
|
7
|
+
* - Slack notifications (future)
|
|
8
|
+
*
|
|
9
|
+
* Architecture:
|
|
10
|
+
* - Tokens stored in .tokens/ directory (gitignored)
|
|
11
|
+
* - Local callback server on configurable port for OAuth redirects
|
|
12
|
+
* - Refresh tokens auto-renewed before expiry
|
|
13
|
+
* - Works alongside API key auth (users can choose either)
|
|
14
|
+
*
|
|
15
|
+
* Flow:
|
|
16
|
+
* 1. User runs `seo-intel auth google` or clicks "Connect" in web wizard
|
|
17
|
+
* 2. Opens browser to Google consent screen
|
|
18
|
+
* 3. Google redirects to localhost:PORT/oauth/callback
|
|
19
|
+
* 4. We exchange code for tokens, store them
|
|
20
|
+
* 5. Subsequent API calls use the stored access token (auto-refresh)
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync, unlinkSync } from 'fs';
|
|
24
|
+
import { join, dirname } from 'path';
|
|
25
|
+
import { fileURLToPath } from 'url';
|
|
26
|
+
import { createServer } from 'http';
|
|
27
|
+
|
|
28
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
29
|
+
const ROOT = join(__dirname, '..');
|
|
30
|
+
const TOKENS_DIR = join(ROOT, '.tokens');
|
|
31
|
+
|
|
32
|
+
// ── Provider Configs ───────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
const PROVIDERS = {
|
|
35
|
+
google: {
|
|
36
|
+
name: 'Google',
|
|
37
|
+
authUrl: 'https://accounts.google.com/o/oauth2/v2/auth',
|
|
38
|
+
tokenUrl: 'https://oauth2.googleapis.com/token',
|
|
39
|
+
scopes: [
|
|
40
|
+
'https://www.googleapis.com/auth/webmasters.readonly', // Search Console
|
|
41
|
+
'https://www.googleapis.com/auth/analytics.readonly', // Analytics (future)
|
|
42
|
+
],
|
|
43
|
+
// Client ID/Secret come from .env or project config
|
|
44
|
+
envClientId: 'GOOGLE_CLIENT_ID',
|
|
45
|
+
envClientSecret: 'GOOGLE_CLIENT_SECRET',
|
|
46
|
+
},
|
|
47
|
+
// Future providers:
|
|
48
|
+
// slack: { ... },
|
|
49
|
+
// github: { ... },
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
// ── Token Storage ──────────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
function ensureTokenDir() {
|
|
55
|
+
if (!existsSync(TOKENS_DIR)) mkdirSync(TOKENS_DIR, { recursive: true });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function tokenPath(provider) {
|
|
59
|
+
return join(TOKENS_DIR, `${provider}.json`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Read stored tokens for a provider.
|
|
64
|
+
* @param {string} provider - e.g. 'google'
|
|
65
|
+
* @returns {{ accessToken, refreshToken, expiresAt, scopes } | null}
|
|
66
|
+
*/
|
|
67
|
+
export function getTokens(provider) {
|
|
68
|
+
try {
|
|
69
|
+
const path = tokenPath(provider);
|
|
70
|
+
if (!existsSync(path)) return null;
|
|
71
|
+
const data = JSON.parse(readFileSync(path, 'utf8'));
|
|
72
|
+
return data;
|
|
73
|
+
} catch {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Save tokens for a provider.
|
|
80
|
+
*/
|
|
81
|
+
function saveTokens(provider, tokens) {
|
|
82
|
+
ensureTokenDir();
|
|
83
|
+
writeFileSync(tokenPath(provider), JSON.stringify({
|
|
84
|
+
...tokens,
|
|
85
|
+
savedAt: Date.now(),
|
|
86
|
+
}, null, 2));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Delete stored tokens (disconnect).
|
|
91
|
+
*/
|
|
92
|
+
export function clearTokens(provider) {
|
|
93
|
+
try {
|
|
94
|
+
const p = tokenPath(provider);
|
|
95
|
+
if (existsSync(p)) unlinkSync(p);
|
|
96
|
+
} catch { /* ok */ }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Check if a provider is connected (has valid-looking tokens).
|
|
101
|
+
*/
|
|
102
|
+
export function isConnected(provider) {
|
|
103
|
+
const tokens = getTokens(provider);
|
|
104
|
+
return !!(tokens?.accessToken && tokens?.refreshToken);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ── Client Credentials ─────────────────────────────────────────────────────
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Get OAuth client credentials from .env.
|
|
111
|
+
* For Google, users create a project at console.cloud.google.com
|
|
112
|
+
* and paste client_id + client_secret.
|
|
113
|
+
*/
|
|
114
|
+
function getClientCredentials(provider) {
|
|
115
|
+
const config = PROVIDERS[provider];
|
|
116
|
+
if (!config) throw new Error(`Unknown OAuth provider: ${provider}`);
|
|
117
|
+
|
|
118
|
+
const clientId = process.env[config.envClientId];
|
|
119
|
+
const clientSecret = process.env[config.envClientSecret];
|
|
120
|
+
|
|
121
|
+
if (!clientId || !clientSecret) {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return { clientId, clientSecret };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ── OAuth Flow ─────────────────────────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
const DEFAULT_CALLBACK_PORT = 9876;
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Build the OAuth authorization URL.
|
|
134
|
+
* @param {string} provider
|
|
135
|
+
* @param {object} [opts]
|
|
136
|
+
* @param {number} [opts.port] - callback port (default 9876)
|
|
137
|
+
* @param {string[]} [opts.scopes] - override default scopes
|
|
138
|
+
* @returns {{ url: string, state: string }}
|
|
139
|
+
*/
|
|
140
|
+
export function getAuthUrl(provider, opts = {}) {
|
|
141
|
+
const config = PROVIDERS[provider];
|
|
142
|
+
if (!config) throw new Error(`Unknown OAuth provider: ${provider}`);
|
|
143
|
+
|
|
144
|
+
const creds = getClientCredentials(provider);
|
|
145
|
+
if (!creds) {
|
|
146
|
+
throw new Error(
|
|
147
|
+
`Missing ${config.envClientId} and ${config.envClientSecret} in .env.\n` +
|
|
148
|
+
` → Set up OAuth credentials at https://console.cloud.google.com/apis/credentials`
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const port = opts.port || DEFAULT_CALLBACK_PORT;
|
|
153
|
+
const redirectUri = `http://localhost:${port}/oauth/callback`;
|
|
154
|
+
const scopes = opts.scopes || config.scopes;
|
|
155
|
+
const state = Math.random().toString(36).slice(2, 14);
|
|
156
|
+
|
|
157
|
+
const params = new URLSearchParams({
|
|
158
|
+
client_id: creds.clientId,
|
|
159
|
+
redirect_uri: redirectUri,
|
|
160
|
+
response_type: 'code',
|
|
161
|
+
scope: scopes.join(' '),
|
|
162
|
+
access_type: 'offline', // get refresh token
|
|
163
|
+
prompt: 'consent', // always show consent to ensure refresh token
|
|
164
|
+
state,
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
url: `${config.authUrl}?${params.toString()}`,
|
|
169
|
+
state,
|
|
170
|
+
redirectUri,
|
|
171
|
+
port,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Exchange authorization code for tokens.
|
|
177
|
+
* @param {string} provider
|
|
178
|
+
* @param {string} code - authorization code from callback
|
|
179
|
+
* @param {string} redirectUri
|
|
180
|
+
* @returns {Promise<{ accessToken, refreshToken, expiresAt, scopes }>}
|
|
181
|
+
*/
|
|
182
|
+
async function exchangeCode(provider, code, redirectUri) {
|
|
183
|
+
const config = PROVIDERS[provider];
|
|
184
|
+
const creds = getClientCredentials(provider);
|
|
185
|
+
|
|
186
|
+
const body = new URLSearchParams({
|
|
187
|
+
code,
|
|
188
|
+
client_id: creds.clientId,
|
|
189
|
+
client_secret: creds.clientSecret,
|
|
190
|
+
redirect_uri: redirectUri,
|
|
191
|
+
grant_type: 'authorization_code',
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
const res = await fetch(config.tokenUrl, {
|
|
195
|
+
method: 'POST',
|
|
196
|
+
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
197
|
+
body: body.toString(),
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
if (!res.ok) {
|
|
201
|
+
const err = await res.text();
|
|
202
|
+
throw new Error(`Token exchange failed: ${res.status} ${err}`);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const data = await res.json();
|
|
206
|
+
|
|
207
|
+
const tokens = {
|
|
208
|
+
accessToken: data.access_token,
|
|
209
|
+
refreshToken: data.refresh_token,
|
|
210
|
+
expiresAt: Date.now() + (data.expires_in * 1000) - 60000, // 1min buffer
|
|
211
|
+
scopes: data.scope?.split(' ') || [],
|
|
212
|
+
tokenType: data.token_type,
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
saveTokens(provider, tokens);
|
|
216
|
+
return tokens;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Refresh an expired access token.
|
|
221
|
+
* @param {string} provider
|
|
222
|
+
* @returns {Promise<string>} new access token
|
|
223
|
+
*/
|
|
224
|
+
export async function refreshAccessToken(provider) {
|
|
225
|
+
const config = PROVIDERS[provider];
|
|
226
|
+
const creds = getClientCredentials(provider);
|
|
227
|
+
const stored = getTokens(provider);
|
|
228
|
+
|
|
229
|
+
if (!stored?.refreshToken) {
|
|
230
|
+
throw new Error(`No refresh token for ${provider}. Re-authenticate with: seo-intel auth ${provider}`);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const body = new URLSearchParams({
|
|
234
|
+
refresh_token: stored.refreshToken,
|
|
235
|
+
client_id: creds.clientId,
|
|
236
|
+
client_secret: creds.clientSecret,
|
|
237
|
+
grant_type: 'refresh_token',
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
const res = await fetch(config.tokenUrl, {
|
|
241
|
+
method: 'POST',
|
|
242
|
+
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
243
|
+
body: body.toString(),
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
if (!res.ok) {
|
|
247
|
+
const err = await res.text();
|
|
248
|
+
throw new Error(`Token refresh failed: ${res.status} ${err}`);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const data = await res.json();
|
|
252
|
+
|
|
253
|
+
const tokens = {
|
|
254
|
+
...stored,
|
|
255
|
+
accessToken: data.access_token,
|
|
256
|
+
expiresAt: Date.now() + (data.expires_in * 1000) - 60000,
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
// Google doesn't always return a new refresh token
|
|
260
|
+
if (data.refresh_token) {
|
|
261
|
+
tokens.refreshToken = data.refresh_token;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
saveTokens(provider, tokens);
|
|
265
|
+
return tokens.accessToken;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Get a valid access token (auto-refreshes if expired).
|
|
270
|
+
* @param {string} provider
|
|
271
|
+
* @returns {Promise<string>} access token ready to use
|
|
272
|
+
*/
|
|
273
|
+
export async function getAccessToken(provider) {
|
|
274
|
+
const tokens = getTokens(provider);
|
|
275
|
+
if (!tokens) {
|
|
276
|
+
throw new Error(`Not connected to ${provider}. Run: seo-intel auth ${provider}`);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Refresh if expired (or within 1min of expiry)
|
|
280
|
+
if (Date.now() >= tokens.expiresAt) {
|
|
281
|
+
return refreshAccessToken(provider);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
return tokens.accessToken;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// ── Local Callback Server ──────────────────────────────────────────────────
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Start a temporary local server to handle the OAuth callback.
|
|
291
|
+
* Returns a promise that resolves with the auth code.
|
|
292
|
+
*
|
|
293
|
+
* @param {string} provider
|
|
294
|
+
* @param {string} expectedState
|
|
295
|
+
* @param {number} port
|
|
296
|
+
* @returns {Promise<{ code: string }>}
|
|
297
|
+
*/
|
|
298
|
+
function startCallbackServer(provider, expectedState, port) {
|
|
299
|
+
return new Promise((resolve, reject) => {
|
|
300
|
+
const server = createServer((req, res) => {
|
|
301
|
+
const url = new URL(req.url, `http://localhost:${port}`);
|
|
302
|
+
|
|
303
|
+
if (url.pathname === '/oauth/callback') {
|
|
304
|
+
const code = url.searchParams.get('code');
|
|
305
|
+
const state = url.searchParams.get('state');
|
|
306
|
+
const error = url.searchParams.get('error');
|
|
307
|
+
|
|
308
|
+
if (error) {
|
|
309
|
+
res.writeHead(200, { 'Content-Type': 'text/html' });
|
|
310
|
+
res.end(`
|
|
311
|
+
<html><body style="font-family:system-ui;text-align:center;padding:60px;">
|
|
312
|
+
<h2 style="color:#e74c3c;">❌ Authorization failed</h2>
|
|
313
|
+
<p>${error}</p>
|
|
314
|
+
<p style="color:#888;">You can close this tab.</p>
|
|
315
|
+
</body></html>
|
|
316
|
+
`);
|
|
317
|
+
server.close();
|
|
318
|
+
reject(new Error(`OAuth denied: ${error}`));
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if (state !== expectedState) {
|
|
323
|
+
res.writeHead(400, { 'Content-Type': 'text/html' });
|
|
324
|
+
res.end('<h2>State mismatch — possible CSRF. Try again.</h2>');
|
|
325
|
+
server.close();
|
|
326
|
+
reject(new Error('OAuth state mismatch'));
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
res.writeHead(200, { 'Content-Type': 'text/html' });
|
|
331
|
+
res.end(`
|
|
332
|
+
<html><body style="font-family:system-ui;text-align:center;padding:60px;">
|
|
333
|
+
<h2 style="color:#2ecc71;">✅ Connected to ${PROVIDERS[provider]?.name || provider}!</h2>
|
|
334
|
+
<p style="color:#888;">You can close this tab and return to the terminal.</p>
|
|
335
|
+
</body></html>
|
|
336
|
+
`);
|
|
337
|
+
|
|
338
|
+
server.close();
|
|
339
|
+
resolve({ code });
|
|
340
|
+
} else {
|
|
341
|
+
res.writeHead(404);
|
|
342
|
+
res.end();
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
server.listen(port, () => {
|
|
347
|
+
// Server ready
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
// Auto-timeout after 3 minutes
|
|
351
|
+
setTimeout(() => {
|
|
352
|
+
server.close();
|
|
353
|
+
reject(new Error('OAuth callback timed out (3 minutes). Try again.'));
|
|
354
|
+
}, 180000);
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// ── Public API: Full OAuth Flow ────────────────────────────────────────────
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Run the full OAuth flow for a provider.
|
|
362
|
+
* Opens browser → waits for callback → exchanges code → stores tokens.
|
|
363
|
+
*
|
|
364
|
+
* @param {string} provider - e.g. 'google'
|
|
365
|
+
* @param {object} [opts]
|
|
366
|
+
* @param {number} [opts.port] - callback port
|
|
367
|
+
* @param {boolean} [opts.openBrowser] - auto-open browser (default true)
|
|
368
|
+
* @returns {Promise<{ success: boolean, provider: string, scopes: string[] }>}
|
|
369
|
+
*/
|
|
370
|
+
export async function startOAuthFlow(provider, opts = {}) {
|
|
371
|
+
const { url, state, redirectUri, port } = getAuthUrl(provider, opts);
|
|
372
|
+
|
|
373
|
+
// Start callback server BEFORE opening browser
|
|
374
|
+
const callbackPromise = startCallbackServer(provider, state, port);
|
|
375
|
+
|
|
376
|
+
// Open browser
|
|
377
|
+
if (opts.openBrowser !== false) {
|
|
378
|
+
const { exec } = await import('child_process');
|
|
379
|
+
const cmd = process.platform === 'darwin' ? 'open' :
|
|
380
|
+
process.platform === 'win32' ? 'start' : 'xdg-open';
|
|
381
|
+
exec(`${cmd} "${url}"`);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Wait for callback
|
|
385
|
+
const { code } = await callbackPromise;
|
|
386
|
+
|
|
387
|
+
// Exchange code for tokens
|
|
388
|
+
const tokens = await exchangeCode(provider, code, redirectUri);
|
|
389
|
+
|
|
390
|
+
return {
|
|
391
|
+
success: true,
|
|
392
|
+
provider,
|
|
393
|
+
scopes: tokens.scopes,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// ── Provider Status ────────────────────────────────────────────────────────
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Get connection status for all providers.
|
|
401
|
+
* Useful for the web wizard and status command.
|
|
402
|
+
*/
|
|
403
|
+
export function getAllConnectionStatus() {
|
|
404
|
+
const statuses = {};
|
|
405
|
+
for (const [key, config] of Object.entries(PROVIDERS)) {
|
|
406
|
+
const tokens = getTokens(key);
|
|
407
|
+
const creds = getClientCredentials(key);
|
|
408
|
+
statuses[key] = {
|
|
409
|
+
name: config.name,
|
|
410
|
+
connected: !!(tokens?.accessToken && tokens?.refreshToken),
|
|
411
|
+
hasCredentials: !!creds,
|
|
412
|
+
expiresAt: tokens?.expiresAt || null,
|
|
413
|
+
scopes: tokens?.scopes || [],
|
|
414
|
+
needsSetup: !creds,
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
return statuses;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Available providers and their required env vars.
|
|
422
|
+
* Shown in setup wizard to guide users.
|
|
423
|
+
*/
|
|
424
|
+
export function getProviderRequirements() {
|
|
425
|
+
return Object.entries(PROVIDERS).map(([key, config]) => ({
|
|
426
|
+
id: key,
|
|
427
|
+
name: config.name,
|
|
428
|
+
envVars: [config.envClientId, config.envClientSecret],
|
|
429
|
+
scopes: config.scopes,
|
|
430
|
+
setupUrl: key === 'google' ? 'https://console.cloud.google.com/apis/credentials' : null,
|
|
431
|
+
}));
|
|
432
|
+
}
|