webmaster-mcp 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +92 -44
- package/dist/core/analysis/index.js +102 -13
- package/dist/core/auth/accounts.d.ts +1 -1
- package/dist/core/auth/accounts.js +16 -4
- package/dist/core/auth/prompt.js +16 -2
- package/dist/core/auth/store.js +28 -17
- package/dist/core/cache/index.js +45 -15
- package/dist/core/errors.js +9 -2
- package/dist/core/normalize/index.js +26 -3
- package/dist/core/scheduler.js +63 -36
- package/dist/core/service.d.ts +1 -1
- package/dist/core/service.js +117 -37
- package/dist/core/validation.js +47 -26
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/mcp/index.js +12 -1
- package/dist/mcp/tools/index.js +90 -9
- package/dist/providers/bing/index.js +71 -17
- package/dist/providers/google/auth.js +87 -24
- package/dist/providers/google/index.js +98 -18
- package/dist/providers/yandex/auth.js +24 -7
- package/dist/providers/yandex/index.js +154 -52
- package/dist/server/context.d.ts +2 -2
- package/dist/server/context.js +21 -7
- package/package.json +5 -1
package/dist/cli/index.js
CHANGED
|
@@ -1,49 +1,97 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { WebmasterError } from '../core/errors.js';
|
|
3
|
+
import { createServer } from '../mcp/index.js';
|
|
4
4
|
import { BingProvider } from '../providers/bing/index.js';
|
|
5
|
+
import { GoogleProvider } from '../providers/google/index.js';
|
|
5
6
|
import { YandexProvider } from '../providers/yandex/index.js';
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
process.stdout.write(`${engine}/${name} disconnected.\n`);
|
|
29
|
-
return;
|
|
30
|
-
} if (action === 'google') {
|
|
31
|
-
await new GoogleProvider(store, name, fetch, args.includes('--write')).authenticate();
|
|
32
|
-
accounts.add('google', name);
|
|
33
|
-
}
|
|
34
|
-
else if (action === 'bing') {
|
|
35
|
-
await new BingProvider(store, name).authenticate();
|
|
36
|
-
accounts.add('bing', name);
|
|
37
|
-
}
|
|
38
|
-
else if (action === 'yandex') {
|
|
39
|
-
await new YandexProvider(store, name).authenticate();
|
|
40
|
-
accounts.add('yandex', name);
|
|
7
|
+
import { createContext } from '../server/context.js';
|
|
8
|
+
function printHelp() {
|
|
9
|
+
process.stdout.write(`webmaster-mcp v0.1.0
|
|
10
|
+
Unified local-first MCP server for Google Search Console, Bing Webmaster Tools, and Yandex Webmaster.
|
|
11
|
+
|
|
12
|
+
Usage:
|
|
13
|
+
webmaster-mcp [command] [options]
|
|
14
|
+
|
|
15
|
+
Commands:
|
|
16
|
+
mcp Start the MCP server over stdio
|
|
17
|
+
sites List all verified properties across connected engines
|
|
18
|
+
auth google [--write] Authenticate with Google Search Console via browser OAuth
|
|
19
|
+
auth bing Authenticate with Bing Webmaster Tools API key
|
|
20
|
+
auth yandex Authenticate with Yandex Webmaster via OAuth token
|
|
21
|
+
auth status Show connection status for all search engines
|
|
22
|
+
auth logout <engine> Disconnect an engine (google | bing | yandex)
|
|
23
|
+
help, --help, -h Show this help message
|
|
24
|
+
|
|
25
|
+
Options:
|
|
26
|
+
--name <account> Account profile name (default: "default")
|
|
27
|
+
--write Request read/write scopes for Google OAuth
|
|
28
|
+
\n`);
|
|
41
29
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
30
|
+
async function main() {
|
|
31
|
+
const args = process.argv.slice(2);
|
|
32
|
+
if (args.length === 0 || args.includes('--help') || args.includes('-h') || args[0] === 'help') {
|
|
33
|
+
if (args.includes('--mcponce-background') || process.env.MCPONCE_BACKGROUND_SERVER === '1') {
|
|
34
|
+
await createServer().run();
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
printHelp();
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const command = args[0];
|
|
41
|
+
if (command === 'mcp' || args.includes('--mcponce-background') || process.env.MCPONCE_BACKGROUND_SERVER === '1') {
|
|
42
|
+
await createServer().run();
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
const { service, store, accounts } = createContext();
|
|
46
|
+
const nameIndex = args.indexOf('--name');
|
|
47
|
+
const name = nameIndex >= 0 ? (args[nameIndex + 1] ?? 'default') : 'default';
|
|
48
|
+
if (!/^[A-Za-z0-9_-]{1,40}$/.test(name))
|
|
49
|
+
throw new Error('Invalid account name.');
|
|
50
|
+
if (command === 'sites') {
|
|
51
|
+
process.stdout.write(`${JSON.stringify(await service.sites(), null, 2)}\n`);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (command !== 'auth')
|
|
55
|
+
throw new Error('Usage: webmaster-mcp [mcp|sites|auth|help]. Run webmaster-mcp --help for details.');
|
|
56
|
+
const action = args[1];
|
|
57
|
+
if (action === 'status') {
|
|
58
|
+
for (const engine of ['google', 'bing', 'yandex'])
|
|
59
|
+
for (const account of accounts.names(engine)) {
|
|
60
|
+
const connected = !!(await store.get(engine, account));
|
|
61
|
+
process.stdout.write(`${engine.padEnd(7)} ${account.padEnd(12)} ${connected ? '✓ connected' : '✗ not connected'}\n`);
|
|
62
|
+
}
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (action === 'logout') {
|
|
66
|
+
const engine = args[2];
|
|
67
|
+
if (!['google', 'bing', 'yandex'].includes(engine))
|
|
68
|
+
throw new Error('Choose google, bing, or yandex.');
|
|
69
|
+
await accounts.remove(engine, name);
|
|
70
|
+
process.stdout.write(`${engine}/${name} disconnected.\n`);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (action === 'google') {
|
|
74
|
+
await new GoogleProvider(store, name, fetch, args.includes('--write')).authenticate();
|
|
75
|
+
accounts.add('google', name);
|
|
76
|
+
}
|
|
77
|
+
else if (action === 'bing') {
|
|
78
|
+
await new BingProvider(store, name).authenticate();
|
|
79
|
+
accounts.add('bing', name);
|
|
80
|
+
}
|
|
81
|
+
else if (action === 'yandex') {
|
|
82
|
+
await new YandexProvider(store, name).authenticate();
|
|
83
|
+
accounts.add('yandex', name);
|
|
84
|
+
}
|
|
85
|
+
else
|
|
86
|
+
throw new Error('Usage: webmaster-mcp auth [google|bing|yandex|status|logout]');
|
|
87
|
+
process.stdout.write(`${action}/${name} connected.\n`);
|
|
46
88
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
}
|
|
89
|
+
main().catch((error) => {
|
|
90
|
+
if (error instanceof WebmasterError) {
|
|
91
|
+
process.stderr.write(`${error.code}: ${error.message}\n`);
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
process.stderr.write(`ERROR: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
95
|
+
}
|
|
96
|
+
process.exitCode = 1;
|
|
97
|
+
});
|
|
@@ -1,14 +1,103 @@
|
|
|
1
1
|
import { aggregate } from '../normalize/index.js';
|
|
2
|
-
export function findCtrOpportunities(rows, config = { minImpressions: 100, maxCtr: 0.02 }) {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
export function
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
export function findCtrOpportunities(rows, config = { minImpressions: 100, maxCtr: 0.02 }) {
|
|
3
|
+
return rows
|
|
4
|
+
.filter((r) => (r.impressions ?? 0) >= config.minImpressions && r.ctr !== undefined && r.ctr <= config.maxCtr)
|
|
5
|
+
.map((metric) => ({
|
|
6
|
+
kind: 'low_ctr',
|
|
7
|
+
metric,
|
|
8
|
+
reason: `${metric.impressions} impressions with ${(metric.ctr * 100).toFixed(2)}% CTR`,
|
|
9
|
+
}));
|
|
10
|
+
}
|
|
11
|
+
export function findNearFirstPage(rows, config = { minImpressions: 50, minPosition: 8, maxPosition: 20 }) {
|
|
12
|
+
return rows
|
|
13
|
+
.filter((r) => (r.impressions ?? 0) >= config.minImpressions &&
|
|
14
|
+
r.position !== undefined &&
|
|
15
|
+
r.position >= config.minPosition &&
|
|
16
|
+
r.position <= config.maxPosition)
|
|
17
|
+
.map((metric) => ({
|
|
18
|
+
kind: 'near_first_page',
|
|
19
|
+
metric,
|
|
20
|
+
reason: `Average position ${metric.position.toFixed(1)} with ${metric.impressions} impressions`,
|
|
21
|
+
}));
|
|
22
|
+
}
|
|
23
|
+
function declines(current, previous, key, fraction) {
|
|
24
|
+
const older = new Map(aggregate(previous, key).map((r) => [`${r.engine}\0${r.site}\0${r[key]}`, r]));
|
|
25
|
+
return aggregate(current, key).flatMap((metric) => {
|
|
26
|
+
const before = older.get(`${metric.engine}\0${metric.site}\0${metric[key]}`);
|
|
27
|
+
if (!before?.clicks || metric.clicks === undefined || metric.clicks > before.clicks * (1 - fraction))
|
|
28
|
+
return [];
|
|
29
|
+
return [
|
|
30
|
+
{
|
|
31
|
+
kind: key === 'page' ? 'declining_page' : 'declining_query',
|
|
32
|
+
metric,
|
|
33
|
+
previous: before,
|
|
34
|
+
reason: `Clicks fell from ${before.clicks} to ${metric.clicks}`,
|
|
35
|
+
},
|
|
36
|
+
];
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
export function findDecliningPages(current, previous, fraction = 0.2) {
|
|
40
|
+
return declines(current, previous, 'page', fraction);
|
|
41
|
+
}
|
|
42
|
+
export function findDecliningQueries(current, previous, fraction = 0.2) {
|
|
43
|
+
return declines(current, previous, 'query', fraction);
|
|
44
|
+
}
|
|
45
|
+
export function findImpressionGrowthWithoutClicks(current, previous, config = { minGrowth: 0.3, maxClickGrowth: 0.05 }) {
|
|
46
|
+
const older = new Map(aggregate(previous, 'query').map((r) => [`${r.engine}\0${r.site}\0${r.query}`, r]));
|
|
47
|
+
return aggregate(current, 'query').flatMap((metric) => {
|
|
48
|
+
const before = older.get(`${metric.engine}\0${metric.site}\0${metric.query}`);
|
|
49
|
+
if (!before?.impressions ||
|
|
50
|
+
metric.impressions === undefined ||
|
|
51
|
+
metric.impressions < before.impressions * (1 + config.minGrowth) ||
|
|
52
|
+
(metric.clicks ?? 0) > (before.clicks ?? 0) * (1 + config.maxClickGrowth))
|
|
53
|
+
return [];
|
|
54
|
+
return [
|
|
55
|
+
{
|
|
56
|
+
kind: 'impression_growth_without_clicks',
|
|
57
|
+
metric,
|
|
58
|
+
previous: before,
|
|
59
|
+
reason: `Impressions grew from ${before.impressions} to ${metric.impressions} while clicks stayed near ${before.clicks ?? 0}`,
|
|
60
|
+
},
|
|
61
|
+
];
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
export function compareEngines(rows) {
|
|
65
|
+
return aggregate(rows, 'engine').map((r) => ({
|
|
66
|
+
engine: r.engine,
|
|
67
|
+
site: r.site,
|
|
68
|
+
clicks: r.clicks,
|
|
69
|
+
impressions: r.impressions,
|
|
70
|
+
ctr: r.ctr,
|
|
71
|
+
position: r.position,
|
|
72
|
+
semantics: 'Provider reported metrics; counting, position, and data freshness can differ.',
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
export function findEngineGaps(rows, engines, minImpressions = 1) {
|
|
76
|
+
const byPage = new Map();
|
|
77
|
+
for (const r of aggregate(rows.filter((x) => !!x.page), 'page'))
|
|
78
|
+
byPage.set(r.page, [...(byPage.get(r.page) ?? []), r]);
|
|
79
|
+
return [...byPage].flatMap(([page, found]) => {
|
|
80
|
+
const visible = found.filter((r) => (r.impressions ?? 0) >= minImpressions).map((r) => r.engine);
|
|
81
|
+
const absent = engines.filter((e) => !found.some((r) => r.engine === e));
|
|
82
|
+
return visible.length && absent.length
|
|
83
|
+
? [
|
|
84
|
+
{
|
|
85
|
+
page,
|
|
86
|
+
visibleIn: visible,
|
|
87
|
+
absentFromResults: absent,
|
|
88
|
+
interpretation: 'No row returned for this engine; this does not prove the URL is not indexed.',
|
|
89
|
+
},
|
|
90
|
+
]
|
|
91
|
+
: [];
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
export function findIndexingGaps(inspections) {
|
|
95
|
+
const byUrl = new Map();
|
|
96
|
+
for (const i of inspections)
|
|
97
|
+
byUrl.set(i.url, [...(byUrl.get(i.url) ?? []), i]);
|
|
98
|
+
return [...byUrl].flatMap(([url, items]) => {
|
|
99
|
+
const indexed = items.filter((i) => i.state === 'indexed').map((i) => i.engine);
|
|
100
|
+
const notIndexed = items.filter((i) => i.state === 'not_indexed').map((i) => i.engine);
|
|
101
|
+
return indexed.length && notIndexed.length ? [{ url, indexed, notIndexed, evidence: items }] : [];
|
|
102
|
+
});
|
|
103
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import type { Cache } from '../cache/index.js';
|
|
1
2
|
import type { Engine } from '../types/index.js';
|
|
2
3
|
import type { CredentialStore } from './store.js';
|
|
3
|
-
import type { Cache } from '../cache/index.js';
|
|
4
4
|
export declare class AccountRegistry {
|
|
5
5
|
private cache;
|
|
6
6
|
private store;
|
|
@@ -5,8 +5,20 @@ export class AccountRegistry {
|
|
|
5
5
|
this.cache = cache;
|
|
6
6
|
this.store = store;
|
|
7
7
|
}
|
|
8
|
-
names(engine) {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
names(engine) {
|
|
9
|
+
const rows = this.cache.db
|
|
10
|
+
.prepare('SELECT account FROM accounts WHERE provider=? ORDER BY account')
|
|
11
|
+
.all(engine);
|
|
12
|
+
const names = rows.map((r) => r.account);
|
|
13
|
+
if (!names.includes('default'))
|
|
14
|
+
names.unshift('default');
|
|
15
|
+
return names;
|
|
16
|
+
}
|
|
17
|
+
add(engine, account) {
|
|
18
|
+
this.cache.db.prepare('INSERT OR IGNORE INTO accounts(provider,account) VALUES (?,?)').run(engine, account);
|
|
19
|
+
}
|
|
20
|
+
async remove(engine, account) {
|
|
21
|
+
await this.store.delete(engine, account);
|
|
22
|
+
this.cache.db.prepare('DELETE FROM accounts WHERE provider=? AND account=?').run(engine, account);
|
|
23
|
+
}
|
|
12
24
|
}
|
package/dist/core/auth/prompt.js
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
1
|
import { createInterface } from 'node:readline';
|
|
2
2
|
import { Writable } from 'node:stream';
|
|
3
|
-
export async function secretPrompt(label) {
|
|
4
|
-
|
|
3
|
+
export async function secretPrompt(label) {
|
|
4
|
+
if (!process.stdin.isTTY)
|
|
5
|
+
throw new Error('Interactive terminal required.');
|
|
6
|
+
process.stderr.write(label);
|
|
7
|
+
const output = new Writable({
|
|
8
|
+
write(_chunk, _encoding, callback) {
|
|
9
|
+
callback();
|
|
10
|
+
},
|
|
11
|
+
});
|
|
12
|
+
const rl = createInterface({ input: process.stdin, output, terminal: true });
|
|
13
|
+
return new Promise((resolve) => rl.question('', (answer) => {
|
|
14
|
+
rl.close();
|
|
15
|
+
process.stderr.write('\n');
|
|
16
|
+
resolve(answer.trim());
|
|
17
|
+
}));
|
|
18
|
+
}
|
package/dist/core/auth/store.js
CHANGED
|
@@ -1,22 +1,33 @@
|
|
|
1
1
|
import { Entry } from '@napi-rs/keyring';
|
|
2
2
|
export class KeychainCredentialStore {
|
|
3
|
-
entry(provider, account) {
|
|
4
|
-
|
|
5
|
-
return provider === 'bing' ? { apiKey: env } : { accessToken: env }; try {
|
|
6
|
-
const value = this.entry(provider, account).getPassword();
|
|
7
|
-
return value ? JSON.parse(value) : null;
|
|
3
|
+
entry(provider, account) {
|
|
4
|
+
return new Entry('webmaster-mcp', `${provider}/${account}`);
|
|
8
5
|
}
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
6
|
+
async get(provider, account = 'default') {
|
|
7
|
+
const env = process.env[`WEBMASTER_${provider.toUpperCase()}_${account.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`] ??
|
|
8
|
+
(account === 'default' ? process.env[`WEBMASTER_${provider.toUpperCase()}_TOKEN`] : undefined);
|
|
9
|
+
if (env)
|
|
10
|
+
return provider === 'bing' ? { apiKey: env } : { accessToken: env };
|
|
11
|
+
try {
|
|
12
|
+
const value = this.entry(provider, account).getPassword();
|
|
13
|
+
return value ? JSON.parse(value) : null;
|
|
14
|
+
}
|
|
15
|
+
catch (e) {
|
|
16
|
+
if (String(e).toLowerCase().includes('no entry'))
|
|
17
|
+
return null;
|
|
18
|
+
throw new Error('OS credential store unavailable. Configure a desktop keyring or use a development environment variable.');
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
async set(provider, account, credentials) {
|
|
22
|
+
this.entry(provider, account).setPassword(JSON.stringify(credentials));
|
|
23
|
+
}
|
|
24
|
+
async delete(provider, account) {
|
|
25
|
+
try {
|
|
26
|
+
this.entry(provider, account).deletePassword();
|
|
27
|
+
}
|
|
28
|
+
catch (e) {
|
|
29
|
+
if (!String(e).toLowerCase().includes('no entry'))
|
|
30
|
+
throw e;
|
|
31
|
+
}
|
|
17
32
|
}
|
|
18
|
-
catch (e) {
|
|
19
|
-
if (!String(e).toLowerCase().includes('no entry'))
|
|
20
|
-
throw e;
|
|
21
|
-
} }
|
|
22
33
|
}
|
package/dist/core/cache/index.js
CHANGED
|
@@ -1,21 +1,51 @@
|
|
|
1
|
-
import Database from 'better-sqlite3';
|
|
2
1
|
import { createHash } from 'node:crypto';
|
|
3
2
|
import { mkdirSync } from 'node:fs';
|
|
4
|
-
import { dirname } from 'node:path';
|
|
5
3
|
import { homedir } from 'node:os';
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
4
|
+
import { dirname } from 'node:path';
|
|
5
|
+
import Database from 'better-sqlite3';
|
|
6
|
+
export function defaultDataPath() {
|
|
7
|
+
if (process.env.WEBMASTER_MCP_DATA_DIR)
|
|
8
|
+
return `${process.env.WEBMASTER_MCP_DATA_DIR}/cache.db`;
|
|
9
|
+
if (process.platform === 'darwin')
|
|
10
|
+
return `${homedir()}/Library/Application Support/webmaster-mcp/cache.db`;
|
|
11
|
+
if (process.platform === 'win32')
|
|
12
|
+
return `${process.env.LOCALAPPDATA ?? homedir()}/webmaster-mcp/cache.db`;
|
|
13
|
+
return `${process.env.XDG_CACHE_HOME ?? `${homedir()}/.cache`}/webmaster-mcp/cache.db`;
|
|
14
|
+
}
|
|
15
|
+
function stable(value) {
|
|
16
|
+
if (Array.isArray(value))
|
|
17
|
+
return value.map(stable);
|
|
18
|
+
if (value && typeof value === 'object')
|
|
19
|
+
return Object.fromEntries(Object.entries(value)
|
|
20
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
21
|
+
.map(([k, v]) => [k, stable(v)]));
|
|
22
|
+
return value;
|
|
23
|
+
}
|
|
24
|
+
export function cacheKey(provider, account, site, operation, params) {
|
|
25
|
+
return createHash('sha256')
|
|
26
|
+
.update(JSON.stringify(stable({ provider, account, site, operation, params })))
|
|
27
|
+
.digest('hex');
|
|
28
|
+
}
|
|
14
29
|
export class Cache {
|
|
15
30
|
db;
|
|
16
|
-
constructor(path = defaultDataPath()) {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
31
|
+
constructor(path = defaultDataPath()) {
|
|
32
|
+
if (path !== ':memory:')
|
|
33
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
34
|
+
this.db = new Database(path);
|
|
35
|
+
this.db.pragma('busy_timeout = 5000');
|
|
36
|
+
this.db.pragma('journal_mode = WAL');
|
|
37
|
+
this.db.exec('CREATE TABLE IF NOT EXISTS accounts(provider TEXT NOT NULL,account TEXT NOT NULL,PRIMARY KEY(provider,account)); CREATE TABLE IF NOT EXISTS cache(key TEXT PRIMARY KEY,value TEXT NOT NULL,expires INTEGER NOT NULL)');
|
|
38
|
+
}
|
|
39
|
+
get(key) {
|
|
40
|
+
const row = this.db.prepare('SELECT value FROM cache WHERE key=? AND expires>?').get(key, Date.now());
|
|
41
|
+
return row ? JSON.parse(row.value) : null;
|
|
42
|
+
}
|
|
43
|
+
set(key, value, ttlMs) {
|
|
44
|
+
this.db
|
|
45
|
+
.prepare('INSERT INTO cache(key,value,expires) VALUES (?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,expires=excluded.expires')
|
|
46
|
+
.run(key, JSON.stringify(value), Date.now() + ttlMs);
|
|
47
|
+
}
|
|
48
|
+
close() {
|
|
49
|
+
this.db.close();
|
|
50
|
+
}
|
|
21
51
|
}
|
package/dist/core/errors.js
CHANGED
|
@@ -10,7 +10,12 @@ export class WebmasterError extends Error {
|
|
|
10
10
|
export function normalizeError(engine, account, error) {
|
|
11
11
|
if (error instanceof WebmasterError)
|
|
12
12
|
return { engine, account, code: error.code, message: error.message };
|
|
13
|
-
return {
|
|
13
|
+
return {
|
|
14
|
+
engine,
|
|
15
|
+
account,
|
|
16
|
+
code: 'UPSTREAM_ERROR',
|
|
17
|
+
message: 'Provider request failed. Check provider status and local configuration.',
|
|
18
|
+
};
|
|
14
19
|
}
|
|
15
20
|
export function httpError(status) {
|
|
16
21
|
if (status === 401)
|
|
@@ -21,4 +26,6 @@ export function httpError(status) {
|
|
|
21
26
|
return new WebmasterError('RATE_LIMITED', 'Provider rate limit reached.', status);
|
|
22
27
|
return new WebmasterError('UPSTREAM_ERROR', `Provider returned HTTP ${status}.`, status);
|
|
23
28
|
}
|
|
24
|
-
export function redact(value) {
|
|
29
|
+
export function redact(value) {
|
|
30
|
+
return value.replace(/(Bearer|OAuth|apikey=|access_token=|refresh_token=)\s*[^\s&"']+/gi, '$1 [REDACTED]');
|
|
31
|
+
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
export function finite(value) {
|
|
1
|
+
export function finite(value) {
|
|
2
|
+
const n = typeof value === 'number' ? value : typeof value === 'string' && value.trim() ? Number(value) : NaN;
|
|
3
|
+
return Number.isFinite(n) ? n : undefined;
|
|
4
|
+
}
|
|
2
5
|
export function metric(input) {
|
|
3
6
|
const clicks = finite(input.clicks), impressions = finite(input.impressions);
|
|
4
|
-
const ctr = finite(input.ctr) ??
|
|
7
|
+
const ctr = finite(input.ctr) ??
|
|
8
|
+
(clicks !== undefined && impressions !== undefined && impressions > 0 ? clicks / impressions : undefined);
|
|
5
9
|
return { ...input, clicks, impressions, ctr, position: finite(input.position) };
|
|
6
10
|
}
|
|
7
11
|
export function aggregate(rows, key) {
|
|
@@ -11,5 +15,24 @@ export function aggregate(rows, key) {
|
|
|
11
15
|
const id = `${row.engine}\0${row.site}\0${value}`;
|
|
12
16
|
groups.set(id, [...(groups.get(id) ?? []), row]);
|
|
13
17
|
}
|
|
14
|
-
return [...groups.values()].map(
|
|
18
|
+
return [...groups.values()].map((group) => {
|
|
19
|
+
const first = group[0];
|
|
20
|
+
const clicks = group.some((r) => r.clicks !== undefined)
|
|
21
|
+
? group.reduce((n, r) => n + (r.clicks ?? 0), 0)
|
|
22
|
+
: undefined;
|
|
23
|
+
const impressions = group.some((r) => r.impressions !== undefined)
|
|
24
|
+
? group.reduce((n, r) => n + (r.impressions ?? 0), 0)
|
|
25
|
+
: undefined;
|
|
26
|
+
const positions = group.filter((r) => r.position !== undefined);
|
|
27
|
+
const weight = positions.reduce((n, r) => n + (r.impressions ?? 1), 0);
|
|
28
|
+
return metric({
|
|
29
|
+
engine: first.engine,
|
|
30
|
+
site: first.site,
|
|
31
|
+
account: first.account,
|
|
32
|
+
[key]: first[key],
|
|
33
|
+
clicks,
|
|
34
|
+
impressions,
|
|
35
|
+
position: weight ? positions.reduce((n, r) => n + r.position * (r.impressions ?? 1), 0) / weight : undefined,
|
|
36
|
+
});
|
|
37
|
+
});
|
|
15
38
|
}
|
package/dist/core/scheduler.js
CHANGED
|
@@ -7,46 +7,73 @@ export class RequestScheduler {
|
|
|
7
7
|
constructor(policy) {
|
|
8
8
|
this.policy = policy;
|
|
9
9
|
}
|
|
10
|
-
async slot() {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
next();
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
10
|
+
async slot() {
|
|
11
|
+
if (this.active >= this.policy.concurrency)
|
|
12
|
+
await new Promise((resolve) => this.queue.push(resolve));
|
|
13
|
+
else
|
|
14
|
+
this.active++;
|
|
15
|
+
const delay = Math.max(0, this.next - Date.now());
|
|
16
|
+
this.next = Math.max(Date.now(), this.next) + this.policy.minimumDelay;
|
|
17
|
+
if (delay)
|
|
18
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
19
|
+
return () => {
|
|
20
|
+
const next = this.queue.shift();
|
|
21
|
+
if (next)
|
|
22
|
+
next();
|
|
23
|
+
else
|
|
24
|
+
this.active--;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
async run(fn) {
|
|
28
|
+
const release = await this.slot();
|
|
29
|
+
try {
|
|
30
|
+
for (let attempt = 0;; attempt++) {
|
|
31
|
+
try {
|
|
32
|
+
return await fn();
|
|
33
|
+
}
|
|
34
|
+
catch (e) {
|
|
35
|
+
const retryable = e instanceof WebmasterError
|
|
36
|
+
? e.code === 'RATE_LIMITED' || e.code === 'NETWORK_ERROR' || (e.status !== undefined && e.status >= 500)
|
|
37
|
+
: e instanceof TypeError;
|
|
38
|
+
if (!retryable || attempt >= this.policy.retries)
|
|
39
|
+
throw e;
|
|
40
|
+
await new Promise((r) => setTimeout(r, Math.min(5000, 250 * 2 ** attempt + Math.random() * 100)));
|
|
41
|
+
}
|
|
28
42
|
}
|
|
29
43
|
}
|
|
44
|
+
finally {
|
|
45
|
+
release();
|
|
46
|
+
}
|
|
30
47
|
}
|
|
31
|
-
finally {
|
|
32
|
-
release();
|
|
33
|
-
} }
|
|
34
48
|
}
|
|
35
|
-
export const schedules = {
|
|
49
|
+
export const schedules = {
|
|
50
|
+
google: { concurrency: 3, minimumDelay: 100, retries: 2 },
|
|
51
|
+
bing: { concurrency: 2, minimumDelay: 250, retries: 2 },
|
|
52
|
+
yandex: { concurrency: 2, minimumDelay: 200, retries: 2 },
|
|
53
|
+
};
|
|
36
54
|
const shared = new Map();
|
|
37
|
-
export function providerScheduler(engine) {
|
|
38
|
-
scheduler =
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
55
|
+
export function providerScheduler(engine) {
|
|
56
|
+
let scheduler = shared.get(engine);
|
|
57
|
+
if (!scheduler) {
|
|
58
|
+
scheduler = new RequestScheduler(schedules[engine]);
|
|
59
|
+
shared.set(engine, scheduler);
|
|
60
|
+
}
|
|
61
|
+
return scheduler;
|
|
43
62
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
63
|
+
export async function jsonFetch(fetcher, url, options = {}) {
|
|
64
|
+
let response;
|
|
65
|
+
try {
|
|
66
|
+
response = await fetcher(url, { ...options, signal: options.signal ?? AbortSignal.timeout(15000) });
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
throw new WebmasterError('NETWORK_ERROR', 'Provider network request failed.');
|
|
70
|
+
}
|
|
71
|
+
if (!response.ok)
|
|
72
|
+
throw httpError(response.status);
|
|
73
|
+
try {
|
|
74
|
+
return await response.json();
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
throw new WebmasterError('UPSTREAM_ERROR', 'Provider returned invalid JSON.');
|
|
78
|
+
}
|
|
49
79
|
}
|
|
50
|
-
catch {
|
|
51
|
-
throw new WebmasterError('UPSTREAM_ERROR', 'Provider returned invalid JSON.');
|
|
52
|
-
} }
|
package/dist/core/service.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { type Cache } from './cache/index.js';
|
|
1
2
|
import type { Capability, Engine, PartialResult, PerformanceQuery, SearchMetric, UrlInspection, WebmasterProvider } from './types/index.js';
|
|
2
|
-
import { Cache } from './cache/index.js';
|
|
3
3
|
export declare class WebmasterService {
|
|
4
4
|
readonly providers: WebmasterProvider[];
|
|
5
5
|
readonly cache: Cache;
|