search-rotation 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -0
- package/dist/abort.js +14 -0
- package/dist/config.js +109 -0
- package/dist/engines/base.js +72 -0
- package/dist/engines/duckduckgo.js +98 -0
- package/dist/engines/exa.js +113 -0
- package/dist/engines/firecrawl.js +78 -0
- package/dist/engines/googlecse.js +41 -0
- package/dist/engines/hosted-mcp.js +24 -0
- package/dist/engines/index.js +13 -0
- package/dist/engines/jina.js +28 -0
- package/dist/engines/parallel.js +74 -0
- package/dist/engines/tavily.js +74 -0
- package/dist/history.js +59 -0
- package/dist/index.js +182 -0
- package/dist/mcp/engine-status.js +58 -0
- package/dist/mcp/http.js +35 -0
- package/dist/mcp/server.js +105 -0
- package/dist/persistence.js +80 -0
- package/dist/quota.js +37 -0
- package/dist/router.js +183 -0
- package/dist/search-time.js +31 -0
- package/dist/status.js +141 -0
- package/dist/types.js +1 -0
- package/dist/usage.js +95 -0
- package/dist/version.js +5 -0
- package/dist/web/app.js +155 -0
- package/dist/web/security.js +116 -0
- package/docs/assets/brand/FONT-LICENSE.txt +93 -0
- package/docs/assets/brand/README.md +18 -0
- package/docs/assets/brand/icon-128.png +0 -0
- package/docs/assets/brand/icon-32.png +0 -0
- package/docs/assets/brand/icon-512.png +0 -0
- package/docs/assets/brand/icon-dark-128.png +0 -0
- package/docs/assets/brand/icon-dark-512.png +0 -0
- package/docs/assets/brand/icon-dark.svg +1 -0
- package/docs/assets/brand/icon-light-128.png +0 -0
- package/docs/assets/brand/icon-light-512.png +0 -0
- package/docs/assets/brand/icon-light.svg +1 -0
- package/docs/assets/brand/icon-mono.svg +1 -0
- package/docs/assets/brand/icon.svg +1 -0
- package/docs/assets/brand/preview.png +0 -0
- package/docs/assets/brand/preview.svg +1 -0
- package/docs/assets/brand/wordmark-dark.svg +1 -0
- package/docs/assets/brand/wordmark-light.svg +1 -0
- package/docs/assets/dashboard-dark.png +0 -0
- package/docs/assets/dashboard-light.png +0 -0
- package/docs/assets/dashboard.png +0 -0
- package/docs/assets/mcp-tools-dark.png +0 -0
- package/docs/assets/mcp-tools-light.png +0 -0
- package/docs/assets/mcp-tools.png +0 -0
- package/docs/assets/search-rotation.svg +52 -0
- package/docs/clients.md +93 -0
- package/docs/operations.md +66 -0
- package/docs/provider-access.md +38 -0
- package/docs/release-notes.md +52 -0
- package/package.json +65 -0
- package/static/app.js +1126 -0
- package/static/engine-logos/README.md +11 -0
- package/static/engine-logos/duckduckgo.png +0 -0
- package/static/engine-logos/exa.png +0 -0
- package/static/engine-logos/firecrawl.png +0 -0
- package/static/engine-logos/google-cse.png +0 -0
- package/static/engine-logos/jina.png +0 -0
- package/static/engine-logos/parallel.png +0 -0
- package/static/engine-logos/tavily.png +0 -0
- package/static/i18n.js +709 -0
- package/static/index.html +95 -0
- package/static/style.css +1258 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
const sleeper = new Int32Array(new SharedArrayBuffer(4));
|
|
5
|
+
/** Synchronous Lamport bakery lock for local processes. Each contender owns a
|
|
6
|
+
* unique directory for its whole lifetime; stale recovery never unlinks a
|
|
7
|
+
* successor's lock. Missing ticket means choosing=true. Only dead PIDs are
|
|
8
|
+
* recovered: slow live owners are never evicted on a wall-clock timeout.
|
|
9
|
+
* Limitation: after a crash, OS PID reuse can make an abandoned contender look
|
|
10
|
+
* alive. Writes then time out safely until that PID exits or the abandoned
|
|
11
|
+
* directory is removed while all search-rotation processes are stopped. */
|
|
12
|
+
export function withFileLock(file, action, timeoutMs = 10_000) {
|
|
13
|
+
const root = `${file}.lock`;
|
|
14
|
+
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
15
|
+
const id = `${process.pid}-${randomUUID()}`;
|
|
16
|
+
const own = path.join(root, id);
|
|
17
|
+
fs.mkdirSync(own, { mode: 0o700 });
|
|
18
|
+
const started = Date.now();
|
|
19
|
+
const ticket = (name) => {
|
|
20
|
+
const dir = path.join(root, name);
|
|
21
|
+
const pid = Number(name.split('-')[0]);
|
|
22
|
+
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
23
|
+
return null;
|
|
24
|
+
try {
|
|
25
|
+
process.kill(pid, 0);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (error.code === 'ESRCH') {
|
|
29
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
return Number(fs.readFileSync(path.join(dir, 'ticket'), 'utf8')) || 0;
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
if (error.code !== 'ENOENT')
|
|
38
|
+
throw error;
|
|
39
|
+
return fs.existsSync(dir) ? 0 : null;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
try {
|
|
43
|
+
let mine = 1;
|
|
44
|
+
for (const name of fs.readdirSync(root))
|
|
45
|
+
if (name !== id)
|
|
46
|
+
mine = Math.max(mine, (ticket(name) ?? 0) + 1);
|
|
47
|
+
atomicWriteFile(path.join(own, 'ticket'), String(mine));
|
|
48
|
+
while (true) {
|
|
49
|
+
let blocked = false;
|
|
50
|
+
for (const name of fs.readdirSync(root)) {
|
|
51
|
+
if (name === id)
|
|
52
|
+
continue;
|
|
53
|
+
const other = ticket(name);
|
|
54
|
+
if (other !== null && (other === 0 || other < mine || (other === mine && name < id))) {
|
|
55
|
+
blocked = true;
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (!blocked)
|
|
60
|
+
return action();
|
|
61
|
+
if (Date.now() - started >= timeoutMs)
|
|
62
|
+
throw new Error(`Persistence lock timeout: ${path.basename(file)}`);
|
|
63
|
+
Atomics.wait(sleeper, 0, 0, 5);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
fs.rmSync(own, { recursive: true, force: true });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/** Unique temp names plus rename keep concurrent readers on complete snapshots. */
|
|
71
|
+
export function atomicWriteFile(file, content) {
|
|
72
|
+
const tmp = `${file}.${process.pid}.${randomUUID()}.tmp`;
|
|
73
|
+
try {
|
|
74
|
+
fs.writeFileSync(tmp, content, { mode: 0o600, flag: 'wx' });
|
|
75
|
+
fs.renameSync(tmp, file);
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
fs.rmSync(tmp, { force: true });
|
|
79
|
+
}
|
|
80
|
+
}
|
package/dist/quota.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export function computeRemainingPct(used, limit, remote) {
|
|
2
|
+
if (remote && typeof remote.limit === 'number' && remote.limit > 0) {
|
|
3
|
+
const remaining = remote.remaining ?? remote.limit - (remote.used ?? 0);
|
|
4
|
+
return Math.max(0, Math.min(1, remaining / remote.limit));
|
|
5
|
+
}
|
|
6
|
+
if (limit > 0) {
|
|
7
|
+
const total = used.consumed ?? Number(used.search) + Number(used.fetch);
|
|
8
|
+
return Number.isFinite(total) ? Math.max(0, Math.min(1, (limit - total) / limit)) : null;
|
|
9
|
+
}
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
export function requestCost(adapter, kind, input) {
|
|
13
|
+
const value = adapter.estimateCost?.(kind, input) ?? adapter.meta.quota?.costs?.[kind] ?? 1;
|
|
14
|
+
return Number.isFinite(value) && value >= 0 ? value : 1;
|
|
15
|
+
}
|
|
16
|
+
export function quotaStatus(adapter, ctx, cfg, usage, remote) {
|
|
17
|
+
const policy = adapter.meta.quota;
|
|
18
|
+
const period = adapter.meta.keyless === 'ip' && !ctx.apiKey ? 'ip' : policy?.period ?? 'month';
|
|
19
|
+
const unit = policy?.unit ?? 'requests';
|
|
20
|
+
if (period === 'ip' || !ctx.apiKey) {
|
|
21
|
+
const count = usage.get(adapter.meta.id);
|
|
22
|
+
// Successful calls observed by this installation, never the shared IP balance.
|
|
23
|
+
return { period, unit: 'requests', limit: null, used: count.search + count.fetch, source: 'local', estimated: false };
|
|
24
|
+
}
|
|
25
|
+
const count = period === 'day' ? usage.getDay(adapter.meta.id, policy?.timeZone) : usage.get(adapter.meta.id);
|
|
26
|
+
const localUsed = unit === 'credits' ? count.consumed ?? count.search + count.fetch : count.search + count.fetch;
|
|
27
|
+
const limit = period === 'day'
|
|
28
|
+
? cfg.settings.dailyLimits?.[adapter.meta.id] ?? policy?.limit ?? 0
|
|
29
|
+
: cfg.settings.monthlyLimits[adapter.meta.id] ?? policy?.limit ?? adapter.meta.monthlyFree;
|
|
30
|
+
if (remote && typeof remote.limit === 'number' && remote.limit > 0 && (remote.used !== undefined || remote.remaining !== undefined)) {
|
|
31
|
+
return { period, unit, limit: remote.limit, used: Math.max(0, remote.used ?? remote.limit - remote.remaining), source: 'remote', estimated: remote.estimated ?? false, timeZone: policy?.timeZone };
|
|
32
|
+
}
|
|
33
|
+
return { period, unit, limit: limit > 0 ? limit : null, used: localUsed, source: limit > 0 ? 'local' : 'unknown', estimated: policy?.estimated ?? true, timeZone: policy?.timeZone };
|
|
34
|
+
}
|
|
35
|
+
export function remainingPct(quota) {
|
|
36
|
+
return quota.limit && quota.used !== null ? Math.max(0, Math.min(1, (quota.limit - quota.used) / quota.limit)) : null;
|
|
37
|
+
}
|
package/dist/router.js
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { quotaStatus, remainingPct, requestCost } from './quota.js';
|
|
2
|
+
import { fetchRemoteQuotaCached, recordRemoteConsumption } from './status.js';
|
|
3
|
+
import { HttpError } from './engines/base.js';
|
|
4
|
+
import { hasTimeFilter, normalizeSearchTime, describeSearchTime } from './search-time.js';
|
|
5
|
+
import { abortable } from './abort.js';
|
|
6
|
+
export class RouterError extends Error {
|
|
7
|
+
attempts;
|
|
8
|
+
constructor(message, attempts) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.attempts = attempts;
|
|
11
|
+
this.name = 'RouterError';
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export class SearchRouter {
|
|
15
|
+
opts;
|
|
16
|
+
rr = new Map();
|
|
17
|
+
health = new Map();
|
|
18
|
+
constructor(opts) {
|
|
19
|
+
this.opts = opts;
|
|
20
|
+
}
|
|
21
|
+
now() { return this.opts.now?.() ?? Date.now(); }
|
|
22
|
+
state(c) {
|
|
23
|
+
const id = c.adapter.meta.id;
|
|
24
|
+
const identity = JSON.stringify([c.ctx.apiKey, c.ctx.extra]);
|
|
25
|
+
let state = this.health.get(id);
|
|
26
|
+
if (!state || state.identity !== identity) {
|
|
27
|
+
state = { identity, failures: 0, until: 0, reserved: 0 };
|
|
28
|
+
this.health.set(id, state);
|
|
29
|
+
}
|
|
30
|
+
return state;
|
|
31
|
+
}
|
|
32
|
+
failure(c, err) {
|
|
33
|
+
const state = this.state(c);
|
|
34
|
+
state.failures++;
|
|
35
|
+
const status = err instanceof HttpError ? err.status : undefined;
|
|
36
|
+
let delay = 0;
|
|
37
|
+
if (status === 429) {
|
|
38
|
+
const header = err.retryAfter;
|
|
39
|
+
const seconds = header?.trim() ? Number(header) : NaN;
|
|
40
|
+
const date = header ? Date.parse(header) : NaN;
|
|
41
|
+
delay = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : Number.isFinite(date) ? Math.max(0, date - this.now()) : 30_000;
|
|
42
|
+
}
|
|
43
|
+
else if (status === 401 || status === 403)
|
|
44
|
+
delay = 60_000;
|
|
45
|
+
else if (state.failures >= 2)
|
|
46
|
+
delay = Math.min(300_000, 30_000 * 2 ** Math.min(4, state.failures - 2));
|
|
47
|
+
state.until = this.now() + delay;
|
|
48
|
+
}
|
|
49
|
+
async chain(kind, signal, prefer, onlyEngine, searchInput) {
|
|
50
|
+
signal.throwIfAborted();
|
|
51
|
+
const cfg = this.opts.getConfig();
|
|
52
|
+
const byId = new Map(this.opts.adapters.map(a => [a.meta.id, a]));
|
|
53
|
+
const order = onlyEngine ? [onlyEngine] : kind === 'fetch' ? cfg.fetchOrder : cfg.engines.map(e => e.id);
|
|
54
|
+
const entries = [];
|
|
55
|
+
for (const id of new Set(order)) {
|
|
56
|
+
const adapter = byId.get(id);
|
|
57
|
+
const e = cfg.engines.find(e => e.id === id);
|
|
58
|
+
if (!adapter || !e || (!onlyEngine && !e.enabled) || !(kind === 'search' ? adapter.search : adapter.fetchUrl))
|
|
59
|
+
continue;
|
|
60
|
+
if (!onlyEngine && !e.apiKey && adapter.meta.keylessCapabilities && !adapter.meta.keylessCapabilities.includes(kind))
|
|
61
|
+
continue;
|
|
62
|
+
const c = { adapter, ctx: { apiKey: e.apiKey, extra: e.extra, signal } };
|
|
63
|
+
if (searchInput && hasTimeFilter(searchInput) && !adapter.supportsSearchTime?.(searchInput, c.ctx))
|
|
64
|
+
continue;
|
|
65
|
+
if (this.state(c).until > this.now())
|
|
66
|
+
continue;
|
|
67
|
+
entries.push(c);
|
|
68
|
+
}
|
|
69
|
+
const rated = await abortable(Promise.all(entries.map(async (c) => {
|
|
70
|
+
const remote = c.adapter.meta.quotaEndpoint ? (await fetchRemoteQuotaCached(c.adapter, c.ctx)).quota : null;
|
|
71
|
+
const pct = remainingPct(quotaStatus(c.adapter, c.ctx, cfg, this.opts.usage, remote));
|
|
72
|
+
return { c, pct };
|
|
73
|
+
})), signal);
|
|
74
|
+
const groups = [
|
|
75
|
+
rated.filter(r => r.pct === null || r.pct > 0.1),
|
|
76
|
+
rated.filter(r => r.pct !== null && r.pct > 0 && r.pct <= 0.1),
|
|
77
|
+
cfg.settings.strictFreeMode ? [] : rated.filter(r => r.pct !== null && r.pct <= 0),
|
|
78
|
+
];
|
|
79
|
+
// Filtered and unfiltered calls must not advance each other's rotation cursor.
|
|
80
|
+
const pool = JSON.stringify([kind, Boolean(searchInput && hasTimeFilter(searchInput)), rated.map(r => r.c.adapter.meta.id).sort()]);
|
|
81
|
+
const turn = prefer ? 0 : (this.rr.get(pool) ?? 0);
|
|
82
|
+
if (!prefer)
|
|
83
|
+
this.rr.set(pool, turn + 1);
|
|
84
|
+
let ordered = groups.flatMap(group => {
|
|
85
|
+
const start = group.length ? turn % group.length : 0;
|
|
86
|
+
return [...group.slice(start), ...group.slice(0, start)];
|
|
87
|
+
}).map(r => r.c);
|
|
88
|
+
if (prefer)
|
|
89
|
+
ordered = [...ordered.filter(c => c.adapter.meta.id === prefer), ...ordered.filter(c => c.adapter.meta.id !== prefer)];
|
|
90
|
+
return ordered;
|
|
91
|
+
}
|
|
92
|
+
recordUsage(c, kind, input, error) {
|
|
93
|
+
try {
|
|
94
|
+
this.opts.usage.record(c.adapter.meta.id, kind, error, requestCost(c.adapter, kind, input), c.adapter.meta.quota?.timeZone);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
console.error('[search-rotation] Verbrauch konnte nicht gespeichert werden; Dateirechte/Sperre prüfen.');
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async execute(kind, input, opts) {
|
|
101
|
+
const start = Date.now();
|
|
102
|
+
const cfg = this.opts.getConfig();
|
|
103
|
+
const timeoutMs = cfg.settings.requestTimeoutMs ?? 60_000;
|
|
104
|
+
const controller = new AbortController();
|
|
105
|
+
const timer = setTimeout(() => controller.abort(new Error(`Zeitlimit nach ${timeoutMs} ms überschritten`)), timeoutMs);
|
|
106
|
+
const signal = opts.signal ? AbortSignal.any([opts.signal, controller.signal]) : controller.signal;
|
|
107
|
+
const attempts = [];
|
|
108
|
+
const rawInput = kind === 'search' ? [input.query, hasTimeFilter(input) ? `[${describeSearchTime(input)}]` : ''].filter(Boolean).join(' ') : input.url;
|
|
109
|
+
try {
|
|
110
|
+
const chain = await this.chain(kind, signal, opts.onlyEngine ?? opts.preferEngine, opts.onlyEngine, kind === 'search' ? input : undefined);
|
|
111
|
+
if (!chain.length)
|
|
112
|
+
throw new RouterError(`Keine aktivierte ${kind === 'search' ? 'Such-Engine' : 'Fetch-Engine'} verfügbar. ${kind === 'search' && hasTimeFilter(input) ? 'Für diesen Zeitfilter ist aktuell keine kompatible Engine verfügbar. ' : ''}Engines, Cooldown und Gratis-Kontingent prüfen.`, attempts);
|
|
113
|
+
for (const c of chain) {
|
|
114
|
+
signal.throwIfAborted();
|
|
115
|
+
let reservation;
|
|
116
|
+
const cost = requestCost(c.adapter, kind, input);
|
|
117
|
+
// Check and reserve synchronously after obtaining the current snapshot.
|
|
118
|
+
if (cfg.settings.strictFreeMode) {
|
|
119
|
+
const remote = c.adapter.meta.quotaEndpoint ? (await fetchRemoteQuotaCached(c.adapter, c.ctx)).quota : null;
|
|
120
|
+
const quota = quotaStatus(c.adapter, c.ctx, cfg, this.opts.usage, remote);
|
|
121
|
+
const health = this.state(c);
|
|
122
|
+
if (quota.limit !== null && quota.used !== null && quota.used + health.reserved + cost > quota.limit)
|
|
123
|
+
continue;
|
|
124
|
+
health.reserved += cost;
|
|
125
|
+
reservation = health;
|
|
126
|
+
}
|
|
127
|
+
const at = Date.now();
|
|
128
|
+
let value;
|
|
129
|
+
try {
|
|
130
|
+
value = await abortable(kind === 'search'
|
|
131
|
+
? c.adapter.search(input, c.ctx)
|
|
132
|
+
: c.adapter.fetchUrl(input, c.ctx), signal);
|
|
133
|
+
signal.throwIfAborted();
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
if (signal.aborted)
|
|
137
|
+
throw signal.reason;
|
|
138
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
139
|
+
this.failure(c, err);
|
|
140
|
+
this.recordUsage(c, kind, input, error);
|
|
141
|
+
attempts.push({ engine: c.adapter.meta.id, ok: false, ms: Date.now() - at, error });
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
finally {
|
|
145
|
+
if (reservation)
|
|
146
|
+
reservation.reserved -= cost;
|
|
147
|
+
}
|
|
148
|
+
const health = this.state(c);
|
|
149
|
+
health.failures = 0;
|
|
150
|
+
health.until = 0;
|
|
151
|
+
this.recordUsage(c, kind, input);
|
|
152
|
+
recordRemoteConsumption(c.adapter, c.ctx, requestCost(c.adapter, kind, input));
|
|
153
|
+
attempts.push({ engine: c.adapter.meta.id, ok: true, ms: Date.now() - at });
|
|
154
|
+
this.opts.history?.record({ kind, input: rawInput, engine: c.adapter.meta.id, ok: true, ms: Date.now() - start, attempts,
|
|
155
|
+
result: typeof value === 'string' ? { chars: value.length, markdown: value } : { count: value.items.length, items: value.items } });
|
|
156
|
+
return { value, engine: c.adapter.meta.id, attempts };
|
|
157
|
+
}
|
|
158
|
+
const message = attempts.length === chain.length
|
|
159
|
+
? `Alle ${attempts.length} ${kind === 'search' ? 'Such-Engines' : 'Fetch-Engines'} fehlgeschlagen.`
|
|
160
|
+
: 'Gratis-Kontingent erschöpft; keine weitere Engine verfügbar.';
|
|
161
|
+
throw new RouterError(message, attempts);
|
|
162
|
+
}
|
|
163
|
+
catch (err) {
|
|
164
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
165
|
+
this.opts.history?.record({ kind, input: rawInput, engine: null, ok: false, ms: Date.now() - start, attempts, error: message });
|
|
166
|
+
throw err instanceof RouterError ? err : new RouterError(message, attempts);
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
clearTimeout(timer);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
async search(input, opts = {}) {
|
|
173
|
+
input = normalizeSearchTime(input, this.now());
|
|
174
|
+
const configured = this.opts.getConfig().settings.defaultNumResults;
|
|
175
|
+
const numResults = input.numResults ?? (configured === null ? undefined : configured ?? 8);
|
|
176
|
+
const { value, engine, attempts } = await this.execute('search', { ...input, numResults }, opts);
|
|
177
|
+
return { ...value, engine, attempts };
|
|
178
|
+
}
|
|
179
|
+
async fetchUrl(input, opts = {}) {
|
|
180
|
+
const { value, engine, attempts } = await this.execute('fetch', input, opts);
|
|
181
|
+
return { markdown: value, engine, attempts };
|
|
182
|
+
}
|
|
183
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export const timeRangeSchema = z.enum(['day', 'week', 'month', 'year']);
|
|
3
|
+
export const dateSchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'Expected YYYY-MM-DD').refine(value => {
|
|
4
|
+
const date = new Date(`${value}T00:00:00.000Z`);
|
|
5
|
+
return Number.isFinite(date.getTime()) && date.toISOString().slice(0, 10) === value;
|
|
6
|
+
}, 'Invalid calendar date');
|
|
7
|
+
export function hasTimeFilter(input) {
|
|
8
|
+
return input.timeRange !== undefined || input.startDate !== undefined || input.endDate !== undefined;
|
|
9
|
+
}
|
|
10
|
+
/** Resolve once per request so every failover receives identical UTC date bounds. */
|
|
11
|
+
export function normalizeSearchTime(input, now) {
|
|
12
|
+
if (input.timeRange !== undefined)
|
|
13
|
+
timeRangeSchema.parse(input.timeRange);
|
|
14
|
+
if (input.startDate !== undefined)
|
|
15
|
+
dateSchema.parse(input.startDate);
|
|
16
|
+
if (input.endDate !== undefined)
|
|
17
|
+
dateSchema.parse(input.endDate);
|
|
18
|
+
if (input.timeRange !== undefined && (input.startDate !== undefined || input.endDate !== undefined)) {
|
|
19
|
+
throw new Error('timeRange cannot be combined with startDate or endDate');
|
|
20
|
+
}
|
|
21
|
+
if (input.startDate && input.endDate && input.startDate > input.endDate) {
|
|
22
|
+
throw new Error('startDate must be on or before endDate');
|
|
23
|
+
}
|
|
24
|
+
if (!input.timeRange)
|
|
25
|
+
return input;
|
|
26
|
+
const days = { day: 1, week: 7, month: 30, year: 365 }[input.timeRange];
|
|
27
|
+
return { ...input, startDate: new Date(now - days * 86_400_000).toISOString().slice(0, 10), endDate: new Date(now).toISOString().slice(0, 10) };
|
|
28
|
+
}
|
|
29
|
+
export function describeSearchTime(input) {
|
|
30
|
+
return [input.timeRange, input.startDate && `from ${input.startDate}`, input.endDate && `through ${input.endDate}`].filter(Boolean).join(' ');
|
|
31
|
+
}
|
package/dist/status.js
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { quotaStatus, remainingPct } from "./quota.js";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { abortable } from "./abort.js";
|
|
4
|
+
const remoteCache = new Map();
|
|
5
|
+
const REMOTE_TTL_MS = 5 * 60 * 1000;
|
|
6
|
+
const pendingQuotas = new Map();
|
|
7
|
+
let cacheGeneration = 0;
|
|
8
|
+
export function maskKey(key) {
|
|
9
|
+
if (!key)
|
|
10
|
+
return "";
|
|
11
|
+
if (key.length <= 10)
|
|
12
|
+
return `${key.slice(0, 2)}…${key.slice(-2)}`;
|
|
13
|
+
return `${key.slice(0, 6)}…${key.slice(-4)}`;
|
|
14
|
+
}
|
|
15
|
+
/** Remote-Quota-Cache leeren (z. B. nach dem Hinterlegen eines neuen Keys). */
|
|
16
|
+
export function clearRemoteQuotaCache() {
|
|
17
|
+
remoteCache.clear();
|
|
18
|
+
// Old callers may finish, but their snapshots must not refill this generation.
|
|
19
|
+
cacheGeneration++;
|
|
20
|
+
pendingQuotas.clear();
|
|
21
|
+
}
|
|
22
|
+
/** Remote-Quota vom Anbieter abrufen, 5 Minuten gecacht. */
|
|
23
|
+
export async function fetchRemoteQuotaCached(adapter, ctx) {
|
|
24
|
+
if (!adapter.remoteQuota || !ctx.apiKey)
|
|
25
|
+
return { quota: null };
|
|
26
|
+
ctx.signal?.throwIfAborted();
|
|
27
|
+
const key = cacheKey(adapter, ctx);
|
|
28
|
+
const cached = remoteCache.get(key);
|
|
29
|
+
if (cached && Date.now() - cached.at < REMOTE_TTL_MS) {
|
|
30
|
+
return { quota: cached.quota, error: cached.error };
|
|
31
|
+
}
|
|
32
|
+
let pending = pendingQuotas.get(key);
|
|
33
|
+
if (!pending) {
|
|
34
|
+
const generation = cacheGeneration;
|
|
35
|
+
const controller = new AbortController();
|
|
36
|
+
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(20_000)]);
|
|
37
|
+
const entry = { controller, waiters: 0, consumed: 0, settled: false, promise: Promise.resolve({ quota: null }) };
|
|
38
|
+
pendingQuotas.set(key, entry);
|
|
39
|
+
entry.promise = (async () => {
|
|
40
|
+
try {
|
|
41
|
+
const snapshot = await abortable(Promise.resolve().then(() => {
|
|
42
|
+
signal.throwIfAborted();
|
|
43
|
+
return adapter.remoteQuota({ ...ctx, signal });
|
|
44
|
+
}), signal);
|
|
45
|
+
// Provider snapshots may predate requests completed while fetching.
|
|
46
|
+
// Conservatively add that consumption instead of restoring spent quota.
|
|
47
|
+
const quota = addConsumption(snapshot, entry.consumed);
|
|
48
|
+
if (cacheGeneration === generation && pendingQuotas.get(key) === entry)
|
|
49
|
+
remoteCache.set(key, { at: Date.now(), quota });
|
|
50
|
+
return { quota };
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
if (controller.signal.aborted)
|
|
54
|
+
throw controller.signal.reason;
|
|
55
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
56
|
+
if (cacheGeneration === generation && pendingQuotas.get(key) === entry)
|
|
57
|
+
remoteCache.set(key, { at: Date.now(), quota: null, error });
|
|
58
|
+
return { quota: null, error };
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
entry.settled = true;
|
|
62
|
+
if (pendingQuotas.get(key) === entry)
|
|
63
|
+
pendingQuotas.delete(key);
|
|
64
|
+
}
|
|
65
|
+
})();
|
|
66
|
+
pending = entry;
|
|
67
|
+
}
|
|
68
|
+
pending.waiters++;
|
|
69
|
+
try {
|
|
70
|
+
return await abortable(pending.promise, ctx.signal);
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
pending.waiters--;
|
|
74
|
+
if (pending.waiters === 0 && !pending.settled) {
|
|
75
|
+
if (pendingQuotas.get(key) === pending)
|
|
76
|
+
pendingQuotas.delete(key);
|
|
77
|
+
pending.controller.abort(new Error("Keine wartenden Quota-Aufrufe mehr"));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
export async function buildStatus(cfg, usage, adapters) {
|
|
82
|
+
const searchPos = new Map(cfg.engines.map((e, i) => [e.id, i]));
|
|
83
|
+
const fetchPos = new Map(cfg.fetchOrder.map((id, i) => [id, i]));
|
|
84
|
+
const rows = adapters.map(async (adapter) => {
|
|
85
|
+
const meta = adapter.meta;
|
|
86
|
+
const e = cfg.engines.find((x) => x.id === meta.id);
|
|
87
|
+
const used = usage.get(meta.id);
|
|
88
|
+
const monthlyLimit = !e?.apiKey ? 0 : cfg.settings.monthlyLimits[meta.id] ?? meta.monthlyFree;
|
|
89
|
+
const { quota, error } = await fetchRemoteQuotaCached(adapter, {
|
|
90
|
+
apiKey: e?.apiKey,
|
|
91
|
+
extra: e?.extra,
|
|
92
|
+
});
|
|
93
|
+
return {
|
|
94
|
+
id: meta.id,
|
|
95
|
+
label: meta.label,
|
|
96
|
+
homepage: meta.homepage,
|
|
97
|
+
signupUrl: meta.signupUrl,
|
|
98
|
+
capabilities: !e?.apiKey && meta.keylessCapabilities ? meta.keylessCapabilities : meta.capabilities,
|
|
99
|
+
supportedCapabilities: meta.capabilities,
|
|
100
|
+
keylessCapabilities: meta.keyless === "ip" ? meta.keylessCapabilities ?? meta.capabilities : [],
|
|
101
|
+
keyless: meta.keyless,
|
|
102
|
+
notes: meta.notes,
|
|
103
|
+
extraFields: meta.extraFields ?? [],
|
|
104
|
+
enabled: e?.enabled ?? true,
|
|
105
|
+
searchPosition: searchPos.get(meta.id) ?? -1,
|
|
106
|
+
fetchPosition: fetchPos.has(meta.id) ? fetchPos.get(meta.id) : -1,
|
|
107
|
+
hasKey: Boolean(e?.apiKey),
|
|
108
|
+
keyMasked: maskKey(e?.apiKey),
|
|
109
|
+
extrasSet: Object.fromEntries((meta.extraFields ?? []).map((f) => [f.key, Boolean(e?.extra?.[f.key])])),
|
|
110
|
+
monthlyLimit,
|
|
111
|
+
used,
|
|
112
|
+
remote: quota,
|
|
113
|
+
remoteError: error,
|
|
114
|
+
remainingPct: remainingPct(quotaStatus(adapter, { apiKey: e?.apiKey }, cfg, usage, quota)),
|
|
115
|
+
quota: quotaStatus(adapter, { apiKey: e?.apiKey }, cfg, usage, quota),
|
|
116
|
+
lastError: used.lastError,
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
return Promise.all(rows);
|
|
120
|
+
}
|
|
121
|
+
function cacheKey(adapter, ctx) {
|
|
122
|
+
return adapter.meta.id + ':' + createHash('sha256').update(JSON.stringify([ctx.apiKey, ctx.extra])).digest('hex');
|
|
123
|
+
}
|
|
124
|
+
/** Account for requests made since the last provider snapshot. */
|
|
125
|
+
export function recordRemoteConsumption(adapter, ctx, units) {
|
|
126
|
+
if (!Number.isFinite(units) || units <= 0)
|
|
127
|
+
return;
|
|
128
|
+
const key = cacheKey(adapter, ctx);
|
|
129
|
+
const pending = pendingQuotas.get(key);
|
|
130
|
+
if (pending)
|
|
131
|
+
pending.consumed += units;
|
|
132
|
+
const entry = remoteCache.get(key);
|
|
133
|
+
if (entry?.quota)
|
|
134
|
+
entry.quota = addConsumption(entry.quota, units);
|
|
135
|
+
}
|
|
136
|
+
function addConsumption(q, units) {
|
|
137
|
+
if (!units || !q.limit || (q.used === undefined && q.remaining === undefined))
|
|
138
|
+
return q;
|
|
139
|
+
const used = (q.used ?? q.limit - q.remaining) + units;
|
|
140
|
+
return { ...q, used, remaining: Math.max(0, q.limit - used), estimated: true };
|
|
141
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/usage.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { atomicWriteFile, withFileLock } from "./persistence.js";
|
|
4
|
+
/** Lokale Verbrauchszähler pro Engine und Kalendermonat (UTC). */
|
|
5
|
+
export class UsageStore {
|
|
6
|
+
file;
|
|
7
|
+
data = {};
|
|
8
|
+
constructor(dir) {
|
|
9
|
+
this.file = path.join(dir, "usage.json");
|
|
10
|
+
this.load();
|
|
11
|
+
}
|
|
12
|
+
load() {
|
|
13
|
+
this.data = {};
|
|
14
|
+
try {
|
|
15
|
+
const raw = JSON.parse(fs.readFileSync(this.file, "utf8"));
|
|
16
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
17
|
+
return;
|
|
18
|
+
for (const [month, engines] of Object.entries(raw)) {
|
|
19
|
+
if (!engines || typeof engines !== "object" || Array.isArray(engines))
|
|
20
|
+
continue;
|
|
21
|
+
const cleanMonth = {};
|
|
22
|
+
for (const [engine, u] of Object.entries(engines)) {
|
|
23
|
+
if (!u || typeof u !== "object")
|
|
24
|
+
continue;
|
|
25
|
+
const rec = u;
|
|
26
|
+
// Krumme Zähler (NaN/Strings) würden pct = NaN erzeugen und die
|
|
27
|
+
// Engine still aus der Rotation werfen — strikt säubern.
|
|
28
|
+
const num = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : 0;
|
|
29
|
+
cleanMonth[engine] = {
|
|
30
|
+
search: num(rec.search),
|
|
31
|
+
fetch: num(rec.fetch),
|
|
32
|
+
errors: num(rec.errors),
|
|
33
|
+
consumed: typeof rec.consumed === "number" && Number.isFinite(rec.consumed) && rec.consumed >= 0
|
|
34
|
+
? rec.consumed : num(rec.search) + num(rec.fetch),
|
|
35
|
+
lastError: typeof rec.lastError === "string" ? rec.lastError : undefined,
|
|
36
|
+
lastUsed: typeof rec.lastUsed === "string" ? rec.lastUsed : undefined,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
this.data[month] = cleanMonth;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
// A missing or malformed snapshot may be initialized for compatibility.
|
|
44
|
+
// Other I/O failures must abort the transaction, never erase counters.
|
|
45
|
+
if (!(error instanceof SyntaxError) && error.code !== "ENOENT")
|
|
46
|
+
throw error;
|
|
47
|
+
this.data = {};
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
monthKey(date = new Date()) {
|
|
51
|
+
return `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}`;
|
|
52
|
+
}
|
|
53
|
+
/** Provider-local calendar day; Google uses America/Los_Angeles. */
|
|
54
|
+
dayKey(date = new Date(), timeZone = "UTC") {
|
|
55
|
+
const parts = new Intl.DateTimeFormat("en-CA", { timeZone, year: "numeric", month: "2-digit", day: "2-digit" }).formatToParts(date);
|
|
56
|
+
const part = (type) => parts.find(p => p.type === type).value;
|
|
57
|
+
return `${part("year")}-${part("month")}-${part("day")}`;
|
|
58
|
+
}
|
|
59
|
+
record(engine, kind, error, consumption = 1, timeZone = "UTC") {
|
|
60
|
+
if (!Number.isFinite(consumption) || consumption < 0)
|
|
61
|
+
throw new Error("Invalid quota consumption");
|
|
62
|
+
const now = new Date();
|
|
63
|
+
const day = this.dayKey(now, timeZone);
|
|
64
|
+
withFileLock(this.file, () => {
|
|
65
|
+
this.load();
|
|
66
|
+
// YYYY-MM monthly records remain unchanged; YYYY-MM-DD adds daily buckets.
|
|
67
|
+
for (const key of [this.monthKey(now), day]) {
|
|
68
|
+
const bucket = (this.data[key] ??= {});
|
|
69
|
+
const e = (bucket[engine] ??= { search: 0, fetch: 0, errors: 0, consumed: 0 });
|
|
70
|
+
if (error) {
|
|
71
|
+
e.errors += 1;
|
|
72
|
+
e.lastError = `${now.toISOString()}: ${error}`;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
e.consumed = (e.consumed ?? e.search + e.fetch) + consumption;
|
|
76
|
+
e[kind] += 1;
|
|
77
|
+
e.lastUsed = now.toISOString();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const cutoff = new Date(now.getTime() - 62 * 86_400_000).toISOString().slice(0, 10);
|
|
81
|
+
for (const key of Object.keys(this.data))
|
|
82
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(key) && key < cutoff)
|
|
83
|
+
delete this.data[key];
|
|
84
|
+
atomicWriteFile(this.file, JSON.stringify(this.data, null, 2));
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
get(engine) {
|
|
88
|
+
this.load();
|
|
89
|
+
return this.data[this.monthKey()]?.[engine] ?? { search: 0, fetch: 0, errors: 0 };
|
|
90
|
+
}
|
|
91
|
+
getDay(engine, timeZone = "UTC") {
|
|
92
|
+
this.load();
|
|
93
|
+
return this.data[this.dayKey(new Date(), timeZone)]?.[engine] ?? { search: 0, fetch: 0, errors: 0 };
|
|
94
|
+
}
|
|
95
|
+
}
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
// Version aus package.json — einzige Quelle, kein doppeltes Pflegen beim Bumpen.
|
|
3
|
+
// Resolution: dist/version.js → ../package.json (liegt im npm-Paket und im Repo immer neben dist).
|
|
4
|
+
const pkg = createRequire(import.meta.url)("../package.json");
|
|
5
|
+
export const VERSION = pkg.version;
|