search-rotation 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -0
- package/dist/abort.js +14 -0
- package/dist/config.js +109 -0
- package/dist/engines/base.js +72 -0
- package/dist/engines/duckduckgo.js +98 -0
- package/dist/engines/exa.js +113 -0
- package/dist/engines/firecrawl.js +78 -0
- package/dist/engines/googlecse.js +41 -0
- package/dist/engines/hosted-mcp.js +24 -0
- package/dist/engines/index.js +13 -0
- package/dist/engines/jina.js +28 -0
- package/dist/engines/parallel.js +74 -0
- package/dist/engines/tavily.js +74 -0
- package/dist/history.js +59 -0
- package/dist/index.js +182 -0
- package/dist/mcp/engine-status.js +58 -0
- package/dist/mcp/http.js +35 -0
- package/dist/mcp/server.js +105 -0
- package/dist/persistence.js +80 -0
- package/dist/quota.js +37 -0
- package/dist/router.js +183 -0
- package/dist/search-time.js +31 -0
- package/dist/status.js +141 -0
- package/dist/types.js +1 -0
- package/dist/usage.js +95 -0
- package/dist/version.js +5 -0
- package/dist/web/app.js +155 -0
- package/dist/web/security.js +116 -0
- package/docs/assets/brand/FONT-LICENSE.txt +93 -0
- package/docs/assets/brand/README.md +18 -0
- package/docs/assets/brand/icon-128.png +0 -0
- package/docs/assets/brand/icon-32.png +0 -0
- package/docs/assets/brand/icon-512.png +0 -0
- package/docs/assets/brand/icon-dark-128.png +0 -0
- package/docs/assets/brand/icon-dark-512.png +0 -0
- package/docs/assets/brand/icon-dark.svg +1 -0
- package/docs/assets/brand/icon-light-128.png +0 -0
- package/docs/assets/brand/icon-light-512.png +0 -0
- package/docs/assets/brand/icon-light.svg +1 -0
- package/docs/assets/brand/icon-mono.svg +1 -0
- package/docs/assets/brand/icon.svg +1 -0
- package/docs/assets/brand/preview.png +0 -0
- package/docs/assets/brand/preview.svg +1 -0
- package/docs/assets/brand/wordmark-dark.svg +1 -0
- package/docs/assets/brand/wordmark-light.svg +1 -0
- package/docs/assets/dashboard-dark.png +0 -0
- package/docs/assets/dashboard-light.png +0 -0
- package/docs/assets/dashboard.png +0 -0
- package/docs/assets/mcp-tools-dark.png +0 -0
- package/docs/assets/mcp-tools-light.png +0 -0
- package/docs/assets/mcp-tools.png +0 -0
- package/docs/assets/search-rotation.svg +52 -0
- package/docs/clients.md +93 -0
- package/docs/operations.md +66 -0
- package/docs/provider-access.md +38 -0
- package/docs/release-notes.md +52 -0
- package/package.json +65 -0
- package/static/app.js +1126 -0
- package/static/engine-logos/README.md +11 -0
- package/static/engine-logos/duckduckgo.png +0 -0
- package/static/engine-logos/exa.png +0 -0
- package/static/engine-logos/firecrawl.png +0 -0
- package/static/engine-logos/google-cse.png +0 -0
- package/static/engine-logos/jina.png +0 -0
- package/static/engine-logos/parallel.png +0 -0
- package/static/engine-logos/tavily.png +0 -0
- package/static/i18n.js +709 -0
- package/static/index.html +95 -0
- package/static/style.css +1258 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Robin Bially (LocalFoundry)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
<picture>
|
|
2
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/brand/wordmark-dark.svg">
|
|
3
|
+
<img src="docs/assets/brand/wordmark-light.svg" width="376" alt="search-rotation">
|
|
4
|
+
</picture>
|
|
5
|
+
|
|
6
|
+
**One MCP server for web search and page extraction across multiple providers.**
|
|
7
|
+
|
|
8
|
+
Built by [Robin Bially](https://github.com/localfoundry) under the LocalFoundry label.
|
|
9
|
+
|
|
10
|
+
Rotate across available quotas, automatically fail over when a provider is unavailable, and return consistent results to your AI assistant. A local dashboard lets you manage API keys, reorder engines, check quotas, and inspect request history.
|
|
11
|
+
|
|
12
|
+
<picture>
|
|
13
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/dashboard-dark.png">
|
|
14
|
+
<img src="docs/assets/dashboard-light.png" alt="search-rotation dashboard showing per-engine call counts, remaining budgets, recent activity, and expanded MCP setup">
|
|
15
|
+
</picture>
|
|
16
|
+
|
|
17
|
+
*Actual dashboard UI, shown in English with illustrative demo data. Calls and provider budgets are displayed separately; activity bars offer time-window and per-engine details.*
|
|
18
|
+
|
|
19
|
+
## Quick start
|
|
20
|
+
|
|
21
|
+
### Homebrew
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
brew install localfoundry/tap/search-rotation
|
|
25
|
+
search-rotation --http --open
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Homebrew installs the required Node.js runtime. For MCP over stdio, configure your
|
|
29
|
+
client to run `search-rotation` with no arguments. See the
|
|
30
|
+
[LocalFoundry tap](https://github.com/localfoundry/homebrew-tap) for upgrades and details.
|
|
31
|
+
|
|
32
|
+
### GitHub / npx
|
|
33
|
+
|
|
34
|
+
Requires **Node.js 20.3+** and **Git**. Install directly from GitHub — **no npm account needed**.
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
npx -y --allow-git=all github:localfoundry/search-rotation-mcp#v0.4.9
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
To preview the dashboard:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
npx -y --allow-git=all github:localfoundry/search-rotation-mcp#v0.4.9 --http --open
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Add your provider keys in the dashboard, then connect your assistant using the **[MCP client setup guide](docs/clients.md)** for Codex, Claude, Cursor, or OpenCode.
|
|
47
|
+
|
|
48
|
+
## What you get
|
|
49
|
+
|
|
50
|
+
- **Search and fetch:** independent rotation for web search and Markdown page extraction.
|
|
51
|
+
- **Automatic failover:** quota-aware ordering, rate-limit cooldowns, and request timeouts.
|
|
52
|
+
- **Local dashboard:** API keys, drag-and-drop engine order, per-engine calls and errors, separate quota balances, and interactive request history.
|
|
53
|
+
- **Local or remote:** MCP over stdio or authenticated Streamable HTTP.
|
|
54
|
+
|
|
55
|
+
**Providers:** Tavily · Firecrawl · Parallel · Exa · Google PSE · Jina Reader · DuckDuckGo HTML. [Keyless access and quota accounting](docs/provider-access.md) vary by provider.
|
|
56
|
+
|
|
57
|
+
## MCP tools
|
|
58
|
+
|
|
59
|
+
`web_search` · `fetch_url` · `engine_status` · `open_dashboard`
|
|
60
|
+
|
|
61
|
+
The dashboard's **MCP Tools** tab explains each tool, its parameters, and copyable example calls.
|
|
62
|
+
|
|
63
|
+
<picture>
|
|
64
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/mcp-tools-dark.png">
|
|
65
|
+
<img src="docs/assets/mcp-tools-light.png" alt="MCP Tools overview with parameters and example calls for all four tools">
|
|
66
|
+
</picture>
|
|
67
|
+
|
|
68
|
+
*Actual dashboard UI, shown in English.*
|
|
69
|
+
|
|
70
|
+
## Search time filters
|
|
71
|
+
|
|
72
|
+
`web_search` accepts the following arguments:
|
|
73
|
+
|
|
74
|
+
| Parameter | Meaning |
|
|
75
|
+
| --- | --- |
|
|
76
|
+
| `query` | Required search text. |
|
|
77
|
+
| `numResults` | Optional result count, 1–20; otherwise the dashboard setting applies. |
|
|
78
|
+
| `engine` | Preferred provider, subject to time-filter support and availability; failover stays enabled. |
|
|
79
|
+
| `timeRange` | `day`, `week`, `month`, or `year`: UTC date window starting 1, 7, 30, or 365 days ago, through today. |
|
|
80
|
+
| `startDate` | Optional lower date bound in `YYYY-MM-DD` format. |
|
|
81
|
+
| `endDate` | Optional upper date bound in `YYYY-MM-DD` format. |
|
|
82
|
+
|
|
83
|
+
Use either `timeRange` or explicit dates. One-sided bounds and equal start/end dates are allowed; invalid calendar dates, reversed bounds, and mixing relative and explicit filters are rejected before any provider request.
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{"name":"web_search","arguments":{"query":"AI inference research","timeRange":"week","numResults":5}}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
```json
|
|
90
|
+
{"name":"web_search","arguments":{"query":"AI inference research","startDate":"2026-08-01","endDate":"2026-08-31"}}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Relative windows are resolved **once per request** into UTC dates, including across failover. These are date filters, not exact rolling 24-hour windows; `month` and `year` mean 30 and 365 days, not calendar arithmetic. Exa receives the start of the first UTC day and the end of the last UTC day.
|
|
94
|
+
|
|
95
|
+
### Provider support and rotation
|
|
96
|
+
|
|
97
|
+
| Provider / access | Relative window | Two date bounds | One date bound |
|
|
98
|
+
| --- | --- | --- | --- |
|
|
99
|
+
| Tavily, with or without key | Yes | Yes | Yes |
|
|
100
|
+
| Firecrawl, with or without key | Yes | Yes | Skipped |
|
|
101
|
+
| Exa, direct API with key | Yes | Yes | Yes |
|
|
102
|
+
| Exa, hosted MCP without key | Skipped | Skipped | Skipped |
|
|
103
|
+
| Parallel, either access mode | Skipped | Skipped | Skipped |
|
|
104
|
+
| Google PSE / DuckDuckGo HTML | Skipped | Skipped | Skipped |
|
|
105
|
+
|
|
106
|
+
This table describes **implemented support in search-rotation**, not every upstream feature. Tavily receives `start_date` / `end_date`, Firecrawl receives a custom `tbs` range, and Exa receives `startPublishedDate` / `endPublishedDate`. Firecrawl is conservatively excluded for one-sided dates; the other skipped paths have no implemented date mapping.
|
|
107
|
+
|
|
108
|
+
The router excludes incompatible providers **before quota checks and rotation**, including when an incompatible `engine` is preferred. Each eligible provider pool has its own rotation cursor, so mixing filtered and unfiltered searches does not starve providers. Existing quota priorities, cooldowns, strict-free rules, and failover apply within that pool. If no compatible provider is available, the search fails explicitly; it never retries without the filter. Searches without time arguments retain access to all otherwise eligible providers.
|
|
109
|
+
|
|
110
|
+
Results show a publication date when supplied by the adapter, and request history includes the requested period. Provider date metadata can be estimated or missing: Tavily filters publication **or update** dates, Exa filters estimated publication dates, and Firecrawl uses its search index's date interpretation. Exact boundary inclusion follows the provider; this is not an independent verification of each page's publication date. See the [Tavily](https://docs.tavily.com/documentation/api-reference/endpoint/search), [Firecrawl](https://docs.firecrawl.dev/api-reference/endpoint/search), and [Exa](https://exa.ai/docs/reference/search) references.
|
|
111
|
+
|
|
112
|
+
After updating, reconnect your MCP client to load the new tool schema.
|
|
113
|
+
|
|
114
|
+
## Releasing
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
VERSION=0.4.9 ./scripts/release.sh --publish
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
The script checks the package (`npm ci`, build, tests, `npm run smoke:package`), bumps the version, points the document pins at it, commits and tags, packs the tarball with its checksum, creates the GitHub release and updates the formula in `localfoundry/homebrew-tap`. Without `--publish` it only prepares the artifacts in `.build/releases`; `--dry-run` checks the prerequisites, `--draft` creates a draft release and `--force` tolerates a dirty tree. The tap is cloned temporarily when `TAP_DIR` is not set, so a fresh checkout is enough.
|
|
121
|
+
|
|
122
|
+
## Learn more
|
|
123
|
+
|
|
124
|
+
[Client setup](docs/clients.md) · [Operations & configuration (DE)](docs/operations.md) · [Releases](https://github.com/localfoundry/search-rotation-mcp/releases) · [CI](https://github.com/localfoundry/search-rotation-mcp/actions/workflows/ci.yml) · [MIT license](LICENSE)
|
package/dist/abort.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** Stop waiting even if a provider fails to honour cancellation. */
|
|
2
|
+
export function abortable(work, signal) {
|
|
3
|
+
if (!signal)
|
|
4
|
+
return work;
|
|
5
|
+
return new Promise((resolve, reject) => {
|
|
6
|
+
const abort = () => reject(signal.reason ?? new Error('Anfrage abgebrochen'));
|
|
7
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
8
|
+
work.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort));
|
|
9
|
+
if (signal.aborted) {
|
|
10
|
+
signal.removeEventListener('abort', abort);
|
|
11
|
+
abort();
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
export function configDir() {
|
|
5
|
+
return process.env.SEARCH_ROTATION_HOME || path.join(os.homedir(), ".config", "search-rotation");
|
|
6
|
+
}
|
|
7
|
+
export function normalizeConfig(raw, d) {
|
|
8
|
+
const known = new Set(d.knownIds);
|
|
9
|
+
const rawEngines = Array.isArray(raw?.engines)
|
|
10
|
+
? raw.engines
|
|
11
|
+
.filter((e) => e && typeof e.id === "string" && known.has(e.id))
|
|
12
|
+
.map((e) => ({
|
|
13
|
+
id: e.id,
|
|
14
|
+
enabled: typeof e.enabled === "boolean" ? e.enabled : d.defaultEnabled[e.id] ?? true,
|
|
15
|
+
apiKey: typeof e.apiKey === "string" && e.apiKey.trim() ? e.apiKey.trim() : undefined,
|
|
16
|
+
extra: e.extra && typeof e.extra === "object" && !Array.isArray(e.extra)
|
|
17
|
+
? Object.fromEntries(Object.entries(e.extra).filter(([, v]) => typeof v === "string"))
|
|
18
|
+
: undefined,
|
|
19
|
+
}))
|
|
20
|
+
: [];
|
|
21
|
+
const have = new Set(rawEngines.map((e) => e.id));
|
|
22
|
+
// Such-Engines in Such-Reihenfolge, danach evtl. fetch-only Engines (z. B. Jina)
|
|
23
|
+
for (const id of d.knownIds) {
|
|
24
|
+
if (!have.has(id))
|
|
25
|
+
rawEngines.push({ id, enabled: d.defaultEnabled[id] ?? true });
|
|
26
|
+
}
|
|
27
|
+
const fetchSet = new Set(d.fetchOrder);
|
|
28
|
+
const fo = Array.isArray(raw?.fetchOrder)
|
|
29
|
+
? raw.fetchOrder.filter((id) => typeof id === "string" && fetchSet.has(id))
|
|
30
|
+
: [];
|
|
31
|
+
for (const id of d.fetchOrder) {
|
|
32
|
+
if (!fo.includes(id))
|
|
33
|
+
fo.push(id);
|
|
34
|
+
}
|
|
35
|
+
const seen = new Set();
|
|
36
|
+
const engines = rawEngines.filter((e) => {
|
|
37
|
+
// Hand-editierte Configs können Dubletten enthalten — der Router würde
|
|
38
|
+
// sonst dieselbe Engine mehrfach pro Suche aufrufen (doppelte Quota!).
|
|
39
|
+
if (seen.has(e.id))
|
|
40
|
+
return false;
|
|
41
|
+
seen.add(e.id);
|
|
42
|
+
return true;
|
|
43
|
+
});
|
|
44
|
+
for (const engine of engines) {
|
|
45
|
+
if (d.requiredKeyIds?.includes(engine.id) && !engine.apiKey)
|
|
46
|
+
engine.enabled = false;
|
|
47
|
+
}
|
|
48
|
+
const s = raw?.settings ?? {};
|
|
49
|
+
const rawLimits = s.monthlyLimits && typeof s.monthlyLimits === "object" && !Array.isArray(s.monthlyLimits)
|
|
50
|
+
? s.monthlyLimits
|
|
51
|
+
: {};
|
|
52
|
+
// Nur endliche Zahlen >= 0 — ein Tippfehler ("abc") würde die Engine sonst
|
|
53
|
+
// unbegrenzt schalten, statt sie zu drosseln.
|
|
54
|
+
const monthlyLimits = {};
|
|
55
|
+
for (const [k, v] of Object.entries(rawLimits)) {
|
|
56
|
+
if (typeof v === "number" && Number.isFinite(v) && v >= 0)
|
|
57
|
+
monthlyLimits[k] = v;
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
version: 1,
|
|
61
|
+
engines,
|
|
62
|
+
fetchOrder: fo,
|
|
63
|
+
settings: {
|
|
64
|
+
port: Number.isInteger(s.port) && s.port >= 1024 && s.port <= 65535 ? s.port : 6277,
|
|
65
|
+
token: typeof s.token === "string" ? s.token : "",
|
|
66
|
+
monthlyLimits,
|
|
67
|
+
dailyLimits: Object.fromEntries(Object.entries(s.dailyLimits ?? {}).filter(([, v]) => typeof v === "number" && Number.isFinite(v) && v >= 0)),
|
|
68
|
+
strictFreeMode: s.strictFreeMode === true,
|
|
69
|
+
defaultNumResults: s.defaultNumResults === null ? null : Number.isInteger(s.defaultNumResults) && s.defaultNumResults >= 1 && s.defaultNumResults <= 20 ? s.defaultNumResults : 8,
|
|
70
|
+
requestTimeoutMs: Number.isInteger(s.requestTimeoutMs) && s.requestTimeoutMs >= 1000 && s.requestTimeoutMs <= 300_000 ? s.requestTimeoutMs : 60_000,
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
export class ConfigStore {
|
|
75
|
+
dir;
|
|
76
|
+
file;
|
|
77
|
+
constructor(dir = configDir()) {
|
|
78
|
+
this.dir = dir;
|
|
79
|
+
this.file = path.join(dir, "config.json");
|
|
80
|
+
}
|
|
81
|
+
load(d) {
|
|
82
|
+
fs.mkdirSync(this.dir, { recursive: true });
|
|
83
|
+
if (!fs.existsSync(this.file)) {
|
|
84
|
+
const cfg = normalizeConfig(null, d);
|
|
85
|
+
this.save(cfg);
|
|
86
|
+
return cfg;
|
|
87
|
+
}
|
|
88
|
+
let raw;
|
|
89
|
+
try {
|
|
90
|
+
raw = JSON.parse(fs.readFileSync(this.file, "utf8"));
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
throw new Error(`Konfigdatei ${this.file} ist kein gültiges JSON: ${err.message}`);
|
|
94
|
+
}
|
|
95
|
+
return normalizeConfig(raw, d);
|
|
96
|
+
}
|
|
97
|
+
save(cfg) {
|
|
98
|
+
fs.mkdirSync(this.dir, { recursive: true });
|
|
99
|
+
const tmp = `${this.file}.tmp`;
|
|
100
|
+
fs.writeFileSync(tmp, JSON.stringify(cfg, null, 2) + "\n", { mode: 0o600 });
|
|
101
|
+
fs.renameSync(tmp, this.file);
|
|
102
|
+
try {
|
|
103
|
+
fs.chmodSync(this.file, 0o600);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
/* chmod optional */
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { VERSION } from "../version.js";
|
|
2
|
+
export const USER_AGENT = `search-rotation/${VERSION}`;
|
|
3
|
+
export class NeedsKeyError extends Error {
|
|
4
|
+
constructor(engineId, signupUrl) {
|
|
5
|
+
super(`${engineId}: kein API-Key konfiguriert — gratis Key: ${signupUrl}`);
|
|
6
|
+
this.name = "NeedsKeyError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export class HttpError extends Error {
|
|
10
|
+
status;
|
|
11
|
+
body;
|
|
12
|
+
url;
|
|
13
|
+
retryAfter;
|
|
14
|
+
constructor(status, body, url, retryAfter) {
|
|
15
|
+
let host = url;
|
|
16
|
+
try {
|
|
17
|
+
host = new URL(url).host;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
/* URL schon im Fehlertext behalten */
|
|
21
|
+
}
|
|
22
|
+
super(`HTTP ${status} von ${host}: ${body.slice(0, 200)}`);
|
|
23
|
+
this.status = status;
|
|
24
|
+
this.body = body;
|
|
25
|
+
this.url = url;
|
|
26
|
+
this.retryAfter = retryAfter;
|
|
27
|
+
this.name = "HttpError";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export async function httpText(url, init = {}, opts = {}) {
|
|
31
|
+
const timeout = AbortSignal.timeout(opts.timeoutMs ?? 20_000);
|
|
32
|
+
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;
|
|
33
|
+
const headers = {
|
|
34
|
+
"user-agent": USER_AGENT,
|
|
35
|
+
// new Headers(...) normalisiert auch Headers-Instanzen und string[][] mit
|
|
36
|
+
// — ein simpler Spread würde sie als leeres Objekt droppen.
|
|
37
|
+
...Object.fromEntries(new Headers(init.headers ?? undefined)),
|
|
38
|
+
};
|
|
39
|
+
const res = await fetch(url, { ...init, signal, headers });
|
|
40
|
+
const body = await res.text();
|
|
41
|
+
if (!res.ok)
|
|
42
|
+
throw new HttpError(res.status, body, url, res.headers.get("retry-after"));
|
|
43
|
+
return body;
|
|
44
|
+
}
|
|
45
|
+
export async function httpJson(url, init = {}, opts = {}) {
|
|
46
|
+
const text = await httpText(url, { ...init, headers: { accept: "application/json", ...Object.fromEntries(new Headers(init.headers ?? undefined)) } }, opts);
|
|
47
|
+
try {
|
|
48
|
+
return JSON.parse(text);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
throw new Error(`Ungültige Antwort von ${safeHost(url)} (kein JSON): ${text.slice(0, 120)}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export function bearer(key) {
|
|
55
|
+
return key ? { authorization: `Bearer ${key}` } : {};
|
|
56
|
+
}
|
|
57
|
+
export function cap(n, fallback = 8, max = 20) {
|
|
58
|
+
const v = typeof n === "number" && Number.isFinite(n) ? Math.round(n) : fallback;
|
|
59
|
+
return Math.min(Math.max(v, 1), Math.max(Math.round(max), 1));
|
|
60
|
+
}
|
|
61
|
+
function safeHost(url) {
|
|
62
|
+
try {
|
|
63
|
+
return new URL(url).host;
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return url;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** Undefined means the provider chooses, including no local result slicing. */
|
|
70
|
+
export function optionalCap(n) {
|
|
71
|
+
return n === undefined ? undefined : cap(n);
|
|
72
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { optionalCap as cap, HttpError } from "./base.js";
|
|
2
|
+
const ENTITIES = {
|
|
3
|
+
amp: "&",
|
|
4
|
+
lt: "<",
|
|
5
|
+
gt: ">",
|
|
6
|
+
quot: '"',
|
|
7
|
+
apos: "'",
|
|
8
|
+
nbsp: " ",
|
|
9
|
+
uuml: "ü",
|
|
10
|
+
ouml: "ö",
|
|
11
|
+
auml: "ä",
|
|
12
|
+
Uuml: "Ü",
|
|
13
|
+
Ouml: "Ö",
|
|
14
|
+
Auml: "Ä",
|
|
15
|
+
szlig: "ß",
|
|
16
|
+
};
|
|
17
|
+
function decodeEntities(s) {
|
|
18
|
+
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, code) => {
|
|
19
|
+
if (ENTITIES[code])
|
|
20
|
+
return ENTITIES[code];
|
|
21
|
+
const lower = code.toLowerCase();
|
|
22
|
+
if (lower.startsWith("#x"))
|
|
23
|
+
return String.fromCodePoint(parseInt(code.slice(2), 16));
|
|
24
|
+
if (code.startsWith("#"))
|
|
25
|
+
return String.fromCodePoint(parseInt(code.slice(1), 10));
|
|
26
|
+
return m;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
function stripTags(s) {
|
|
30
|
+
return decodeEntities(s.replace(/<[^>]*>/g, ""))
|
|
31
|
+
.replace(/\s+/g, " ")
|
|
32
|
+
.trim();
|
|
33
|
+
}
|
|
34
|
+
/** Best-effort-Parser für https://html.duckduckgo.com/html/ (inoffiziell). */
|
|
35
|
+
export function parseDdgHtml(html) {
|
|
36
|
+
const items = [];
|
|
37
|
+
const linkRe = /<a[^>]+class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
|
|
38
|
+
const snippetRe = /<a[^>]+class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/g;
|
|
39
|
+
const links = [...html.matchAll(linkRe)];
|
|
40
|
+
for (const [i, m] of links.slice(0, 20).entries()) {
|
|
41
|
+
// Ein Snippet gehört nur zum Bereich nach diesem Treffer und vor dem nächsten.
|
|
42
|
+
const block = html.slice(m.index + m[0].length, links[i + 1]?.index ?? html.length);
|
|
43
|
+
snippetRe.lastIndex = 0;
|
|
44
|
+
const snippet = snippetRe.exec(block)?.[1];
|
|
45
|
+
let raw = decodeEntities(m[1]);
|
|
46
|
+
const uddg = raw.match(/[?&]uddg=([^&]+)/);
|
|
47
|
+
if (uddg) {
|
|
48
|
+
try {
|
|
49
|
+
raw = decodeURIComponent(uddg[1]);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
raw = uddg[1];
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
items.push({ title: stripTags(m[2]) || "(ohne Titel)", url: raw, snippet: snippet === undefined ? undefined : stripTags(snippet) });
|
|
56
|
+
}
|
|
57
|
+
return items;
|
|
58
|
+
}
|
|
59
|
+
async function search(input, ctx) {
|
|
60
|
+
const timeout = AbortSignal.timeout(20_000);
|
|
61
|
+
const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
|
|
62
|
+
const res = await fetch("https://html.duckduckgo.com/html/", {
|
|
63
|
+
method: "POST",
|
|
64
|
+
headers: {
|
|
65
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
66
|
+
accept: "text/html",
|
|
67
|
+
"accept-language": "en",
|
|
68
|
+
},
|
|
69
|
+
body: new URLSearchParams({ q: input.query }),
|
|
70
|
+
redirect: "follow",
|
|
71
|
+
signal,
|
|
72
|
+
});
|
|
73
|
+
if (!res.ok) {
|
|
74
|
+
throw new HttpError(res.status, await res.text().catch(() => ""), "https://html.duckduckgo.com/html/");
|
|
75
|
+
}
|
|
76
|
+
const html = await res.text();
|
|
77
|
+
const items = parseDdgHtml(html).slice(0, cap(input.numResults));
|
|
78
|
+
if (items.length === 0) {
|
|
79
|
+
throw new Error("duckduckgo: keine Ergebnisse (möglicherweise blockiert — Bot-Detection)");
|
|
80
|
+
}
|
|
81
|
+
return { items };
|
|
82
|
+
}
|
|
83
|
+
export const DUCKDUCKGO = {
|
|
84
|
+
meta: {
|
|
85
|
+
id: "duckduckgo",
|
|
86
|
+
label: "DuckDuckGo (HTML)",
|
|
87
|
+
homepage: "https://duckduckgo.com",
|
|
88
|
+
signupUrl: "https://duckduckgo.com",
|
|
89
|
+
keyless: "ip",
|
|
90
|
+
defaultEnabled: false,
|
|
91
|
+
capabilities: ["search"],
|
|
92
|
+
monthlyFree: 0,
|
|
93
|
+
quota: { period: "ip", unit: "requests", estimated: true },
|
|
94
|
+
quotaEndpoint: false,
|
|
95
|
+
notes: "Inoffizieller HTML-Scrape: kein Kontingent, aber Bot-Detection möglich. Als letzte Instanz gedacht.",
|
|
96
|
+
},
|
|
97
|
+
search,
|
|
98
|
+
};
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { optionalCap as cap, httpJson } from "./base.js";
|
|
2
|
+
import { withHostedMcp } from "./hosted-mcp.js";
|
|
3
|
+
const SIGNUP = "https://dashboard.exa.ai";
|
|
4
|
+
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
|
|
5
|
+
/**
|
|
6
|
+
* Keyless-Fallback: Exas gehosteter MCP-Server ist ohne Key IP-basiert
|
|
7
|
+
* nutzbar und bietet Suche (web_search_exa) und Fetch (web_fetch_exa).
|
|
8
|
+
* Pro Aufruf eine eigene Client-Session — einfach und robust.
|
|
9
|
+
*/
|
|
10
|
+
function mcpText(result) {
|
|
11
|
+
return (result?.content ?? [])
|
|
12
|
+
.filter((c) => c?.type === "text" && typeof c.text === "string")
|
|
13
|
+
.map((c) => c.text)
|
|
14
|
+
.join("\n\n");
|
|
15
|
+
}
|
|
16
|
+
function parseExaMcpSearch(text) {
|
|
17
|
+
const items = [];
|
|
18
|
+
for (const block of text.split(/\n-{3,}\n/)) {
|
|
19
|
+
const url = block.match(/^URL:\s*(\S+)$/m)?.[1]?.trim();
|
|
20
|
+
if (!url)
|
|
21
|
+
continue;
|
|
22
|
+
const title = block.match(/^Title:\s*(.+)$/m)?.[1]?.trim() || url;
|
|
23
|
+
const hl = block.indexOf("Highlights:");
|
|
24
|
+
const snippet = hl >= 0
|
|
25
|
+
? block
|
|
26
|
+
.slice(hl + "Highlights:".length)
|
|
27
|
+
.replace(/\s+/g, " ")
|
|
28
|
+
.trim()
|
|
29
|
+
.slice(0, 500)
|
|
30
|
+
: undefined;
|
|
31
|
+
items.push({ title, url, snippet });
|
|
32
|
+
}
|
|
33
|
+
return items;
|
|
34
|
+
}
|
|
35
|
+
async function directSearch(input, ctx) {
|
|
36
|
+
const j = await httpJson("https://api.exa.ai/search", {
|
|
37
|
+
method: "POST",
|
|
38
|
+
headers: { "content-type": "application/json", "x-api-key": ctx.apiKey },
|
|
39
|
+
body: JSON.stringify({
|
|
40
|
+
query: input.query,
|
|
41
|
+
startPublishedDate: input.startDate ? `${input.startDate}T00:00:00.000Z` : undefined,
|
|
42
|
+
endPublishedDate: input.endDate ? `${input.endDate}T23:59:59.999Z` : undefined,
|
|
43
|
+
numResults: cap(input.numResults),
|
|
44
|
+
type: "auto",
|
|
45
|
+
contents: { text: { maxCharacters: 400 } },
|
|
46
|
+
}),
|
|
47
|
+
}, { signal: ctx.signal });
|
|
48
|
+
const items = (Array.isArray(j?.results) ? j.results : []).map((r) => ({
|
|
49
|
+
title: r?.title || r?.url || "(ohne Titel)",
|
|
50
|
+
url: String(r?.url ?? ""),
|
|
51
|
+
snippet: typeof r?.text === "string" ? r.text.slice(0, 500) : undefined,
|
|
52
|
+
published: typeof r?.publishedDate === "string" ? r.publishedDate : undefined,
|
|
53
|
+
}));
|
|
54
|
+
return { items };
|
|
55
|
+
}
|
|
56
|
+
async function directFetch(input, ctx) {
|
|
57
|
+
const j = await httpJson("https://api.exa.ai/contents", {
|
|
58
|
+
method: "POST",
|
|
59
|
+
headers: { "content-type": "application/json", "x-api-key": ctx.apiKey },
|
|
60
|
+
body: JSON.stringify({ ids: [input.url], text: { maxCharacters: 50_000 } }),
|
|
61
|
+
}, { signal: ctx.signal, timeoutMs: 45_000 });
|
|
62
|
+
const text = j?.results?.[0]?.text;
|
|
63
|
+
if (typeof text !== "string" || !text.trim())
|
|
64
|
+
throw new Error("exa: kein Inhalt extrahiert");
|
|
65
|
+
return text;
|
|
66
|
+
}
|
|
67
|
+
async function search(input, ctx) {
|
|
68
|
+
// Mit Key: direkte API. Ohne Key: gehosteter Exa-MCP (IP-basiert, gratis).
|
|
69
|
+
if (!ctx.apiKey) {
|
|
70
|
+
const text = await withHostedMcp(EXA_MCP_URL, ctx, 30_000, async (client, signal) => {
|
|
71
|
+
const res = await client.callTool({ name: "web_search_exa", arguments: { query: input.query, numResults: cap(input.numResults) } }, undefined, { timeout: 30_000, signal });
|
|
72
|
+
if (res?.isError)
|
|
73
|
+
throw new Error(`exa-mcp: ${mcpText(res).slice(0, 200)}`);
|
|
74
|
+
return mcpText(res);
|
|
75
|
+
});
|
|
76
|
+
const items = parseExaMcpSearch(text);
|
|
77
|
+
if (items.length === 0)
|
|
78
|
+
throw new Error("exa-mcp: keine Ergebnisse");
|
|
79
|
+
return { items };
|
|
80
|
+
}
|
|
81
|
+
return directSearch(input, ctx);
|
|
82
|
+
}
|
|
83
|
+
async function fetchUrl(input, ctx) {
|
|
84
|
+
if (!ctx.apiKey) {
|
|
85
|
+
const text = await withHostedMcp(EXA_MCP_URL, ctx, 45_000, async (client, signal) => {
|
|
86
|
+
const res = await client.callTool({ name: "web_fetch_exa", arguments: { urls: [input.url], maxCharacters: 50_000 } }, undefined, { timeout: 45_000, signal });
|
|
87
|
+
if (res?.isError)
|
|
88
|
+
throw new Error(`exa-mcp: ${mcpText(res).slice(0, 200)}`);
|
|
89
|
+
return mcpText(res);
|
|
90
|
+
});
|
|
91
|
+
if (!text.trim())
|
|
92
|
+
throw new Error("exa-mcp: kein Inhalt extrahiert");
|
|
93
|
+
return text;
|
|
94
|
+
}
|
|
95
|
+
return directFetch(input, ctx);
|
|
96
|
+
}
|
|
97
|
+
export const EXA = {
|
|
98
|
+
meta: {
|
|
99
|
+
id: "exa",
|
|
100
|
+
label: "Exa",
|
|
101
|
+
homepage: "https://exa.ai",
|
|
102
|
+
signupUrl: SIGNUP,
|
|
103
|
+
keyless: "ip",
|
|
104
|
+
capabilities: ["search", "fetch"],
|
|
105
|
+
monthlyFree: 0,
|
|
106
|
+
quota: { period: "month", unit: "requests", estimated: true },
|
|
107
|
+
quotaEndpoint: false,
|
|
108
|
+
notes: "Ohne Key: automatisch über gehosteten Exa-MCP (mcp.exa.ai, IP-basiert limitiert) — Suche und Fetch. Mit Key: direkte API. Nutzung lokal gezählt; tatsächliches Guthaben und Anbieter-Gesamtverbrauch unbekannt.",
|
|
109
|
+
},
|
|
110
|
+
supportsSearchTime: (_input, ctx) => Boolean(ctx.apiKey),
|
|
111
|
+
search,
|
|
112
|
+
fetchUrl,
|
|
113
|
+
};
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { bearer, optionalCap as cap, httpJson } from "./base.js";
|
|
2
|
+
const BASE = "https://api.firecrawl.dev/v2";
|
|
3
|
+
async function search(input, ctx) {
|
|
4
|
+
const usDate = (date) => `${date.slice(5, 7)}/${date.slice(8, 10)}/${date.slice(0, 4)}`;
|
|
5
|
+
const tbs = input.startDate && input.endDate
|
|
6
|
+
? `cdr:1,cd_min:${usDate(input.startDate)},cd_max:${usDate(input.endDate)}` : undefined;
|
|
7
|
+
const j = await httpJson(`${BASE}/search`, {
|
|
8
|
+
method: "POST",
|
|
9
|
+
headers: { "content-type": "application/json", ...bearer(ctx.apiKey) },
|
|
10
|
+
body: JSON.stringify({ query: input.query, limit: cap(input.numResults), tbs }),
|
|
11
|
+
}, { signal: ctx.signal, timeoutMs: 45_000 });
|
|
12
|
+
if (j && j.success === false)
|
|
13
|
+
throw new Error(`firecrawl: ${j.error ?? "unbekannter Fehler"}`);
|
|
14
|
+
const arr = j?.data?.web ?? (Array.isArray(j?.data) ? j.data : []);
|
|
15
|
+
const items = arr.map((d) => ({
|
|
16
|
+
title: d?.title || d?.url || "(ohne Titel)",
|
|
17
|
+
url: String(d?.url ?? ""),
|
|
18
|
+
snippet: typeof d?.description === "string"
|
|
19
|
+
? d.description.slice(0, 500)
|
|
20
|
+
: typeof d?.markdown === "string"
|
|
21
|
+
? d.markdown.slice(0, 300)
|
|
22
|
+
: undefined,
|
|
23
|
+
}));
|
|
24
|
+
return { items };
|
|
25
|
+
}
|
|
26
|
+
async function fetchUrl(input, ctx) {
|
|
27
|
+
const j = await httpJson(`${BASE}/scrape`, {
|
|
28
|
+
method: "POST",
|
|
29
|
+
headers: { "content-type": "application/json", ...bearer(ctx.apiKey) },
|
|
30
|
+
body: JSON.stringify({ url: input.url, formats: ["markdown"], onlyMainContent: true }),
|
|
31
|
+
}, { signal: ctx.signal, timeoutMs: 45_000 });
|
|
32
|
+
if (j && j.success === false)
|
|
33
|
+
throw new Error(`firecrawl: ${j.error ?? "unbekannter Fehler"}`);
|
|
34
|
+
const md = j?.data?.markdown;
|
|
35
|
+
if (typeof md !== "string" || !md.trim())
|
|
36
|
+
throw new Error("firecrawl: kein Inhalt gescraped");
|
|
37
|
+
return md;
|
|
38
|
+
}
|
|
39
|
+
async function remoteQuota(ctx) {
|
|
40
|
+
const j = await httpJson(`${BASE}/team/credit-usage`, { headers: bearer(ctx.apiKey) }, { signal: ctx.signal });
|
|
41
|
+
// Response: { success, data: { remainingCredits, planCredits, billingPeriodStart, billingPeriodEnd } }
|
|
42
|
+
const d = j?.data ?? {};
|
|
43
|
+
const limit = pickNum(d.planCredits);
|
|
44
|
+
const remaining = pickNum(d.remainingCredits);
|
|
45
|
+
return {
|
|
46
|
+
limit,
|
|
47
|
+
remaining,
|
|
48
|
+
used: limit !== undefined && remaining !== undefined ? limit - remaining : undefined,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function pickNum(...vals) {
|
|
52
|
+
for (const v of vals) {
|
|
53
|
+
if (typeof v === "number" && Number.isFinite(v))
|
|
54
|
+
return v;
|
|
55
|
+
}
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
export const FIRECRAWL = {
|
|
59
|
+
meta: {
|
|
60
|
+
id: "firecrawl",
|
|
61
|
+
label: "Firecrawl",
|
|
62
|
+
homepage: "https://www.firecrawl.dev",
|
|
63
|
+
signupUrl: "https://www.firecrawl.dev/signin",
|
|
64
|
+
keyless: "ip",
|
|
65
|
+
capabilities: ["search", "fetch"],
|
|
66
|
+
monthlyFree: 1000,
|
|
67
|
+
quota: { period: "month", unit: "credits", limit: 1000, estimated: true, costs: { search: 2, fetch: 1 } },
|
|
68
|
+
quotaEndpoint: true,
|
|
69
|
+
notes: "Gratis-Guthaben mit Konto; kein garantierter monatlicher Reset. Quota per API abrufbar; Reset nach Billing-Periode (Kontostand zählt Remote), nicht am Kalendermonat. Ohne Key nur wenige IP-basierte Requests.",
|
|
70
|
+
},
|
|
71
|
+
// Documented Firecrawl v2 default: 10 results (docs.firecrawl.dev/api-reference/endpoint/search).
|
|
72
|
+
// Cost remains an estimate; do not substitute our dashboard default here.
|
|
73
|
+
estimateCost: (kind, input) => kind === "search" ? 2 * Math.ceil((cap(input.numResults) ?? 10) / 10) : 1,
|
|
74
|
+
supportsSearchTime: input => Boolean(input.startDate && input.endDate),
|
|
75
|
+
search,
|
|
76
|
+
fetchUrl,
|
|
77
|
+
remoteQuota,
|
|
78
|
+
};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { optionalCap as cap, httpJson, NeedsKeyError } from "./base.js";
|
|
2
|
+
const SIGNUP = "https://developers.google.com/custom-search/v1/overview";
|
|
3
|
+
async function search(input, ctx) {
|
|
4
|
+
if (!ctx.apiKey)
|
|
5
|
+
throw new NeedsKeyError("google-cse", SIGNUP);
|
|
6
|
+
const cx = ctx.extra?.cx;
|
|
7
|
+
if (!cx)
|
|
8
|
+
throw new Error("google-cse: CX (Search Engine ID) fehlt — im Dashboard unter Extras hinterlegen.");
|
|
9
|
+
const url = new URL("https://www.googleapis.com/customsearch/v1");
|
|
10
|
+
url.searchParams.set("key", ctx.apiKey);
|
|
11
|
+
url.searchParams.set("cx", cx);
|
|
12
|
+
url.searchParams.set("q", input.query);
|
|
13
|
+
if (input.numResults !== undefined)
|
|
14
|
+
url.searchParams.set("num", String(Math.min(cap(input.numResults), 10)));
|
|
15
|
+
const j = await httpJson(url.toString(), {}, { signal: ctx.signal });
|
|
16
|
+
if (j?.error?.message)
|
|
17
|
+
throw new Error(`google-cse: ${j.error.message}`);
|
|
18
|
+
const items = (Array.isArray(j?.items) ? j.items : []).map((r) => ({
|
|
19
|
+
title: r?.title || "(ohne Titel)",
|
|
20
|
+
url: String(r?.link ?? ""),
|
|
21
|
+
snippet: typeof r?.snippet === "string" ? r.snippet : undefined,
|
|
22
|
+
}));
|
|
23
|
+
return { items };
|
|
24
|
+
}
|
|
25
|
+
export const GOOGLE_CSE = {
|
|
26
|
+
meta: {
|
|
27
|
+
id: "google-cse",
|
|
28
|
+
label: "Google PSE",
|
|
29
|
+
homepage: "https://www.google.com",
|
|
30
|
+
signupUrl: SIGNUP,
|
|
31
|
+
keyless: "no",
|
|
32
|
+
capabilities: ["search"],
|
|
33
|
+
monthlyFree: 3000,
|
|
34
|
+
quota: { period: "day", unit: "requests", limit: 100, timeZone: "America/Los_Angeles", estimated: false },
|
|
35
|
+
quotaEndpoint: false,
|
|
36
|
+
notes: "Für bestehende Kunden: 100 Queries pro Pacific-Tag. API-Key und Search Engine ID (CX) erforderlich. Für Neukunden geschlossen; Abschaltung am 1. Januar 2027.",
|
|
37
|
+
extraFields: [{ key: "cx", label: "Search Engine ID (CX)" }],
|
|
38
|
+
defaultEnabled: false,
|
|
39
|
+
},
|
|
40
|
+
search,
|
|
41
|
+
};
|