pi-web-kit 0.1.6 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +44 -6
- package/extensions/index.ts +202 -14
- package/package.json +9 -6
- package/src/cache.ts +13 -4
- package/src/config.ts +26 -9
- package/src/providers/context7.ts +111 -0
- package/src/providers/exa-mcp.ts +13 -2
- package/src/providers/exa.ts +29 -3
- package/src/providers/firecrawl.ts +18 -2
- package/src/providers/index.ts +9 -0
- package/src/providers/tinyfish.ts +5 -1
- package/src/types.ts +72 -1
- package/src/urls.ts +10 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.1 - 2026-06-14
|
|
4
|
+
|
|
3
5
|
All notable changes to this project will be documented in this file.
|
|
4
6
|
|
|
5
7
|
This project follows the spirit of [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and uses semantic versioning for releases.
|
|
6
8
|
|
|
7
9
|
## [Unreleased]
|
|
8
10
|
|
|
11
|
+
## [0.2.0] - 2026-06-13
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- Add `library_search` and `library_docs` tools backed by Context7.
|
|
16
|
+
- Add `code_search` backed by Exa Code.
|
|
17
|
+
- Gate optional developer-search tool registration on API key availability.
|
|
18
|
+
|
|
9
19
|
## [0.1.6] - 2026-06-06
|
|
10
20
|
|
|
11
21
|
### Changed
|
package/README.md
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
# pi-web-kit
|
|
2
2
|
|
|
3
|
-
Context-efficient web
|
|
3
|
+
Context-efficient web and developer search tools for [Pi](https://pi.dev): `web_search`, `web_fetch`, `library_search`, `library_docs`, and `code_search`.
|
|
4
4
|
|
|
5
|
-
`pi-web-kit` provides provider-backed search
|
|
5
|
+
`pi-web-kit` provides provider-backed search, page fetching, library docs lookup, and code-context search with bounded output, chunked reads, URL validation, and an in-memory fetch cache designed for agent workflows.
|
|
6
6
|
|
|
7
7
|
## Features
|
|
8
8
|
|
|
9
9
|
- `web_search` for current/external web information, including multi-query searches.
|
|
10
10
|
- `web_fetch` for reading one or more URLs, with `offset` / `limit` chunk reads for long pages.
|
|
11
|
-
-
|
|
11
|
+
- `library_search` and `library_docs` for library resolution and current, version-aware documentation/code examples.
|
|
12
|
+
- `code_search` for practical examples and implementation context.
|
|
13
|
+
- Multiple provider backends: Exa MCP, Exa API, TinyFish, Brave Search, Firecrawl, markdown.new, Context7, and Exa Code.
|
|
12
14
|
- Provider-tailored tool schemas at Pi startup/reload.
|
|
13
15
|
- URL validation: HTTP(S)-only, no embedded credentials, fragment stripping, duplicate removal, and length/count limits.
|
|
14
16
|
- In-memory fetch cache with TTL, LRU eviction, max entry count, max byte count, and cache keys based on provider/config/fetch-affecting options.
|
|
@@ -96,7 +98,8 @@ PI_OFFLINE=1 # disables install/update telemetry
|
|
|
96
98
|
PI_TELEMETRY=0 # disables install/update telemetry
|
|
97
99
|
PI_WEB_KIT_PROVIDER_SEARCH=exa_mcp|exa|tinyfish|brave|firecrawl
|
|
98
100
|
PI_WEB_KIT_PROVIDER_FETCH=exa_mcp|exa|tinyfish|markdown_new|firecrawl
|
|
99
|
-
EXA_API_KEY=...
|
|
101
|
+
EXA_API_KEY=... # enables Exa provider and code_search
|
|
102
|
+
CONTEXT7_API_KEY=... # enables library_search and library_docs
|
|
100
103
|
TINYFISH_API_KEY=...
|
|
101
104
|
BRAVE_SEARCH_API_KEY=...
|
|
102
105
|
FIRECRAWL_API_KEY=...
|
|
@@ -118,7 +121,9 @@ Example:
|
|
|
118
121
|
"provider_search": "firecrawl",
|
|
119
122
|
"provider_fetch": "markdown_new",
|
|
120
123
|
"apiKeys": {
|
|
121
|
-
"firecrawl": "..."
|
|
124
|
+
"firecrawl": "...",
|
|
125
|
+
"context7": "...",
|
|
126
|
+
"exa": "..."
|
|
122
127
|
},
|
|
123
128
|
"markdownNew": {
|
|
124
129
|
"method": "auto",
|
|
@@ -165,6 +170,39 @@ Fetches page content with the active fetch provider. Results are cached in memor
|
|
|
165
170
|
|
|
166
171
|
Provider-specific parameters are exposed only for the configured provider, such as TinyFish `format`, markdown.new `method` / `retainImages`, or Firecrawl `format`, `waitFor`, `mobile`, `location`, and `maxAge`.
|
|
167
172
|
|
|
173
|
+
### `library_search`
|
|
174
|
+
|
|
175
|
+
Resolves packages, frameworks, SDKs, APIs, CLIs, and libraries to canonical library IDs.
|
|
176
|
+
|
|
177
|
+
| Parameter | Type | Description |
|
|
178
|
+
|---|---|---|
|
|
179
|
+
| `libraryName` | string | Library/package/framework name to search for. |
|
|
180
|
+
| `query` | string | Optional user task/question for relevance ranking. |
|
|
181
|
+
| `fast` | boolean | Skip LLM reranking for lower latency. |
|
|
182
|
+
| `limit` | integer | Maximum libraries to return. Range: 1-20. Default: 10. |
|
|
183
|
+
|
|
184
|
+
### `library_docs`
|
|
185
|
+
|
|
186
|
+
Fetches current docs and code snippets for a library. Provide `libraryId`, or provide `libraryName` and the tool resolves the best match first.
|
|
187
|
+
|
|
188
|
+
| Parameter | Type | Description |
|
|
189
|
+
|---|---|---|
|
|
190
|
+
| `libraryId` | string | Canonical library ID, such as `/vercel/next.js`. |
|
|
191
|
+
| `libraryName` | string | Library name to resolve when `libraryId` is not known. |
|
|
192
|
+
| `query` | string | Specific docs question or coding task. |
|
|
193
|
+
| `version` | string | Optional version/tag to pin, appended as `@version`. |
|
|
194
|
+
| `fast` | boolean | Skip LLM reranking for lower latency. |
|
|
195
|
+
| `limit` | integer | Maximum code and info snippets to return. Range: 1-20. Default: 10. |
|
|
196
|
+
|
|
197
|
+
### `code_search`
|
|
198
|
+
|
|
199
|
+
Finds practical code examples, implementation context, setup snippets, migrations, usage patterns, and error-message research.
|
|
200
|
+
|
|
201
|
+
| Parameter | Type | Description |
|
|
202
|
+
|---|---|---|
|
|
203
|
+
| `query` | string | Code-context query. |
|
|
204
|
+
| `tokensNum` | `"dynamic"` or integer | Output token target. Integer range: 50-100000. Default: `"dynamic"`. |
|
|
205
|
+
|
|
168
206
|
## Cache and limits
|
|
169
207
|
|
|
170
208
|
`web_fetch` uses an in-memory cache for the current Pi process.
|
|
@@ -183,7 +221,7 @@ Cache keys include the provider, canonical URL, fetch-affecting parameters, rele
|
|
|
183
221
|
|
|
184
222
|
## Privacy and security
|
|
185
223
|
|
|
186
|
-
`pi-web-kit` sends search queries and fetched URLs to the configured provider. Fetch providers may also receive provider-specific options. API keys are read from environment variables or local config files and are used only for provider requests.
|
|
224
|
+
`pi-web-kit` sends search queries and fetched URLs to the configured provider. Developer-search tools send library/doc queries to Context7 and code-context queries to Exa when those tools are enabled. Fetch providers may also receive provider-specific options. API keys are read from environment variables or local config files and are used only for provider requests.
|
|
187
225
|
|
|
188
226
|
The extension rejects non-HTTP(S) URLs and URLs with embedded username/password credentials. Provider responses are not sandboxed; they are returned to Pi as tool output.
|
|
189
227
|
|
package/extensions/index.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { fetchCache, type CachedPage } from "../src/cache.js";
|
|
|
5
5
|
import { resolveConfig } from "../src/config.js";
|
|
6
6
|
import { DEFAULT_FETCH_LIMIT, DEFAULT_NUM_RESULTS, MAX_LIMIT, MAX_NUM_RESULTS, MAX_OFFSET, MAX_QUERY_COUNT, MAX_URL_COUNT, MULTI_FETCH_LIMIT } from "../src/limits.js";
|
|
7
7
|
import { truncateText } from "../src/http.js";
|
|
8
|
-
import { createFetchProvider, createSearchProvider } from "../src/providers/index.js";
|
|
8
|
+
import { createCodeSearchProvider, createContext7Provider, createFetchProvider, createSearchProvider } from "../src/providers/index.js";
|
|
9
9
|
import { mapFetchResults } from "../src/providers/fallback.js";
|
|
10
10
|
import type { FetchProviderName, SearchProviderName, WebFetchResult } from "../src/types.js";
|
|
11
11
|
import { canonicalWebUrl, normalizeUrlInput } from "../src/urls.js";
|
|
@@ -55,8 +55,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
55
55
|
emitProgress(onUpdate, progress);
|
|
56
56
|
}
|
|
57
57
|
const result = { provider: config.provider_search, queries: grouped };
|
|
58
|
-
|
|
59
|
-
return { content: [{ type: "text", text }], details: boundedDetails(result) };
|
|
58
|
+
return jsonToolResult(result);
|
|
60
59
|
},
|
|
61
60
|
renderCall(args, theme) {
|
|
62
61
|
return new Text(renderWebCall("search", args as Record<string, any>, theme), 0, 0);
|
|
@@ -85,13 +84,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
85
84
|
assertProviderUnchanged("web_fetch", startupConfig.provider_fetch, config.provider_fetch);
|
|
86
85
|
const progress = createProgress("fetch", config.provider_fetch, urls);
|
|
87
86
|
const result = await fetchWithCache(config.provider_fetch, params, urls, signal, config, (event) => {
|
|
88
|
-
|
|
89
|
-
else if (event.status === "done") markProgressDone(progress, event.url, event.note);
|
|
90
|
-
else if (event.status === "error") markProgressError(progress, event.url, event.error);
|
|
87
|
+
updateFetchProgress(progress, event);
|
|
91
88
|
emitProgress(onUpdate, progress);
|
|
92
89
|
});
|
|
93
|
-
|
|
94
|
-
return { content: [{ type: "text", text }], details: boundedDetails(result) };
|
|
90
|
+
return jsonToolResult(result);
|
|
95
91
|
},
|
|
96
92
|
renderCall(args, theme) {
|
|
97
93
|
return new Text(renderWebCall("fetch", args as Record<string, any>, theme), 0, 0);
|
|
@@ -100,6 +96,69 @@ export default function (pi: ExtensionAPI) {
|
|
|
100
96
|
return renderWebResult("fetch", result, options, theme, context);
|
|
101
97
|
},
|
|
102
98
|
});
|
|
99
|
+
|
|
100
|
+
if (startupConfig.apiKeys.context7) {
|
|
101
|
+
pi.registerTool({
|
|
102
|
+
name: "library_search",
|
|
103
|
+
label: "Library Search",
|
|
104
|
+
description: "Resolve library, package, framework, SDK, API, or CLI names to canonical library IDs.",
|
|
105
|
+
promptSnippet: "Resolve a library name to a canonical library ID before querying docs.",
|
|
106
|
+
promptGuidelines: ["Use library_search when a library/framework/package is ambiguous or you need a canonical library ID."],
|
|
107
|
+
parameters: buildLibrarySearchSchema(),
|
|
108
|
+
async execute(_toolCallId, rawParams, signal, _onUpdate, ctx) {
|
|
109
|
+
const params = rawParams as Record<string, any>;
|
|
110
|
+
const libraryName = requiredString(params.libraryName, "libraryName");
|
|
111
|
+
const query = optionalString(params.query, "query") ?? libraryName;
|
|
112
|
+
const limit = parseInteger(params.limit, 10, "limit", 1, MAX_NUM_RESULTS);
|
|
113
|
+
const provider = createContext7Provider(runtimeConfig(pi, ctx.cwd));
|
|
114
|
+
const result = await provider.searchLibraries({ libraryName, query, fast: params.fast === true, limit }, signal);
|
|
115
|
+
return jsonToolResult(result);
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
pi.registerTool({
|
|
120
|
+
name: "library_docs",
|
|
121
|
+
label: "Library Docs",
|
|
122
|
+
description: "Fetch current, version-aware documentation and code examples for a library.",
|
|
123
|
+
promptSnippet: "Get current library documentation and code examples.",
|
|
124
|
+
promptGuidelines: ["Use library_docs for current APIs, framework behavior, SDK examples, package docs, and version-specific library questions."],
|
|
125
|
+
parameters: buildLibraryDocsSchema(),
|
|
126
|
+
async execute(_toolCallId, rawParams, signal, _onUpdate, ctx) {
|
|
127
|
+
const params = rawParams as Record<string, any>;
|
|
128
|
+
const query = requiredString(params.query, "query");
|
|
129
|
+
const limit = parseInteger(params.limit, 10, "limit", 1, MAX_NUM_RESULTS);
|
|
130
|
+
const provider = createContext7Provider(runtimeConfig(pi, ctx.cwd));
|
|
131
|
+
let libraryId = optionalString(params.libraryId, "libraryId");
|
|
132
|
+
if (!libraryId) {
|
|
133
|
+
const libraryName = requiredString(params.libraryName, "libraryName");
|
|
134
|
+
const resolved = await provider.searchLibraries({ libraryName, query, fast: params.fast === true, limit: 1 }, signal);
|
|
135
|
+
libraryId = resolved.results[0]?.id;
|
|
136
|
+
if (!libraryId) throw new Error(`No library found for '${libraryName}'. Try library_search with a more specific name.`);
|
|
137
|
+
}
|
|
138
|
+
const result = await provider.getDocs({ libraryId, query, version: optionalString(params.version, "version"), type: "json", fast: params.fast === true, limit }, signal);
|
|
139
|
+
return jsonToolResult(result);
|
|
140
|
+
},
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (startupConfig.apiKeys.exa) {
|
|
145
|
+
pi.registerTool({
|
|
146
|
+
name: "code_search",
|
|
147
|
+
label: "Code Search",
|
|
148
|
+
description: "Find practical code examples, usage patterns, setup snippets, migrations, and error context.",
|
|
149
|
+
promptSnippet: "Find real-world code examples, usage patterns, migrations, and error context.",
|
|
150
|
+
promptGuidelines: ["Use code_search for real-world code examples, GitHub/open-source usage patterns, API syntax examples, setup snippets, migrations, and error messages."],
|
|
151
|
+
parameters: buildCodeSearchSchema(),
|
|
152
|
+
async execute(_toolCallId, rawParams, signal, _onUpdate, ctx) {
|
|
153
|
+
const params = rawParams as Record<string, any>;
|
|
154
|
+
const query = requiredString(params.query, "query");
|
|
155
|
+
const tokensNum = parseTokensNum(params.tokensNum);
|
|
156
|
+
const provider = createCodeSearchProvider(runtimeConfig(pi, ctx.cwd));
|
|
157
|
+
const result = await provider.searchCode({ query, tokensNum }, signal);
|
|
158
|
+
return jsonToolResult(result);
|
|
159
|
+
},
|
|
160
|
+
});
|
|
161
|
+
}
|
|
103
162
|
}
|
|
104
163
|
|
|
105
164
|
type ProgressKind = "search" | "fetch";
|
|
@@ -152,12 +211,39 @@ export function buildFetchSchema(provider: FetchProviderName) {
|
|
|
152
211
|
return Type.Object(props, { additionalProperties: false });
|
|
153
212
|
}
|
|
154
213
|
|
|
155
|
-
function
|
|
156
|
-
return
|
|
214
|
+
export function buildLibrarySearchSchema() {
|
|
215
|
+
return Type.Object({
|
|
216
|
+
libraryName: Type.String({ description: "Library, package, framework, SDK, API, CLI, or product name", minLength: 1, maxLength: 500 }),
|
|
217
|
+
query: Type.Optional(Type.String({ description: "User task/question used for relevance ranking", minLength: 1, maxLength: 500 })),
|
|
218
|
+
fast: Type.Optional(Type.Boolean({ description: "Skip LLM reranking for lower latency" })),
|
|
219
|
+
limit: Type.Optional(int("Maximum libraries to return", 1, MAX_NUM_RESULTS)),
|
|
220
|
+
}, { additionalProperties: false });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function buildLibraryDocsSchema() {
|
|
224
|
+
return Type.Object({
|
|
225
|
+
libraryId: Type.Optional(Type.String({ description: "Canonical library ID, for example /vercel/next.js", minLength: 1, maxLength: 500 })),
|
|
226
|
+
libraryName: Type.Optional(Type.String({ description: "Library name to resolve when libraryId is not known", minLength: 1, maxLength: 500 })),
|
|
227
|
+
query: Type.String({ description: "Specific docs question or coding task", minLength: 1, maxLength: 500 }),
|
|
228
|
+
version: Type.Optional(Type.String({ description: "Optional version/tag to pin, appended as @version", minLength: 1, maxLength: 100 })),
|
|
229
|
+
fast: Type.Optional(Type.Boolean({ description: "Skip LLM reranking for lower latency" })),
|
|
230
|
+
limit: Type.Optional(int("Maximum code and info snippets to return", 1, MAX_NUM_RESULTS)),
|
|
231
|
+
}, { additionalProperties: false });
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export function buildCodeSearchSchema() {
|
|
235
|
+
return Type.Object({
|
|
236
|
+
query: Type.String({ description: "Code-context query for examples, APIs, setup, migrations, or errors", minLength: 1, maxLength: 2000 }),
|
|
237
|
+
tokensNum: Type.Optional(Type.Union([Type.Literal("dynamic"), int("Target response tokens", 50, MAX_LIMIT)])),
|
|
238
|
+
}, { additionalProperties: false });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function buildSearchDescription(_provider: SearchProviderName): string {
|
|
242
|
+
return "Search the web. Use query or queries; returns compact results grouped by query.";
|
|
157
243
|
}
|
|
158
244
|
|
|
159
|
-
function buildFetchDescription(
|
|
160
|
-
return
|
|
245
|
+
function buildFetchDescription(_provider: FetchProviderName): string {
|
|
246
|
+
return "Fetch URL content. Results are cached by URL/options; use offset/limit to read long pages in chunks.";
|
|
161
247
|
}
|
|
162
248
|
|
|
163
249
|
|
|
@@ -249,12 +335,37 @@ export function buildCacheKey(provider: FetchProviderName, url: string, params:
|
|
|
249
335
|
return `${provider}\0${scope}\0${JSON.stringify(affecting)}\0${canonical}`;
|
|
250
336
|
}
|
|
251
337
|
|
|
338
|
+
function runtimeConfig(pi: ExtensionAPI, cwd: string) {
|
|
339
|
+
return resolveConfig({ providerSearch: pi.getFlag("web-provider-search"), providerFetch: pi.getFlag("web-provider-fetch") }, cwd);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function jsonToolResult(result: unknown) {
|
|
343
|
+
const text = truncateText(JSON.stringify(result, null, 2));
|
|
344
|
+
return { content: [{ type: "text" as const, text }], details: boundedDetails(result) };
|
|
345
|
+
}
|
|
346
|
+
|
|
252
347
|
function parseInteger(value: unknown, defaultValue: number, name: string, min: number, max: number): number {
|
|
253
348
|
if (value == null) return defaultValue;
|
|
254
349
|
if (typeof value !== "number" || !Number.isInteger(value) || !Number.isFinite(value) || value < min || value > max) throw new Error(`${name} must be a finite integer between ${min} and ${max}.`);
|
|
255
350
|
return value;
|
|
256
351
|
}
|
|
257
352
|
|
|
353
|
+
function requiredString(value: unknown, name: string): string {
|
|
354
|
+
if (typeof value !== "string" || !value.trim()) throw new Error(`${name} must be a non-empty string.`);
|
|
355
|
+
return value.trim();
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function optionalString(value: unknown, name: string): string | undefined {
|
|
359
|
+
if (value == null) return undefined;
|
|
360
|
+
return requiredString(value, name);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function parseTokensNum(value: unknown): "dynamic" | number {
|
|
364
|
+
if (value == null) return "dynamic";
|
|
365
|
+
if (value === "dynamic") return "dynamic";
|
|
366
|
+
return parseInteger(value, 0, "tokensNum", 50, MAX_LIMIT);
|
|
367
|
+
}
|
|
368
|
+
|
|
258
369
|
function fetchConfigDefaults(provider: FetchProviderName, config?: any): Record<string, unknown> {
|
|
259
370
|
if (provider === "markdown_new") return { method: config?.markdownNew?.method ?? "auto", retainImages: config?.markdownNew?.retainImages ?? false };
|
|
260
371
|
if (provider === "firecrawl") return { onlyMainContent: true, format: "markdown" };
|
|
@@ -277,11 +388,74 @@ function assertProviderUnchanged(tool: string, startup: string, runtime: string)
|
|
|
277
388
|
}
|
|
278
389
|
|
|
279
390
|
function boundedDetails(value: any): unknown {
|
|
280
|
-
if (value?.queries && Array.isArray(value.queries)) return
|
|
281
|
-
if (value?.results && Array.isArray(value.results)) return
|
|
391
|
+
if (value?.queries && Array.isArray(value.queries)) return searchDetails(value);
|
|
392
|
+
if (value?.provider === "context7" && value?.results && Array.isArray(value.results)) return librarySearchDetails(value);
|
|
393
|
+
if (value?.provider === "context7" && value?.codeSnippets && value?.infoSnippets) return libraryDocsDetails(value);
|
|
394
|
+
if (value?.provider === "exa" && typeof value.response === "string") return codeSearchDetails(value);
|
|
395
|
+
if (value?.results && Array.isArray(value.results)) return fetchDetails(value);
|
|
282
396
|
return value;
|
|
283
397
|
}
|
|
284
398
|
|
|
399
|
+
function searchDetails(value: any) {
|
|
400
|
+
return {
|
|
401
|
+
provider: value.provider,
|
|
402
|
+
queries: value.queries.map((q: any) => ({
|
|
403
|
+
query: q.query,
|
|
404
|
+
resultCount: (q.results ?? []).length,
|
|
405
|
+
results: (q.results ?? []).map((r: any) => ({ title: r.title, url: r.url, siteName: r.siteName, position: r.position })),
|
|
406
|
+
})),
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function librarySearchDetails(value: any) {
|
|
411
|
+
return {
|
|
412
|
+
provider: value.provider,
|
|
413
|
+
libraryName: value.libraryName,
|
|
414
|
+
resultCount: value.results.length,
|
|
415
|
+
results: value.results.map((r: any) => ({ id: r.id, title: r.title, state: r.state, trustScore: r.trustScore, versions: r.versions })),
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
function libraryDocsDetails(value: any) {
|
|
420
|
+
const codeSources = value.codeSnippets.map((s: any) => s.codeId);
|
|
421
|
+
const infoSources = value.infoSnippets.map((s: any) => s.pageId);
|
|
422
|
+
return {
|
|
423
|
+
provider: value.provider,
|
|
424
|
+
libraryId: value.libraryId,
|
|
425
|
+
query: value.query,
|
|
426
|
+
codeSnippetCount: value.codeSnippets.length,
|
|
427
|
+
infoSnippetCount: value.infoSnippets.length,
|
|
428
|
+
sources: [...codeSources, ...infoSources].filter(Boolean).slice(0, MAX_NUM_RESULTS),
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function codeSearchDetails(value: any) {
|
|
433
|
+
return {
|
|
434
|
+
provider: value.provider,
|
|
435
|
+
query: value.query,
|
|
436
|
+
resultsCount: value.resultsCount,
|
|
437
|
+
outputTokens: value.outputTokens,
|
|
438
|
+
requestId: value.requestId,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function fetchDetails(value: any) {
|
|
443
|
+
return {
|
|
444
|
+
provider: value.provider,
|
|
445
|
+
results: value.results.map((r: any) => ({
|
|
446
|
+
url: r.url,
|
|
447
|
+
fetchedUrl: r.fetchedUrl,
|
|
448
|
+
title: r.title,
|
|
449
|
+
format: r.format,
|
|
450
|
+
cached: r.cached,
|
|
451
|
+
refreshed: r.refreshed,
|
|
452
|
+
cacheKey: r.cacheKey,
|
|
453
|
+
range: r.range,
|
|
454
|
+
error: r.error,
|
|
455
|
+
})),
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
|
|
285
459
|
function createProgress(kind: ProgressKind, provider: string, labels: string[]): WebProgress {
|
|
286
460
|
return { kind, provider, total: labels.length, completed: 0, items: labels.map((label) => ({ label, status: "pending" })) };
|
|
287
461
|
}
|
|
@@ -307,6 +481,20 @@ function markProgressError(progress: WebProgress, label: string, error?: string)
|
|
|
307
481
|
progress.completed++;
|
|
308
482
|
}
|
|
309
483
|
|
|
484
|
+
function updateFetchProgress(progress: WebProgress, event: FetchProgressEvent) {
|
|
485
|
+
switch (event.status) {
|
|
486
|
+
case "current":
|
|
487
|
+
markProgressCurrent(progress, event.url);
|
|
488
|
+
break;
|
|
489
|
+
case "done":
|
|
490
|
+
markProgressDone(progress, event.url, event.note);
|
|
491
|
+
break;
|
|
492
|
+
case "error":
|
|
493
|
+
markProgressError(progress, event.url, event.error);
|
|
494
|
+
break;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
310
498
|
function emitProgress(onUpdate: ((patch: any) => void) | undefined, progress: WebProgress) {
|
|
311
499
|
const verb = progress.kind === "search" ? "Searching web" : "Fetching pages";
|
|
312
500
|
onUpdate?.({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-web-kit",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Context-efficient web search and fetch tools for Pi.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -20,6 +20,9 @@
|
|
|
20
20
|
"pi",
|
|
21
21
|
"web-search",
|
|
22
22
|
"web-fetch",
|
|
23
|
+
"library-docs",
|
|
24
|
+
"code-search",
|
|
25
|
+
"context7",
|
|
23
26
|
"exa",
|
|
24
27
|
"firecrawl"
|
|
25
28
|
],
|
|
@@ -55,12 +58,12 @@
|
|
|
55
58
|
"typebox": "*"
|
|
56
59
|
},
|
|
57
60
|
"devDependencies": {
|
|
58
|
-
"@earendil-works/pi-ai": "^0.
|
|
59
|
-
"@earendil-works/pi-coding-agent": "^0.
|
|
60
|
-
"@earendil-works/pi-tui": "^0.
|
|
61
|
+
"@earendil-works/pi-ai": "^0.78.0",
|
|
62
|
+
"@earendil-works/pi-coding-agent": "^0.78.0",
|
|
63
|
+
"@earendil-works/pi-tui": "^0.78.0",
|
|
61
64
|
"@types/node": "^25.6.2",
|
|
62
|
-
"tsx": "^4.
|
|
63
|
-
"typebox": "^1.1.
|
|
65
|
+
"tsx": "^4.22.4",
|
|
66
|
+
"typebox": "^1.1.39",
|
|
64
67
|
"typescript": "^6.0.3"
|
|
65
68
|
},
|
|
66
69
|
"publishConfig": {
|
package/src/cache.ts
CHANGED
|
@@ -22,7 +22,10 @@ export class FetchCache {
|
|
|
22
22
|
constructor(private opts = { maxEntries: FETCH_CACHE_MAX_ENTRIES, maxBytes: FETCH_CACHE_MAX_BYTES, ttlMs: FETCH_CACHE_TTL_MS }) {}
|
|
23
23
|
|
|
24
24
|
get totalCachedBytes() { return this.totalBytes; }
|
|
25
|
-
get size() {
|
|
25
|
+
get size() {
|
|
26
|
+
this.evictExpired();
|
|
27
|
+
return this.entries.size;
|
|
28
|
+
}
|
|
26
29
|
|
|
27
30
|
get(key: string, now = Date.now()): CachedPage | undefined {
|
|
28
31
|
const entry = this.entries.get(key);
|
|
@@ -47,11 +50,15 @@ export class FetchCache {
|
|
|
47
50
|
return stored;
|
|
48
51
|
}
|
|
49
52
|
|
|
50
|
-
clear() {
|
|
53
|
+
clear() {
|
|
54
|
+
this.entries.clear();
|
|
55
|
+
this.totalBytes = 0;
|
|
56
|
+
}
|
|
51
57
|
|
|
52
58
|
private delete(key: string) {
|
|
53
59
|
const old = this.entries.get(key);
|
|
54
|
-
if (old)
|
|
60
|
+
if (!old) return;
|
|
61
|
+
this.totalBytes -= old.page.bytes;
|
|
55
62
|
this.entries.delete(key);
|
|
56
63
|
}
|
|
57
64
|
|
|
@@ -65,7 +72,9 @@ export class FetchCache {
|
|
|
65
72
|
}
|
|
66
73
|
|
|
67
74
|
private evictExpired(now = Date.now()) {
|
|
68
|
-
for (const [key, entry] of this.entries)
|
|
75
|
+
for (const [key, entry] of this.entries) {
|
|
76
|
+
if (now - entry.page.fetchedAt > this.opts.ttlMs) this.delete(key);
|
|
77
|
+
}
|
|
69
78
|
}
|
|
70
79
|
}
|
|
71
80
|
|
package/src/config.ts
CHANGED
|
@@ -5,6 +5,19 @@ import type { FetchProviderName, SearchProviderName, WebKitConfig } from "./type
|
|
|
5
5
|
|
|
6
6
|
const SEARCH = ["exa_mcp", "exa", "tinyfish", "brave", "firecrawl"] as const;
|
|
7
7
|
const FETCH = ["exa_mcp", "exa", "tinyfish", "markdown_new", "firecrawl"] as const;
|
|
8
|
+
const DEFAULT_CONFIG: WebKitConfig = {
|
|
9
|
+
provider_search: "exa_mcp",
|
|
10
|
+
provider_fetch: "exa_mcp",
|
|
11
|
+
apiKeys: {},
|
|
12
|
+
markdownNew: { method: "auto", retainImages: false },
|
|
13
|
+
};
|
|
14
|
+
const PROVIDER_ENV_NAMES = {
|
|
15
|
+
exa: "EXA_API_KEY",
|
|
16
|
+
tinyfish: "TINYFISH_API_KEY",
|
|
17
|
+
brave: "BRAVE_SEARCH_API_KEY",
|
|
18
|
+
firecrawl: "FIRECRAWL_API_KEY",
|
|
19
|
+
context7: "CONTEXT7_API_KEY",
|
|
20
|
+
} as const;
|
|
8
21
|
|
|
9
22
|
type PartialConfig = Partial<Omit<WebKitConfig, "apiKeys" | "markdownNew">> & {
|
|
10
23
|
apiKeys?: Partial<WebKitConfig["apiKeys"]>;
|
|
@@ -12,12 +25,7 @@ type PartialConfig = Partial<Omit<WebKitConfig, "apiKeys" | "markdownNew">> & {
|
|
|
12
25
|
};
|
|
13
26
|
|
|
14
27
|
export function resolveConfig(flags: { providerSearch?: unknown; providerFetch?: unknown } = {}, cwd = process.cwd(), env = process.env): WebKitConfig {
|
|
15
|
-
let cfg
|
|
16
|
-
provider_search: "exa_mcp",
|
|
17
|
-
provider_fetch: "exa_mcp",
|
|
18
|
-
apiKeys: {},
|
|
19
|
-
markdownNew: { method: "auto", retainImages: false },
|
|
20
|
-
};
|
|
28
|
+
let cfg = DEFAULT_CONFIG;
|
|
21
29
|
cfg = merge(cfg, {
|
|
22
30
|
provider_search: env.PI_WEB_KIT_PROVIDER_SEARCH as SearchProviderName | undefined,
|
|
23
31
|
provider_fetch: env.PI_WEB_KIT_PROVIDER_FETCH as FetchProviderName | undefined,
|
|
@@ -26,11 +34,13 @@ export function resolveConfig(flags: { providerSearch?: unknown; providerFetch?:
|
|
|
26
34
|
tinyfish: env.TINYFISH_API_KEY,
|
|
27
35
|
brave: env.BRAVE_SEARCH_API_KEY,
|
|
28
36
|
firecrawl: env.FIRECRAWL_API_KEY,
|
|
37
|
+
context7: env.CONTEXT7_API_KEY,
|
|
29
38
|
},
|
|
30
39
|
});
|
|
31
40
|
const home = env.HOME ?? homedir();
|
|
32
41
|
for (const path of [join(home, ".pi/agent/pi-web-kit.json"), join(cwd, ".pi-web-kit.json")]) {
|
|
33
|
-
|
|
42
|
+
const fileConfig = readConfigFile(path);
|
|
43
|
+
if (fileConfig) cfg = merge(cfg, fileConfig);
|
|
34
44
|
}
|
|
35
45
|
cfg = merge(cfg, {
|
|
36
46
|
provider_search: flags.providerSearch as SearchProviderName | undefined,
|
|
@@ -63,9 +73,16 @@ export function validateFetchProvider(name: string): asserts name is FetchProvid
|
|
|
63
73
|
if (!FETCH.includes(name as FetchProviderName)) throw new Error(`Unknown fetch provider '${name}'. Expected one of: ${FETCH.join(", ")}.`);
|
|
64
74
|
}
|
|
65
75
|
|
|
66
|
-
|
|
76
|
+
function readConfigFile(path: string): PartialConfig | undefined {
|
|
77
|
+
try {
|
|
78
|
+
return JSON.parse(readFileSync(path, "utf8")) as PartialConfig;
|
|
79
|
+
} catch (e: any) {
|
|
80
|
+
if (e?.code !== "ENOENT") throw e;
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
67
84
|
|
|
68
|
-
export function requireKey(config: WebKitConfig, provider: "exa" | "tinyfish" | "brave" | "firecrawl"): string {
|
|
85
|
+
export function requireKey(config: WebKitConfig, provider: "exa" | "tinyfish" | "brave" | "firecrawl" | "context7"): string {
|
|
69
86
|
const key = config.apiKeys[provider];
|
|
70
87
|
const envName = PROVIDER_ENV_NAMES[provider];
|
|
71
88
|
if (!key) throw new Error(`${provider} provider requires ${envName} or apiKeys.${provider} in .pi-web-kit.json / ~/.pi/agent/pi-web-kit.json.`);
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { requestJson } from "../http.js";
|
|
2
|
+
import type { Context7ContextInput, Context7DocsResult, Context7LibrarySearchInput, Context7LibrarySearchResult, WebKitConfig } from "../types.js";
|
|
3
|
+
import { requireKey } from "../config.js";
|
|
4
|
+
|
|
5
|
+
const BASE_URL = "https://context7.com/api/v2/";
|
|
6
|
+
|
|
7
|
+
export class Context7Provider {
|
|
8
|
+
private key: string;
|
|
9
|
+
constructor(config: WebKitConfig) { this.key = requireKey(config, "context7"); }
|
|
10
|
+
|
|
11
|
+
async searchLibraries(input: Context7LibrarySearchInput, signal?: AbortSignal): Promise<Context7LibrarySearchResult> {
|
|
12
|
+
const query = input.query ?? input.libraryName;
|
|
13
|
+
const params = new URLSearchParams({ libraryName: input.libraryName, query });
|
|
14
|
+
addFastParam(params, input.fast);
|
|
15
|
+
const data = await requestJson<any>(buildApiUrl("libs/search", params), {
|
|
16
|
+
headers: this.headers(),
|
|
17
|
+
signal,
|
|
18
|
+
timeoutMs: 30_000,
|
|
19
|
+
});
|
|
20
|
+
const limit = input.limit ?? 10;
|
|
21
|
+
return {
|
|
22
|
+
provider: "context7",
|
|
23
|
+
libraryName: input.libraryName,
|
|
24
|
+
query,
|
|
25
|
+
searchFilterApplied: data.searchFilterApplied,
|
|
26
|
+
results: (data.results ?? []).slice(0, limit).map(toLibraryResult).filter((r: any) => r.id),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async getDocs(input: Context7ContextInput, signal?: AbortSignal): Promise<Context7DocsResult> {
|
|
31
|
+
const libraryId = withVersion(input.libraryId, input.version);
|
|
32
|
+
const params = new URLSearchParams({ libraryId, query: input.query, type: input.type ?? "json" });
|
|
33
|
+
addFastParam(params, input.fast);
|
|
34
|
+
const data = await requestJson<any>(buildApiUrl("context", params), {
|
|
35
|
+
headers: this.headers(),
|
|
36
|
+
signal,
|
|
37
|
+
timeoutMs: 45_000,
|
|
38
|
+
});
|
|
39
|
+
const limit = input.limit ?? 10;
|
|
40
|
+
return {
|
|
41
|
+
provider: "context7",
|
|
42
|
+
libraryId,
|
|
43
|
+
query: input.query,
|
|
44
|
+
codeSnippets: (data.codeSnippets ?? []).slice(0, limit).map(toCodeSnippet),
|
|
45
|
+
infoSnippets: (data.infoSnippets ?? []).slice(0, limit).map(toInfoSnippet),
|
|
46
|
+
rules: data.rules,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private headers(): HeadersInit {
|
|
51
|
+
return { authorization: `Bearer ${this.key}` };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function addFastParam(params: URLSearchParams, fast?: boolean) {
|
|
56
|
+
if (fast != null) params.set("fast", String(fast));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function buildApiUrl(pathname: "libs/search" | "context", params: URLSearchParams): string {
|
|
60
|
+
const url = new URL(pathname, BASE_URL);
|
|
61
|
+
for (const [key, value] of params) url.searchParams.set(key, value);
|
|
62
|
+
return url.toString();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function toLibraryResult(r: any) {
|
|
66
|
+
return {
|
|
67
|
+
id: r.id,
|
|
68
|
+
title: r.title,
|
|
69
|
+
description: r.description,
|
|
70
|
+
branch: r.branch,
|
|
71
|
+
lastUpdateDate: r.lastUpdateDate,
|
|
72
|
+
state: r.state,
|
|
73
|
+
totalTokens: r.totalTokens,
|
|
74
|
+
totalSnippets: r.totalSnippets,
|
|
75
|
+
stars: r.stars,
|
|
76
|
+
trustScore: r.trustScore,
|
|
77
|
+
benchmarkScore: r.benchmarkScore,
|
|
78
|
+
versions: r.versions,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function toCodeSnippet(s: any) {
|
|
83
|
+
return {
|
|
84
|
+
codeTitle: s.codeTitle,
|
|
85
|
+
codeDescription: s.codeDescription,
|
|
86
|
+
codeLanguage: s.codeLanguage,
|
|
87
|
+
codeTokens: s.codeTokens,
|
|
88
|
+
codeId: s.codeId,
|
|
89
|
+
pageTitle: s.pageTitle,
|
|
90
|
+
sourceFile: s.sourceFile,
|
|
91
|
+
isDynamic: s.isDynamic,
|
|
92
|
+
codeList: s.codeList,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function toInfoSnippet(s: any) {
|
|
97
|
+
return {
|
|
98
|
+
pageId: s.pageId,
|
|
99
|
+
breadcrumb: s.breadcrumb,
|
|
100
|
+
content: s.content,
|
|
101
|
+
contentTokens: s.contentTokens,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function withVersion(libraryId: string, version?: string): string {
|
|
106
|
+
const id = libraryId.trim();
|
|
107
|
+
const v = version?.trim();
|
|
108
|
+
if (!v) return id;
|
|
109
|
+
if (id.endsWith(`@${v}`) || id.endsWith(`/${v}`)) return id;
|
|
110
|
+
return `${id}@${v}`;
|
|
111
|
+
}
|
package/src/providers/exa-mcp.ts
CHANGED
|
@@ -73,18 +73,29 @@ function textFromContent(result: any): string {
|
|
|
73
73
|
function normalizeSearch(result: any) {
|
|
74
74
|
const structured = result?.structuredContent ?? result?.result ?? result;
|
|
75
75
|
const list = structured.results ?? structured.data ?? structured.items;
|
|
76
|
-
if (Array.isArray(list)) return list.map(
|
|
76
|
+
if (Array.isArray(list)) return list.map(toSearchResult).filter((r: any) => r.url);
|
|
77
77
|
const text = textFromContent(result);
|
|
78
78
|
const urls = [...text.matchAll(/https?:\/\/[^\s)\]}>"']+/g)].map((m) => m[0]);
|
|
79
79
|
return [...new Set(urls)].map((url, i) => ({ url, snippet: i === 0 ? text.slice(0, 1000) : undefined, position: i + 1 }));
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
function toSearchResult(r: any, index: number) {
|
|
83
|
+
return {
|
|
84
|
+
title: r.title,
|
|
85
|
+
url: r.url,
|
|
86
|
+
snippet: asSnippet(r.snippet ?? r.text ?? r.summary ?? r.highlights),
|
|
87
|
+
siteName: r.siteName,
|
|
88
|
+
position: index + 1,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
82
92
|
function normalizeFetch(result: any, urls: string[]) {
|
|
83
93
|
const structured = result?.structuredContent ?? result?.result ?? result;
|
|
84
94
|
const list = structured.results ?? structured.data ?? structured.pages;
|
|
85
95
|
if (Array.isArray(list)) return urls.map((url, i) => {
|
|
86
96
|
const r = list.find((x: any) => urlsMatch(x.url, url)) ?? list[i];
|
|
87
|
-
|
|
97
|
+
if (!r) return { url, error: "No content returned by Exa MCP." };
|
|
98
|
+
return { url, title: r.title, content: r.markdown ?? r.text ?? r.content ?? r.html, format: "markdown" as const, metadata: r, error: r.error };
|
|
88
99
|
});
|
|
89
100
|
const text = textFromContent(result);
|
|
90
101
|
return urls.length <= 1 ? [{ url: urls[0] ?? "", content: text, format: "markdown" as const }] : urls.map((url) => ({ url, content: text, format: "markdown" as const }));
|
package/src/providers/exa.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { asSnippet, normalizeUrls, requestJson } from "../http.js";
|
|
2
2
|
import { DEFAULT_NUM_RESULTS } from "../limits.js";
|
|
3
3
|
import { urlsMatch } from "../urls.js";
|
|
4
|
-
import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
|
|
4
|
+
import type { ExaCodeInput, ExaCodeResult, FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
|
|
5
5
|
import { requireKey } from "../config.js";
|
|
6
6
|
import { applyExaFetchFallbacks } from "./fallback.js";
|
|
7
7
|
|
|
@@ -9,6 +9,10 @@ export class ExaProvider implements SearchProvider, FetchProvider {
|
|
|
9
9
|
private key: string;
|
|
10
10
|
constructor(private config: WebKitConfig) { this.key = requireKey(config, "exa"); }
|
|
11
11
|
|
|
12
|
+
private headers() {
|
|
13
|
+
return { "content-type": "application/json", "x-api-key": this.key };
|
|
14
|
+
}
|
|
15
|
+
|
|
12
16
|
async search(input: SearchInput, signal?: AbortSignal) {
|
|
13
17
|
const body = {
|
|
14
18
|
query: input.query,
|
|
@@ -25,7 +29,7 @@ export class ExaProvider implements SearchProvider, FetchProvider {
|
|
|
25
29
|
};
|
|
26
30
|
const data = await requestJson<any>("https://api.exa.ai/search", {
|
|
27
31
|
method: "POST",
|
|
28
|
-
headers:
|
|
32
|
+
headers: this.headers(),
|
|
29
33
|
body: JSON.stringify(body),
|
|
30
34
|
signal,
|
|
31
35
|
timeoutMs: 30000,
|
|
@@ -48,7 +52,7 @@ export class ExaProvider implements SearchProvider, FetchProvider {
|
|
|
48
52
|
if (urls.length === 0) return { provider: "exa", results: [] };
|
|
49
53
|
const data = await requestJson<any>("https://api.exa.ai/contents", {
|
|
50
54
|
method: "POST",
|
|
51
|
-
headers:
|
|
55
|
+
headers: this.headers(),
|
|
52
56
|
body: JSON.stringify({ urls, text: true, highlights: false }),
|
|
53
57
|
signal,
|
|
54
58
|
timeoutMs: 45000,
|
|
@@ -61,4 +65,26 @@ export class ExaProvider implements SearchProvider, FetchProvider {
|
|
|
61
65
|
}) };
|
|
62
66
|
return applyExaFetchFallbacks(this.config, input, urls, primary, signal);
|
|
63
67
|
}
|
|
68
|
+
|
|
69
|
+
async searchCode(input: ExaCodeInput, signal?: AbortSignal): Promise<ExaCodeResult> {
|
|
70
|
+
const data = await requestJson<any>("https://api.exa.ai/context", {
|
|
71
|
+
method: "POST",
|
|
72
|
+
headers: this.headers(),
|
|
73
|
+
body: JSON.stringify({
|
|
74
|
+
query: input.query,
|
|
75
|
+
tokensNum: input.tokensNum ?? "dynamic",
|
|
76
|
+
}),
|
|
77
|
+
signal,
|
|
78
|
+
timeoutMs: 45_000,
|
|
79
|
+
});
|
|
80
|
+
return {
|
|
81
|
+
provider: "exa",
|
|
82
|
+
query: data.query ?? input.query,
|
|
83
|
+
response: data.response ?? "",
|
|
84
|
+
resultsCount: data.resultsCount,
|
|
85
|
+
searchTime: data.searchTime,
|
|
86
|
+
outputTokens: data.outputTokens,
|
|
87
|
+
requestId: data.requestId,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
64
90
|
}
|
|
@@ -48,8 +48,8 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
|
|
|
48
48
|
}),
|
|
49
49
|
});
|
|
50
50
|
const d = data.data ?? data;
|
|
51
|
-
const selected =
|
|
52
|
-
const content =
|
|
51
|
+
const selected = selectFirecrawlContent(d, format);
|
|
52
|
+
const content = stringifyContent(selected);
|
|
53
53
|
return { url: d.url ?? url, content, format, title: d.metadata?.title, metadata: d.metadata ?? d };
|
|
54
54
|
} catch (e) {
|
|
55
55
|
return { url, error: e instanceof Error ? e.message : String(e) };
|
|
@@ -58,3 +58,19 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
|
|
|
58
58
|
return { provider: "firecrawl" as const, results };
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
|
+
|
|
62
|
+
function selectFirecrawlContent(data: any, format: string): unknown {
|
|
63
|
+
switch (format) {
|
|
64
|
+
case "html":
|
|
65
|
+
return data.html;
|
|
66
|
+
case "json":
|
|
67
|
+
return data.json;
|
|
68
|
+
default:
|
|
69
|
+
return data.markdown;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function stringifyContent(value: unknown): string | undefined {
|
|
74
|
+
if (typeof value === "string") return value;
|
|
75
|
+
return value == null ? undefined : JSON.stringify(value, null, 2);
|
|
76
|
+
}
|
package/src/providers/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { validateFetchProvider, validateSearchProvider } from "../config.js";
|
|
2
2
|
import type { FetchProvider, SearchProvider, WebKitConfig } from "../types.js";
|
|
3
3
|
import { BraveProvider } from "./brave.js";
|
|
4
|
+
import { Context7Provider } from "./context7.js";
|
|
4
5
|
import { ExaMcpProvider } from "./exa-mcp.js";
|
|
5
6
|
import { ExaProvider } from "./exa.js";
|
|
6
7
|
import { FirecrawlProvider } from "./firecrawl.js";
|
|
@@ -28,3 +29,11 @@ export function createFetchProvider(config: WebKitConfig): FetchProvider {
|
|
|
28
29
|
case "firecrawl": return new FirecrawlProvider(config);
|
|
29
30
|
}
|
|
30
31
|
}
|
|
32
|
+
|
|
33
|
+
export function createContext7Provider(config: WebKitConfig): Context7Provider {
|
|
34
|
+
return new Context7Provider(config);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function createCodeSearchProvider(config: WebKitConfig): ExaProvider {
|
|
38
|
+
return new ExaProvider(config);
|
|
39
|
+
}
|
|
@@ -34,7 +34,11 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
|
|
|
34
34
|
const r = list.find((x: any) => (x.url ?? x.source_url) === url) ?? list[i];
|
|
35
35
|
if (!r) return { url, error: "No content returned by TinyFish." };
|
|
36
36
|
const content = r.text ?? r.content ?? r.markdown ?? r.html;
|
|
37
|
-
return { url, content:
|
|
37
|
+
return { url, content: stringifyContent(content), format: input.format ?? r.format ?? "markdown", title: r.title, metadata: r, error: r.error };
|
|
38
38
|
}) };
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
|
+
|
|
42
|
+
function stringifyContent(value: unknown): string | undefined {
|
|
43
|
+
return typeof value === "string" ? value : JSON.stringify(value, null, 2);
|
|
44
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -5,7 +5,7 @@ export type FetchFormat = "markdown" | "html" | "json";
|
|
|
5
5
|
export interface WebKitConfig {
|
|
6
6
|
provider_search: SearchProviderName;
|
|
7
7
|
provider_fetch: FetchProviderName;
|
|
8
|
-
apiKeys: Partial<Record<"exa" | "tinyfish" | "brave" | "firecrawl", string>>;
|
|
8
|
+
apiKeys: Partial<Record<"exa" | "tinyfish" | "brave" | "firecrawl" | "context7", string>>;
|
|
9
9
|
markdownNew: { method: "auto" | "ai" | "browser"; retainImages: boolean };
|
|
10
10
|
}
|
|
11
11
|
|
|
@@ -38,6 +38,77 @@ export interface WebFetchResult {
|
|
|
38
38
|
results: Array<{ url: string; content?: string; format?: FetchFormat; title?: string; metadata?: Record<string, unknown>; error?: string }>;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
export interface Context7LibrarySearchInput {
|
|
42
|
+
libraryName: string;
|
|
43
|
+
query?: string;
|
|
44
|
+
fast?: boolean;
|
|
45
|
+
limit?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface Context7LibrarySearchResult {
|
|
49
|
+
provider: "context7";
|
|
50
|
+
libraryName: string;
|
|
51
|
+
query: string;
|
|
52
|
+
searchFilterApplied?: boolean;
|
|
53
|
+
results: Array<{
|
|
54
|
+
id: string;
|
|
55
|
+
title?: string;
|
|
56
|
+
description?: string;
|
|
57
|
+
branch?: string;
|
|
58
|
+
lastUpdateDate?: string;
|
|
59
|
+
state?: string;
|
|
60
|
+
totalTokens?: number;
|
|
61
|
+
totalSnippets?: number;
|
|
62
|
+
stars?: number;
|
|
63
|
+
trustScore?: number;
|
|
64
|
+
benchmarkScore?: number;
|
|
65
|
+
versions?: string[];
|
|
66
|
+
}>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface Context7ContextInput {
|
|
70
|
+
libraryId: string;
|
|
71
|
+
query: string;
|
|
72
|
+
version?: string;
|
|
73
|
+
type?: "json";
|
|
74
|
+
fast?: boolean;
|
|
75
|
+
limit?: number;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface Context7DocsResult {
|
|
79
|
+
provider: "context7";
|
|
80
|
+
libraryId: string;
|
|
81
|
+
query: string;
|
|
82
|
+
codeSnippets: Array<{
|
|
83
|
+
codeTitle?: string;
|
|
84
|
+
codeDescription?: string;
|
|
85
|
+
codeLanguage?: string;
|
|
86
|
+
codeTokens?: number;
|
|
87
|
+
codeId?: string;
|
|
88
|
+
pageTitle?: string;
|
|
89
|
+
sourceFile?: string;
|
|
90
|
+
isDynamic?: boolean;
|
|
91
|
+
codeList?: Array<{ language: string; code: string }>;
|
|
92
|
+
}>;
|
|
93
|
+
infoSnippets: Array<{ pageId?: string; breadcrumb?: string; content: string; contentTokens?: number }>;
|
|
94
|
+
rules?: unknown;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface ExaCodeInput {
|
|
98
|
+
query: string;
|
|
99
|
+
tokensNum?: "dynamic" | number;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface ExaCodeResult {
|
|
103
|
+
provider: "exa";
|
|
104
|
+
query: string;
|
|
105
|
+
response: string;
|
|
106
|
+
resultsCount?: number;
|
|
107
|
+
searchTime?: number;
|
|
108
|
+
outputTokens?: number;
|
|
109
|
+
requestId?: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
41
112
|
export interface SearchProvider {
|
|
42
113
|
search(input: SearchInput, signal?: AbortSignal): Promise<WebSearchResult>;
|
|
43
114
|
}
|
package/src/urls.ts
CHANGED
|
@@ -17,10 +17,14 @@ export function normalizeWebUrl(value: string): string {
|
|
|
17
17
|
export function canonicalWebUrl(value: string): string {
|
|
18
18
|
const parsed = new URL(normalizeWebUrl(value));
|
|
19
19
|
parsed.hostname = parsed.hostname.toLowerCase();
|
|
20
|
-
if ((parsed
|
|
20
|
+
if (isDefaultPort(parsed)) parsed.port = "";
|
|
21
21
|
return parsed.toString();
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
function isDefaultPort(url: URL): boolean {
|
|
25
|
+
return (url.protocol === "https:" && url.port === "443") || (url.protocol === "http:" && url.port === "80");
|
|
26
|
+
}
|
|
27
|
+
|
|
24
28
|
export function normalizeUrlInput(input: { url?: string; urls?: string[] }, maxCount = MAX_URL_COUNT): string[] {
|
|
25
29
|
const raw = [...(Array.isArray(input.urls) ? input.urls : []), ...(input.url ? [input.url] : [])].map((url) => String(url).trim()).filter(Boolean);
|
|
26
30
|
const unique = [...new Set(raw.map(normalizeWebUrl))];
|
|
@@ -34,9 +38,12 @@ export function urlsMatch(a: string | undefined, b: string | undefined): boolean
|
|
|
34
38
|
const ca = canonicalWebUrl(a);
|
|
35
39
|
const cb = canonicalWebUrl(b);
|
|
36
40
|
if (ca === cb) return true;
|
|
37
|
-
|
|
38
|
-
return trimSlash(ca) === trimSlash(cb);
|
|
41
|
+
return trimTrailingSlash(ca) === trimTrailingSlash(cb);
|
|
39
42
|
} catch {
|
|
40
43
|
return a === b;
|
|
41
44
|
}
|
|
42
45
|
}
|
|
46
|
+
|
|
47
|
+
function trimTrailingSlash(value: string): string {
|
|
48
|
+
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
49
|
+
}
|