wave-agent-sdk 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +23 -1
- package/dist/agent.js +34 -4
- package/dist/managers/aiManager.d.ts +36 -0
- package/dist/managers/aiManager.js +240 -8
- package/dist/managers/messageManager.d.ts +1 -3
- package/dist/managers/messageManager.js +19 -9
- package/dist/managers/slashCommandManager.js +28 -0
- package/dist/managers/subagentManager.d.ts +0 -2
- package/dist/services/session.js +2 -9
- package/dist/telemetry/instrumentation.d.ts +1 -1
- package/dist/telemetry/instrumentation.js +57 -28
- package/dist/tools/bashTool.js +16 -6
- package/dist/tools/grepTool.js +11 -3
- package/dist/types/index.d.ts +3 -0
- package/dist/types/messaging.d.ts +2 -0
- package/dist/utils/containerSetup.js +0 -3
- package/dist/utils/messageOperations.d.ts +10 -1
- package/dist/utils/messageOperations.js +19 -1
- package/dist/utils/notificationXml.js +5 -0
- package/dist/utils/ripgrep.d.ts +4 -4
- package/dist/utils/ripgrep.js +4 -27
- package/package.json +20 -8
- package/bin/rg +0 -79
- package/dist/managers/forkedAgentManager.d.ts +0 -50
- package/dist/managers/forkedAgentManager.js +0 -130
- package/scripts/install_ripgrep.js +0 -111
- package/scripts/postinstall.js +0 -38
- package/src/agent.ts +0 -1271
- package/src/constants/subagents.ts +0 -4
- package/src/constants/toolLimits.ts +0 -15
- package/src/constants/tools.ts +0 -24
- package/src/core/plugin.ts +0 -237
- package/src/core/session.ts +0 -9
- package/src/index.ts +0 -38
- package/src/managers/MemoryRuleManager.ts +0 -198
- package/src/managers/aiManager.ts +0 -2429
- package/src/managers/backgroundTaskManager.ts +0 -480
- package/src/managers/bangManager.ts +0 -119
- package/src/managers/cronManager.ts +0 -401
- package/src/managers/foregroundTaskManager.ts +0 -30
- package/src/managers/forkedAgentManager.ts +0 -216
- package/src/managers/hookManager.ts +0 -1089
- package/src/managers/liveConfigManager.ts +0 -445
- package/src/managers/lspManager.ts +0 -470
- package/src/managers/mcpManager.ts +0 -1010
- package/src/managers/messageManager.ts +0 -1180
- package/src/managers/messageQueue.ts +0 -157
- package/src/managers/permissionManager.ts +0 -1140
- package/src/managers/planManager.ts +0 -112
- package/src/managers/pluginManager.ts +0 -330
- package/src/managers/pluginScopeManager.ts +0 -124
- package/src/managers/reversionManager.ts +0 -178
- package/src/managers/skillManager.ts +0 -578
- package/src/managers/slashCommandManager.ts +0 -564
- package/src/managers/subagentManager.ts +0 -915
- package/src/managers/toolManager.ts +0 -489
- package/src/managers/workflowManager.ts +0 -493
- package/src/prompts/autoMemory.ts +0 -33
- package/src/prompts/autoMemoryExtraction.ts +0 -146
- package/src/prompts/index.ts +0 -506
- package/src/prompts/planModeReminders.ts +0 -126
- package/src/services/GitService.ts +0 -131
- package/src/services/MarketplaceService.ts +0 -1052
- package/src/services/MemoryRuleService.ts +0 -71
- package/src/services/aiService.ts +0 -919
- package/src/services/authService.ts +0 -565
- package/src/services/autoMemoryService.ts +0 -280
- package/src/services/configurationService.ts +0 -1520
- package/src/services/fileWatcher.ts +0 -319
- package/src/services/hook.ts +0 -348
- package/src/services/initializationService.ts +0 -357
- package/src/services/interactionService.ts +0 -229
- package/src/services/jsonlHandler.ts +0 -295
- package/src/services/memory.ts +0 -234
- package/src/services/pluginLoader.ts +0 -252
- package/src/services/remoteSettingsService.ts +0 -366
- package/src/services/reversionService.ts +0 -122
- package/src/services/session.ts +0 -922
- package/src/services/taskManager.ts +0 -321
- package/src/telemetry/events.ts +0 -65
- package/src/telemetry/instrumentation.ts +0 -499
- package/src/telemetry/sessionTracing.ts +0 -348
- package/src/tools/agentTool.ts +0 -322
- package/src/tools/askUserQuestion.ts +0 -160
- package/src/tools/bashTool.ts +0 -621
- package/src/tools/buildTool.ts +0 -61
- package/src/tools/cronCreateTool.ts +0 -161
- package/src/tools/cronDeleteTool.ts +0 -51
- package/src/tools/cronListTool.ts +0 -47
- package/src/tools/editTool.ts +0 -313
- package/src/tools/enterPlanMode.ts +0 -124
- package/src/tools/enterWorktreeTool.ts +0 -190
- package/src/tools/exitPlanMode.ts +0 -139
- package/src/tools/exitWorktreeTool.ts +0 -236
- package/src/tools/globTool.ts +0 -167
- package/src/tools/grepTool.ts +0 -399
- package/src/tools/lspTool.ts +0 -883
- package/src/tools/readTool.ts +0 -426
- package/src/tools/skillTool.ts +0 -248
- package/src/tools/taskManagementTools.ts +0 -574
- package/src/tools/taskStopTool.ts +0 -78
- package/src/tools/types.ts +0 -137
- package/src/tools/webFetchTool.ts +0 -373
- package/src/tools/workflowTool.ts +0 -205
- package/src/tools/writeTool.ts +0 -267
- package/src/types/agent.ts +0 -128
- package/src/types/auth.ts +0 -19
- package/src/types/commands.ts +0 -30
- package/src/types/config.ts +0 -43
- package/src/types/configuration.ts +0 -168
- package/src/types/core.ts +0 -105
- package/src/types/cron.ts +0 -13
- package/src/types/environment.ts +0 -106
- package/src/types/fileSearch.ts +0 -4
- package/src/types/history.ts +0 -7
- package/src/types/hooks.ts +0 -256
- package/src/types/index.ts +0 -41
- package/src/types/lsp.ts +0 -98
- package/src/types/marketplace.ts +0 -71
- package/src/types/mcp.ts +0 -43
- package/src/types/memoryRule.ts +0 -31
- package/src/types/messaging.ts +0 -114
- package/src/types/permissions.ts +0 -80
- package/src/types/plugins.ts +0 -39
- package/src/types/processes.ts +0 -99
- package/src/types/reversion.ts +0 -29
- package/src/types/session.ts +0 -12
- package/src/types/skills.ts +0 -108
- package/src/types/tasks.ts +0 -13
- package/src/types/telemetry.ts +0 -98
- package/src/types/tools.ts +0 -44
- package/src/types/workflow.ts +0 -6
- package/src/utils/abortUtils.ts +0 -118
- package/src/utils/atomicWrite.ts +0 -61
- package/src/utils/bashParser.ts +0 -899
- package/src/utils/cacheControlUtils.ts +0 -426
- package/src/utils/commandArgumentParser.ts +0 -161
- package/src/utils/commandPathResolver.ts +0 -88
- package/src/utils/configPaths.ts +0 -220
- package/src/utils/configValidator.ts +0 -100
- package/src/utils/constants.ts +0 -39
- package/src/utils/container.ts +0 -92
- package/src/utils/containerSetup.ts +0 -371
- package/src/utils/convertMessagesForAPI.ts +0 -326
- package/src/utils/cronTasks.ts +0 -128
- package/src/utils/cronTasksLock.ts +0 -182
- package/src/utils/cronToHuman.ts +0 -99
- package/src/utils/customCommands.ts +0 -95
- package/src/utils/editUtils.ts +0 -20
- package/src/utils/fileFormat.ts +0 -40
- package/src/utils/fileSearch.ts +0 -151
- package/src/utils/fileUtils.ts +0 -223
- package/src/utils/gitUtils.ts +0 -315
- package/src/utils/globalLogger.ts +0 -128
- package/src/utils/groupMessagesByApiRound.ts +0 -121
- package/src/utils/hookMatcher.ts +0 -176
- package/src/utils/markdownParser.ts +0 -317
- package/src/utils/mcpUtils.ts +0 -138
- package/src/utils/messageOperations.ts +0 -641
- package/src/utils/modelCapabilities.ts +0 -30
- package/src/utils/nameGenerator.ts +0 -99
- package/src/utils/notificationXml.ts +0 -57
- package/src/utils/openaiClient.ts +0 -251
- package/src/utils/parseCronExpression.ts +0 -78
- package/src/utils/path.ts +0 -72
- package/src/utils/pathEncoder.ts +0 -400
- package/src/utils/pathSafety.ts +0 -40
- package/src/utils/promptHistory.ts +0 -168
- package/src/utils/ripgrep.ts +0 -35
- package/src/utils/shellResolver.ts +0 -190
- package/src/utils/skillParser.ts +0 -246
- package/src/utils/stringUtils.ts +0 -156
- package/src/utils/subagentParser.ts +0 -279
- package/src/utils/taskReminder.ts +0 -96
- package/src/utils/tokenCalculation.ts +0 -43
- package/src/utils/tokenEstimate.ts +0 -34
- package/src/utils/toolResultStorage.ts +0 -117
- package/src/utils/worktreeSession.ts +0 -26
- package/src/utils/worktreeUtils.ts +0 -786
- package/src/workflow/budgetTracker.ts +0 -34
- package/src/workflow/concurrencyLimiter.ts +0 -47
- package/src/workflow/journal.ts +0 -95
- package/src/workflow/progressReporter.ts +0 -141
- package/src/workflow/runState.ts +0 -65
- package/src/workflow/scriptRuntime.ts +0 -274
- package/src/workflow/structuredOutput.ts +0 -123
- package/src/workflow/types.ts +0 -101
- package/src/workflow/workflowApis.ts +0 -410
- package/vendor/ripgrep/linux-aarch64/rg +0 -0
- package/vendor/ripgrep/linux-x86_64/rg +0 -0
- package/vendor/ripgrep/macos-aarch64/rg +0 -0
- package/vendor/ripgrep/macos-x86_64/rg +0 -0
- package/vendor/ripgrep/windows-aarch64/rg.exe +0 -0
- package/vendor/ripgrep/windows-x86_64/rg.exe +0 -0
package/src/tools/types.ts
DELETED
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tool plugin interface definitions
|
|
3
|
-
*/
|
|
4
|
-
|
|
5
|
-
import { ChatCompletionFunctionTool } from "openai/resources.js";
|
|
6
|
-
import type {
|
|
7
|
-
PermissionMode,
|
|
8
|
-
PermissionCallback,
|
|
9
|
-
} from "../types/permissions.js";
|
|
10
|
-
|
|
11
|
-
import type { SubagentConfiguration } from "../utils/subagentParser.js";
|
|
12
|
-
import type { SkillMetadata } from "../types/skills.js";
|
|
13
|
-
|
|
14
|
-
export interface ToolPlugin {
|
|
15
|
-
name: string;
|
|
16
|
-
config: ChatCompletionFunctionTool;
|
|
17
|
-
execute: (
|
|
18
|
-
args: Record<string, unknown>,
|
|
19
|
-
context: ToolContext,
|
|
20
|
-
) => Promise<ToolResult>;
|
|
21
|
-
formatCompactParams?: (
|
|
22
|
-
params: Record<string, unknown>,
|
|
23
|
-
context: ToolContext,
|
|
24
|
-
) => string;
|
|
25
|
-
/**
|
|
26
|
-
* Function to provide a prompt to be added to the tool description
|
|
27
|
-
*/
|
|
28
|
-
prompt?: (args?: {
|
|
29
|
-
availableSubagents?: SubagentConfiguration[];
|
|
30
|
-
availableSkills?: SkillMetadata[];
|
|
31
|
-
workdir?: string;
|
|
32
|
-
isSubagent?: boolean;
|
|
33
|
-
}) => string;
|
|
34
|
-
/**
|
|
35
|
-
* Whether this tool is safe to run in parallel with other tools.
|
|
36
|
-
* Default (undefined) = true (parallel). Set to false for tools that
|
|
37
|
-
* perform read-modify-write on shared resources (e.g. Edit, Write).
|
|
38
|
-
*/
|
|
39
|
-
isConcurrencySafe?: boolean;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export interface ToolResult {
|
|
43
|
-
success: boolean;
|
|
44
|
-
content: string;
|
|
45
|
-
error?: string;
|
|
46
|
-
// Short output, used to display summary information in collapsed state
|
|
47
|
-
shortResult?: string;
|
|
48
|
-
// File path for operations that affect files
|
|
49
|
-
filePath?: string;
|
|
50
|
-
// Optional starting line number for file operations
|
|
51
|
-
startLineNumber?: number;
|
|
52
|
-
// Image data, for supporting multimedia content
|
|
53
|
-
images?: Array<{
|
|
54
|
-
data: string; // base64 encoded image data
|
|
55
|
-
mediaType?: string; // Image media type, such as "image/png"
|
|
56
|
-
}>;
|
|
57
|
-
// Whether the tool was manually backgrounded by the user (e.g. via Ctrl-B)
|
|
58
|
-
isManuallyBackgrounded?: boolean;
|
|
59
|
-
// Optional metadata for the tool result
|
|
60
|
-
metadata?: Record<string, unknown>;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export interface ToolContext {
|
|
64
|
-
abortSignal?: AbortSignal;
|
|
65
|
-
backgroundTaskManager?: import("../managers/backgroundTaskManager.js").BackgroundTaskManager;
|
|
66
|
-
workdir: string;
|
|
67
|
-
/** Tool manager instance for tool discovery (used by ToolSearchTool) */
|
|
68
|
-
toolManager?: import("../managers/toolManager.js").ToolManager;
|
|
69
|
-
/** Permission mode for this tool execution */
|
|
70
|
-
permissionMode?: PermissionMode;
|
|
71
|
-
/** Custom permission callback */
|
|
72
|
-
canUseToolCallback?: PermissionCallback;
|
|
73
|
-
/** Permission manager instance for permission checks */
|
|
74
|
-
permissionManager?: import("../managers/permissionManager.js").PermissionManager;
|
|
75
|
-
/** MCP manager instance for calling MCP tools */
|
|
76
|
-
mcpManager?: import("../managers/mcpManager.js").McpManager;
|
|
77
|
-
/** LSP manager instance for code intelligence */
|
|
78
|
-
lspManager?: import("../types/lsp.js").ILspManager;
|
|
79
|
-
/** Reversion manager instance for file snapshots */
|
|
80
|
-
reversionManager?: import("../managers/reversionManager.js").ReversionManager;
|
|
81
|
-
/** Current message ID for associating snapshots */
|
|
82
|
-
messageId?: string;
|
|
83
|
-
/** Foreground task manager for backgrounding tasks */
|
|
84
|
-
foregroundTaskManager?: import("../types/processes.js").IForegroundTaskManager;
|
|
85
|
-
/** Task manager instance for task management */
|
|
86
|
-
taskManager: import("../services/taskManager.js").TaskManager;
|
|
87
|
-
/** Subagent manager instance for agent delegation */
|
|
88
|
-
subagentManager?: import("../managers/subagentManager.js").SubagentManager;
|
|
89
|
-
/** Skill manager instance for skill invocation */
|
|
90
|
-
skillManager?: import("../managers/skillManager.js").SkillManager;
|
|
91
|
-
/** Cron manager instance for scheduling tasks */
|
|
92
|
-
cronManager?: import("../managers/cronManager.js").CronManager;
|
|
93
|
-
/** AI manager instance for AI operations */
|
|
94
|
-
aiManager?: import("../managers/aiManager.js").AIManager;
|
|
95
|
-
/** AI service instance for AI operations */
|
|
96
|
-
aiService?: typeof import("../services/aiService.js");
|
|
97
|
-
/** Message manager instance for message operations */
|
|
98
|
-
messageManager?: import("../managers/messageManager.js").MessageManager;
|
|
99
|
-
/** Current session ID */
|
|
100
|
-
sessionId?: string;
|
|
101
|
-
/** The ID of the current tool call */
|
|
102
|
-
toolCallId?: string;
|
|
103
|
-
/** Callback to update the short result of the current tool block */
|
|
104
|
-
onShortResultUpdate?: (shortResult: string) => void;
|
|
105
|
-
/** Callback to update the full result of the current tool block */
|
|
106
|
-
onResultUpdate?: (result: string) => void;
|
|
107
|
-
/** Limits for file reading operations */
|
|
108
|
-
fileReadingLimits?: {
|
|
109
|
-
maxSizeBytes: number;
|
|
110
|
-
maxTokens: number;
|
|
111
|
-
};
|
|
112
|
-
/** State of files read in the current session for deduplication */
|
|
113
|
-
readFileState?: Map<
|
|
114
|
-
string,
|
|
115
|
-
{
|
|
116
|
-
mtime: number;
|
|
117
|
-
hash: string;
|
|
118
|
-
offset?: number; // undefined = full read or Edit/Write entry (never dedup)
|
|
119
|
-
limit?: number;
|
|
120
|
-
}
|
|
121
|
-
>;
|
|
122
|
-
/** Hook manager instance for executing hooks */
|
|
123
|
-
hookManager?: import("../managers/hookManager.js").HookManager;
|
|
124
|
-
/** Callback to notify when the current working directory changes */
|
|
125
|
-
onCwdChange?: (newCwd: string) => void;
|
|
126
|
-
/** Original working directory (before any cd changes) for CWD reset */
|
|
127
|
-
originalWorkdir?: string;
|
|
128
|
-
/** Workflow manager instance for workflow orchestration */
|
|
129
|
-
workflowManager?: import("../managers/workflowManager.js").WorkflowManager;
|
|
130
|
-
/**
|
|
131
|
-
* Per-session merged environment (OS env overlaid with the settings env
|
|
132
|
-
* snapshot) for this session. Tools that spawn subprocesses (Bash, hooks)
|
|
133
|
-
* should merge this on top of `process.env` so settings `env` vars reach
|
|
134
|
-
* the subprocess without polluting other sessions in one stdio process.
|
|
135
|
-
*/
|
|
136
|
-
sessionEnv?: Record<string, string>;
|
|
137
|
-
}
|
|
@@ -1,373 +0,0 @@
|
|
|
1
|
-
import TurndownService from "turndown";
|
|
2
|
-
import { LRUCache } from "lru-cache";
|
|
3
|
-
import { WEB_FETCH_TOOL_NAME } from "../constants/tools.js";
|
|
4
|
-
import type { ToolPlugin, ToolResult, ToolContext } from "./types.js";
|
|
5
|
-
import { logger } from "../utils/globalLogger.js";
|
|
6
|
-
|
|
7
|
-
// --- Security Limits ---
|
|
8
|
-
const MAX_HTTP_CONTENT_LENGTH = 10 * 1024 * 1024; // 10MB
|
|
9
|
-
const FETCH_TIMEOUT_MS = 60_000; // 60s
|
|
10
|
-
const MAX_REDIRECTS = 10;
|
|
11
|
-
const MAX_MARKDOWN_LENGTH = 100_000;
|
|
12
|
-
const USER_AGENT = "Wave-User (+https://github.com/netease-lcap/wave-agent)";
|
|
13
|
-
|
|
14
|
-
// --- Cache (LRU with 15min TTL, 50MB max) ---
|
|
15
|
-
const CACHE_TTL = 15 * 60 * 1000; // 15 minutes
|
|
16
|
-
const CACHE_MAX_BYTES = 50 * 1024 * 1024; // 50MB
|
|
17
|
-
|
|
18
|
-
interface CacheEntry {
|
|
19
|
-
bytes: number;
|
|
20
|
-
code: number;
|
|
21
|
-
codeText: string;
|
|
22
|
-
content: string;
|
|
23
|
-
contentType: string;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const cache = new LRUCache<string, CacheEntry>({
|
|
27
|
-
ttl: CACHE_TTL,
|
|
28
|
-
maxSize: CACHE_MAX_BYTES,
|
|
29
|
-
sizeCalculation: (entry) => entry.bytes,
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
// --- Helpers ---
|
|
33
|
-
|
|
34
|
-
function formatSize(bytes: number): string {
|
|
35
|
-
if (bytes < 1024) return `${bytes}B`;
|
|
36
|
-
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
|
|
37
|
-
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
type URLValidationResult = { valid: true } | { valid: false; error: string };
|
|
41
|
-
|
|
42
|
-
function validateURL(url: string): URLValidationResult {
|
|
43
|
-
if (url.length > 2000) {
|
|
44
|
-
return {
|
|
45
|
-
valid: false,
|
|
46
|
-
error: "URL exceeds maximum length of 2000 characters",
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
let parsed: URL;
|
|
51
|
-
try {
|
|
52
|
-
parsed = new URL(url);
|
|
53
|
-
} catch (error) {
|
|
54
|
-
return {
|
|
55
|
-
valid: false,
|
|
56
|
-
error: `Invalid URL: ${error instanceof Error ? error.message : String(error)}`,
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
if (parsed.username || parsed.password) {
|
|
61
|
-
return { valid: false, error: "URL must not contain username or password" };
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
const hostParts = parsed.hostname.split(".");
|
|
65
|
-
if (hostParts.length < 2) {
|
|
66
|
-
return {
|
|
67
|
-
valid: false,
|
|
68
|
-
error: "URL hostname must have at least two parts (e.g., example.com)",
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
return { valid: true };
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
function isPermittedRedirect(
|
|
76
|
-
originalUrl: string,
|
|
77
|
-
redirectUrl: string,
|
|
78
|
-
): boolean {
|
|
79
|
-
try {
|
|
80
|
-
const original = new URL(originalUrl);
|
|
81
|
-
const redirect = new URL(redirectUrl);
|
|
82
|
-
const origHost = original.host;
|
|
83
|
-
const redirHost = redirect.host;
|
|
84
|
-
|
|
85
|
-
// Same host
|
|
86
|
-
if (origHost === redirHost) return true;
|
|
87
|
-
|
|
88
|
-
// www. variation (e.g., example.com <-> www.example.com)
|
|
89
|
-
const bareOrig = origHost.replace(/^www\./, "");
|
|
90
|
-
const bareRedir = redirHost.replace(/^www\./, "");
|
|
91
|
-
if (bareOrig === bareRedir) return true;
|
|
92
|
-
|
|
93
|
-
return false;
|
|
94
|
-
} catch {
|
|
95
|
-
return false;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// --- Tool ---
|
|
100
|
-
|
|
101
|
-
export const webFetchTool: ToolPlugin = {
|
|
102
|
-
name: WEB_FETCH_TOOL_NAME,
|
|
103
|
-
config: {
|
|
104
|
-
type: "function",
|
|
105
|
-
function: {
|
|
106
|
-
name: WEB_FETCH_TOOL_NAME,
|
|
107
|
-
description: `IMPORTANT: WebFetch WILL FAIL for authenticated or private URLs. Before using this tool, check if the URL points to an authenticated service (e.g. Google Docs, Confluence, Jira, GitHub). If so, look for a specialized MCP tool that provides authenticated access.
|
|
108
|
-
|
|
109
|
-
- Fetches content from a specified URL and processes it using an AI model
|
|
110
|
-
- Takes a URL and a prompt as input
|
|
111
|
-
- Fetches the URL content, converts HTML to markdown
|
|
112
|
-
- Processes the content with the prompt using a small, fast model
|
|
113
|
-
- Returns the model's response about the content
|
|
114
|
-
- Use this tool when you need to retrieve and analyze web content
|
|
115
|
-
|
|
116
|
-
Usage notes:
|
|
117
|
-
- IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions.
|
|
118
|
-
- The URL must be a fully-formed valid URL
|
|
119
|
-
- HTTP URLs will be automatically upgraded to HTTPS
|
|
120
|
-
- Content exceeding ${formatSize(MAX_MARKDOWN_LENGTH)} will be truncated
|
|
121
|
-
- The prompt should describe what information you want to extract from the page
|
|
122
|
-
- This tool is read-only and does not modify any files
|
|
123
|
-
- Results may be summarized if the content is very large
|
|
124
|
-
- Includes an LRU cache with a 15-minute TTL for faster responses when repeatedly accessing the same URL
|
|
125
|
-
- When a URL redirects to a different host, the tool will inform you and provide the redirect URL in a special format. You should then make a new WebFetch request with the redirect URL to fetch the content.
|
|
126
|
-
- For GitHub URLs, prefer using the gh CLI via Bash instead (e.g., gh pr view, gh issue view, gh api).`,
|
|
127
|
-
parameters: {
|
|
128
|
-
type: "object",
|
|
129
|
-
properties: {
|
|
130
|
-
url: {
|
|
131
|
-
type: "string",
|
|
132
|
-
description: "The URL to fetch content from",
|
|
133
|
-
format: "uri",
|
|
134
|
-
},
|
|
135
|
-
prompt: {
|
|
136
|
-
type: "string",
|
|
137
|
-
description: "The prompt to run on the fetched content",
|
|
138
|
-
},
|
|
139
|
-
},
|
|
140
|
-
required: ["url", "prompt"],
|
|
141
|
-
additionalProperties: false,
|
|
142
|
-
},
|
|
143
|
-
},
|
|
144
|
-
},
|
|
145
|
-
execute: async (
|
|
146
|
-
args: Record<string, unknown>,
|
|
147
|
-
context: ToolContext,
|
|
148
|
-
): Promise<ToolResult> => {
|
|
149
|
-
let url = args.url as string;
|
|
150
|
-
const prompt = args.prompt as string;
|
|
151
|
-
|
|
152
|
-
if (!url || !prompt) {
|
|
153
|
-
return {
|
|
154
|
-
success: false,
|
|
155
|
-
content: "",
|
|
156
|
-
error: "Both url and prompt parameters are required",
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// Upgrade HTTP to HTTPS
|
|
161
|
-
if (url.startsWith("http://")) {
|
|
162
|
-
url = "https://" + url.substring(7);
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Validate URL
|
|
166
|
-
const validation = validateURL(url);
|
|
167
|
-
if (!validation.valid) {
|
|
168
|
-
return {
|
|
169
|
-
success: false,
|
|
170
|
-
content: "",
|
|
171
|
-
error: validation.error,
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
try {
|
|
176
|
-
const cached = cache.get(url);
|
|
177
|
-
if (cached) {
|
|
178
|
-
const markdown = cached.content;
|
|
179
|
-
return processWithAI(
|
|
180
|
-
url,
|
|
181
|
-
prompt,
|
|
182
|
-
markdown,
|
|
183
|
-
cached.code,
|
|
184
|
-
cached.codeText,
|
|
185
|
-
context,
|
|
186
|
-
);
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
// Fetch with redirect following
|
|
190
|
-
const result = await fetchWithRedirects(url, context.abortSignal);
|
|
191
|
-
|
|
192
|
-
if (result.kind === "redirect") {
|
|
193
|
-
return {
|
|
194
|
-
success: true,
|
|
195
|
-
content: `REDIRECT_TO: ${result.redirectUrl}\nThe URL redirected to a different host. Please make a new WebFetch request with this redirect URL if you wish to continue.`,
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
if (result.kind === "error") {
|
|
200
|
-
return {
|
|
201
|
-
success: false,
|
|
202
|
-
content: "",
|
|
203
|
-
error: result.error,
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
const { response, finalUrl } = result;
|
|
208
|
-
|
|
209
|
-
if (!response.ok) {
|
|
210
|
-
return {
|
|
211
|
-
success: false,
|
|
212
|
-
content: "",
|
|
213
|
-
error: `Failed to fetch URL: ${response.status} ${response.statusText}`,
|
|
214
|
-
};
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const contentLengthHeader = response.headers.get("content-length");
|
|
218
|
-
const contentLength = contentLengthHeader
|
|
219
|
-
? parseInt(contentLengthHeader, 10)
|
|
220
|
-
: null;
|
|
221
|
-
if (contentLength !== null && contentLength > MAX_HTTP_CONTENT_LENGTH) {
|
|
222
|
-
return {
|
|
223
|
-
success: false,
|
|
224
|
-
content: "",
|
|
225
|
-
error: `Content too large: ${formatSize(contentLength)} exceeds limit of ${formatSize(MAX_HTTP_CONTENT_LENGTH)}`,
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
const html = await response.text();
|
|
230
|
-
const turndownService = new TurndownService();
|
|
231
|
-
let markdown = turndownService.turndown(html);
|
|
232
|
-
|
|
233
|
-
const markdownBytes = new TextEncoder().encode(markdown).length;
|
|
234
|
-
|
|
235
|
-
// Truncate if too large
|
|
236
|
-
if (markdown.length > MAX_MARKDOWN_LENGTH) {
|
|
237
|
-
markdown =
|
|
238
|
-
markdown.substring(0, MAX_MARKDOWN_LENGTH) +
|
|
239
|
-
`[Content truncated at ${MAX_MARKDOWN_LENGTH} characters due to length limit.]`;
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
// Store in LRU cache
|
|
243
|
-
cache.set(finalUrl, {
|
|
244
|
-
bytes: markdownBytes,
|
|
245
|
-
code: response.status,
|
|
246
|
-
codeText: response.statusText,
|
|
247
|
-
content: markdown,
|
|
248
|
-
contentType: response.headers.get("content-type") || "",
|
|
249
|
-
});
|
|
250
|
-
|
|
251
|
-
return processWithAI(
|
|
252
|
-
finalUrl,
|
|
253
|
-
prompt,
|
|
254
|
-
markdown,
|
|
255
|
-
response.status,
|
|
256
|
-
response.statusText,
|
|
257
|
-
context,
|
|
258
|
-
markdownBytes,
|
|
259
|
-
);
|
|
260
|
-
} catch (error) {
|
|
261
|
-
logger.error(`WebFetch failed for ${url}:`, error);
|
|
262
|
-
return {
|
|
263
|
-
success: false,
|
|
264
|
-
content: "",
|
|
265
|
-
error: `WebFetch failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
266
|
-
};
|
|
267
|
-
}
|
|
268
|
-
},
|
|
269
|
-
formatCompactParams: (params: Record<string, unknown>) => {
|
|
270
|
-
return `Fetch ${params.url}`;
|
|
271
|
-
},
|
|
272
|
-
};
|
|
273
|
-
|
|
274
|
-
// --- Fetch with redirect following ---
|
|
275
|
-
|
|
276
|
-
async function fetchWithRedirects(
|
|
277
|
-
initialUrl: string,
|
|
278
|
-
abortSignal?: AbortSignal,
|
|
279
|
-
redirectCount = 0,
|
|
280
|
-
): Promise<
|
|
281
|
-
| { kind: "response"; response: Response; finalUrl: string }
|
|
282
|
-
| { kind: "redirect"; redirectUrl: string }
|
|
283
|
-
| { kind: "error"; error: string }
|
|
284
|
-
> {
|
|
285
|
-
if (redirectCount >= MAX_REDIRECTS) {
|
|
286
|
-
return {
|
|
287
|
-
kind: "error",
|
|
288
|
-
error: `Too many redirects (max ${MAX_REDIRECTS})`,
|
|
289
|
-
};
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
const controller = new AbortController();
|
|
293
|
-
const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
294
|
-
|
|
295
|
-
// Forward the context's abort signal if provided
|
|
296
|
-
if (abortSignal) {
|
|
297
|
-
abortSignal.addEventListener("abort", () => controller.abort(), {
|
|
298
|
-
once: true,
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
let response: Response;
|
|
303
|
-
try {
|
|
304
|
-
response = await fetch(initialUrl, {
|
|
305
|
-
redirect: "manual",
|
|
306
|
-
signal: controller.signal,
|
|
307
|
-
headers: {
|
|
308
|
-
"User-Agent": USER_AGENT,
|
|
309
|
-
Accept: "text/markdown, text/html, */*",
|
|
310
|
-
},
|
|
311
|
-
});
|
|
312
|
-
} finally {
|
|
313
|
-
clearTimeout(timeoutId);
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
if (response.status >= 300 && response.status < 400) {
|
|
317
|
-
const location = response.headers.get("location");
|
|
318
|
-
if (location) {
|
|
319
|
-
const redirectUrl = new URL(location, initialUrl).toString();
|
|
320
|
-
|
|
321
|
-
if (!isPermittedRedirect(initialUrl, redirectUrl)) {
|
|
322
|
-
return { kind: "redirect", redirectUrl };
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
// Follow permitted redirect recursively
|
|
326
|
-
return fetchWithRedirects(redirectUrl, abortSignal, redirectCount + 1);
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
return { kind: "response", response, finalUrl: initialUrl };
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// --- AI Processing ---
|
|
334
|
-
|
|
335
|
-
async function processWithAI(
|
|
336
|
-
url: string,
|
|
337
|
-
prompt: string,
|
|
338
|
-
markdown: string,
|
|
339
|
-
statusCode: number,
|
|
340
|
-
statusText: string,
|
|
341
|
-
context: ToolContext,
|
|
342
|
-
contentSize?: number,
|
|
343
|
-
): Promise<ToolResult> {
|
|
344
|
-
if (!context.aiManager || !context.aiService) {
|
|
345
|
-
return {
|
|
346
|
-
success: false,
|
|
347
|
-
content: markdown,
|
|
348
|
-
error: "AI Manager or AI Service not available for processing content",
|
|
349
|
-
};
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
const modelConfig = context.aiManager.getModelConfig();
|
|
353
|
-
const fastModel = modelConfig.fastModel;
|
|
354
|
-
|
|
355
|
-
const aiResponse = await context.aiService.processWebContent({
|
|
356
|
-
gatewayConfig: context.aiManager.getGatewayConfig(),
|
|
357
|
-
modelConfig: modelConfig,
|
|
358
|
-
content: markdown,
|
|
359
|
-
prompt: prompt,
|
|
360
|
-
model: fastModel,
|
|
361
|
-
abortSignal: context.abortSignal,
|
|
362
|
-
});
|
|
363
|
-
|
|
364
|
-
const sizeStr =
|
|
365
|
-
contentSize !== undefined ? formatSize(contentSize) : "unknown size";
|
|
366
|
-
const statusStr = `${statusCode} ${statusText}`.trim();
|
|
367
|
-
|
|
368
|
-
return {
|
|
369
|
-
success: true,
|
|
370
|
-
content: aiResponse.content || "",
|
|
371
|
-
shortResult: `Received ${sizeStr} (${statusStr}) from ${url}`,
|
|
372
|
-
};
|
|
373
|
-
}
|