vibecore 0.2.0a1__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vibecore/agents/default.py +6 -11
- vibecore/agents/{task_agent.py → task.py} +2 -6
- vibecore/auth/__init__.py +15 -0
- vibecore/auth/config.py +38 -0
- vibecore/auth/interceptor.py +141 -0
- vibecore/auth/manager.py +173 -0
- vibecore/auth/models.py +54 -0
- vibecore/auth/oauth_flow.py +129 -0
- vibecore/auth/pkce.py +29 -0
- vibecore/auth/storage.py +111 -0
- vibecore/auth/token_manager.py +131 -0
- vibecore/cli.py +117 -9
- vibecore/flow.py +105 -0
- vibecore/handlers/stream_handler.py +11 -0
- vibecore/main.py +28 -6
- vibecore/models/anthropic_auth.py +226 -0
- vibecore/settings.py +61 -5
- vibecore/tools/task/executor.py +1 -1
- vibecore/tools/webfetch/__init__.py +7 -0
- vibecore/tools/webfetch/executor.py +127 -0
- vibecore/tools/webfetch/models.py +22 -0
- vibecore/tools/webfetch/tools.py +46 -0
- vibecore/tools/websearch/__init__.py +5 -0
- vibecore/tools/websearch/base.py +27 -0
- vibecore/tools/websearch/ddgs/__init__.py +5 -0
- vibecore/tools/websearch/ddgs/backend.py +64 -0
- vibecore/tools/websearch/executor.py +43 -0
- vibecore/tools/websearch/models.py +20 -0
- vibecore/tools/websearch/tools.py +49 -0
- vibecore/widgets/tool_message_factory.py +24 -0
- vibecore/widgets/tool_messages.py +219 -0
- vibecore/widgets/tool_messages.tcss +94 -0
- {vibecore-0.2.0a1.dist-info → vibecore-0.3.0.dist-info}/METADATA +107 -1
- {vibecore-0.2.0a1.dist-info → vibecore-0.3.0.dist-info}/RECORD +37 -15
- vibecore-0.3.0.dist-info/entry_points.txt +2 -0
- vibecore-0.2.0a1.dist-info/entry_points.txt +0 -2
- {vibecore-0.2.0a1.dist-info → vibecore-0.3.0.dist-info}/WHEEL +0 -0
- {vibecore-0.2.0a1.dist-info → vibecore-0.3.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Anthropic model with Pro/Max authentication support."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Literal, overload
|
|
5
|
+
|
|
6
|
+
import litellm
|
|
7
|
+
from agents.agent_output import AgentOutputSchemaBase
|
|
8
|
+
from agents.handoffs import Handoff
|
|
9
|
+
from agents.items import TResponseInputItem
|
|
10
|
+
from agents.model_settings import ModelSettings
|
|
11
|
+
from agents.models.interface import ModelTracing
|
|
12
|
+
from agents.tool import Tool
|
|
13
|
+
from agents.tracing.span_data import GenerationSpanData
|
|
14
|
+
from agents.tracing.spans import Span
|
|
15
|
+
from openai import AsyncStream
|
|
16
|
+
from openai.types.chat import ChatCompletionChunk
|
|
17
|
+
from openai.types.responses import Response
|
|
18
|
+
|
|
19
|
+
from vibecore.auth.config import ANTHROPIC_CONFIG
|
|
20
|
+
from vibecore.auth.interceptor import AnthropicRequestInterceptor
|
|
21
|
+
from vibecore.auth.manager import AnthropicAuthManager
|
|
22
|
+
from vibecore.auth.storage import SecureAuthStorage
|
|
23
|
+
from vibecore.models.anthropic import AnthropicModel, _transform_messages_for_cache
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AnthropicProMaxModel(AnthropicModel):
|
|
29
|
+
"""Anthropic model with Pro/Max authentication and Claude Code spoofing."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, model_name: str, base_url: str | None = None, api_key: str | None = None, use_auth: bool = True):
|
|
32
|
+
"""
|
|
33
|
+
Initialize AnthropicProMaxModel.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
model_name: Name of the model.
|
|
37
|
+
base_url: Optional base URL override.
|
|
38
|
+
api_key: Optional API key (ignored if Pro/Max auth is active).
|
|
39
|
+
use_auth: Whether to use Pro/Max authentication.
|
|
40
|
+
"""
|
|
41
|
+
super().__init__(model_name, base_url, api_key)
|
|
42
|
+
self.use_auth = use_auth
|
|
43
|
+
self.auth_manager: AnthropicAuthManager | None = None
|
|
44
|
+
self.interceptor: AnthropicRequestInterceptor | None = None
|
|
45
|
+
|
|
46
|
+
if self.use_auth:
|
|
47
|
+
self._initialize_auth()
|
|
48
|
+
|
|
49
|
+
def _initialize_auth(self) -> None:
|
|
50
|
+
"""Initialize authentication components."""
|
|
51
|
+
storage = SecureAuthStorage()
|
|
52
|
+
self.auth_manager = AnthropicAuthManager()
|
|
53
|
+
self.interceptor = AnthropicRequestInterceptor(storage)
|
|
54
|
+
|
|
55
|
+
# Check if authenticated (we'll check async later when needed)
|
|
56
|
+
logger.info("AnthropicProMaxModel initialized with authentication support")
|
|
57
|
+
|
|
58
|
+
async def _inject_system_prompt(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
59
|
+
"""
|
|
60
|
+
Inject Claude Code identity into system messages.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
messages: Original messages.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
Messages with Claude Code identity injected.
|
|
67
|
+
"""
|
|
68
|
+
if not self.use_auth or not self.interceptor:
|
|
69
|
+
return messages
|
|
70
|
+
|
|
71
|
+
# Check if using Pro/Max auth
|
|
72
|
+
storage = SecureAuthStorage()
|
|
73
|
+
auth = await storage.load("anthropic")
|
|
74
|
+
if not auth or auth.type != "oauth":
|
|
75
|
+
return messages # Only inject for Pro/Max users
|
|
76
|
+
|
|
77
|
+
# Find or create system message
|
|
78
|
+
messages_copy = messages.copy()
|
|
79
|
+
system_index = next((i for i, msg in enumerate(messages_copy) if msg.get("role") == "system"), None)
|
|
80
|
+
|
|
81
|
+
if system_index is not None:
|
|
82
|
+
# Prepend Claude Code identity to existing system message
|
|
83
|
+
current_content = messages_copy[system_index].get("content", "")
|
|
84
|
+
messages_copy[system_index]["content"] = f"{ANTHROPIC_CONFIG.CLAUDE_CODE_IDENTITY}\n\n{current_content}"
|
|
85
|
+
else:
|
|
86
|
+
# Add new system message at the beginning
|
|
87
|
+
messages_copy.insert(0, {"role": "system", "content": ANTHROPIC_CONFIG.CLAUDE_CODE_IDENTITY})
|
|
88
|
+
|
|
89
|
+
return messages_copy
|
|
90
|
+
|
|
91
|
+
async def _apply_auth_headers(self, headers: dict[str, str]) -> dict[str, str]:
|
|
92
|
+
"""
|
|
93
|
+
Apply authentication and Claude Code headers.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
headers: Original headers.
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
Modified headers with auth and Claude Code spoofing.
|
|
100
|
+
"""
|
|
101
|
+
if not self.use_auth or not self.interceptor:
|
|
102
|
+
return headers
|
|
103
|
+
|
|
104
|
+
# Use interceptor to apply auth and Claude Code headers
|
|
105
|
+
modified_request = await self.interceptor.intercept_request(url="https://api.anthropic.com", headers=headers)
|
|
106
|
+
|
|
107
|
+
return modified_request.get("headers", headers)
|
|
108
|
+
|
|
109
|
+
@overload
|
|
110
|
+
async def _fetch_response(
|
|
111
|
+
self,
|
|
112
|
+
system_instructions: str | None,
|
|
113
|
+
input: str | list[TResponseInputItem],
|
|
114
|
+
model_settings: ModelSettings,
|
|
115
|
+
tools: list[Tool],
|
|
116
|
+
output_schema: AgentOutputSchemaBase | None,
|
|
117
|
+
handoffs: list[Handoff],
|
|
118
|
+
span: Span[GenerationSpanData],
|
|
119
|
+
tracing: ModelTracing,
|
|
120
|
+
stream: Literal[True],
|
|
121
|
+
prompt: Any | None = None,
|
|
122
|
+
) -> tuple[Response, AsyncStream[ChatCompletionChunk]]: ...
|
|
123
|
+
|
|
124
|
+
@overload
|
|
125
|
+
async def _fetch_response(
|
|
126
|
+
self,
|
|
127
|
+
system_instructions: str | None,
|
|
128
|
+
input: str | list[TResponseInputItem],
|
|
129
|
+
model_settings: ModelSettings,
|
|
130
|
+
tools: list[Tool],
|
|
131
|
+
output_schema: AgentOutputSchemaBase | None,
|
|
132
|
+
handoffs: list[Handoff],
|
|
133
|
+
span: Span[GenerationSpanData],
|
|
134
|
+
tracing: ModelTracing,
|
|
135
|
+
stream: Literal[False],
|
|
136
|
+
prompt: Any | None = None,
|
|
137
|
+
) -> Any: ...
|
|
138
|
+
|
|
139
|
+
async def _fetch_response(
|
|
140
|
+
self,
|
|
141
|
+
system_instructions: str | None,
|
|
142
|
+
input: str | list[TResponseInputItem],
|
|
143
|
+
model_settings: ModelSettings,
|
|
144
|
+
tools: list[Tool],
|
|
145
|
+
output_schema: AgentOutputSchemaBase | None,
|
|
146
|
+
handoffs: list[Handoff],
|
|
147
|
+
span: Span[GenerationSpanData],
|
|
148
|
+
tracing: ModelTracing,
|
|
149
|
+
stream: bool = False,
|
|
150
|
+
prompt: Any | None = None,
|
|
151
|
+
) -> Any | tuple[Response, AsyncStream[ChatCompletionChunk]]:
|
|
152
|
+
"""Override _fetch_response to add auth and Claude Code support."""
|
|
153
|
+
# Store the original litellm.acompletion function
|
|
154
|
+
original_acompletion = litellm.acompletion
|
|
155
|
+
|
|
156
|
+
async def _intercepting_acompletion(*args, **kwargs):
|
|
157
|
+
"""Intercept litellm.acompletion calls to transform messages and headers."""
|
|
158
|
+
# Only transform for this Anthropic model
|
|
159
|
+
if kwargs.get("model") == self.model:
|
|
160
|
+
if "messages" in kwargs:
|
|
161
|
+
messages = kwargs["messages"]
|
|
162
|
+
logger.debug(f"Intercepting Anthropic API call with {len(messages)} messages")
|
|
163
|
+
|
|
164
|
+
# Add Claude Code identity to system prompt
|
|
165
|
+
messages = await self._inject_system_prompt(messages)
|
|
166
|
+
|
|
167
|
+
# Transform messages to add cache_control
|
|
168
|
+
messages = _transform_messages_for_cache(messages)
|
|
169
|
+
kwargs["messages"] = messages
|
|
170
|
+
|
|
171
|
+
# Apply auth headers if available
|
|
172
|
+
if self.use_auth and self.interceptor:
|
|
173
|
+
# Get existing headers or create new dict
|
|
174
|
+
headers = kwargs.get("extra_headers", {})
|
|
175
|
+
|
|
176
|
+
# Apply auth and Claude Code headers
|
|
177
|
+
headers = await self._apply_auth_headers(headers)
|
|
178
|
+
|
|
179
|
+
# Update kwargs
|
|
180
|
+
kwargs["extra_headers"] = headers
|
|
181
|
+
|
|
182
|
+
# For Pro/Max users, prevent API key from being added
|
|
183
|
+
storage = SecureAuthStorage()
|
|
184
|
+
auth = await storage.load("anthropic")
|
|
185
|
+
if auth and auth.type == "oauth":
|
|
186
|
+
# CRITICAL: Set api_key to None to prevent litellm from adding x-api-key header
|
|
187
|
+
# when using OAuth authentication
|
|
188
|
+
kwargs["api_key"] = None
|
|
189
|
+
|
|
190
|
+
# Call the original function with transformed kwargs
|
|
191
|
+
return await original_acompletion(*args, **kwargs)
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
# Temporarily replace litellm.acompletion
|
|
195
|
+
litellm.acompletion = _intercepting_acompletion
|
|
196
|
+
|
|
197
|
+
# Call the parent's implementation
|
|
198
|
+
if stream:
|
|
199
|
+
return await super()._fetch_response(
|
|
200
|
+
system_instructions=system_instructions,
|
|
201
|
+
input=input,
|
|
202
|
+
model_settings=model_settings,
|
|
203
|
+
tools=tools,
|
|
204
|
+
output_schema=output_schema,
|
|
205
|
+
handoffs=handoffs,
|
|
206
|
+
span=span,
|
|
207
|
+
tracing=tracing,
|
|
208
|
+
stream=True,
|
|
209
|
+
prompt=prompt,
|
|
210
|
+
)
|
|
211
|
+
else:
|
|
212
|
+
return await super()._fetch_response(
|
|
213
|
+
system_instructions=system_instructions,
|
|
214
|
+
input=input,
|
|
215
|
+
model_settings=model_settings,
|
|
216
|
+
tools=tools,
|
|
217
|
+
output_schema=output_schema,
|
|
218
|
+
handoffs=handoffs,
|
|
219
|
+
span=span,
|
|
220
|
+
tracing=tracing,
|
|
221
|
+
stream=False,
|
|
222
|
+
prompt=prompt,
|
|
223
|
+
)
|
|
224
|
+
finally:
|
|
225
|
+
# Always restore the original function
|
|
226
|
+
litellm.acompletion = original_acompletion
|
vibecore/settings.py
CHANGED
|
@@ -4,14 +4,28 @@ import os
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Literal
|
|
6
6
|
|
|
7
|
-
from agents import Model, OpenAIChatCompletionsModel
|
|
7
|
+
from agents import Model, ModelSettings, OpenAIChatCompletionsModel
|
|
8
8
|
from agents.models.multi_provider import MultiProvider
|
|
9
|
-
from
|
|
9
|
+
from openai.types import Reasoning
|
|
10
|
+
from pydantic import BaseModel, Field, field_validator
|
|
10
11
|
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, YamlConfigSettingsSource
|
|
11
12
|
|
|
12
13
|
from vibecore.models import AnthropicModel
|
|
13
14
|
|
|
14
15
|
|
|
16
|
+
class AuthSettings(BaseModel):
|
|
17
|
+
"""Configuration for authentication."""
|
|
18
|
+
|
|
19
|
+
use_pro_max: bool = Field(
|
|
20
|
+
default=False,
|
|
21
|
+
description="Use Anthropic Pro/Max authentication if available",
|
|
22
|
+
)
|
|
23
|
+
auto_refresh: bool = Field(
|
|
24
|
+
default=True,
|
|
25
|
+
description="Automatically refresh OAuth tokens",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
15
29
|
class SessionSettings(BaseModel):
|
|
16
30
|
"""Configuration for session storage."""
|
|
17
31
|
|
|
@@ -92,9 +106,10 @@ class Settings(BaseSettings):
|
|
|
92
106
|
default_model: str = Field(
|
|
93
107
|
# default="o3",
|
|
94
108
|
# default="gpt-4.1",
|
|
109
|
+
default="gpt-5",
|
|
95
110
|
# default="qwen3-30b-a3b-mlx@8bit",
|
|
96
111
|
# default="mistralai/devstral-small-2507",
|
|
97
|
-
default="anthropic/claude-sonnet-4-20250514",
|
|
112
|
+
# default="anthropic/claude-sonnet-4-20250514",
|
|
98
113
|
# default="anthropic/claude-3-5-haiku-20241022",
|
|
99
114
|
# default="litellm/deepseek/deepseek-chat",
|
|
100
115
|
description="Default model to use for agents (e.g., 'gpt-4.1', 'o3-mini', 'anthropic/claude-sonnet-4')",
|
|
@@ -109,6 +124,24 @@ class Settings(BaseSettings):
|
|
|
109
124
|
default=None,
|
|
110
125
|
description="Default reasoning effort level for agents (null, 'minimal', 'low', 'medium', 'high')",
|
|
111
126
|
)
|
|
127
|
+
reasoning_summary: Literal["auto", "concise", "detailed"] | None = Field(
|
|
128
|
+
default="auto",
|
|
129
|
+
description="Reasoning summary mode ('auto', 'concise', 'detailed', or 'off')",
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
@field_validator("reasoning_summary", mode="before")
|
|
133
|
+
@classmethod
|
|
134
|
+
def validate_reasoning_summary(cls, v):
|
|
135
|
+
"""Convert string 'null' to None for reasoning_summary field."""
|
|
136
|
+
if v == "off" or v == "":
|
|
137
|
+
return None
|
|
138
|
+
return v
|
|
139
|
+
|
|
140
|
+
# Authentication configuration
|
|
141
|
+
auth: AuthSettings = Field(
|
|
142
|
+
default_factory=AuthSettings,
|
|
143
|
+
description="Authentication configuration",
|
|
144
|
+
)
|
|
112
145
|
|
|
113
146
|
# Session configuration
|
|
114
147
|
session: SessionSettings = Field(
|
|
@@ -122,22 +155,45 @@ class Settings(BaseSettings):
|
|
|
122
155
|
description="List of MCP servers to connect to",
|
|
123
156
|
)
|
|
124
157
|
|
|
158
|
+
rich_tool_names: list[str] = Field(
|
|
159
|
+
default_factory=list,
|
|
160
|
+
description="List of tools to render with RichToolMessage (temporary settings)",
|
|
161
|
+
)
|
|
162
|
+
|
|
125
163
|
@property
|
|
126
164
|
def model(self) -> str | Model:
|
|
127
165
|
"""Get the configured model.
|
|
128
166
|
|
|
129
|
-
Returns an
|
|
167
|
+
Returns an AnthropicProMaxModel instance if auth is enabled and model is Anthropic,
|
|
168
|
+
returns an AnthropicModel instance if the model name starts with 'anthropic/',
|
|
130
169
|
returns a OpenAIChatCompletionsModel instance if there is a custom base URL set,
|
|
131
170
|
otherwise returns the model name as a plain string (for OpenAI/LiteLLM models).
|
|
132
171
|
"""
|
|
133
172
|
custom_base = "OPENAI_BASE_URL" in os.environ
|
|
134
173
|
if self.default_model.startswith("anthropic/"):
|
|
135
|
-
|
|
174
|
+
# Check if Pro/Max auth should be used
|
|
175
|
+
if self.auth.use_pro_max:
|
|
176
|
+
from vibecore.models.anthropic_auth import AnthropicProMaxModel
|
|
177
|
+
|
|
178
|
+
return AnthropicProMaxModel(self.default_model, use_auth=True)
|
|
179
|
+
else:
|
|
180
|
+
return AnthropicModel(self.default_model)
|
|
136
181
|
elif custom_base:
|
|
137
182
|
openai_provider = MultiProvider().openai_provider
|
|
138
183
|
return OpenAIChatCompletionsModel(self.default_model, openai_provider._get_client())
|
|
139
184
|
return self.default_model
|
|
140
185
|
|
|
186
|
+
@property
|
|
187
|
+
def default_model_settings(self) -> ModelSettings:
|
|
188
|
+
"""Get the default model settings."""
|
|
189
|
+
return ModelSettings(
|
|
190
|
+
include_usage=True,
|
|
191
|
+
reasoning=Reasoning(
|
|
192
|
+
summary=self.reasoning_summary,
|
|
193
|
+
effort=self.reasoning_effort,
|
|
194
|
+
),
|
|
195
|
+
)
|
|
196
|
+
|
|
141
197
|
@classmethod
|
|
142
198
|
def settings_customise_sources(
|
|
143
199
|
cls,
|
vibecore/tools/task/executor.py
CHANGED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Webfetch execution logic for fetching and converting web content."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from urllib.parse import urlparse
|
|
5
|
+
|
|
6
|
+
import html2text
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from .models import WebFetchParams
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
async def fetch_url(params: WebFetchParams) -> str:
|
|
13
|
+
"""Fetch a URL and convert its content to Markdown.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
params: WebFetch parameters including URL and options
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
Markdown-formatted content or error message
|
|
20
|
+
"""
|
|
21
|
+
try:
|
|
22
|
+
# Validate URL
|
|
23
|
+
parsed = urlparse(params.url)
|
|
24
|
+
if not parsed.scheme:
|
|
25
|
+
return f"Error: Invalid URL - missing scheme (http:// or https://): {params.url}"
|
|
26
|
+
if not parsed.netloc:
|
|
27
|
+
return f"Error: Invalid URL - missing domain: {params.url}"
|
|
28
|
+
if parsed.scheme not in ["http", "https"]:
|
|
29
|
+
return f"Error: Unsupported URL scheme: {parsed.scheme}"
|
|
30
|
+
|
|
31
|
+
# Configure headers
|
|
32
|
+
headers = {
|
|
33
|
+
"User-Agent": params.user_agent
|
|
34
|
+
or "Mozilla/5.0 (compatible; Vibecore/1.0; +https://github.com/serialx/vibecore)",
|
|
35
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,"
|
|
36
|
+
"text/plain;q=0.8,application/json;q=0.7,*/*;q=0.5",
|
|
37
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
38
|
+
"Accept-Encoding": "gzip, deflate",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
# Fetch the URL
|
|
42
|
+
async with httpx.AsyncClient(
|
|
43
|
+
timeout=httpx.Timeout(params.timeout),
|
|
44
|
+
follow_redirects=params.follow_redirects,
|
|
45
|
+
) as client:
|
|
46
|
+
response = await client.get(params.url, headers=headers)
|
|
47
|
+
response.raise_for_status()
|
|
48
|
+
|
|
49
|
+
# Get content type
|
|
50
|
+
content_type = response.headers.get("content-type", "").lower()
|
|
51
|
+
|
|
52
|
+
# Handle different content types
|
|
53
|
+
if "application/json" in content_type:
|
|
54
|
+
# Pretty-print JSON as Markdown code block
|
|
55
|
+
try:
|
|
56
|
+
json_data = response.json()
|
|
57
|
+
content = f"```json\n{json.dumps(json_data, indent=2)}\n```"
|
|
58
|
+
except json.JSONDecodeError:
|
|
59
|
+
content = response.text[: params.max_length]
|
|
60
|
+
|
|
61
|
+
elif "text/html" in content_type or "application/xhtml" in content_type:
|
|
62
|
+
# Convert HTML to Markdown
|
|
63
|
+
html_content = response.text[: params.max_length]
|
|
64
|
+
|
|
65
|
+
# Configure html2text
|
|
66
|
+
h = html2text.HTML2Text()
|
|
67
|
+
h.ignore_links = False
|
|
68
|
+
h.ignore_images = False
|
|
69
|
+
h.ignore_emphasis = False
|
|
70
|
+
h.body_width = 0 # Don't wrap lines
|
|
71
|
+
h.skip_internal_links = False
|
|
72
|
+
h.inline_links = True
|
|
73
|
+
h.wrap_links = False
|
|
74
|
+
h.wrap_list_items = False
|
|
75
|
+
h.ul_item_mark = "-"
|
|
76
|
+
h.emphasis_mark = "*"
|
|
77
|
+
h.strong_mark = "**"
|
|
78
|
+
|
|
79
|
+
content = h.handle(html_content)
|
|
80
|
+
|
|
81
|
+
# Clean up excessive newlines
|
|
82
|
+
while "\n\n\n" in content:
|
|
83
|
+
content = content.replace("\n\n\n", "\n\n")
|
|
84
|
+
content = content.strip()
|
|
85
|
+
|
|
86
|
+
elif "text/plain" in content_type or "text/" in content_type:
|
|
87
|
+
# Plain text - return as is
|
|
88
|
+
content = response.text[: params.max_length]
|
|
89
|
+
|
|
90
|
+
else:
|
|
91
|
+
# Unknown content type - try to handle as text
|
|
92
|
+
content = response.text[: params.max_length]
|
|
93
|
+
if not content:
|
|
94
|
+
return f"Error: Unable to extract text content from {content_type}"
|
|
95
|
+
|
|
96
|
+
# Add metadata
|
|
97
|
+
metadata = [
|
|
98
|
+
f"# Content from {params.url}",
|
|
99
|
+
f"**Status Code:** {response.status_code}",
|
|
100
|
+
f"**Content Type:** {content_type.split(';')[0] if content_type else 'unknown'}",
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
# Add redirect info if applicable
|
|
104
|
+
if response.history:
|
|
105
|
+
metadata.append(f"**Redirected:** {len(response.history)} time(s)")
|
|
106
|
+
metadata.append(f"**Final URL:** {response.url}")
|
|
107
|
+
|
|
108
|
+
metadata.append("") # Empty line before content
|
|
109
|
+
|
|
110
|
+
# Check if content was truncated
|
|
111
|
+
if len(response.text) > params.max_length:
|
|
112
|
+
metadata.append(f"*Note: Content truncated to {params.max_length} characters*")
|
|
113
|
+
metadata.append("")
|
|
114
|
+
|
|
115
|
+
# Combine metadata and content
|
|
116
|
+
full_content = "\n".join(metadata) + "\n" + content
|
|
117
|
+
|
|
118
|
+
return full_content
|
|
119
|
+
|
|
120
|
+
except httpx.TimeoutException:
|
|
121
|
+
return f"Error: Request timed out after {params.timeout} seconds"
|
|
122
|
+
except httpx.HTTPStatusError as e:
|
|
123
|
+
return f"Error: HTTP {e.response.status_code} - {e.response.reason_phrase}"
|
|
124
|
+
except httpx.RequestError as e:
|
|
125
|
+
return f"Error: Failed to connect to {params.url}: {e!s}"
|
|
126
|
+
except Exception as e:
|
|
127
|
+
return f"Error: Unexpected error while fetching {params.url}: {e!s}"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Data models for the webfetch tool."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class WebFetchParams(BaseModel):
|
|
7
|
+
"""Parameters for fetching web content."""
|
|
8
|
+
|
|
9
|
+
url: str = Field(description="The URL to fetch content from")
|
|
10
|
+
timeout: int = Field(default=30, description="Request timeout in seconds")
|
|
11
|
+
user_agent: str | None = Field(
|
|
12
|
+
default=None,
|
|
13
|
+
description="Optional custom User-Agent header",
|
|
14
|
+
)
|
|
15
|
+
follow_redirects: bool = Field(
|
|
16
|
+
default=True,
|
|
17
|
+
description="Whether to follow HTTP redirects",
|
|
18
|
+
)
|
|
19
|
+
max_length: int = Field(
|
|
20
|
+
default=1000000, # ~1MB of text
|
|
21
|
+
description="Maximum content length to fetch in characters",
|
|
22
|
+
)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Webfetch tool for Vibecore agents."""
|
|
2
|
+
|
|
3
|
+
from agents import RunContextWrapper, function_tool
|
|
4
|
+
|
|
5
|
+
from vibecore.context import VibecoreContext
|
|
6
|
+
|
|
7
|
+
from .executor import fetch_url
|
|
8
|
+
from .models import WebFetchParams
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@function_tool
|
|
12
|
+
async def webfetch(
|
|
13
|
+
ctx: RunContextWrapper[VibecoreContext],
|
|
14
|
+
url: str,
|
|
15
|
+
timeout: int = 30,
|
|
16
|
+
follow_redirects: bool = True,
|
|
17
|
+
) -> str:
|
|
18
|
+
"""Fetch content from a URL and convert it to Markdown format.
|
|
19
|
+
|
|
20
|
+
This tool fetches web content and converts it to clean, readable Markdown.
|
|
21
|
+
It handles HTML pages, JSON APIs, and plain text content appropriately.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
ctx: The run context wrapper
|
|
25
|
+
url: The URL to fetch content from (must include http:// or https://)
|
|
26
|
+
timeout: Request timeout in seconds (default: 30)
|
|
27
|
+
follow_redirects: Whether to follow HTTP redirects (default: True)
|
|
28
|
+
|
|
29
|
+
Returns:
|
|
30
|
+
Markdown-formatted content from the URL, including metadata about the
|
|
31
|
+
request (status code, content type, etc.) or an error message if the
|
|
32
|
+
fetch fails.
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
- Fetch a webpage: url="https://example.com"
|
|
36
|
+
- Fetch JSON API: url="https://api.example.com/data"
|
|
37
|
+
- Fetch with timeout: url="https://slow-site.com", timeout=60
|
|
38
|
+
- Don't follow redirects: url="https://short.link/abc", follow_redirects=False
|
|
39
|
+
"""
|
|
40
|
+
params = WebFetchParams(
|
|
41
|
+
url=url,
|
|
42
|
+
timeout=timeout,
|
|
43
|
+
follow_redirects=follow_redirects,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return await fetch_url(params)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Base classes for websearch backends."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
from .models import SearchParams
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class WebSearchBackend(ABC):
|
|
9
|
+
"""Abstract base class for web search backends."""
|
|
10
|
+
|
|
11
|
+
@abstractmethod
|
|
12
|
+
async def search(self, params: SearchParams) -> str:
|
|
13
|
+
"""Perform a web search.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
params: Search parameters
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
JSON string containing search results or error message
|
|
20
|
+
"""
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def name(self) -> str:
|
|
26
|
+
"""Return the name of this backend."""
|
|
27
|
+
pass
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""DuckDuckGo search backend implementation."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from ddgs import DDGS
|
|
6
|
+
|
|
7
|
+
from ..base import WebSearchBackend
|
|
8
|
+
from ..models import SearchParams, SearchResult
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DDGSBackend(WebSearchBackend):
|
|
12
|
+
"""DuckDuckGo search backend using ddgs library."""
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def name(self) -> str:
|
|
16
|
+
"""Return the name of this backend."""
|
|
17
|
+
return "DuckDuckGo"
|
|
18
|
+
|
|
19
|
+
async def search(self, params: SearchParams) -> str:
|
|
20
|
+
"""Perform a web search using ddgs.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
params: Search parameters
|
|
24
|
+
|
|
25
|
+
Returns:
|
|
26
|
+
JSON string containing search results or error message
|
|
27
|
+
"""
|
|
28
|
+
try:
|
|
29
|
+
# Create DDGS instance
|
|
30
|
+
ddgs = DDGS()
|
|
31
|
+
|
|
32
|
+
# Perform search (synchronous call)
|
|
33
|
+
# ddgs.text() expects 'query' as first positional argument
|
|
34
|
+
raw_results = ddgs.text(
|
|
35
|
+
query=params.query,
|
|
36
|
+
region=params.region,
|
|
37
|
+
safesearch=params.safesearch,
|
|
38
|
+
max_results=params.max_results,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# Convert to our model format
|
|
42
|
+
results = []
|
|
43
|
+
for r in raw_results:
|
|
44
|
+
result = SearchResult(
|
|
45
|
+
title=r.get("title", ""),
|
|
46
|
+
href=r.get("href", ""),
|
|
47
|
+
body=r.get("body", ""),
|
|
48
|
+
)
|
|
49
|
+
results.append(result.model_dump())
|
|
50
|
+
|
|
51
|
+
if not results:
|
|
52
|
+
return json.dumps({"success": False, "message": "No search results found", "results": []})
|
|
53
|
+
|
|
54
|
+
return json.dumps(
|
|
55
|
+
{
|
|
56
|
+
"success": True,
|
|
57
|
+
"message": f"Found {len(results)} result{'s' if len(results) != 1 else ''}",
|
|
58
|
+
"query": params.query,
|
|
59
|
+
"results": results,
|
|
60
|
+
}
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
except Exception as e:
|
|
64
|
+
return json.dumps({"success": False, "message": f"Search failed: {e!s}", "results": []})
|