dataify-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataify_mcp/__init__.py +37 -0
- dataify_mcp/_version.py +3 -0
- dataify_mcp/client/__init__.py +7 -0
- dataify_mcp/client/_base.py +223 -0
- dataify_mcp/client/_http.py +205 -0
- dataify_mcp/client/_protocol.py +204 -0
- dataify_mcp/client/_sse.py +319 -0
- dataify_mcp/tools/__init__.py +90 -0
- dataify_mcp/tools/amazon.py +137 -0
- dataify_mcp/tools/bing.py +141 -0
- dataify_mcp/tools/facebook.py +63 -0
- dataify_mcp/tools/glassdoor.py +41 -0
- dataify_mcp/tools/google_scraper.py +121 -0
- dataify_mcp/tools/google_serp.py +320 -0
- dataify_mcp/tools/indeed.py +41 -0
- dataify_mcp/tools/instagram.py +52 -0
- dataify_mcp/tools/linkedin.py +41 -0
- dataify_mcp/tools/other_scrapers.py +137 -0
- dataify_mcp/tools/other_search.py +90 -0
- dataify_mcp/tools/reddit.py +41 -0
- dataify_mcp/tools/task_status.py +188 -0
- dataify_mcp/tools/tiktok.py +81 -0
- dataify_mcp/tools/twitter.py +41 -0
- dataify_mcp/tools/user.py +79 -0
- dataify_mcp/tools/web_unlocker.py +53 -0
- dataify_mcp/tools/youtube.py +163 -0
- dataify_mcp/types/__init__.py +39 -0
- dataify_mcp/types/_enums.py +220 -0
- dataify_mcp/types/_errors.py +73 -0
- dataify_mcp/types/_mcp.py +89 -0
- dataify_sdk-0.1.0.dist-info/METADATA +154 -0
- dataify_sdk-0.1.0.dist-info/RECORD +34 -0
- dataify_sdk-0.1.0.dist-info/WHEEL +4 -0
- dataify_sdk-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"""SSE (Server-Sent Events) transport for MCP.
|
|
2
|
+
|
|
3
|
+
Implements the SSE transport: GET ``/sse`` for server-to-client events,
|
|
4
|
+
POST ``/message?sessionId=...`` for client-to-server requests.
|
|
5
|
+
|
|
6
|
+
Session auth bridging: the initial ``/sse`` connection carries ``?token=``.
|
|
7
|
+
The server stores the auth parameters in Redis keyed by sessionId (24h TTL).
|
|
8
|
+
Subsequent ``/message`` requests only need ``?sessionId=`` — the server
|
|
9
|
+
reads auth from Redis.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import json
|
|
16
|
+
import logging
|
|
17
|
+
import uuid
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import httpx
|
|
22
|
+
|
|
23
|
+
from dataify_mcp.types._errors import (
|
|
24
|
+
AuthenticationError,
|
|
25
|
+
ConnectionError,
|
|
26
|
+
ProtocolError,
|
|
27
|
+
ServerError,
|
|
28
|
+
TimeoutError,
|
|
29
|
+
)
|
|
30
|
+
from dataify_mcp.types._mcp import JSONRPCError, JSONRPCRequest, JSONRPCResponse
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
HEADER_SESSION_ID = "Mcp-Session-Id"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class SSETransport:
|
|
39
|
+
"""SSE transport that uses ``/sse`` + ``/message`` endpoints.
|
|
40
|
+
|
|
41
|
+
Parameters
|
|
42
|
+
----------
|
|
43
|
+
base_url:
|
|
44
|
+
Base URL of the MCP server, e.g. ``http://localhost:7780``.
|
|
45
|
+
token:
|
|
46
|
+
Dataify API token passed as ``?token=`` on the initial SSE connection.
|
|
47
|
+
tool_codes:
|
|
48
|
+
Optional comma-separated tool codes passed as ``?tools=``.
|
|
49
|
+
timeout:
|
|
50
|
+
HTTP request timeout in seconds (default 30).
|
|
51
|
+
reconnect_delay:
|
|
52
|
+
Initial delay in seconds before reconnecting on SSE disconnect (default 2).
|
|
53
|
+
max_reconnect_delay:
|
|
54
|
+
Maximum delay in seconds for exponential backoff (default 60).
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
base_url: str
|
|
58
|
+
token: str
|
|
59
|
+
tool_codes: str | None = None
|
|
60
|
+
timeout: float = 30.0
|
|
61
|
+
reconnect_delay: float = 2.0
|
|
62
|
+
max_reconnect_delay: float = 60.0
|
|
63
|
+
|
|
64
|
+
# --- internal state -------------------------------------------------
|
|
65
|
+
|
|
66
|
+
_http_client: httpx.AsyncClient | None = None
|
|
67
|
+
_session_id: str | None = None
|
|
68
|
+
_pending: dict[int, asyncio.Future[JSONRPCResponse]] | None = None
|
|
69
|
+
_sse_task: asyncio.Task[None] | None = None
|
|
70
|
+
_stop_event: asyncio.Event | None = None
|
|
71
|
+
|
|
72
|
+
# --- URL construction -----------------------------------------------
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def sse_url(self) -> str:
|
|
76
|
+
"""SSE endpoint URL with auth params."""
|
|
77
|
+
url = f"{self.base_url.rstrip('/')}/sse?token={self.token}"
|
|
78
|
+
if self.tool_codes:
|
|
79
|
+
url += f"&tools={self.tool_codes}"
|
|
80
|
+
return url
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def message_url(self) -> str:
|
|
84
|
+
"""Message endpoint URL."""
|
|
85
|
+
session = self._session_id or ""
|
|
86
|
+
return f"{self.base_url.rstrip('/')}/message?sessionId={session}"
|
|
87
|
+
|
|
88
|
+
# --- lifecycle ------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
async def open(self) -> None:
|
|
91
|
+
"""Open the transport: create HTTP client and start the SSE listener."""
|
|
92
|
+
if self._http_client is not None:
|
|
93
|
+
return
|
|
94
|
+
|
|
95
|
+
self._http_client = httpx.AsyncClient(timeout=httpx.Timeout(self.timeout))
|
|
96
|
+
self._pending = {}
|
|
97
|
+
self._stop_event = asyncio.Event()
|
|
98
|
+
self._sse_task = asyncio.create_task(self._listen_sse())
|
|
99
|
+
|
|
100
|
+
async def close(self) -> None:
|
|
101
|
+
"""Close the transport: stop SSE listener and clean up."""
|
|
102
|
+
if self._stop_event:
|
|
103
|
+
self._stop_event.set()
|
|
104
|
+
|
|
105
|
+
if self._sse_task:
|
|
106
|
+
self._sse_task.cancel()
|
|
107
|
+
try:
|
|
108
|
+
await self._sse_task
|
|
109
|
+
except asyncio.CancelledError:
|
|
110
|
+
pass
|
|
111
|
+
self._sse_task = None
|
|
112
|
+
|
|
113
|
+
if self._http_client is not None:
|
|
114
|
+
await self._http_client.aclose()
|
|
115
|
+
self._http_client = None
|
|
116
|
+
|
|
117
|
+
# Resolve any pending futures as errors
|
|
118
|
+
if self._pending:
|
|
119
|
+
for fut in self._pending.values():
|
|
120
|
+
if not fut.done():
|
|
121
|
+
fut.set_exception(ConnectionError("Transport closed"))
|
|
122
|
+
self._pending.clear()
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def is_open(self) -> bool:
|
|
126
|
+
"""Check whether the transport is currently open."""
|
|
127
|
+
return (
|
|
128
|
+
self._http_client is not None
|
|
129
|
+
and not self._http_client.is_closed
|
|
130
|
+
and self._stop_event is not None
|
|
131
|
+
and not self._stop_event.is_set()
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# --- request / response ---------------------------------------------
|
|
135
|
+
|
|
136
|
+
async def send_request(self, request: JSONRPCRequest) -> JSONRPCResponse:
|
|
137
|
+
"""Send a JSON-RPC request via POST to ``/message`` and wait for the SSE response.
|
|
138
|
+
|
|
139
|
+
Creates a Future keyed by the request id, posts the request, and
|
|
140
|
+
waits for the SSE listener to resolve the future.
|
|
141
|
+
"""
|
|
142
|
+
if self._http_client is None or self._pending is None:
|
|
143
|
+
raise ConnectionError("Transport is not open. Call open() first.")
|
|
144
|
+
|
|
145
|
+
if self._session_id is None:
|
|
146
|
+
raise ConnectionError(
|
|
147
|
+
"SSE session not yet established. Wait for the SSE listener to connect."
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
# Create a future to bridge POST → SSE
|
|
151
|
+
fut: asyncio.Future[JSONRPCResponse] = asyncio.get_event_loop().create_future()
|
|
152
|
+
self._pending[request.id] = fut
|
|
153
|
+
|
|
154
|
+
try:
|
|
155
|
+
body = {
|
|
156
|
+
"jsonrpc": request.jsonrpc,
|
|
157
|
+
"id": request.id,
|
|
158
|
+
"method": request.method,
|
|
159
|
+
"params": request.params or {},
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
logger.debug("SSE request id=%d method=%s", request.id, request.method)
|
|
163
|
+
|
|
164
|
+
http_resp = await self._http_client.post(
|
|
165
|
+
self.message_url,
|
|
166
|
+
json=body,
|
|
167
|
+
headers={"Content-Type": "application/json"},
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
if http_resp.status_code >= 500:
|
|
171
|
+
raise ServerError(f"Server returned HTTP {http_resp.status_code}")
|
|
172
|
+
|
|
173
|
+
# The response may come back synchronously or via SSE.
|
|
174
|
+
# If it's a direct response, use it immediately.
|
|
175
|
+
if http_resp.status_code == 200 and http_resp.text.strip():
|
|
176
|
+
try:
|
|
177
|
+
data = http_resp.json()
|
|
178
|
+
resp = JSONRPCResponse(
|
|
179
|
+
jsonrpc=data.get("jsonrpc", "2.0"),
|
|
180
|
+
id=data.get("id", request.id),
|
|
181
|
+
result=data.get("result"),
|
|
182
|
+
error=JSONRPCError(**data["error"]) if "error" in data else None,
|
|
183
|
+
)
|
|
184
|
+
if not fut.done():
|
|
185
|
+
fut.set_result(resp)
|
|
186
|
+
return resp
|
|
187
|
+
except json.JSONDecodeError:
|
|
188
|
+
pass # response will come via SSE
|
|
189
|
+
|
|
190
|
+
# Wait for the SSE event to deliver the response
|
|
191
|
+
return await asyncio.wait_for(fut, timeout=self.timeout)
|
|
192
|
+
|
|
193
|
+
except asyncio.TimeoutError:
|
|
194
|
+
raise TimeoutError(
|
|
195
|
+
f"Request id={request.id} method={request.method} timed out "
|
|
196
|
+
f"after {self.timeout}s"
|
|
197
|
+
)
|
|
198
|
+
except httpx.NetworkError as exc:
|
|
199
|
+
raise ConnectionError(f"Network error: {exc}") from exc
|
|
200
|
+
finally:
|
|
201
|
+
self._pending.pop(request.id, None)
|
|
202
|
+
|
|
203
|
+
async def send_notification(self, request: JSONRPCRequest) -> None:
|
|
204
|
+
"""Send a JSON-RPC notification (no response expected)."""
|
|
205
|
+
if self._http_client is None:
|
|
206
|
+
raise ConnectionError("Transport is not open.")
|
|
207
|
+
|
|
208
|
+
body = {
|
|
209
|
+
"jsonrpc": request.jsonrpc,
|
|
210
|
+
"method": request.method,
|
|
211
|
+
"params": request.params or {},
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
await self._http_client.post(
|
|
216
|
+
self.message_url,
|
|
217
|
+
json=body,
|
|
218
|
+
headers={"Content-Type": "application/json"},
|
|
219
|
+
)
|
|
220
|
+
except httpx.NetworkError as exc:
|
|
221
|
+
logger.warning("SSE notification %s failed: %s", request.method, exc)
|
|
222
|
+
|
|
223
|
+
# --- SSE listener ---------------------------------------------------
|
|
224
|
+
|
|
225
|
+
async def _listen_sse(self) -> None:
|
|
226
|
+
"""Long-running task that consumes SSE events from ``/sse``.
|
|
227
|
+
|
|
228
|
+
Handles:
|
|
229
|
+
* ``endpoint`` event — captures the sessionId for subsequent /message calls.
|
|
230
|
+
* ``message`` event — delivers JSON-RPC responses, resolving pending futures.
|
|
231
|
+
* Reconnection with exponential backoff on disconnect.
|
|
232
|
+
"""
|
|
233
|
+
if self._stop_event is None or self._http_client is None:
|
|
234
|
+
return
|
|
235
|
+
|
|
236
|
+
delay = self.reconnect_delay
|
|
237
|
+
|
|
238
|
+
while not self._stop_event.is_set():
|
|
239
|
+
try:
|
|
240
|
+
async with self._http_client.stream("GET", self.sse_url) as resp:
|
|
241
|
+
if resp.status_code == 401 or resp.status_code == 403:
|
|
242
|
+
raise AuthenticationError(
|
|
243
|
+
"SSE authentication failed. Check your API token."
|
|
244
|
+
)
|
|
245
|
+
if resp.status_code >= 500:
|
|
246
|
+
raise ServerError(f"SSE server error: HTTP {resp.status_code}")
|
|
247
|
+
|
|
248
|
+
delay = self.reconnect_delay # reset backoff on successful connect
|
|
249
|
+
|
|
250
|
+
async for line in resp.aiter_lines():
|
|
251
|
+
if self._stop_event.is_set():
|
|
252
|
+
return
|
|
253
|
+
|
|
254
|
+
if not line:
|
|
255
|
+
continue
|
|
256
|
+
|
|
257
|
+
if line.startswith("data:"):
|
|
258
|
+
data_str = line[5:].strip()
|
|
259
|
+
await self._handle_sse_data(data_str)
|
|
260
|
+
elif line.startswith("event:"):
|
|
261
|
+
event_type = line[6:].strip()
|
|
262
|
+
if event_type == "endpoint":
|
|
263
|
+
# The next data line will contain the session endpoint
|
|
264
|
+
pass
|
|
265
|
+
|
|
266
|
+
except asyncio.CancelledError:
|
|
267
|
+
return
|
|
268
|
+
except AuthenticationError:
|
|
269
|
+
raise # don't reconnect on auth failures
|
|
270
|
+
except Exception as exc:
|
|
271
|
+
logger.warning("SSE stream error: %s. Reconnecting in %.1fs...", exc, delay)
|
|
272
|
+
try:
|
|
273
|
+
await asyncio.wait_for(
|
|
274
|
+
self._stop_event.wait(), timeout=delay
|
|
275
|
+
)
|
|
276
|
+
return # stop was set
|
|
277
|
+
except asyncio.TimeoutError:
|
|
278
|
+
pass # time to reconnect
|
|
279
|
+
|
|
280
|
+
# Exponential backoff
|
|
281
|
+
delay = min(delay * 2, self.max_reconnect_delay)
|
|
282
|
+
|
|
283
|
+
async def _handle_sse_data(self, data_str: str) -> None:
|
|
284
|
+
"""Parse an SSE data line and dispatch to pending futures."""
|
|
285
|
+
if not data_str or self._pending is None:
|
|
286
|
+
return
|
|
287
|
+
|
|
288
|
+
try:
|
|
289
|
+
data = json.loads(data_str)
|
|
290
|
+
except json.JSONDecodeError:
|
|
291
|
+
return
|
|
292
|
+
|
|
293
|
+
# endpoint event: contains the session URI
|
|
294
|
+
if isinstance(data, str) and "sessionId=" in data:
|
|
295
|
+
# Extract sessionId from the URL
|
|
296
|
+
import urllib.parse
|
|
297
|
+
parsed = urllib.parse.urlparse(data)
|
|
298
|
+
qs = urllib.parse.parse_qs(parsed.query)
|
|
299
|
+
session_ids = qs.get("sessionId", [])
|
|
300
|
+
if session_ids:
|
|
301
|
+
self._session_id = session_ids[0]
|
|
302
|
+
logger.debug("SSE session established: %s", self._session_id)
|
|
303
|
+
return
|
|
304
|
+
|
|
305
|
+
if not isinstance(data, dict):
|
|
306
|
+
return
|
|
307
|
+
|
|
308
|
+
# It's a JSON-RPC response
|
|
309
|
+
resp_id = data.get("id")
|
|
310
|
+
if resp_id is not None and resp_id in self._pending:
|
|
311
|
+
resp = JSONRPCResponse(
|
|
312
|
+
jsonrpc=data.get("jsonrpc", "2.0"),
|
|
313
|
+
id=resp_id,
|
|
314
|
+
result=data.get("result"),
|
|
315
|
+
error=JSONRPCError(**data["error"]) if "error" in data else None,
|
|
316
|
+
)
|
|
317
|
+
fut = self._pending[resp_id]
|
|
318
|
+
if not fut.done():
|
|
319
|
+
fut.set_result(resp)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Typed tool wrappers for Dataify MCP tools.
|
|
2
|
+
|
|
3
|
+
Each sub-module wraps a category of tools with Pydantic models for
|
|
4
|
+
type-safe parameter passing. These modules are **auto-generated**
|
|
5
|
+
by ``scripts/codegen.py`` from a running Dataify MCP server.
|
|
6
|
+
|
|
7
|
+
Categories
|
|
8
|
+
----------
|
|
9
|
+
- ``task_status`` — web_unlock_task, web_unlock_statistics, scraper_task_list, etc.
|
|
10
|
+
- ``user`` — query_user_info, query_user_balance, query_user_api_keys, etc.
|
|
11
|
+
- ``web_unlocker`` — request_web_unlocker
|
|
12
|
+
- ``google_serp`` — google_search, google_news, google_images, google_maps, etc.
|
|
13
|
+
- ``google_scraper`` — google_map_details, google_map_comment, google_shopping_info, etc.
|
|
14
|
+
- ``bing`` — bing_search, bing_images, bing_maps, bing_news, etc.
|
|
15
|
+
- ``other_search`` — yandex_search, duckduckgo_search
|
|
16
|
+
- ``amazon`` — scrape_amazon_product, scrape_amazon_comment, etc.
|
|
17
|
+
- ``youtube`` — scrape_youtube_video, scrape_youtube_comment, etc.
|
|
18
|
+
- ``tiktok`` — scrape_tiktok_posts, scrape_tiktok_profiles, etc.
|
|
19
|
+
- ``facebook`` — scrape_facebook_post, scrape_facebook_profile, etc.
|
|
20
|
+
- ``instagram`` — scrape_instagram_profiles, etc.
|
|
21
|
+
- ``reddit`` — scrape_reddit_posts, etc.
|
|
22
|
+
- ``twitter`` — scrape_twitter_post, etc.
|
|
23
|
+
- ``linkedin`` — scrape_linkedin_company_information, etc.
|
|
24
|
+
- ``glassdoor`` — scrape_glassdoor_company, etc.
|
|
25
|
+
- ``indeed`` — scrape_indeed_companies_info, etc.
|
|
26
|
+
- ``other_scrapers`` — airbnb, booking, crunchbase, ebay, github, walmart, zillow
|
|
27
|
+
|
|
28
|
+
Regenerate with::
|
|
29
|
+
|
|
30
|
+
python scripts/codegen.py --server http://localhost:7780 --token YOUR_TOKEN
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from dataify_mcp.tools import (
|
|
36
|
+
amazon,
|
|
37
|
+
bing,
|
|
38
|
+
facebook,
|
|
39
|
+
glassdoor,
|
|
40
|
+
google_scraper,
|
|
41
|
+
google_serp,
|
|
42
|
+
indeed,
|
|
43
|
+
instagram,
|
|
44
|
+
linkedin,
|
|
45
|
+
other_scrapers,
|
|
46
|
+
other_search,
|
|
47
|
+
reddit,
|
|
48
|
+
task_status,
|
|
49
|
+
tiktok,
|
|
50
|
+
twitter,
|
|
51
|
+
user,
|
|
52
|
+
web_unlocker,
|
|
53
|
+
youtube,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
_ALL_MODULES = [
|
|
57
|
+
task_status,
|
|
58
|
+
user,
|
|
59
|
+
web_unlocker,
|
|
60
|
+
google_serp,
|
|
61
|
+
google_scraper,
|
|
62
|
+
bing,
|
|
63
|
+
other_search,
|
|
64
|
+
amazon,
|
|
65
|
+
youtube,
|
|
66
|
+
tiktok,
|
|
67
|
+
facebook,
|
|
68
|
+
instagram,
|
|
69
|
+
reddit,
|
|
70
|
+
twitter,
|
|
71
|
+
linkedin,
|
|
72
|
+
glassdoor,
|
|
73
|
+
indeed,
|
|
74
|
+
other_scrapers,
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def attach_all(client_cls: type) -> None:
|
|
79
|
+
"""Attach all typed tool methods to a DataifyClient subclass or instance.
|
|
80
|
+
|
|
81
|
+
Called automatically by ``DataifyClient.__init__`` so users don't
|
|
82
|
+
need to invoke this manually.
|
|
83
|
+
|
|
84
|
+
Parameters
|
|
85
|
+
----------
|
|
86
|
+
client_cls:
|
|
87
|
+
The ``DataifyClient`` class (not an instance).
|
|
88
|
+
"""
|
|
89
|
+
for mod in _ALL_MODULES:
|
|
90
|
+
mod._attach(client_cls)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Amazon scraper tools.
|
|
2
|
+
|
|
3
|
+
Wraps all Amazon-related MCP tools: product (5 spider IDs), comment,
|
|
4
|
+
seller, product list, and global product.
|
|
5
|
+
|
|
6
|
+
Auto-generated. Regenerate with: python scripts/codegen.py
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, Field
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
# scrape_amazon_product
|
|
18
|
+
# ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ScrapeAmazonProductParams(BaseModel):
|
|
22
|
+
"""Parameters for ``scrape_amazon_product``.
|
|
23
|
+
|
|
24
|
+
采集 Amazon 产品详情。支持 5 种采集方式: by-asin, by-url,
|
|
25
|
+
by-keywords, by-category-url, by-best-sellers。
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
spider_id: str = Field(
|
|
29
|
+
default="amazon_product_by-asin",
|
|
30
|
+
description="采集器标识: amazon_product_by-asin / by-url / by-keywords / by-category-url / by-best-sellers",
|
|
31
|
+
)
|
|
32
|
+
asin: str | None = Field(default="B0BZYCJK89", description="ASIN (用于 by-asin)")
|
|
33
|
+
url: str | None = Field(default="", description="产品/类别/畅销商品 URL (用于 by-url/by-category-url/by-best-sellers)")
|
|
34
|
+
category_url: str | None = Field(default="", description="畅销类别 URL (用于 by-best-sellers)")
|
|
35
|
+
keyword: str | None = Field(default="coffee", description="搜索关键词 (用于 by-keywords)")
|
|
36
|
+
page_turning: str | None = Field(default="", description="采集页数 (用于 by-keywords/by-category-url/by-best-sellers)")
|
|
37
|
+
lowest_price: str | None = Field(default="20", description="最低价格 (用于 by-keywords)")
|
|
38
|
+
highest_price: str | None = Field(default="50", description="最高价格 (用于 by-keywords)")
|
|
39
|
+
sort_by: str | None = Field(default="畅销排行", description="排序方式 (用于 by-category-url)")
|
|
40
|
+
collect_subcategories: str | None = Field(default="", description="收集子类别 (用于 by-category-url)")
|
|
41
|
+
zip_code: str | None = Field(default="94107", description="邮政编码 (用于 by-url)")
|
|
42
|
+
file_name: str | None = Field(default="{{TasksID}}", description="Builder file_name")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def scrape_amazon_product(self, params: ScrapeAmazonProductParams | None = None) -> Any:
|
|
46
|
+
"""Call ``scrape_amazon_product`` — Amazon 产品详情采集。
|
|
47
|
+
|
|
48
|
+
通过 ASIN、URL、关键词、类别 URL 或畅销排行榜采集 Amazon 产品详情。
|
|
49
|
+
"""
|
|
50
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
51
|
+
return await self.call_tool("scrape_amazon_product", arguments)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
# scrape_amazon_comment
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class ScrapeAmazonCommentParams(BaseModel):
|
|
60
|
+
"""Parameters for ``scrape_amazon_comment`` — Amazon 评论采集。"""
|
|
61
|
+
|
|
62
|
+
url: str = Field(..., description="Amazon 产品评论页面 URL")
|
|
63
|
+
file_name: str | None = Field(default="{{TasksID}}", description="Builder file_name")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
async def scrape_amazon_comment(self, params: ScrapeAmazonCommentParams | None = None) -> Any:
|
|
67
|
+
"""Call ``scrape_amazon_comment`` — Amazon 评论采集。"""
|
|
68
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
69
|
+
return await self.call_tool("scrape_amazon_comment", arguments)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# ---------------------------------------------------------------------------
|
|
73
|
+
# scrape_amazon_seller
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ScrapeAmazonSellerParams(BaseModel):
|
|
78
|
+
"""Parameters for ``scrape_amazon_seller`` — Amazon 卖家信息采集。"""
|
|
79
|
+
|
|
80
|
+
url: str = Field(..., description="Amazon 卖家页面 URL")
|
|
81
|
+
file_name: str | None = Field(default="{{TasksID}}", description="Builder file_name")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
async def scrape_amazon_seller(self, params: ScrapeAmazonSellerParams | None = None) -> Any:
|
|
85
|
+
"""Call ``scrape_amazon_seller`` — Amazon 卖家信息采集。"""
|
|
86
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
87
|
+
return await self.call_tool("scrape_amazon_seller", arguments)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
# scrape_amazon_product_list
|
|
92
|
+
# ---------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class ScrapeAmazonProductListParams(BaseModel):
|
|
96
|
+
"""Parameters for ``scrape_amazon_product_list`` — Amazon 产品列表采集。"""
|
|
97
|
+
|
|
98
|
+
url: str = Field(..., description="Amazon 产品列表/搜索结果 URL")
|
|
99
|
+
file_name: str | None = Field(default="{{TasksID}}", description="Builder file_name")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
async def scrape_amazon_product_list(self, params: ScrapeAmazonProductListParams | None = None) -> Any:
|
|
103
|
+
"""Call ``scrape_amazon_product_list`` — Amazon 产品列表采集。"""
|
|
104
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
105
|
+
return await self.call_tool("scrape_amazon_product_list", arguments)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
# scrape_amazon_global_product
|
|
110
|
+
# ---------------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ScrapeAmazonGlobalProductParams(BaseModel):
|
|
114
|
+
"""Parameters for ``scrape_amazon_global_product`` — Amazon 全球产品采集。"""
|
|
115
|
+
|
|
116
|
+
url: str = Field(..., description="Amazon 全球站点产品 URL")
|
|
117
|
+
file_name: str | None = Field(default="{{TasksID}}", description="Builder file_name")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
async def scrape_amazon_global_product(self, params: ScrapeAmazonGlobalProductParams | None = None) -> Any:
|
|
121
|
+
"""Call ``scrape_amazon_global_product`` — Amazon 全球产品采集。"""
|
|
122
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
123
|
+
return await self.call_tool("scrape_amazon_global_product", arguments)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
# Attach methods to DataifyClient
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _attach(client_cls: type) -> None:
|
|
132
|
+
"""Attach all Amazon scraper methods to the client class."""
|
|
133
|
+
client_cls.scrape_amazon_product = scrape_amazon_product
|
|
134
|
+
client_cls.scrape_amazon_comment = scrape_amazon_comment
|
|
135
|
+
client_cls.scrape_amazon_seller = scrape_amazon_seller
|
|
136
|
+
client_cls.scrape_amazon_product_list = scrape_amazon_product_list
|
|
137
|
+
client_cls.scrape_amazon_global_product = scrape_amazon_global_product
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Bing search tools.
|
|
2
|
+
|
|
3
|
+
Wraps all Bing search variant MCP tools: search, images, maps, news,
|
|
4
|
+
shopping, and videos.
|
|
5
|
+
|
|
6
|
+
Auto-generated. Regenerate with: python scripts/codegen.py
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, Field
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
# bing_search
|
|
18
|
+
# ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BingSearchParams(BaseModel):
|
|
22
|
+
"""Parameters for ``bing_search`` — Bing 网页搜索。
|
|
23
|
+
|
|
24
|
+
通过 Bing 搜索公开网页信息,按位置、国家/地区、语言等条件获取结果。
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
model_config = {"protected_namespaces": ()}
|
|
28
|
+
|
|
29
|
+
q: str = Field(default="Pizza", description="搜索查询内容")
|
|
30
|
+
json: str | None = Field(default="1", description="输出格式: 1 JSON, 2 JSON+HTML, 3 HTML, 4 Light JSON")
|
|
31
|
+
location: str | None = Field(default="", description="搜索发起的地理位置")
|
|
32
|
+
lat: str | None = Field(default="", description="GPS 纬度")
|
|
33
|
+
lon: str | None = Field(default="", description="GPS 经度")
|
|
34
|
+
mkt: str | None = Field(default="", description="显示语言 <语言代码>-<国家/地区代码>,如 en-US")
|
|
35
|
+
cc: str | None = Field(default="", description="国家/地区代码 (两位)")
|
|
36
|
+
first: str | None = Field(default="0", description="结果偏移量")
|
|
37
|
+
safeSearch: str | None = Field(default="", description="成人内容过滤: Off, Moderate, Strict")
|
|
38
|
+
filters: str | None = Field(default="", description="高级过滤选项")
|
|
39
|
+
no_cache: str | None = Field(default="false", description="是否跳过缓存")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
async def bing_search(self, params: BingSearchParams | None = None) -> Any:
|
|
43
|
+
"""Call ``bing_search`` — Bing 网页搜索。"""
|
|
44
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
45
|
+
return await self.call_tool("bing_search", arguments)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# ---------------------------------------------------------------------------
|
|
49
|
+
# bing_images
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class BingImagesParams(BaseModel):
|
|
54
|
+
"""Parameters for ``bing_images`` — Bing 图片搜索。"""
|
|
55
|
+
|
|
56
|
+
model_config = {"protected_namespaces": ()}
|
|
57
|
+
|
|
58
|
+
q: str = Field(default="Pizza", description="搜索查询内容")
|
|
59
|
+
json: str | None = Field(default="1", description="输出格式")
|
|
60
|
+
mkt: str | None = Field(default="", description="显示语言")
|
|
61
|
+
cc: str | None = Field(default="", description="国家/地区代码")
|
|
62
|
+
first: str | None = Field(default="0", description="结果偏移量")
|
|
63
|
+
safeSearch: str | None = Field(default="", description="成人内容过滤")
|
|
64
|
+
no_cache: str | None = Field(default="false", description="是否跳过缓存")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
async def bing_images(self, params: BingImagesParams | None = None) -> Any:
|
|
68
|
+
"""Call ``bing_images`` — Bing 图片搜索。"""
|
|
69
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
70
|
+
return await self.call_tool("bing_images", arguments)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
# bing_maps, bing_news, bing_shopping, bing_videos
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _BingVariantParams(BaseModel):
|
|
79
|
+
"""Shared params for Bing variants (maps, news, shopping, videos)."""
|
|
80
|
+
|
|
81
|
+
model_config = {"protected_namespaces": ()}
|
|
82
|
+
|
|
83
|
+
q: str = Field(default="Pizza", description="搜索查询内容")
|
|
84
|
+
json: str | None = Field(default="1", description="输出格式")
|
|
85
|
+
mkt: str | None = Field(default="", description="显示语言")
|
|
86
|
+
cc: str | None = Field(default="", description="国家/地区代码")
|
|
87
|
+
first: str | None = Field(default="0", description="结果偏移量")
|
|
88
|
+
safeSearch: str | None = Field(default="", description="成人内容过滤")
|
|
89
|
+
no_cache: str | None = Field(default="false", description="是否跳过缓存")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class BingMapsParams(_BingVariantParams):
|
|
93
|
+
"""Parameters for ``bing_maps`` — Bing 地图搜索。"""
|
|
94
|
+
pass
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class BingNewsParams(_BingVariantParams):
|
|
98
|
+
"""Parameters for ``bing_news`` — Bing 新闻搜索。"""
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class BingShoppingParams(_BingVariantParams):
|
|
103
|
+
"""Parameters for ``bing_shopping`` — Bing 购物搜索。"""
|
|
104
|
+
pass
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class BingVideosParams(_BingVariantParams):
|
|
108
|
+
"""Parameters for ``bing_videos`` — Bing 视频搜索。"""
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_BING_TOOLS = {
|
|
113
|
+
"bing_maps": ("Bing 地图搜索", BingMapsParams),
|
|
114
|
+
"bing_news": ("Bing 新闻搜索", BingNewsParams),
|
|
115
|
+
"bing_shopping": ("Bing 购物搜索", BingShoppingParams),
|
|
116
|
+
"bing_videos": ("Bing 视频搜索", BingVideosParams),
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _make_bing_method(tool_name: str, desc: str, param_cls: type[BaseModel]):
|
|
121
|
+
async def _method(self, params: param_cls | None = None) -> Any: # type: ignore[valid-type]
|
|
122
|
+
arguments = params.model_dump(exclude_none=True) if params else {}
|
|
123
|
+
return await self.call_tool(tool_name, arguments)
|
|
124
|
+
|
|
125
|
+
_method.__name__ = tool_name
|
|
126
|
+
_method.__doc__ = f"""Call ``{tool_name}`` — {desc}."""
|
|
127
|
+
return _method
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ---------------------------------------------------------------------------
|
|
131
|
+
# Attach methods to DataifyClient
|
|
132
|
+
# ---------------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _attach(client_cls: type) -> None:
|
|
136
|
+
"""Attach all Bing methods to the client class."""
|
|
137
|
+
client_cls.bing_search = bing_search
|
|
138
|
+
client_cls.bing_images = bing_images
|
|
139
|
+
|
|
140
|
+
for tool_name, (desc, param_cls) in _BING_TOOLS.items():
|
|
141
|
+
setattr(client_cls, tool_name, _make_bing_method(tool_name, desc, param_cls))
|