langchain-dataxapi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_dataxapi/__init__.py +55 -0
- langchain_dataxapi/_client.py +226 -0
- langchain_dataxapi/endpoint_tools.py +495 -0
- langchain_dataxapi/py.typed +0 -0
- langchain_dataxapi/toolkits.py +55 -0
- langchain_dataxapi/tools.py +258 -0
- langchain_dataxapi-0.1.0.dist-info/METADATA +232 -0
- langchain_dataxapi-0.1.0.dist-info/RECORD +10 -0
- langchain_dataxapi-0.1.0.dist-info/WHEEL +4 -0
- langchain_dataxapi-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""LangChain tools for DataXAPI: social media and market data for agents.
|
|
2
|
+
|
|
3
|
+
``DataXAPIFind`` finds the endpoint for a need (free), ``DataXAPICall`` calls any endpoint by ID, and
|
|
4
|
+
one tool per popular endpoint calls it with typed arguments. ``DataXAPIToolkit`` gives them all.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from importlib import metadata
|
|
8
|
+
|
|
9
|
+
from langchain_dataxapi._client import DataXAPIClient, DataXAPIError
|
|
10
|
+
from langchain_dataxapi.endpoint_tools import (
|
|
11
|
+
ENDPOINT_TOOLS,
|
|
12
|
+
DataXAPIDouyinKeywordTrend,
|
|
13
|
+
DataXAPIDouyinVideoComments,
|
|
14
|
+
DataXAPIDouyinVideoMetrics,
|
|
15
|
+
DataXAPIDouyinVideoSearch,
|
|
16
|
+
DataXAPITikTokVideoSearch,
|
|
17
|
+
DataXAPIWeChatArticle,
|
|
18
|
+
DataXAPIWeChatChannelsVideoSearch,
|
|
19
|
+
DataXAPIWeChatOfficialAccountArticles,
|
|
20
|
+
DataXAPIWeChatSearch,
|
|
21
|
+
DataXAPIXiaohongshuNoteComments,
|
|
22
|
+
DataXAPIXiaohongshuNoteSearch,
|
|
23
|
+
DataXAPIYouTubeTranscript,
|
|
24
|
+
)
|
|
25
|
+
from langchain_dataxapi.toolkits import DataXAPIToolkit
|
|
26
|
+
from langchain_dataxapi.tools import DataXAPIBaseTool, DataXAPICall, DataXAPIEndpointTool, DataXAPIFind
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
__version__ = metadata.version("langchain-dataxapi")
|
|
30
|
+
except metadata.PackageNotFoundError:
|
|
31
|
+
__version__ = ""
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"ENDPOINT_TOOLS",
|
|
35
|
+
"DataXAPIBaseTool",
|
|
36
|
+
"DataXAPICall",
|
|
37
|
+
"DataXAPIClient",
|
|
38
|
+
"DataXAPIDouyinKeywordTrend",
|
|
39
|
+
"DataXAPIDouyinVideoComments",
|
|
40
|
+
"DataXAPIDouyinVideoMetrics",
|
|
41
|
+
"DataXAPIDouyinVideoSearch",
|
|
42
|
+
"DataXAPIEndpointTool",
|
|
43
|
+
"DataXAPIError",
|
|
44
|
+
"DataXAPIFind",
|
|
45
|
+
"DataXAPITikTokVideoSearch",
|
|
46
|
+
"DataXAPIToolkit",
|
|
47
|
+
"DataXAPIWeChatArticle",
|
|
48
|
+
"DataXAPIWeChatChannelsVideoSearch",
|
|
49
|
+
"DataXAPIWeChatOfficialAccountArticles",
|
|
50
|
+
"DataXAPIWeChatSearch",
|
|
51
|
+
"DataXAPIXiaohongshuNoteComments",
|
|
52
|
+
"DataXAPIXiaohongshuNoteSearch",
|
|
53
|
+
"DataXAPIYouTubeTranscript",
|
|
54
|
+
"__version__",
|
|
55
|
+
]
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""A small DataXAPI HTTP client, sync and async, shared by every tool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import uuid
|
|
7
|
+
from importlib import metadata
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
DEFAULT_BASE_URL = "https://api.dataxapi.com"
|
|
13
|
+
DEFAULT_TIMEOUT = 90.0
|
|
14
|
+
SOURCE = "langchain"
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
_VERSION = metadata.version("langchain-dataxapi")
|
|
18
|
+
except metadata.PackageNotFoundError: # running from a source checkout
|
|
19
|
+
_VERSION = "0"
|
|
20
|
+
USER_AGENT = f"langchain-dataxapi/{_VERSION}"
|
|
21
|
+
|
|
22
|
+
# Endpoint IDs look like "douyin.videos.comments.list".
|
|
23
|
+
ENDPOINT_ID_RE = re.compile(r"^[a-z0-9_]+(\.[a-z0-9_]+)+$")
|
|
24
|
+
# A request carrying one of these goes as POST with a JSON body.
|
|
25
|
+
BODY_PARAMS = frozenset({"ids", "keywords"})
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DataXAPIError(Exception):
|
|
29
|
+
"""An error answer from DataXAPI, or a failure to reach it."""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
code: str,
|
|
34
|
+
message: str,
|
|
35
|
+
*,
|
|
36
|
+
status: int | None = None,
|
|
37
|
+
request_id: str | None = None,
|
|
38
|
+
charged: bool | None = False,
|
|
39
|
+
retry: str | None = None,
|
|
40
|
+
) -> None:
|
|
41
|
+
super().__init__(f"{code}: {message}")
|
|
42
|
+
self.code = code
|
|
43
|
+
self.message = message
|
|
44
|
+
self.status = status
|
|
45
|
+
self.request_id = request_id
|
|
46
|
+
self.charged = charged
|
|
47
|
+
self.retry = retry
|
|
48
|
+
|
|
49
|
+
def describe(self) -> str:
|
|
50
|
+
"""The error as one paragraph an agent can act on."""
|
|
51
|
+
message = self.message if self.message.endswith((".", "。", "!", "?")) else self.message + "."
|
|
52
|
+
parts = [f"DataXAPI error {self.code}: {message}"]
|
|
53
|
+
if self.charged is False:
|
|
54
|
+
parts.append("Nothing was charged.")
|
|
55
|
+
elif self.charged is True:
|
|
56
|
+
parts.append("This answer was charged as a call.")
|
|
57
|
+
elif self.request_id:
|
|
58
|
+
parts.append(f"Whether it was charged is not known; look the call up at /v1/calls/{self.request_id}.")
|
|
59
|
+
if self.retry == "yes":
|
|
60
|
+
parts.append("Retrying shortly may work.")
|
|
61
|
+
elif self.retry == "after_fix":
|
|
62
|
+
parts.append("Fix the request before trying again.")
|
|
63
|
+
elif self.retry == "no":
|
|
64
|
+
parts.append("Do not retry the same request.")
|
|
65
|
+
elif self.retry:
|
|
66
|
+
parts.append(f"Retry: {self.retry}.")
|
|
67
|
+
if self.request_id:
|
|
68
|
+
parts.append(f"Request ID: {self.request_id}.")
|
|
69
|
+
return " ".join(parts)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def endpoint_path(endpoint_id: str) -> str:
|
|
73
|
+
if not ENDPOINT_ID_RE.match(endpoint_id or ""):
|
|
74
|
+
raise DataXAPIError(
|
|
75
|
+
"invalid_endpoint",
|
|
76
|
+
"The endpoint ID must look like 'douyin.videos.comments.list'; find one with dataxapi_find.",
|
|
77
|
+
)
|
|
78
|
+
return "/v1/" + endpoint_id.replace(".", "/")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
|
82
|
+
"""Drop empty values so optional arguments left out are not sent."""
|
|
83
|
+
return {k: v for k, v in params.items() if v is not None and v != "" and v != []}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def to_query(params: dict[str, Any]) -> dict[str, str]:
|
|
87
|
+
query: dict[str, str] = {}
|
|
88
|
+
for key, value in params.items():
|
|
89
|
+
if isinstance(value, bool):
|
|
90
|
+
query[key] = "true" if value else "false"
|
|
91
|
+
elif isinstance(value, (list, tuple)):
|
|
92
|
+
query[key] = ",".join(str(v) for v in value)
|
|
93
|
+
else:
|
|
94
|
+
query[key] = str(value)
|
|
95
|
+
return query
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def summarize(envelope: dict[str, Any]) -> dict[str, Any]:
|
|
99
|
+
"""What an agent needs from an answer: the data, the next page, the charge and any warnings."""
|
|
100
|
+
meta = envelope.get("meta") or {}
|
|
101
|
+
out: dict[str, Any] = {
|
|
102
|
+
"endpoint": meta.get("capability"),
|
|
103
|
+
"data": envelope.get("data"),
|
|
104
|
+
"page": envelope.get("page"),
|
|
105
|
+
"billing": meta.get("billing"),
|
|
106
|
+
"request_id": envelope.get("request_id"),
|
|
107
|
+
}
|
|
108
|
+
if envelope.get("warnings"):
|
|
109
|
+
out["warnings"] = envelope["warnings"]
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _error(response: httpx.Response) -> DataXAPIError:
|
|
114
|
+
try:
|
|
115
|
+
payload = response.json()
|
|
116
|
+
except ValueError:
|
|
117
|
+
payload = None
|
|
118
|
+
if isinstance(payload, dict) and isinstance(payload.get("code"), str):
|
|
119
|
+
return DataXAPIError(
|
|
120
|
+
payload["code"],
|
|
121
|
+
str(payload.get("message") or "The request failed."),
|
|
122
|
+
status=response.status_code,
|
|
123
|
+
request_id=payload.get("request_id"),
|
|
124
|
+
charged=payload.get("charged", False),
|
|
125
|
+
retry=payload.get("retry"),
|
|
126
|
+
)
|
|
127
|
+
return DataXAPIError(
|
|
128
|
+
"http_error",
|
|
129
|
+
f"HTTP {response.status_code} without a DataXAPI answer.",
|
|
130
|
+
status=response.status_code,
|
|
131
|
+
charged=None,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _result(response: httpx.Response) -> dict[str, Any]:
|
|
136
|
+
if response.is_success:
|
|
137
|
+
try:
|
|
138
|
+
payload = response.json()
|
|
139
|
+
except ValueError:
|
|
140
|
+
payload = None
|
|
141
|
+
if isinstance(payload, dict):
|
|
142
|
+
return payload
|
|
143
|
+
raise _error(response)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _connection_error(exc: Exception) -> DataXAPIError:
|
|
147
|
+
# The exception's name only: its text could carry the URL, never the key, but keep it short.
|
|
148
|
+
return DataXAPIError("connection_error", f"Could not reach DataXAPI ({type(exc).__name__}).", charged=None)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class DataXAPIClient:
|
|
152
|
+
"""Calls DataXAPI. One page per call; each data call is charged on its own, errors are not."""
|
|
153
|
+
|
|
154
|
+
def __init__(
|
|
155
|
+
self,
|
|
156
|
+
api_key: str | None,
|
|
157
|
+
*,
|
|
158
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
159
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
160
|
+
transport: httpx.BaseTransport | httpx.AsyncBaseTransport | None = None,
|
|
161
|
+
) -> None:
|
|
162
|
+
self._api_key = api_key or None
|
|
163
|
+
self._base_url = base_url.rstrip("/")
|
|
164
|
+
self._timeout = timeout
|
|
165
|
+
self._transport = transport
|
|
166
|
+
|
|
167
|
+
def _headers(self, *, needs_key: bool) -> dict[str, str]:
|
|
168
|
+
if needs_key and not self._api_key:
|
|
169
|
+
raise DataXAPIError(
|
|
170
|
+
"unauthorized",
|
|
171
|
+
"No DataXAPI API key. Set DATAXAPI_API_KEY or pass api_key; "
|
|
172
|
+
"create one at https://dataxapi.com/console/keys/.",
|
|
173
|
+
charged=False,
|
|
174
|
+
retry="after_fix",
|
|
175
|
+
)
|
|
176
|
+
headers = {"Accept": "application/json", "User-Agent": USER_AGENT, "X-Source": SOURCE}
|
|
177
|
+
if self._api_key:
|
|
178
|
+
headers["Authorization"] = f"Bearer {self._api_key}"
|
|
179
|
+
if needs_key:
|
|
180
|
+
# A fresh key per call: a call made again on purpose is a new call.
|
|
181
|
+
headers["Idempotency-Key"] = str(uuid.uuid4())
|
|
182
|
+
return headers
|
|
183
|
+
|
|
184
|
+
@staticmethod
|
|
185
|
+
def _plan(endpoint_id: str, params: dict[str, Any] | None) -> tuple[str, str, dict[str, Any]]:
|
|
186
|
+
path = endpoint_path(endpoint_id)
|
|
187
|
+
given = clean_params(params or {})
|
|
188
|
+
if BODY_PARAMS.intersection(given):
|
|
189
|
+
return "POST", path, {"json": given}
|
|
190
|
+
return "GET", path, {"params": to_query(given)}
|
|
191
|
+
|
|
192
|
+
def call(self, endpoint_id: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
193
|
+
"""Fetch one page from one endpoint."""
|
|
194
|
+
method, path, args = self._plan(endpoint_id, params)
|
|
195
|
+
return self._send(method, path, needs_key=True, **args)
|
|
196
|
+
|
|
197
|
+
async def acall(self, endpoint_id: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
198
|
+
method, path, args = self._plan(endpoint_id, params)
|
|
199
|
+
return await self._asend(method, path, needs_key=True, **args)
|
|
200
|
+
|
|
201
|
+
def find(self, query: str, platform: str | None = None) -> dict[str, Any]:
|
|
202
|
+
"""Endpoints for a need described in plain words, with price and parameters. Free, no key."""
|
|
203
|
+
query_params = to_query(clean_params({"query": query, "platform": platform}))
|
|
204
|
+
return self._send("GET", "/v1/catalog/search", needs_key=False, params=query_params)
|
|
205
|
+
|
|
206
|
+
async def afind(self, query: str, platform: str | None = None) -> dict[str, Any]:
|
|
207
|
+
query_params = to_query(clean_params({"query": query, "platform": platform}))
|
|
208
|
+
return await self._asend("GET", "/v1/catalog/search", needs_key=False, params=query_params)
|
|
209
|
+
|
|
210
|
+
def _send(self, method: str, path: str, *, needs_key: bool, **kwargs: Any) -> dict[str, Any]:
|
|
211
|
+
headers = self._headers(needs_key=needs_key)
|
|
212
|
+
try:
|
|
213
|
+
with httpx.Client(transport=self._transport, timeout=self._timeout) as http: # type: ignore[arg-type]
|
|
214
|
+
response = http.request(method, self._base_url + path, headers=headers, **kwargs)
|
|
215
|
+
except httpx.HTTPError as exc:
|
|
216
|
+
raise _connection_error(exc) from None
|
|
217
|
+
return _result(response)
|
|
218
|
+
|
|
219
|
+
async def _asend(self, method: str, path: str, *, needs_key: bool, **kwargs: Any) -> dict[str, Any]:
|
|
220
|
+
headers = self._headers(needs_key=needs_key)
|
|
221
|
+
try:
|
|
222
|
+
async with httpx.AsyncClient(transport=self._transport, timeout=self._timeout) as http: # type: ignore[arg-type]
|
|
223
|
+
response = await http.request(method, self._base_url + path, headers=headers, **kwargs)
|
|
224
|
+
except httpx.HTTPError as exc:
|
|
225
|
+
raise _connection_error(exc) from None
|
|
226
|
+
return _result(response)
|