twittrapi-haystack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- haystack_integrations/components/websearch/twittrapi/__init__.py +3 -0
- haystack_integrations/components/websearch/twittrapi/py.typed +0 -0
- haystack_integrations/components/websearch/twittrapi/tweet_search.py +179 -0
- haystack_integrations/tools/twittrapi/__init__.py +3 -0
- haystack_integrations/tools/twittrapi/toolset.py +324 -0
- twittrapi_haystack-0.1.0.dist-info/METADATA +159 -0
- twittrapi_haystack-0.1.0.dist-info/RECORD +9 -0
- twittrapi_haystack-0.1.0.dist-info/WHEEL +4 -0
- twittrapi_haystack-0.1.0.dist-info/licenses/LICENSE +21 -0
|
File without changes
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Haystack component that searches X (Twitter) through TwittrAPI and returns Documents."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
|
+
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
9
|
+
|
|
10
|
+
from twittrapi import TwittrAPI, TwittrAPIError
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
SearchCategory = Literal["Top", "Latest", "People", "Photos", "Videos"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def tweet_to_document(tweet: dict[str, Any]) -> Document:
|
|
18
|
+
"""Turns one simple-format tweet into a Document: the text as content, the rest as meta."""
|
|
19
|
+
author = tweet.get("author") or {}
|
|
20
|
+
counts = tweet.get("counts") or {}
|
|
21
|
+
username = author.get("username")
|
|
22
|
+
url = tweet.get("url") or (
|
|
23
|
+
f"https://x.com/{username}/status/{tweet['id']}" if username and tweet.get("id") else None
|
|
24
|
+
)
|
|
25
|
+
meta = {
|
|
26
|
+
"id": tweet.get("id"),
|
|
27
|
+
"url": url,
|
|
28
|
+
"created_at": tweet.get("createdAt"),
|
|
29
|
+
"lang": tweet.get("lang"),
|
|
30
|
+
"author_id": author.get("id"),
|
|
31
|
+
"author_username": username,
|
|
32
|
+
"author_name": author.get("name"),
|
|
33
|
+
"likes": counts.get("likes", 0),
|
|
34
|
+
"retweets": counts.get("retweets", 0),
|
|
35
|
+
"replies": counts.get("replies", 0),
|
|
36
|
+
"quotes": counts.get("quotes", 0),
|
|
37
|
+
"views": counts.get("views", 0),
|
|
38
|
+
"conversation_id": tweet.get("conversationId"),
|
|
39
|
+
}
|
|
40
|
+
return Document(content=tweet.get("text") or "", meta={k: v for k, v in meta.items() if v is not None})
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def user_to_document(user: dict[str, Any]) -> Document:
|
|
44
|
+
"""Turns one simple-format user (category People) into a Document."""
|
|
45
|
+
counts = user.get("counts") or {}
|
|
46
|
+
username = user.get("username")
|
|
47
|
+
content = f"{user.get('name') or username} (@{username})"
|
|
48
|
+
if user.get("description"):
|
|
49
|
+
content += f": {user['description']}"
|
|
50
|
+
meta = {
|
|
51
|
+
"id": user.get("id"),
|
|
52
|
+
"url": f"https://x.com/{username}" if username else None,
|
|
53
|
+
"username": username,
|
|
54
|
+
"name": user.get("name"),
|
|
55
|
+
"followers": counts.get("followers", 0),
|
|
56
|
+
"following": counts.get("following", 0),
|
|
57
|
+
"verified": user.get("verified"),
|
|
58
|
+
"created_at": user.get("createdAt"),
|
|
59
|
+
}
|
|
60
|
+
return Document(content=content, meta={k: v for k, v in meta.items() if v is not None})
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@component
|
|
64
|
+
class TwittrAPITweetSearch:
|
|
65
|
+
"""
|
|
66
|
+
Searches X (Twitter) through [TwittrAPI](https://www.twittrapi.com) and returns the results as Documents.
|
|
67
|
+
|
|
68
|
+
The query takes the same operators as the X search box (`from:`, `lang:`, `since:`,
|
|
69
|
+
`min_faves:`, `"exact phrase"`, `-filter:replies`, ...). Each tweet becomes one Document: the
|
|
70
|
+
full text as `content`, the author, URL, time and engagement counts in `meta`. With
|
|
71
|
+
`category="People"` the results are accounts instead of tweets.
|
|
72
|
+
|
|
73
|
+
The output has the same `documents` and `links` keys as the other web-search components,
|
|
74
|
+
plus `next_cursor` for the next page.
|
|
75
|
+
|
|
76
|
+
Usage example:
|
|
77
|
+
```python
|
|
78
|
+
from haystack_integrations.components.websearch.twittrapi import TwittrAPITweetSearch
|
|
79
|
+
|
|
80
|
+
search = TwittrAPITweetSearch(top_k=20) # reads TWITTRAPI_API_KEY
|
|
81
|
+
result = search.run(query="haystack lang:en -filter:replies")
|
|
82
|
+
for doc in result["documents"]:
|
|
83
|
+
print(doc.meta["author_username"], doc.content)
|
|
84
|
+
```
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
api_key: Secret = Secret.from_env_var("TWITTRAPI_API_KEY"),
|
|
90
|
+
category: SearchCategory = "Latest",
|
|
91
|
+
top_k: int | None = 20,
|
|
92
|
+
base_url: str | None = None,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""
|
|
95
|
+
Creates a TwittrAPITweetSearch component.
|
|
96
|
+
|
|
97
|
+
:param api_key: TwittrAPI key, sent as `X-API-KEY`. Defaults to the `TWITTRAPI_API_KEY`
|
|
98
|
+
environment variable.
|
|
99
|
+
:param category: Search tab: `"Latest"` (newest first), `"Top"`, `"People"`,
|
|
100
|
+
`"Photos"` or `"Videos"`. Can be changed per `run`.
|
|
101
|
+
:param top_k: Results to request per page (1 to 100) and the most Documents returned.
|
|
102
|
+
`None` uses the API default of 20 and keeps everything X returns.
|
|
103
|
+
:param base_url: Override the API base URL. Rarely needed.
|
|
104
|
+
"""
|
|
105
|
+
if top_k is not None and not 1 <= top_k <= 100:
|
|
106
|
+
raise ValueError("top_k must be between 1 and 100, or None.")
|
|
107
|
+
self.api_key = api_key
|
|
108
|
+
self.category = category
|
|
109
|
+
self.top_k = top_k
|
|
110
|
+
self.base_url = base_url
|
|
111
|
+
self._client: TwittrAPI | None = None
|
|
112
|
+
|
|
113
|
+
def warm_up(self) -> None:
|
|
114
|
+
"""Creates the API client."""
|
|
115
|
+
if self._client is None:
|
|
116
|
+
kwargs: dict[str, Any] = {"api_key": self.api_key.resolve_value()}
|
|
117
|
+
if self.base_url:
|
|
118
|
+
kwargs["base_url"] = self.base_url
|
|
119
|
+
self._client = TwittrAPI(**kwargs)
|
|
120
|
+
|
|
121
|
+
def to_dict(self) -> dict[str, Any]:
|
|
122
|
+
"""Serializes the component to a dictionary."""
|
|
123
|
+
return default_to_dict(
|
|
124
|
+
self,
|
|
125
|
+
api_key=self.api_key.to_dict(),
|
|
126
|
+
category=self.category,
|
|
127
|
+
top_k=self.top_k,
|
|
128
|
+
base_url=self.base_url,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
@classmethod
|
|
132
|
+
def from_dict(cls, data: dict[str, Any]) -> TwittrAPITweetSearch:
|
|
133
|
+
"""Deserializes the component from a dictionary."""
|
|
134
|
+
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
|
135
|
+
return default_from_dict(cls, data)
|
|
136
|
+
|
|
137
|
+
@component.output_types(documents=list[Document], links=list[str], next_cursor=str | None)
|
|
138
|
+
def run(
|
|
139
|
+
self,
|
|
140
|
+
query: str,
|
|
141
|
+
category: SearchCategory | None = None,
|
|
142
|
+
cursor: str | None = None,
|
|
143
|
+
) -> dict[str, Any]:
|
|
144
|
+
"""
|
|
145
|
+
Searches X and returns one page of results.
|
|
146
|
+
|
|
147
|
+
:param query: Search query in X search syntax, e.g. `from:deepset_ai lang:en`.
|
|
148
|
+
:param category: Overrides the search tab set at init.
|
|
149
|
+
:param cursor: `next_cursor` from a previous run, to get the next page.
|
|
150
|
+
:returns: A dictionary with:
|
|
151
|
+
- `documents`: one Document per tweet (or per account for `People`).
|
|
152
|
+
- `links`: the URL of each result, in the same order.
|
|
153
|
+
- `next_cursor`: cursor for the next page, `None` on the last page.
|
|
154
|
+
:raises TwittrAPIError: when the API answers with an error (invalid key, no balance,
|
|
155
|
+
bad query).
|
|
156
|
+
"""
|
|
157
|
+
self.warm_up()
|
|
158
|
+
assert self._client is not None
|
|
159
|
+
tab = category or self.category
|
|
160
|
+
page = self._client.search.tweets(
|
|
161
|
+
query,
|
|
162
|
+
category=tab,
|
|
163
|
+
count=self.top_k,
|
|
164
|
+
cursor=cursor,
|
|
165
|
+
response_format="simple",
|
|
166
|
+
)
|
|
167
|
+
items = page.get("data") if isinstance(page, dict) else None
|
|
168
|
+
rows = [row for row in (items or []) if isinstance(row, dict) and not row.get("unavailable")]
|
|
169
|
+
if self.top_k is not None:
|
|
170
|
+
rows = rows[: self.top_k]
|
|
171
|
+
to_doc = user_to_document if tab == "People" else tweet_to_document
|
|
172
|
+
documents = [to_doc(row) for row in rows]
|
|
173
|
+
links = [doc.meta["url"] for doc in documents if doc.meta.get("url")]
|
|
174
|
+
next_cursor = page.get("nextCursor") if isinstance(page, dict) and page.get("hasNextPage") else None
|
|
175
|
+
logger.debug("TwittrAPI search returned {count} documents", count=len(documents))
|
|
176
|
+
return {"documents": documents, "links": links, "next_cursor": next_cursor}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
__all__ = ["TwittrAPIError", "TwittrAPITweetSearch", "tweet_to_document", "user_to_document"]
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""Haystack Toolset with read-only X (Twitter) tools backed by TwittrAPI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from haystack.core.serialization import generate_qualified_class_name
|
|
11
|
+
from haystack.tools import Tool, Toolset
|
|
12
|
+
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
13
|
+
|
|
14
|
+
from twittrapi import TwittrAPI, TwittrAPIError
|
|
15
|
+
|
|
16
|
+
DEFAULT_MAX_CHARS = 20_000
|
|
17
|
+
|
|
18
|
+
_NUMERIC = {"type": "string", "pattern": r"^\d{1,30}$"}
|
|
19
|
+
_CURSOR = {
|
|
20
|
+
"type": "string",
|
|
21
|
+
"description": "Pagination cursor. Leave out for the first page; pass next_cursor from a "
|
|
22
|
+
"previous result to get the next page.",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _count(what: str) -> dict[str, Any]:
|
|
27
|
+
return {
|
|
28
|
+
"type": "integer",
|
|
29
|
+
"minimum": 1,
|
|
30
|
+
"maximum": 100,
|
|
31
|
+
"description": f"How many {what} to request per page (default 20).",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
_USER_TARGET = {
|
|
36
|
+
"user_id": {**_NUMERIC, "description": "Numeric X user ID. Preferred over username."},
|
|
37
|
+
"username": {"type": "string", "description": 'X handle, e.g. "deepset_ai" (a leading @ is fine).'},
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _schema(properties: dict[str, Any], required: list[str] | None = None) -> dict[str, Any]:
|
|
42
|
+
schema: dict[str, Any] = {"type": "object", "properties": properties}
|
|
43
|
+
if required:
|
|
44
|
+
schema["required"] = required
|
|
45
|
+
return schema
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _user_target(args: dict[str, Any]) -> dict[str, Any]:
|
|
49
|
+
"""Returns user_id or username for the SDK's per-user methods (the ID wins)."""
|
|
50
|
+
if args.get("user_id"):
|
|
51
|
+
return {"user_id": args["user_id"]}
|
|
52
|
+
handle = str(args.get("username") or "").strip().lstrip("@").strip()
|
|
53
|
+
if handle:
|
|
54
|
+
return {"username": handle}
|
|
55
|
+
raise ValueError("Provide either user_id or username.")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _page(args: dict[str, Any]) -> dict[str, Any]:
|
|
59
|
+
return {"count": args.get("count"), "cursor": args.get("cursor")}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _truncate(text: str, max_chars: int) -> str:
|
|
63
|
+
if len(text) <= max_chars:
|
|
64
|
+
return text
|
|
65
|
+
return (
|
|
66
|
+
text[:max_chars] + f"\n\n[TRUNCATED: showing the first {max_chars} of {len(text)} characters. The JSON above "
|
|
67
|
+
"is incomplete. Request a smaller count or use the cursor to see the rest.]"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def format_result(data: Any, max_chars: int, paginated: bool) -> str:
|
|
72
|
+
"""Compact JSON for the model, cut at max_chars. Paginated results print the cursor first,
|
|
73
|
+
outside the budget, so the agent can keep paging even when the JSON is cut."""
|
|
74
|
+
text = _truncate(json.dumps(data, ensure_ascii=False, separators=(",", ":")), max_chars)
|
|
75
|
+
if not paginated or not isinstance(data, dict):
|
|
76
|
+
return text
|
|
77
|
+
cursor = data.get("nextCursor") if data.get("hasNextPage") else None
|
|
78
|
+
if not cursor:
|
|
79
|
+
return f"next_cursor: none (last page)\n\n{text}"
|
|
80
|
+
return f"next_cursor: {cursor}\nPass next_cursor as `cursor` to get the next page.\n\n{text}"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class _Spec:
|
|
85
|
+
name: str
|
|
86
|
+
description: str
|
|
87
|
+
parameters: dict[str, Any]
|
|
88
|
+
call: Callable[[TwittrAPI, dict[str, Any]], Any]
|
|
89
|
+
paginated: bool = False
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
_SPECS: list[_Spec] = [
|
|
93
|
+
_Spec(
|
|
94
|
+
"search_tweets",
|
|
95
|
+
"Search X (Twitter) like the search box. Supports X search operators in the query, e.g. "
|
|
96
|
+
'from:user, to:user, "exact phrase", lang:en, since:2026-01-01, until:2026-02-01, '
|
|
97
|
+
"min_faves:100, filter:media, -filter:replies. category Latest gives newest first; "
|
|
98
|
+
"People returns accounts instead of tweets. Paginated.",
|
|
99
|
+
_schema(
|
|
100
|
+
{
|
|
101
|
+
"query": {"type": "string", "description": "Search query in X search syntax."},
|
|
102
|
+
"category": {
|
|
103
|
+
"type": "string",
|
|
104
|
+
"enum": ["Top", "Latest", "People", "Photos", "Videos"],
|
|
105
|
+
"description": "Search tab (default Top).",
|
|
106
|
+
},
|
|
107
|
+
"count": _count("results"),
|
|
108
|
+
"cursor": _CURSOR,
|
|
109
|
+
},
|
|
110
|
+
["query"],
|
|
111
|
+
),
|
|
112
|
+
lambda api, a: api.search.tweets(a["query"], category=a.get("category"), **_page(a), response_format="simple"),
|
|
113
|
+
paginated=True,
|
|
114
|
+
),
|
|
115
|
+
_Spec(
|
|
116
|
+
"get_user_by_username",
|
|
117
|
+
"Get an X user's profile by handle: name, bio, follower and following counts, "
|
|
118
|
+
"verification, created date and the numeric user ID other tools take.",
|
|
119
|
+
_schema({"username": _USER_TARGET["username"]}, ["username"]),
|
|
120
|
+
lambda api, a: api.users.get_by_username(_user_target(a)["username"], response_format="simple"),
|
|
121
|
+
),
|
|
122
|
+
_Spec(
|
|
123
|
+
"get_user_by_id",
|
|
124
|
+
"Get an X user's profile by numeric user ID.",
|
|
125
|
+
_schema({"user_id": _USER_TARGET["user_id"]}, ["user_id"]),
|
|
126
|
+
lambda api, a: api.users.get_by_id(a["user_id"], response_format="simple"),
|
|
127
|
+
),
|
|
128
|
+
_Spec(
|
|
129
|
+
"get_user_tweets",
|
|
130
|
+
"Get a user's recent tweets, newest first. Takes user_id or username. Paginated.",
|
|
131
|
+
_schema({**_USER_TARGET, "count": _count("tweets"), "cursor": _CURSOR}),
|
|
132
|
+
lambda api, a: api.users.tweets(**_user_target(a), **_page(a), response_format="simple"),
|
|
133
|
+
paginated=True,
|
|
134
|
+
),
|
|
135
|
+
_Spec(
|
|
136
|
+
"get_followers",
|
|
137
|
+
"Get accounts that follow a user. Takes user_id or username. Paginated.",
|
|
138
|
+
_schema({**_USER_TARGET, "count": _count("accounts"), "cursor": _CURSOR}),
|
|
139
|
+
lambda api, a: api.users.followers(**_user_target(a), **_page(a), response_format="simple"),
|
|
140
|
+
paginated=True,
|
|
141
|
+
),
|
|
142
|
+
_Spec(
|
|
143
|
+
"get_following",
|
|
144
|
+
"Get accounts a user follows. Takes user_id or username. Paginated.",
|
|
145
|
+
_schema({**_USER_TARGET, "count": _count("accounts"), "cursor": _CURSOR}),
|
|
146
|
+
lambda api, a: api.users.following(**_user_target(a), **_page(a), response_format="simple"),
|
|
147
|
+
paginated=True,
|
|
148
|
+
),
|
|
149
|
+
_Spec(
|
|
150
|
+
"get_tweet",
|
|
151
|
+
"Get a tweet by ID with full text, author, media, engagement counts and its reply "
|
|
152
|
+
"thread. Use cursor to load more replies.",
|
|
153
|
+
_schema({"tweet_id": {**_NUMERIC, "description": "Numeric tweet ID."}, "cursor": _CURSOR}, ["tweet_id"]),
|
|
154
|
+
lambda api, a: api.tweets.get(a["tweet_id"], cursor=a.get("cursor"), response_format="simple"),
|
|
155
|
+
paginated=True,
|
|
156
|
+
),
|
|
157
|
+
_Spec(
|
|
158
|
+
"get_tweet_retweets",
|
|
159
|
+
"Get the accounts that retweeted a tweet. Paginated.",
|
|
160
|
+
_schema(
|
|
161
|
+
{
|
|
162
|
+
"tweet_id": {**_NUMERIC, "description": "Numeric tweet ID."},
|
|
163
|
+
"count": _count("accounts"),
|
|
164
|
+
"cursor": _CURSOR,
|
|
165
|
+
},
|
|
166
|
+
["tweet_id"],
|
|
167
|
+
),
|
|
168
|
+
lambda api, a: api.tweets.retweeters(a["tweet_id"], **_page(a), response_format="simple"),
|
|
169
|
+
paginated=True,
|
|
170
|
+
),
|
|
171
|
+
_Spec(
|
|
172
|
+
"get_tweet_quotes",
|
|
173
|
+
"Get tweets that quote a given tweet. Paginated.",
|
|
174
|
+
_schema(
|
|
175
|
+
{
|
|
176
|
+
"tweet_id": {**_NUMERIC, "description": "Numeric tweet ID."},
|
|
177
|
+
"count": _count("tweets"),
|
|
178
|
+
"cursor": _CURSOR,
|
|
179
|
+
},
|
|
180
|
+
["tweet_id"],
|
|
181
|
+
),
|
|
182
|
+
lambda api, a: api.tweets.quotes(a["tweet_id"], **_page(a), response_format="simple"),
|
|
183
|
+
paginated=True,
|
|
184
|
+
),
|
|
185
|
+
_Spec(
|
|
186
|
+
"get_list_tweets",
|
|
187
|
+
"Get the latest tweets from an X List. Paginated.",
|
|
188
|
+
_schema(
|
|
189
|
+
{"list_id": {**_NUMERIC, "description": "Numeric List ID."}, "count": _count("tweets"), "cursor": _CURSOR},
|
|
190
|
+
["list_id"],
|
|
191
|
+
),
|
|
192
|
+
lambda api, a: api.lists.tweets(a["list_id"], **_page(a), response_format="simple"),
|
|
193
|
+
paginated=True,
|
|
194
|
+
),
|
|
195
|
+
_Spec(
|
|
196
|
+
"get_community_tweets",
|
|
197
|
+
"Get tweets posted in an X Community. Paginated.",
|
|
198
|
+
_schema(
|
|
199
|
+
{
|
|
200
|
+
"community_id": {**_NUMERIC, "description": "Numeric Community ID."},
|
|
201
|
+
"count": _count("tweets"),
|
|
202
|
+
"cursor": _CURSOR,
|
|
203
|
+
},
|
|
204
|
+
["community_id"],
|
|
205
|
+
),
|
|
206
|
+
lambda api, a: api.communities.tweets(a["community_id"], **_page(a), response_format="simple"),
|
|
207
|
+
paginated=True,
|
|
208
|
+
),
|
|
209
|
+
_Spec(
|
|
210
|
+
"get_trends",
|
|
211
|
+
"Get trending topics for a location. location_id is a WOEID: 1 is worldwide, "
|
|
212
|
+
"23424977 the United States; get_trend_locations lists them all.",
|
|
213
|
+
_schema({"location_id": {**_NUMERIC, "description": "WOEID, default 1 (worldwide)."}}),
|
|
214
|
+
lambda api, a: api.trends.get(a.get("location_id") or 1, response_format="simple"),
|
|
215
|
+
),
|
|
216
|
+
_Spec(
|
|
217
|
+
"get_trend_locations",
|
|
218
|
+
"List the locations (WOEIDs) that have trends.",
|
|
219
|
+
_schema({}),
|
|
220
|
+
lambda api, a: api.trends.locations(response_format="simple"),
|
|
221
|
+
),
|
|
222
|
+
]
|
|
223
|
+
|
|
224
|
+
TOOL_NAMES: list[str] = [spec.name for spec in _SPECS]
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class TwittrAPIToolset(Toolset):
|
|
228
|
+
"""
|
|
229
|
+
Read-only X (Twitter) tools for Haystack Agents, backed by [TwittrAPI](https://www.twittrapi.com).
|
|
230
|
+
|
|
231
|
+
Tools: search_tweets, get_user_by_username, get_user_by_id, get_user_tweets,
|
|
232
|
+
get_followers, get_following, get_tweet, get_tweet_retweets, get_tweet_quotes,
|
|
233
|
+
get_list_tweets, get_community_tweets, get_trends, get_trend_locations.
|
|
234
|
+
|
|
235
|
+
Results are the API's compact simple format (flat camelCase JSON), cut at `max_chars`.
|
|
236
|
+
Paginated tools print `next_cursor` first so the agent can ask for the next page. API errors
|
|
237
|
+
(bad ID, no balance) come back to the agent as the tool result instead of raising.
|
|
238
|
+
|
|
239
|
+
Usage example:
|
|
240
|
+
```python
|
|
241
|
+
from haystack.components.agents import Agent
|
|
242
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
243
|
+
from haystack.dataclasses import ChatMessage
|
|
244
|
+
from haystack_integrations.tools.twittrapi import TwittrAPIToolset
|
|
245
|
+
|
|
246
|
+
agent = Agent(
|
|
247
|
+
chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"),
|
|
248
|
+
tools=TwittrAPIToolset(), # reads TWITTRAPI_API_KEY
|
|
249
|
+
)
|
|
250
|
+
result = agent.run(messages=[ChatMessage.from_user("What is @deepset_ai posting about this week?")])
|
|
251
|
+
print(result["messages"][-1].text)
|
|
252
|
+
```
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
def __init__(
|
|
256
|
+
self,
|
|
257
|
+
api_key: Secret = Secret.from_env_var("TWITTRAPI_API_KEY"),
|
|
258
|
+
tool_names: list[str] | None = None,
|
|
259
|
+
max_chars: int = DEFAULT_MAX_CHARS,
|
|
260
|
+
base_url: str | None = None,
|
|
261
|
+
) -> None:
|
|
262
|
+
"""
|
|
263
|
+
Creates the toolset.
|
|
264
|
+
|
|
265
|
+
:param api_key: TwittrAPI key, sent as `X-API-KEY`. Defaults to the `TWITTRAPI_API_KEY`
|
|
266
|
+
environment variable.
|
|
267
|
+
:param tool_names: Only include these tools. `None` includes all of them.
|
|
268
|
+
:param max_chars: Most characters of JSON a tool returns; longer output is cut with a marker.
|
|
269
|
+
:param base_url: Override the API base URL. Rarely needed.
|
|
270
|
+
"""
|
|
271
|
+
unknown = sorted(set(tool_names or []) - set(TOOL_NAMES))
|
|
272
|
+
if unknown:
|
|
273
|
+
raise ValueError(f"Unknown TwittrAPI tools: {', '.join(unknown)}. Available: {', '.join(TOOL_NAMES)}")
|
|
274
|
+
if max_chars < 100:
|
|
275
|
+
raise ValueError("max_chars must be at least 100.")
|
|
276
|
+
self.api_key = api_key
|
|
277
|
+
self.tool_names = list(tool_names) if tool_names is not None else None
|
|
278
|
+
self.max_chars = max_chars
|
|
279
|
+
self.base_url = base_url
|
|
280
|
+
self._client: TwittrAPI | None = None
|
|
281
|
+
selected = [spec for spec in _SPECS if tool_names is None or spec.name in tool_names]
|
|
282
|
+
super().__init__(tools=[self._make_tool(spec) for spec in selected])
|
|
283
|
+
|
|
284
|
+
def _get_client(self) -> TwittrAPI:
|
|
285
|
+
if self._client is None:
|
|
286
|
+
kwargs: dict[str, Any] = {"api_key": self.api_key.resolve_value()}
|
|
287
|
+
if self.base_url:
|
|
288
|
+
kwargs["base_url"] = self.base_url
|
|
289
|
+
self._client = TwittrAPI(**kwargs)
|
|
290
|
+
return self._client
|
|
291
|
+
|
|
292
|
+
def _make_tool(self, spec: _Spec) -> Tool:
|
|
293
|
+
def invoke(**kwargs: Any) -> str:
|
|
294
|
+
try:
|
|
295
|
+
data = spec.call(self._get_client(), kwargs)
|
|
296
|
+
except TwittrAPIError as exc:
|
|
297
|
+
return f"TwittrAPI error: {exc}"
|
|
298
|
+
except (ValueError, TypeError) as exc:
|
|
299
|
+
return f"Invalid arguments: {exc}"
|
|
300
|
+
return format_result(data, self.max_chars, spec.paginated)
|
|
301
|
+
|
|
302
|
+
return Tool(name=spec.name, description=spec.description, parameters=spec.parameters, function=invoke)
|
|
303
|
+
|
|
304
|
+
def to_dict(self) -> dict[str, Any]:
|
|
305
|
+
"""Serializes the toolset by its settings, not by its tools."""
|
|
306
|
+
return {
|
|
307
|
+
"type": generate_qualified_class_name(type(self)),
|
|
308
|
+
"data": {
|
|
309
|
+
"api_key": self.api_key.to_dict(),
|
|
310
|
+
"tool_names": self.tool_names,
|
|
311
|
+
"max_chars": self.max_chars,
|
|
312
|
+
"base_url": self.base_url,
|
|
313
|
+
},
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
@classmethod
|
|
317
|
+
def from_dict(cls, data: dict[str, Any]) -> TwittrAPIToolset:
|
|
318
|
+
"""Rebuilds the toolset from its settings."""
|
|
319
|
+
inner = dict(data["data"])
|
|
320
|
+
deserialize_secrets_inplace(inner, keys=["api_key"])
|
|
321
|
+
return cls(**inner)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
__all__ = ["TOOL_NAMES", "TwittrAPIToolset", "format_result"]
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: twittrapi-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack integration for TwittrAPI: search X (Twitter) into Documents and give Agents read-only X tools.
|
|
5
|
+
Project-URL: Homepage, https://www.twittrapi.com
|
|
6
|
+
Project-URL: Documentation, https://www.twittrapi.com/docs
|
|
7
|
+
Project-URL: Source, https://github.com/twittrapi/twittrapi-haystack
|
|
8
|
+
Project-URL: Issues, https://github.com/twittrapi/twittrapi-haystack/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/twittrapi/twittrapi-haystack/blob/main/CHANGELOG.md
|
|
10
|
+
Author: TwittrAPI
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: agents,haystack,haystack-ai,rag,tweets,twitter,twitter api,twitter search,twittrapi,x
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: haystack-ai>=2.14.0
|
|
28
|
+
Requires-Dist: twittrapi<2,>=1.4.0
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# twittrapi-haystack
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/twittrapi-haystack/)
|
|
34
|
+
[](LICENSE)
|
|
35
|
+
|
|
36
|
+
[Haystack](https://haystack.deepset.ai) integration for [TwittrAPI](https://www.twittrapi.com), a
|
|
37
|
+
pay-per-call REST API for public X (Twitter) data. No X developer account needed.
|
|
38
|
+
|
|
39
|
+
- **`TwittrAPITweetSearch`**: a component that searches X and returns each tweet as a Haystack
|
|
40
|
+
`Document`, with the same `documents` and `links` outputs as the web-search components.
|
|
41
|
+
- **`TwittrAPIToolset`**: read-only X tools for Haystack Agents (search, profiles, timelines,
|
|
42
|
+
followers, tweets with replies, quotes, retweets, Lists, Communities, trends).
|
|
43
|
+
|
|
44
|
+
## Installation
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install twittrapi-haystack
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Create an API key at [twittrapi.com](https://www.twittrapi.com) and set it as `TWITTRAPI_API_KEY`.
|
|
51
|
+
New accounts get test credit.
|
|
52
|
+
|
|
53
|
+
## TwittrAPITweetSearch
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from haystack_integrations.components.websearch.twittrapi import TwittrAPITweetSearch
|
|
57
|
+
|
|
58
|
+
search = TwittrAPITweetSearch(top_k=20) # reads TWITTRAPI_API_KEY
|
|
59
|
+
result = search.run(query="haystack lang:en -filter:replies")
|
|
60
|
+
|
|
61
|
+
for doc in result["documents"]:
|
|
62
|
+
print(doc.meta["author_username"], doc.meta["likes"], doc.content)
|
|
63
|
+
print(result["next_cursor"]) # pass as cursor= for the next page
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The query takes the operators of the X search box: `from:user`, `to:user`, `"exact phrase"`,
|
|
67
|
+
`lang:en`, `since:2026-01-01`, `until:2026-02-01`, `min_faves:100`, `filter:media`,
|
|
68
|
+
`-filter:replies` and more.
|
|
69
|
+
|
|
70
|
+
**Parameters**
|
|
71
|
+
|
|
72
|
+
- `api_key`: TwittrAPI key. Defaults to the `TWITTRAPI_API_KEY` environment variable.
|
|
73
|
+
- `category`: `"Latest"` (default, newest first), `"Top"`, `"People"` (accounts instead of tweets),
|
|
74
|
+
`"Photos"` or `"Videos"`. Can be changed per `run`.
|
|
75
|
+
- `top_k`: results per page, 1 to 100 (default 20).
|
|
76
|
+
|
|
77
|
+
**Outputs**
|
|
78
|
+
|
|
79
|
+
- `documents`: one `Document` per tweet. `content` is the full text; `meta` holds `id`, `url`,
|
|
80
|
+
`created_at`, `lang`, `author_id`, `author_username`, `author_name`, `likes`, `retweets`,
|
|
81
|
+
`replies`, `quotes`, `views` and `conversation_id`.
|
|
82
|
+
- `links`: the tweet URLs, in the same order.
|
|
83
|
+
- `next_cursor`: cursor for the next page, `None` on the last page.
|
|
84
|
+
|
|
85
|
+
### In a pipeline
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from haystack import Pipeline
|
|
89
|
+
from haystack.components.builders import ChatPromptBuilder
|
|
90
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
91
|
+
from haystack.dataclasses import ChatMessage
|
|
92
|
+
from haystack_integrations.components.websearch.twittrapi import TwittrAPITweetSearch
|
|
93
|
+
|
|
94
|
+
template = [
|
|
95
|
+
ChatMessage.from_user(
|
|
96
|
+
"Summarize what people say about {{ topic }} on X.\n"
|
|
97
|
+
"{% for doc in documents %}- @{{ doc.meta.author_username }}: {{ doc.content }}\n{% endfor %}"
|
|
98
|
+
)
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
pipe = Pipeline()
|
|
102
|
+
pipe.add_component("search", TwittrAPITweetSearch(top_k=30))
|
|
103
|
+
pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["topic", "documents"]))
|
|
104
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
|
|
105
|
+
pipe.connect("search.documents", "prompt.documents")
|
|
106
|
+
pipe.connect("prompt.prompt", "llm.messages")
|
|
107
|
+
|
|
108
|
+
result = pipe.run({"search": {"query": "haystack lang:en"}, "prompt": {"topic": "Haystack"}})
|
|
109
|
+
print(result["llm"]["replies"][0].text)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## TwittrAPIToolset
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from haystack.components.agents import Agent
|
|
116
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
117
|
+
from haystack.dataclasses import ChatMessage
|
|
118
|
+
from haystack_integrations.tools.twittrapi import TwittrAPIToolset
|
|
119
|
+
|
|
120
|
+
agent = Agent(
|
|
121
|
+
chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"),
|
|
122
|
+
tools=TwittrAPIToolset(), # reads TWITTRAPI_API_KEY
|
|
123
|
+
)
|
|
124
|
+
result = agent.run(messages=[ChatMessage.from_user("What has @deepset_ai posted this week?")])
|
|
125
|
+
print(result["messages"][-1].text)
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
| Tool | What it does |
|
|
129
|
+
| --- | --- |
|
|
130
|
+
| `search_tweets` | Search X with search operators; `People` returns accounts |
|
|
131
|
+
| `get_user_by_username` / `get_user_by_id` | A profile with counts and the numeric user ID |
|
|
132
|
+
| `get_user_tweets` | A user's recent tweets |
|
|
133
|
+
| `get_followers` / `get_following` | Followers and followed accounts |
|
|
134
|
+
| `get_tweet` | A tweet with its reply thread |
|
|
135
|
+
| `get_tweet_retweets` / `get_tweet_quotes` | Who retweeted a tweet, and the tweets quoting it |
|
|
136
|
+
| `get_list_tweets` | Latest tweets from an X List |
|
|
137
|
+
| `get_community_tweets` | Tweets in an X Community |
|
|
138
|
+
| `get_trends` / `get_trend_locations` | Trending topics by location (WOEID) |
|
|
139
|
+
|
|
140
|
+
Pick a subset with `TwittrAPIToolset(tool_names=["search_tweets", "get_user_tweets"])`. Results are
|
|
141
|
+
TwittrAPI's compact [simple format](https://www.twittrapi.com/docs/simple-format) (flat JSON), cut at
|
|
142
|
+
`max_chars` (default 20,000). Paginated tools print `next_cursor` first so the agent can page on. API
|
|
143
|
+
errors, such as an empty balance, are returned to the agent as the tool result.
|
|
144
|
+
|
|
145
|
+
## Pricing
|
|
146
|
+
|
|
147
|
+
Every call is billed to your prepaid TwittrAPI balance. Prices are on
|
|
148
|
+
[twittrapi.com/pricing](https://www.twittrapi.com/pricing).
|
|
149
|
+
|
|
150
|
+
## Links
|
|
151
|
+
|
|
152
|
+
- [TwittrAPI docs](https://www.twittrapi.com/docs)
|
|
153
|
+
- [Python SDK](https://github.com/twittrapi/twittrapi-python)
|
|
154
|
+
- [MCP server](https://github.com/twittrapi/twittrapi-mcp)
|
|
155
|
+
- Support: support@twittrapi.com or [GitHub issues](https://github.com/twittrapi/twittrapi-haystack/issues)
|
|
156
|
+
|
|
157
|
+
## License
|
|
158
|
+
|
|
159
|
+
MIT
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
haystack_integrations/components/websearch/twittrapi/__init__.py,sha256=UT4asxBXRyE9y7515Sva8W5QqPnNQGRei_YJhiRsF1k,161
|
|
2
|
+
haystack_integrations/components/websearch/twittrapi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
haystack_integrations/components/websearch/twittrapi/tweet_search.py,sha256=yDgyoGB1sXn3L_pKB7bwOLCetpB5C-JDuRhYEkJmBZA,7411
|
|
4
|
+
haystack_integrations/tools/twittrapi/__init__.py,sha256=eGmmt2rHHsENqa1zZgJ2xjlo2ne5e8TPoD3Is48qi6Y,96
|
|
5
|
+
haystack_integrations/tools/twittrapi/toolset.py,sha256=w0MQC_7LJUgTFnGLCBb5w27Y6Jt0Y_XNMnRZcBQcATU,12877
|
|
6
|
+
twittrapi_haystack-0.1.0.dist-info/METADATA,sha256=YDFC03AqHuorVJB-DHpuNv5cZ9G_IcONU89DzvNTPVc,6740
|
|
7
|
+
twittrapi_haystack-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
8
|
+
twittrapi_haystack-0.1.0.dist-info/licenses/LICENSE,sha256=-GJ5-iBXn7frztmb-DL8pU5HC57TKnyojHFqNkuqneU,1066
|
|
9
|
+
twittrapi_haystack-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 TwittrAPI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|