socialscrape-haystack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,37 @@
1
+ """Haystack integration for SocialScrape: social media data for pipelines and agents.
2
+
3
+ `SocialScrapeFetcher` runs any endpoint and returns Documents, `SocialScrapeTranscriptFetcher` turns video URLs
4
+ into transcript Documents, and `SocialScrapeToolset` gives a Haystack Agent tools for profiles, posts,
5
+ comments, transcripts and search across platforms.
6
+ """
7
+
8
+ from importlib import metadata
9
+
10
+ from ._catalog import Endpoint, endpoints
11
+ from .fetcher import SocialScrapeFetcher
12
+ from .toolset import SocialScrapeToolset
13
+ from .transcripts import SocialScrapeTranscriptFetcher
14
+
15
+ # Newer Haystack only loads a pipeline YAML whose classes come from trusted modules (`haystack`,
16
+ # `haystack_integrations`). Whoever imports this package trusts it, so say so, or Pipeline.from_dict and
17
+ # Pipeline.loads would refuse the components here. Older Haystack has no allowlist.
18
+ try:
19
+ from haystack.core.serialization import allow_deserialization_module
20
+
21
+ allow_deserialization_module("socialscrape_haystack")
22
+ except ImportError: # pragma: no cover
23
+ pass
24
+
25
+ try:
26
+ __version__ = metadata.version("socialscrape-haystack")
27
+ except metadata.PackageNotFoundError:
28
+ __version__ = ""
29
+
30
+ __all__ = [
31
+ "Endpoint",
32
+ "SocialScrapeFetcher",
33
+ "SocialScrapeToolset",
34
+ "SocialScrapeTranscriptFetcher",
35
+ "__version__",
36
+ "endpoints",
37
+ ]
@@ -0,0 +1,257 @@
1
+ """The endpoint catalogue this package ships, and the routing the tools share.
2
+
3
+ `endpoints.json` is generated from the SocialScrape API reference: every data
4
+ endpoint that is live, with its parameters. Nothing here makes a request, so
5
+ looking an endpoint up or checking parameters is free.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from collections.abc import Mapping
12
+ from dataclasses import dataclass
13
+ from functools import lru_cache
14
+ from importlib import resources
15
+ from typing import Any
16
+ from urllib.parse import urlsplit
17
+
18
+ # The jobs the cross-platform tools do, and the endpoint that does each one per
19
+ # platform. Every id is checked against endpoints.json by the tests.
20
+ ROLES: dict[str, dict[str, str]] = {
21
+ "profile": {
22
+ "apple_music": "apple_music.artist",
23
+ "bluesky": "bluesky.profile",
24
+ "cosmos": "cosmos.profile",
25
+ "facebook": "facebook.profile",
26
+ "github": "github.user",
27
+ "instagram": "instagram.profile",
28
+ "kwai": "kwai.profile",
29
+ "linkedin": "linkedin.profile",
30
+ "snapchat": "snapchat.profile",
31
+ "soundcloud": "soundcloud.artist",
32
+ "spotify": "spotify.artist",
33
+ "threads": "threads.profile",
34
+ "tiktok": "tiktok.profile",
35
+ "truthsocial": "truthsocial.profile",
36
+ "twitch": "twitch.profile",
37
+ "twitter": "twitter.profile",
38
+ "youtube": "youtube.channel",
39
+ },
40
+ "posts": {
41
+ "bluesky": "bluesky.profile.posts",
42
+ "cosmos": "cosmos.profile.posts",
43
+ "facebook": "facebook.profile.posts",
44
+ "instagram": "instagram.profile.posts",
45
+ "kwai": "kwai.profile.posts",
46
+ "linkedin": "linkedin.company.posts",
47
+ "rumble": "rumble.channel.videos",
48
+ "telegram": "telegram.channel.posts",
49
+ "threads": "threads.profile.posts",
50
+ "tiktok": "tiktok.profile.videos",
51
+ "truthsocial": "truthsocial.profile.posts",
52
+ "twitch": "twitch.user.videos",
53
+ "twitter": "twitter.profile.tweets",
54
+ "youtube": "youtube.channel.videos",
55
+ },
56
+ "post": {
57
+ "bluesky": "bluesky.post",
58
+ "cosmos": "cosmos.post",
59
+ "facebook": "facebook.post",
60
+ "instagram": "instagram.post",
61
+ "kick": "kick.clip",
62
+ "kwai": "kwai.post",
63
+ "linkedin": "linkedin.post",
64
+ "pinterest": "pinterest.pin",
65
+ "reddit": "reddit.post",
66
+ "rumble": "rumble.video",
67
+ "snapchat": "snapchat.spotlight",
68
+ "telegram": "telegram.post",
69
+ "threads": "threads.post",
70
+ "tiktok": "tiktok.video",
71
+ "truthsocial": "truthsocial.post",
72
+ "twitch": "twitch.clip",
73
+ "twitter": "twitter.tweet",
74
+ "youtube": "youtube.video",
75
+ },
76
+ "comments": {
77
+ "facebook": "facebook.post.comments",
78
+ "instagram": "instagram.post.comments",
79
+ "reddit": "reddit.post.comments",
80
+ "rumble": "rumble.video.comments",
81
+ "snapchat": "snapchat.spotlight.comments",
82
+ "tiktok": "tiktok.video.comments",
83
+ "youtube": "youtube.video.comments",
84
+ },
85
+ "search": {
86
+ "apple_music": "apple_music.search",
87
+ "cosmos": "cosmos.search",
88
+ "facebook": "facebook.search.videos",
89
+ "google": "google.search",
90
+ "instagram": "instagram.search.reels",
91
+ "linkedin": "linkedin.search.posts",
92
+ "pinterest": "pinterest.search",
93
+ "reddit": "reddit.search",
94
+ "rumble": "rumble.search",
95
+ "spotify": "spotify.search",
96
+ "threads": "threads.search",
97
+ "tiktok": "tiktok.search.keyword",
98
+ "twitter": "twitter.search.posts",
99
+ "youtube": "youtube.search",
100
+ },
101
+ }
102
+
103
+ TRANSCRIPT = "transcript"
104
+ IDENTIFIERS = ("handle", "url", "id")
105
+
106
+
107
+ @dataclass(frozen=True)
108
+ class Endpoint:
109
+ """One live endpoint: what it is called, what it takes, what it costs."""
110
+
111
+ id: str
112
+ platform: str
113
+ title: str
114
+ path: str
115
+ summary: str
116
+ description: str
117
+ response: str
118
+ paginated: bool
119
+ credits: str
120
+ params: tuple[Mapping[str, Any], ...]
121
+ one_of: tuple[tuple[str, ...], ...]
122
+ example: Mapping[str, Any]
123
+ common: bool # takes trim, raw and max_age_days (every data endpoint does; the account ones do not)
124
+
125
+ @property
126
+ def names(self) -> tuple[str, ...]:
127
+ """Parameter names of its own, without the ones every endpoint takes."""
128
+ return tuple(p["name"] for p in self.params)
129
+
130
+ @property
131
+ def required(self) -> tuple[str, ...]:
132
+ return tuple(p["name"] for p in self.params if p["required"])
133
+
134
+ @property
135
+ def hosts(self) -> tuple[str, ...]:
136
+ """Hosts its `url` parameter accepts, when it has one."""
137
+ for p in self.params:
138
+ if p["name"] == "url":
139
+ return tuple(p.get("hosts") or ())
140
+ return ()
141
+
142
+ def signature(self) -> str:
143
+ """`handle | id (one needed), sort, cursor`: the parameters in one line."""
144
+ grouped = {name for group in self.one_of for name in group}
145
+ bits = [" | ".join(group) + " (one needed)" for group in self.one_of]
146
+ bits += [name + " (required)" for name in self.required if name not in grouped]
147
+ bits += [name for name in self.names if name not in grouped and name not in self.required]
148
+ return ", ".join(bits) or "none"
149
+
150
+
151
+ @lru_cache(maxsize=1)
152
+ def _load() -> tuple[dict[str, Endpoint], tuple[Mapping[str, Any], ...]]:
153
+ text = resources.files(__package__).joinpath("endpoints.json").read_text(encoding="utf-8")
154
+ doc = json.loads(text)
155
+ endpoints = {
156
+ row["id"]: Endpoint(
157
+ id=row["id"], platform=row["platform"], title=row["title"], path=row["path"], summary=row["summary"],
158
+ description=row["description"], response=row["response"], paginated=row["paginated"],
159
+ credits=row["credits"], params=tuple(row["params"]),
160
+ one_of=tuple(tuple(group) for group in row["one_of"]), example=row["example"],
161
+ common=row["common"],
162
+ )
163
+ for row in doc["endpoints"]
164
+ }
165
+ return endpoints, tuple(doc["common"])
166
+
167
+
168
+ def endpoints() -> dict[str, Endpoint]:
169
+ """Every endpoint in the catalogue, by id, in the order of the API reference."""
170
+ return _load()[0]
171
+
172
+
173
+ def common_params() -> tuple[Mapping[str, Any], ...]:
174
+ """The parameters every data endpoint takes: trim, raw, max_age_days."""
175
+ return _load()[1]
176
+
177
+
178
+ def platforms() -> dict[str, str]:
179
+ """Platform key to display name, in the order of the API reference."""
180
+ out: dict[str, str] = {}
181
+ for ep in endpoints().values():
182
+ out.setdefault(ep.platform, ep.title)
183
+ return out
184
+
185
+
186
+ def get_endpoint(endpoint_id: str) -> Endpoint:
187
+ """The endpoint with this id, or a ValueError that says where to look."""
188
+ found = endpoints().get(endpoint_id)
189
+ if found is None:
190
+ near = [i for i in endpoints() if i.startswith(endpoint_id.split(".")[0] + ".")][:12]
191
+ hint = f" Ids that start the same way: {', '.join(near)}." if near else ""
192
+ raise ValueError(f"unknown endpoint id {endpoint_id!r}.{hint} List them with socialscrape_list_endpoints.")
193
+ return found
194
+
195
+
196
+ def check_params(ep: Endpoint, params: Mapping[str, Any]) -> dict[str, Any]:
197
+ """The parameters to send, with None dropped, or a ValueError naming what is wrong.
198
+
199
+ Checked before any request is made: an unknown name, a missing required
200
+ parameter, or none of a group where one is needed.
201
+ """
202
+ sent = {k: v for k, v in params.items() if v is not None}
203
+ allowed = set(ep.names) | ({p["name"] for p in common_params()} if ep.common else set())
204
+ unknown = sorted(set(sent) - allowed)
205
+ if unknown:
206
+ raise ValueError(f"{ep.id} does not take {', '.join(unknown)}. It takes: {ep.signature()}.")
207
+ missing = [name for name in ep.required if name not in sent]
208
+ if missing:
209
+ raise ValueError(f"{ep.id} needs {', '.join(missing)}. It takes: {ep.signature()}.")
210
+ for group in ep.one_of:
211
+ if not any(name in sent for name in group):
212
+ raise ValueError(f"{ep.id} needs one of {', '.join(group)}. It takes: {ep.signature()}.")
213
+ return sent
214
+
215
+
216
+ def _host_matches(host: str, allowed: tuple[str, ...]) -> bool:
217
+ return any(host == h or host.endswith("." + h) for h in allowed)
218
+
219
+
220
+ def platform_of_url(role: str, url: str) -> str | None:
221
+ """Which platform of a role a URL belongs to, read from its host."""
222
+ host = (urlsplit(url if "//" in url else "https://" + url).hostname or "").lower()
223
+ if not host:
224
+ return None
225
+ for platform, endpoint_id in ROLES[role].items():
226
+ if _host_matches(host, endpoints()[endpoint_id].hosts):
227
+ return platform
228
+ return None
229
+
230
+
231
+ def route(role: str, platform: str | None, *, handle: str | None = None, url: str | None = None,
232
+ id: str | None = None) -> tuple[Endpoint, dict[str, Any]]:
233
+ """The endpoint that does `role` on a platform, and the one identifier to send it.
234
+
235
+ Without a platform the URL's host decides. Endpoints differ in what they
236
+ look a thing up by, so an identifier the endpoint does not take is an
237
+ error that says which ones it does.
238
+ """
239
+ table = ROLES[role]
240
+ if not platform:
241
+ if not url:
242
+ raise ValueError(f"pass a platform ({', '.join(sorted(table))}), or a url it can be read from")
243
+ platform = platform_of_url(role, url)
244
+ if platform is None:
245
+ raise ValueError(f"no platform recognised in {url!r}; pass platform, one of {', '.join(sorted(table))}")
246
+ key = platform.strip().lower().replace("-", "_")
247
+ key = {"x": "twitter"}.get(key, key)
248
+ if key not in table:
249
+ raise ValueError(f"no {role} lookup for {platform!r}; platforms: {', '.join(sorted(table))}")
250
+ ep = endpoints()[table[key]]
251
+ given = {name: value for name, value in (("handle", handle), ("url", url), ("id", id)) if value}
252
+ for name in IDENTIFIERS:
253
+ if name in given and name in ep.names:
254
+ return ep, {name: given[name]}
255
+ takes = " or ".join(name for name in IDENTIFIERS if name in ep.names)
256
+ got = ", ".join(given) or "nothing"
257
+ raise ValueError(f"{ep.id} looks things up by {takes}; got {got}")
@@ -0,0 +1,57 @@
1
+ """How the components and the toolset reach the API: one place that owns the SocialScrape client."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ from importlib import metadata
7
+ from typing import Any
8
+
9
+ from socialscrape import AsyncSocialScrape, Response, SocialScrape
10
+ from socialscrape import __version__ as _sdk_version
11
+
12
+ from ._catalog import check_params, get_endpoint
13
+
14
+ try:
15
+ _version = metadata.version("socialscrape-haystack")
16
+ except metadata.PackageNotFoundError: # a source checkout that was never installed
17
+ _version = "0"
18
+
19
+ USER_AGENT = f"socialscrape-haystack/{_version} socialscrape-python/{_sdk_version}"
20
+
21
+
22
+ class Session:
23
+ """Makes the SocialScrape client on first use and keeps it for later calls.
24
+
25
+ Pass `client` (and `async_client`) to use ones you configured yourself, for
26
+ example with your own timeouts or `httpx` client. Otherwise the key is
27
+ `api_key`, or SOCIALSCRAPE_API_KEY when it is left out.
28
+ """
29
+
30
+ def __init__(self, *, api_key: str | None = None, base_url: str | None = None, client: SocialScrape | None = None,
31
+ async_client: AsyncSocialScrape | None = None) -> None:
32
+ self._api_key = api_key
33
+ self._base_url = base_url
34
+ self._client = client
35
+ self._async_client = async_client
36
+
37
+ def _options(self) -> dict[str, Any]:
38
+ return {"api_key": self._api_key, "base_url": self._base_url, "default_headers": {"user-agent": USER_AGENT}}
39
+
40
+ def client(self) -> SocialScrape:
41
+ if self._client is None:
42
+ self._client = SocialScrape(**self._options())
43
+ return self._client
44
+
45
+ def call(self, endpoint_id: str, params: Mapping[str, Any]) -> Response[Any]:
46
+ """Run one endpoint. Parameters are checked first, so a bad call costs no request."""
47
+ ep = get_endpoint(endpoint_id)
48
+ return self.client().request("GET", ep.path, query=check_params(ep, params))
49
+
50
+ async def acall(self, endpoint_id: str, params: Mapping[str, Any]) -> Response[Any]:
51
+ ep = get_endpoint(endpoint_id)
52
+ query = check_params(ep, params)
53
+ if self._async_client is not None:
54
+ return await self._async_client.request("GET", ep.path, query=query)
55
+ # A client of our own lives for one call: it cannot outlast the event loop it was made in.
56
+ async with AsyncSocialScrape(**self._options()) as client:
57
+ return await client.request("GET", ep.path, query=query)
@@ -0,0 +1,79 @@
1
+ """Turn API objects into Haystack Documents: the object's text is the content, the rest is metadata."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Mapping
7
+ from typing import Any
8
+
9
+ from haystack import Document
10
+
11
+ # Which fields of an object are its text, by the object's type. Anything else goes to meta.
12
+ _TEXT_FIELDS: dict[str, tuple[str, ...]] = {
13
+ "post": ("title", "text"),
14
+ "comment": ("text",),
15
+ "review": ("text",),
16
+ "transcript": ("text",),
17
+ "profile": ("name", "headline", "bio"),
18
+ "community": ("name", "title", "description"),
19
+ "collection": ("title", "description"),
20
+ "ad": ("title", "body"),
21
+ "product": ("title", "description"),
22
+ "repo": ("full_name", "description"),
23
+ "search_result": ("title", "snippet"),
24
+ "link_page": ("name", "bio"),
25
+ "event": ("title", "description"),
26
+ }
27
+ _FALLBACK_TEXT = ("title", "name", "text", "description")
28
+ _SCALARS = (str, int, float, bool)
29
+
30
+
31
+ def _meta(obj: Mapping[str, Any], text_fields: tuple[str, ...]) -> dict[str, Any]:
32
+ """Flat meta a document store accepts: the object's scalar fields, its counts and who made it."""
33
+ meta: dict[str, Any] = {}
34
+ for key, value in obj.items():
35
+ if key in text_fields or key == "extra":
36
+ continue
37
+ if key == "source" and isinstance(value, str):
38
+ meta[f"{obj.get('object') or 'item'}_source"] = value # a transcript's `source` is captions or ai
39
+ elif isinstance(value, _SCALARS):
40
+ meta[key] = value
41
+ elif key == "stats" and isinstance(value, Mapping):
42
+ meta.update({name: count for name, count in value.items() if isinstance(count, _SCALARS)})
43
+ elif key in ("author", "owner", "advertiser", "seller", "host") and isinstance(value, Mapping):
44
+ who = value.get("handle") or value.get("name")
45
+ if who:
46
+ meta[key] = who
47
+ if isinstance(value.get("url"), str):
48
+ meta[f"{key}_url"] = value["url"]
49
+ if "url" in meta:
50
+ meta["source"] = meta["url"]
51
+ return meta
52
+
53
+
54
+ def to_document(obj: Any, endpoint_id: str) -> Document:
55
+ """One API object as a Document. Meta has `endpoint`, `platform` and, when it has one, `url` and `source`."""
56
+ if not isinstance(obj, Mapping):
57
+ return Document(content=json.dumps(obj, ensure_ascii=False), meta={"endpoint": endpoint_id})
58
+ kind = obj.get("object")
59
+ fields = _TEXT_FIELDS.get(kind, _FALLBACK_TEXT) if isinstance(kind, str) else _FALLBACK_TEXT
60
+ parts = [obj[name].strip() for name in fields if isinstance(obj.get(name), str) and obj[name].strip()]
61
+ if parts:
62
+ content, used = "\n\n".join(parts), fields
63
+ else: # nothing that reads as text (a credit balance, a contribution calendar): keep the whole object
64
+ content, used = json.dumps(obj, ensure_ascii=False), ()
65
+ meta = _meta(obj, used)
66
+ meta["endpoint"] = endpoint_id
67
+ ident = obj.get("id") or obj.get("video_id")
68
+ doc_id = f"{obj.get('platform')}:{kind}:{ident}" if ident and obj.get("platform") and kind else None
69
+ if doc_id:
70
+ return Document(id=doc_id, content=content, meta=meta)
71
+ return Document(content=content, meta=meta)
72
+
73
+
74
+ def items_of(data: Any) -> tuple[list[Any], str | None]:
75
+ """The items of an answer and the cursor for the next page: a list page has them, any other object is one item."""
76
+ if isinstance(data, Mapping) and data.get("object") == "list":
77
+ cursor = data.get("cursor") if data.get("has_more") is not False else None
78
+ return list(data.get("items") or []), cursor or None
79
+ return [data], None