socialscrape-haystack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- socialscrape_haystack/__init__.py +37 -0
- socialscrape_haystack/_catalog.py +257 -0
- socialscrape_haystack/_client.py +57 -0
- socialscrape_haystack/_documents.py +79 -0
- socialscrape_haystack/endpoints.json +198 -0
- socialscrape_haystack/fetcher.py +174 -0
- socialscrape_haystack/py.typed +0 -0
- socialscrape_haystack/toolset.py +405 -0
- socialscrape_haystack/transcripts.py +146 -0
- socialscrape_haystack-0.1.0.dist-info/METADATA +179 -0
- socialscrape_haystack-0.1.0.dist-info/RECORD +13 -0
- socialscrape_haystack-0.1.0.dist-info/WHEEL +4 -0
- socialscrape_haystack-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Haystack integration for SocialScrape: social media data for pipelines and agents.
|
|
2
|
+
|
|
3
|
+
`SocialScrapeFetcher` runs any endpoint and returns Documents, `SocialScrapeTranscriptFetcher` turns video URLs
|
|
4
|
+
into transcript Documents, and `SocialScrapeToolset` gives a Haystack Agent tools for profiles, posts,
|
|
5
|
+
comments, transcripts and search across platforms.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from importlib import metadata
|
|
9
|
+
|
|
10
|
+
from ._catalog import Endpoint, endpoints
|
|
11
|
+
from .fetcher import SocialScrapeFetcher
|
|
12
|
+
from .toolset import SocialScrapeToolset
|
|
13
|
+
from .transcripts import SocialScrapeTranscriptFetcher
|
|
14
|
+
|
|
15
|
+
# Newer Haystack only loads a pipeline YAML whose classes come from trusted modules (`haystack`,
|
|
16
|
+
# `haystack_integrations`). Whoever imports this package trusts it, so say so, or Pipeline.from_dict and
|
|
17
|
+
# Pipeline.loads would refuse the components here. Older Haystack has no allowlist.
|
|
18
|
+
try:
|
|
19
|
+
from haystack.core.serialization import allow_deserialization_module
|
|
20
|
+
|
|
21
|
+
allow_deserialization_module("socialscrape_haystack")
|
|
22
|
+
except ImportError: # pragma: no cover
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
__version__ = metadata.version("socialscrape-haystack")
|
|
27
|
+
except metadata.PackageNotFoundError:
|
|
28
|
+
__version__ = ""
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"Endpoint",
|
|
32
|
+
"SocialScrapeFetcher",
|
|
33
|
+
"SocialScrapeToolset",
|
|
34
|
+
"SocialScrapeTranscriptFetcher",
|
|
35
|
+
"__version__",
|
|
36
|
+
"endpoints",
|
|
37
|
+
]
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""The endpoint catalogue this package ships, and the routing the tools share.
|
|
2
|
+
|
|
3
|
+
`endpoints.json` is generated from the SocialScrape API reference: every data
|
|
4
|
+
endpoint that is live, with its parameters. Nothing here makes a request, so
|
|
5
|
+
looking an endpoint up or checking parameters is free.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from functools import lru_cache
|
|
14
|
+
from importlib import resources
|
|
15
|
+
from typing import Any
|
|
16
|
+
from urllib.parse import urlsplit
|
|
17
|
+
|
|
18
|
+
# The jobs the cross-platform tools do, and the endpoint that does each one per
|
|
19
|
+
# platform. Every id is checked against endpoints.json by the tests.
|
|
20
|
+
ROLES: dict[str, dict[str, str]] = {
|
|
21
|
+
"profile": {
|
|
22
|
+
"apple_music": "apple_music.artist",
|
|
23
|
+
"bluesky": "bluesky.profile",
|
|
24
|
+
"cosmos": "cosmos.profile",
|
|
25
|
+
"facebook": "facebook.profile",
|
|
26
|
+
"github": "github.user",
|
|
27
|
+
"instagram": "instagram.profile",
|
|
28
|
+
"kwai": "kwai.profile",
|
|
29
|
+
"linkedin": "linkedin.profile",
|
|
30
|
+
"snapchat": "snapchat.profile",
|
|
31
|
+
"soundcloud": "soundcloud.artist",
|
|
32
|
+
"spotify": "spotify.artist",
|
|
33
|
+
"threads": "threads.profile",
|
|
34
|
+
"tiktok": "tiktok.profile",
|
|
35
|
+
"truthsocial": "truthsocial.profile",
|
|
36
|
+
"twitch": "twitch.profile",
|
|
37
|
+
"twitter": "twitter.profile",
|
|
38
|
+
"youtube": "youtube.channel",
|
|
39
|
+
},
|
|
40
|
+
"posts": {
|
|
41
|
+
"bluesky": "bluesky.profile.posts",
|
|
42
|
+
"cosmos": "cosmos.profile.posts",
|
|
43
|
+
"facebook": "facebook.profile.posts",
|
|
44
|
+
"instagram": "instagram.profile.posts",
|
|
45
|
+
"kwai": "kwai.profile.posts",
|
|
46
|
+
"linkedin": "linkedin.company.posts",
|
|
47
|
+
"rumble": "rumble.channel.videos",
|
|
48
|
+
"telegram": "telegram.channel.posts",
|
|
49
|
+
"threads": "threads.profile.posts",
|
|
50
|
+
"tiktok": "tiktok.profile.videos",
|
|
51
|
+
"truthsocial": "truthsocial.profile.posts",
|
|
52
|
+
"twitch": "twitch.user.videos",
|
|
53
|
+
"twitter": "twitter.profile.tweets",
|
|
54
|
+
"youtube": "youtube.channel.videos",
|
|
55
|
+
},
|
|
56
|
+
"post": {
|
|
57
|
+
"bluesky": "bluesky.post",
|
|
58
|
+
"cosmos": "cosmos.post",
|
|
59
|
+
"facebook": "facebook.post",
|
|
60
|
+
"instagram": "instagram.post",
|
|
61
|
+
"kick": "kick.clip",
|
|
62
|
+
"kwai": "kwai.post",
|
|
63
|
+
"linkedin": "linkedin.post",
|
|
64
|
+
"pinterest": "pinterest.pin",
|
|
65
|
+
"reddit": "reddit.post",
|
|
66
|
+
"rumble": "rumble.video",
|
|
67
|
+
"snapchat": "snapchat.spotlight",
|
|
68
|
+
"telegram": "telegram.post",
|
|
69
|
+
"threads": "threads.post",
|
|
70
|
+
"tiktok": "tiktok.video",
|
|
71
|
+
"truthsocial": "truthsocial.post",
|
|
72
|
+
"twitch": "twitch.clip",
|
|
73
|
+
"twitter": "twitter.tweet",
|
|
74
|
+
"youtube": "youtube.video",
|
|
75
|
+
},
|
|
76
|
+
"comments": {
|
|
77
|
+
"facebook": "facebook.post.comments",
|
|
78
|
+
"instagram": "instagram.post.comments",
|
|
79
|
+
"reddit": "reddit.post.comments",
|
|
80
|
+
"rumble": "rumble.video.comments",
|
|
81
|
+
"snapchat": "snapchat.spotlight.comments",
|
|
82
|
+
"tiktok": "tiktok.video.comments",
|
|
83
|
+
"youtube": "youtube.video.comments",
|
|
84
|
+
},
|
|
85
|
+
"search": {
|
|
86
|
+
"apple_music": "apple_music.search",
|
|
87
|
+
"cosmos": "cosmos.search",
|
|
88
|
+
"facebook": "facebook.search.videos",
|
|
89
|
+
"google": "google.search",
|
|
90
|
+
"instagram": "instagram.search.reels",
|
|
91
|
+
"linkedin": "linkedin.search.posts",
|
|
92
|
+
"pinterest": "pinterest.search",
|
|
93
|
+
"reddit": "reddit.search",
|
|
94
|
+
"rumble": "rumble.search",
|
|
95
|
+
"spotify": "spotify.search",
|
|
96
|
+
"threads": "threads.search",
|
|
97
|
+
"tiktok": "tiktok.search.keyword",
|
|
98
|
+
"twitter": "twitter.search.posts",
|
|
99
|
+
"youtube": "youtube.search",
|
|
100
|
+
},
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
TRANSCRIPT = "transcript"
|
|
104
|
+
IDENTIFIERS = ("handle", "url", "id")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True)
|
|
108
|
+
class Endpoint:
|
|
109
|
+
"""One live endpoint: what it is called, what it takes, what it costs."""
|
|
110
|
+
|
|
111
|
+
id: str
|
|
112
|
+
platform: str
|
|
113
|
+
title: str
|
|
114
|
+
path: str
|
|
115
|
+
summary: str
|
|
116
|
+
description: str
|
|
117
|
+
response: str
|
|
118
|
+
paginated: bool
|
|
119
|
+
credits: str
|
|
120
|
+
params: tuple[Mapping[str, Any], ...]
|
|
121
|
+
one_of: tuple[tuple[str, ...], ...]
|
|
122
|
+
example: Mapping[str, Any]
|
|
123
|
+
common: bool # takes trim, raw and max_age_days (every data endpoint does; the account ones do not)
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def names(self) -> tuple[str, ...]:
|
|
127
|
+
"""Parameter names of its own, without the ones every endpoint takes."""
|
|
128
|
+
return tuple(p["name"] for p in self.params)
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def required(self) -> tuple[str, ...]:
|
|
132
|
+
return tuple(p["name"] for p in self.params if p["required"])
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def hosts(self) -> tuple[str, ...]:
|
|
136
|
+
"""Hosts its `url` parameter accepts, when it has one."""
|
|
137
|
+
for p in self.params:
|
|
138
|
+
if p["name"] == "url":
|
|
139
|
+
return tuple(p.get("hosts") or ())
|
|
140
|
+
return ()
|
|
141
|
+
|
|
142
|
+
def signature(self) -> str:
|
|
143
|
+
"""`handle | id (one needed), sort, cursor`: the parameters in one line."""
|
|
144
|
+
grouped = {name for group in self.one_of for name in group}
|
|
145
|
+
bits = [" | ".join(group) + " (one needed)" for group in self.one_of]
|
|
146
|
+
bits += [name + " (required)" for name in self.required if name not in grouped]
|
|
147
|
+
bits += [name for name in self.names if name not in grouped and name not in self.required]
|
|
148
|
+
return ", ".join(bits) or "none"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@lru_cache(maxsize=1)
|
|
152
|
+
def _load() -> tuple[dict[str, Endpoint], tuple[Mapping[str, Any], ...]]:
|
|
153
|
+
text = resources.files(__package__).joinpath("endpoints.json").read_text(encoding="utf-8")
|
|
154
|
+
doc = json.loads(text)
|
|
155
|
+
endpoints = {
|
|
156
|
+
row["id"]: Endpoint(
|
|
157
|
+
id=row["id"], platform=row["platform"], title=row["title"], path=row["path"], summary=row["summary"],
|
|
158
|
+
description=row["description"], response=row["response"], paginated=row["paginated"],
|
|
159
|
+
credits=row["credits"], params=tuple(row["params"]),
|
|
160
|
+
one_of=tuple(tuple(group) for group in row["one_of"]), example=row["example"],
|
|
161
|
+
common=row["common"],
|
|
162
|
+
)
|
|
163
|
+
for row in doc["endpoints"]
|
|
164
|
+
}
|
|
165
|
+
return endpoints, tuple(doc["common"])
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def endpoints() -> dict[str, Endpoint]:
|
|
169
|
+
"""Every endpoint in the catalogue, by id, in the order of the API reference."""
|
|
170
|
+
return _load()[0]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def common_params() -> tuple[Mapping[str, Any], ...]:
|
|
174
|
+
"""The parameters every data endpoint takes: trim, raw, max_age_days."""
|
|
175
|
+
return _load()[1]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def platforms() -> dict[str, str]:
|
|
179
|
+
"""Platform key to display name, in the order of the API reference."""
|
|
180
|
+
out: dict[str, str] = {}
|
|
181
|
+
for ep in endpoints().values():
|
|
182
|
+
out.setdefault(ep.platform, ep.title)
|
|
183
|
+
return out
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def get_endpoint(endpoint_id: str) -> Endpoint:
|
|
187
|
+
"""The endpoint with this id, or a ValueError that says where to look."""
|
|
188
|
+
found = endpoints().get(endpoint_id)
|
|
189
|
+
if found is None:
|
|
190
|
+
near = [i for i in endpoints() if i.startswith(endpoint_id.split(".")[0] + ".")][:12]
|
|
191
|
+
hint = f" Ids that start the same way: {', '.join(near)}." if near else ""
|
|
192
|
+
raise ValueError(f"unknown endpoint id {endpoint_id!r}.{hint} List them with socialscrape_list_endpoints.")
|
|
193
|
+
return found
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def check_params(ep: Endpoint, params: Mapping[str, Any]) -> dict[str, Any]:
|
|
197
|
+
"""The parameters to send, with None dropped, or a ValueError naming what is wrong.
|
|
198
|
+
|
|
199
|
+
Checked before any request is made: an unknown name, a missing required
|
|
200
|
+
parameter, or none of a group where one is needed.
|
|
201
|
+
"""
|
|
202
|
+
sent = {k: v for k, v in params.items() if v is not None}
|
|
203
|
+
allowed = set(ep.names) | ({p["name"] for p in common_params()} if ep.common else set())
|
|
204
|
+
unknown = sorted(set(sent) - allowed)
|
|
205
|
+
if unknown:
|
|
206
|
+
raise ValueError(f"{ep.id} does not take {', '.join(unknown)}. It takes: {ep.signature()}.")
|
|
207
|
+
missing = [name for name in ep.required if name not in sent]
|
|
208
|
+
if missing:
|
|
209
|
+
raise ValueError(f"{ep.id} needs {', '.join(missing)}. It takes: {ep.signature()}.")
|
|
210
|
+
for group in ep.one_of:
|
|
211
|
+
if not any(name in sent for name in group):
|
|
212
|
+
raise ValueError(f"{ep.id} needs one of {', '.join(group)}. It takes: {ep.signature()}.")
|
|
213
|
+
return sent
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _host_matches(host: str, allowed: tuple[str, ...]) -> bool:
|
|
217
|
+
return any(host == h or host.endswith("." + h) for h in allowed)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def platform_of_url(role: str, url: str) -> str | None:
|
|
221
|
+
"""Which platform of a role a URL belongs to, read from its host."""
|
|
222
|
+
host = (urlsplit(url if "//" in url else "https://" + url).hostname or "").lower()
|
|
223
|
+
if not host:
|
|
224
|
+
return None
|
|
225
|
+
for platform, endpoint_id in ROLES[role].items():
|
|
226
|
+
if _host_matches(host, endpoints()[endpoint_id].hosts):
|
|
227
|
+
return platform
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def route(role: str, platform: str | None, *, handle: str | None = None, url: str | None = None,
|
|
232
|
+
id: str | None = None) -> tuple[Endpoint, dict[str, Any]]:
|
|
233
|
+
"""The endpoint that does `role` on a platform, and the one identifier to send it.
|
|
234
|
+
|
|
235
|
+
Without a platform the URL's host decides. Endpoints differ in what they
|
|
236
|
+
look a thing up by, so an identifier the endpoint does not take is an
|
|
237
|
+
error that says which ones it does.
|
|
238
|
+
"""
|
|
239
|
+
table = ROLES[role]
|
|
240
|
+
if not platform:
|
|
241
|
+
if not url:
|
|
242
|
+
raise ValueError(f"pass a platform ({', '.join(sorted(table))}), or a url it can be read from")
|
|
243
|
+
platform = platform_of_url(role, url)
|
|
244
|
+
if platform is None:
|
|
245
|
+
raise ValueError(f"no platform recognised in {url!r}; pass platform, one of {', '.join(sorted(table))}")
|
|
246
|
+
key = platform.strip().lower().replace("-", "_")
|
|
247
|
+
key = {"x": "twitter"}.get(key, key)
|
|
248
|
+
if key not in table:
|
|
249
|
+
raise ValueError(f"no {role} lookup for {platform!r}; platforms: {', '.join(sorted(table))}")
|
|
250
|
+
ep = endpoints()[table[key]]
|
|
251
|
+
given = {name: value for name, value in (("handle", handle), ("url", url), ("id", id)) if value}
|
|
252
|
+
for name in IDENTIFIERS:
|
|
253
|
+
if name in given and name in ep.names:
|
|
254
|
+
return ep, {name: given[name]}
|
|
255
|
+
takes = " or ".join(name for name in IDENTIFIERS if name in ep.names)
|
|
256
|
+
got = ", ".join(given) or "nothing"
|
|
257
|
+
raise ValueError(f"{ep.id} looks things up by {takes}; got {got}")
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""How the components and the toolset reach the API: one place that owns the SocialScrape client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from importlib import metadata
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from socialscrape import AsyncSocialScrape, Response, SocialScrape
|
|
10
|
+
from socialscrape import __version__ as _sdk_version
|
|
11
|
+
|
|
12
|
+
from ._catalog import check_params, get_endpoint
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
_version = metadata.version("socialscrape-haystack")
|
|
16
|
+
except metadata.PackageNotFoundError: # a source checkout that was never installed
|
|
17
|
+
_version = "0"
|
|
18
|
+
|
|
19
|
+
USER_AGENT = f"socialscrape-haystack/{_version} socialscrape-python/{_sdk_version}"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Session:
|
|
23
|
+
"""Makes the SocialScrape client on first use and keeps it for later calls.
|
|
24
|
+
|
|
25
|
+
Pass `client` (and `async_client`) to use ones you configured yourself, for
|
|
26
|
+
example with your own timeouts or `httpx` client. Otherwise the key is
|
|
27
|
+
`api_key`, or SOCIALSCRAPE_API_KEY when it is left out.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, *, api_key: str | None = None, base_url: str | None = None, client: SocialScrape | None = None,
|
|
31
|
+
async_client: AsyncSocialScrape | None = None) -> None:
|
|
32
|
+
self._api_key = api_key
|
|
33
|
+
self._base_url = base_url
|
|
34
|
+
self._client = client
|
|
35
|
+
self._async_client = async_client
|
|
36
|
+
|
|
37
|
+
def _options(self) -> dict[str, Any]:
|
|
38
|
+
return {"api_key": self._api_key, "base_url": self._base_url, "default_headers": {"user-agent": USER_AGENT}}
|
|
39
|
+
|
|
40
|
+
def client(self) -> SocialScrape:
|
|
41
|
+
if self._client is None:
|
|
42
|
+
self._client = SocialScrape(**self._options())
|
|
43
|
+
return self._client
|
|
44
|
+
|
|
45
|
+
def call(self, endpoint_id: str, params: Mapping[str, Any]) -> Response[Any]:
|
|
46
|
+
"""Run one endpoint. Parameters are checked first, so a bad call costs no request."""
|
|
47
|
+
ep = get_endpoint(endpoint_id)
|
|
48
|
+
return self.client().request("GET", ep.path, query=check_params(ep, params))
|
|
49
|
+
|
|
50
|
+
async def acall(self, endpoint_id: str, params: Mapping[str, Any]) -> Response[Any]:
|
|
51
|
+
ep = get_endpoint(endpoint_id)
|
|
52
|
+
query = check_params(ep, params)
|
|
53
|
+
if self._async_client is not None:
|
|
54
|
+
return await self._async_client.request("GET", ep.path, query=query)
|
|
55
|
+
# A client of our own lives for one call: it cannot outlast the event loop it was made in.
|
|
56
|
+
async with AsyncSocialScrape(**self._options()) as client:
|
|
57
|
+
return await client.request("GET", ep.path, query=query)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Turn API objects into Haystack Documents: the object's text is the content, the rest is metadata."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from haystack import Document
|
|
10
|
+
|
|
11
|
+
# Which fields of an object are its text, by the object's type. Anything else goes to meta.
|
|
12
|
+
_TEXT_FIELDS: dict[str, tuple[str, ...]] = {
|
|
13
|
+
"post": ("title", "text"),
|
|
14
|
+
"comment": ("text",),
|
|
15
|
+
"review": ("text",),
|
|
16
|
+
"transcript": ("text",),
|
|
17
|
+
"profile": ("name", "headline", "bio"),
|
|
18
|
+
"community": ("name", "title", "description"),
|
|
19
|
+
"collection": ("title", "description"),
|
|
20
|
+
"ad": ("title", "body"),
|
|
21
|
+
"product": ("title", "description"),
|
|
22
|
+
"repo": ("full_name", "description"),
|
|
23
|
+
"search_result": ("title", "snippet"),
|
|
24
|
+
"link_page": ("name", "bio"),
|
|
25
|
+
"event": ("title", "description"),
|
|
26
|
+
}
|
|
27
|
+
_FALLBACK_TEXT = ("title", "name", "text", "description")
|
|
28
|
+
_SCALARS = (str, int, float, bool)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _meta(obj: Mapping[str, Any], text_fields: tuple[str, ...]) -> dict[str, Any]:
|
|
32
|
+
"""Flat meta a document store accepts: the object's scalar fields, its counts and who made it."""
|
|
33
|
+
meta: dict[str, Any] = {}
|
|
34
|
+
for key, value in obj.items():
|
|
35
|
+
if key in text_fields or key == "extra":
|
|
36
|
+
continue
|
|
37
|
+
if key == "source" and isinstance(value, str):
|
|
38
|
+
meta[f"{obj.get('object') or 'item'}_source"] = value # a transcript's `source` is captions or ai
|
|
39
|
+
elif isinstance(value, _SCALARS):
|
|
40
|
+
meta[key] = value
|
|
41
|
+
elif key == "stats" and isinstance(value, Mapping):
|
|
42
|
+
meta.update({name: count for name, count in value.items() if isinstance(count, _SCALARS)})
|
|
43
|
+
elif key in ("author", "owner", "advertiser", "seller", "host") and isinstance(value, Mapping):
|
|
44
|
+
who = value.get("handle") or value.get("name")
|
|
45
|
+
if who:
|
|
46
|
+
meta[key] = who
|
|
47
|
+
if isinstance(value.get("url"), str):
|
|
48
|
+
meta[f"{key}_url"] = value["url"]
|
|
49
|
+
if "url" in meta:
|
|
50
|
+
meta["source"] = meta["url"]
|
|
51
|
+
return meta
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def to_document(obj: Any, endpoint_id: str) -> Document:
|
|
55
|
+
"""One API object as a Document. Meta has `endpoint`, `platform` and, when it has one, `url` and `source`."""
|
|
56
|
+
if not isinstance(obj, Mapping):
|
|
57
|
+
return Document(content=json.dumps(obj, ensure_ascii=False), meta={"endpoint": endpoint_id})
|
|
58
|
+
kind = obj.get("object")
|
|
59
|
+
fields = _TEXT_FIELDS.get(kind, _FALLBACK_TEXT) if isinstance(kind, str) else _FALLBACK_TEXT
|
|
60
|
+
parts = [obj[name].strip() for name in fields if isinstance(obj.get(name), str) and obj[name].strip()]
|
|
61
|
+
if parts:
|
|
62
|
+
content, used = "\n\n".join(parts), fields
|
|
63
|
+
else: # nothing that reads as text (a credit balance, a contribution calendar): keep the whole object
|
|
64
|
+
content, used = json.dumps(obj, ensure_ascii=False), ()
|
|
65
|
+
meta = _meta(obj, used)
|
|
66
|
+
meta["endpoint"] = endpoint_id
|
|
67
|
+
ident = obj.get("id") or obj.get("video_id")
|
|
68
|
+
doc_id = f"{obj.get('platform')}:{kind}:{ident}" if ident and obj.get("platform") and kind else None
|
|
69
|
+
if doc_id:
|
|
70
|
+
return Document(id=doc_id, content=content, meta=meta)
|
|
71
|
+
return Document(content=content, meta=meta)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def items_of(data: Any) -> tuple[list[Any], str | None]:
|
|
75
|
+
"""The items of an answer and the cursor for the next page: a list page has them, any other object is one item."""
|
|
76
|
+
if isinstance(data, Mapping) and data.get("object") == "list":
|
|
77
|
+
cursor = data.get("cursor") if data.get("has_more") is not False else None
|
|
78
|
+
return list(data.get("items") or []), cursor or None
|
|
79
|
+
return [data], None
|