rrss-sdk-wrapper 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rrss/__init__.py +50 -0
- rrss/adapters/base.py +80 -0
- rrss/adapters/instagram.py +123 -0
- rrss/adapters/threads.py +62 -0
- rrss/adapters/tiktok.py +76 -0
- rrss/adapters/x.py +224 -0
- rrss/adapters/x_stub.py +5 -0
- rrss/auth/__init__.py +19 -0
- rrss/cache.py +209 -0
- rrss/cli.py +78 -0
- rrss/client.py +35 -0
- rrss/errors.py +31 -0
- rrss/gateway.py +306 -0
- rrss/models.py +61 -0
- rrss/normalize/__init__.py +1 -0
- rrss/normalize/instagram.py +82 -0
- rrss/normalize/threads.py +82 -0
- rrss/normalize/tiktok.py +51 -0
- rrss/normalize/x.py +192 -0
- rrss/ratelimit.py +30 -0
- rrss/serialize.py +216 -0
- rrss/stream.py +198 -0
- rrss_sdk_wrapper-0.2.0.dist-info/METADATA +173 -0
- rrss_sdk_wrapper-0.2.0.dist-info/RECORD +26 -0
- rrss_sdk_wrapper-0.2.0.dist-info/WHEEL +4 -0
- rrss_sdk_wrapper-0.2.0.dist-info/entry_points.txt +2 -0
rrss/models.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import Generic, Literal, TypeVar
|
|
6
|
+
|
|
7
|
+
Platform = Literal["instagram", "threads", "tiktok", "facebook", "x"]
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class MediaItem:
|
|
14
|
+
type: Literal["image", "video"]
|
|
15
|
+
url: str
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class User:
|
|
20
|
+
id: str
|
|
21
|
+
platform: Platform
|
|
22
|
+
username: str | None = None
|
|
23
|
+
display_name: str | None = None
|
|
24
|
+
bio: str | None = None
|
|
25
|
+
profile_url: str | None = None
|
|
26
|
+
avatar_url: str | None = None
|
|
27
|
+
follower_count: int | None = None
|
|
28
|
+
following_count: int | None = None
|
|
29
|
+
is_verified: bool | None = None
|
|
30
|
+
raw: dict[str, object] | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Post:
|
|
35
|
+
id: str
|
|
36
|
+
author_id: str
|
|
37
|
+
platform: Platform
|
|
38
|
+
text: str | None = None
|
|
39
|
+
created_at: datetime | None = None
|
|
40
|
+
url: str | None = None
|
|
41
|
+
media: list[MediaItem] = field(default_factory=list)
|
|
42
|
+
like_count: int | None = None
|
|
43
|
+
comment_count: int | None = None
|
|
44
|
+
raw: dict[str, object] | None = None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class Comment:
|
|
49
|
+
id: str
|
|
50
|
+
post_id: str
|
|
51
|
+
text: str
|
|
52
|
+
platform: Platform
|
|
53
|
+
author_id: str | None = None
|
|
54
|
+
created_at: datetime | None = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class CursorPage(Generic[T]):
|
|
59
|
+
items: list[T]
|
|
60
|
+
has_more: bool
|
|
61
|
+
next_cursor: str | None = None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Platform-specific normalization helpers."""
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from rrss.models import Comment, MediaItem, Post, User
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _get(obj: Any, key: str, default: Any = None) -> Any:
|
|
10
|
+
if isinstance(obj, dict):
|
|
11
|
+
return obj.get(key, default)
|
|
12
|
+
return getattr(obj, key, default)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def user_from_instagrapi(raw: Any, *, include_raw: bool) -> User:
|
|
16
|
+
return User(
|
|
17
|
+
id=str(_get(raw, "pk") or _get(raw, "id")),
|
|
18
|
+
username=_get(raw, "username"),
|
|
19
|
+
display_name=_get(raw, "full_name"),
|
|
20
|
+
bio=_get(raw, "biography"),
|
|
21
|
+
profile_url=None,
|
|
22
|
+
avatar_url=_get(raw, "profile_pic_url_hd") or _get(raw, "profile_pic_url"),
|
|
23
|
+
follower_count=_get(raw, "follower_count"),
|
|
24
|
+
following_count=_get(raw, "following_count"),
|
|
25
|
+
is_verified=_get(raw, "is_verified"),
|
|
26
|
+
platform="instagram",
|
|
27
|
+
raw=raw if include_raw else None,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def post_from_instagrapi(raw: Any, *, include_raw: bool) -> Post:
|
|
32
|
+
caption = _get(raw, "caption_text")
|
|
33
|
+
if caption is None:
|
|
34
|
+
cap_obj = _get(raw, "caption")
|
|
35
|
+
if cap_obj is not None:
|
|
36
|
+
caption = _get(cap_obj, "text")
|
|
37
|
+
taken = _get(raw, "taken_at")
|
|
38
|
+
created: datetime | None = None
|
|
39
|
+
if taken is not None:
|
|
40
|
+
created = datetime.fromtimestamp(int(taken), tz=UTC)
|
|
41
|
+
code = _get(raw, "code")
|
|
42
|
+
url = f"https://www.instagram.com/p/{code}/" if code else None
|
|
43
|
+
media: list[MediaItem] = []
|
|
44
|
+
video_url = _get(raw, "video_url")
|
|
45
|
+
thumb = _get(raw, "thumbnail_url")
|
|
46
|
+
if video_url:
|
|
47
|
+
media.append(MediaItem(type="video", url=str(video_url)))
|
|
48
|
+
elif thumb:
|
|
49
|
+
media.append(MediaItem(type="image", url=str(thumb)))
|
|
50
|
+
user_field = _get(raw, "user")
|
|
51
|
+
if isinstance(user_field, dict):
|
|
52
|
+
author_id = str(user_field.get("pk", ""))
|
|
53
|
+
else:
|
|
54
|
+
author_id = str(_get(raw, "user_id", ""))
|
|
55
|
+
return Post(
|
|
56
|
+
id=str(_get(raw, "pk") or _get(raw, "id")),
|
|
57
|
+
author_id=author_id,
|
|
58
|
+
text=caption,
|
|
59
|
+
created_at=created,
|
|
60
|
+
url=url,
|
|
61
|
+
media=media,
|
|
62
|
+
like_count=_get(raw, "like_count"),
|
|
63
|
+
comment_count=_get(raw, "comment_count"),
|
|
64
|
+
platform="instagram",
|
|
65
|
+
raw=raw if include_raw else None,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def comment_from_instagrapi(raw: Any, *, post_id: str, include_raw: bool) -> Comment:
|
|
70
|
+
del include_raw
|
|
71
|
+
created: datetime | None = None
|
|
72
|
+
ts = _get(raw, "created_at_utc") or _get(raw, "created_at")
|
|
73
|
+
if ts is not None:
|
|
74
|
+
created = datetime.fromtimestamp(int(ts), tz=UTC)
|
|
75
|
+
return Comment(
|
|
76
|
+
id=str(_get(raw, "pk") or _get(raw, "id")),
|
|
77
|
+
post_id=post_id,
|
|
78
|
+
author_id=str(_get(raw, "user_id")) if _get(raw, "user_id") is not None else None,
|
|
79
|
+
text=str(_get(raw, "text", "")),
|
|
80
|
+
created_at=created,
|
|
81
|
+
platform="instagram",
|
|
82
|
+
)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from rrss.models import CursorPage, MediaItem, Post, User
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def user_from_sidecar(username: str, data: dict[str, Any], *, include_raw: bool) -> User:
|
|
10
|
+
return User(
|
|
11
|
+
id=str(data.get("user_id", data.get("id", username))),
|
|
12
|
+
username=username,
|
|
13
|
+
display_name=data.get("full_name"),
|
|
14
|
+
bio=data.get("biography"),
|
|
15
|
+
profile_url=(
|
|
16
|
+
(data.get("bio_links") or [{}])[0].get("url") if data.get("bio_links") else None
|
|
17
|
+
),
|
|
18
|
+
avatar_url=data.get("profile_pic_url"),
|
|
19
|
+
follower_count=data.get("follower_count"),
|
|
20
|
+
following_count=data.get("following_count"),
|
|
21
|
+
is_verified=data.get("is_verified"),
|
|
22
|
+
platform="threads",
|
|
23
|
+
raw=data if include_raw else None,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _post_from_thread_item(item: dict[str, Any], author_id: str, *, include_raw: bool) -> Post:
|
|
28
|
+
post = item.get("post") or item
|
|
29
|
+
pid = str(post.get("pk") or post.get("code") or post.get("id", ""))
|
|
30
|
+
caption = post.get("caption") or {}
|
|
31
|
+
text = caption.get("text") if isinstance(caption, dict) else caption
|
|
32
|
+
media: list[MediaItem] = []
|
|
33
|
+
for key in ("carousel_media", "image_versions2"):
|
|
34
|
+
block = post.get(key)
|
|
35
|
+
if isinstance(block, list):
|
|
36
|
+
for entry in block:
|
|
37
|
+
if not isinstance(entry, dict):
|
|
38
|
+
continue
|
|
39
|
+
url = entry.get("url") or entry.get("download_url")
|
|
40
|
+
if url:
|
|
41
|
+
media.append(MediaItem(type="image", url=str(url)))
|
|
42
|
+
elif isinstance(block, dict):
|
|
43
|
+
candidates = block.get("candidates") or []
|
|
44
|
+
if candidates:
|
|
45
|
+
media.append(MediaItem(type="image", url=str(candidates[0].get("url", ""))))
|
|
46
|
+
taken = post.get("taken_at")
|
|
47
|
+
created = datetime.fromtimestamp(int(taken), tz=UTC) if taken else None
|
|
48
|
+
return Post(
|
|
49
|
+
id=pid,
|
|
50
|
+
author_id=author_id,
|
|
51
|
+
text=str(text) if text is not None else None,
|
|
52
|
+
created_at=created,
|
|
53
|
+
url=None,
|
|
54
|
+
media=media,
|
|
55
|
+
like_count=post.get("like_count"),
|
|
56
|
+
comment_count=post.get("reply_count"),
|
|
57
|
+
platform="threads",
|
|
58
|
+
raw=post if include_raw else None,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def threads_page_from_sidecar(
|
|
63
|
+
username: str,
|
|
64
|
+
data: dict[str, Any] | list[Any],
|
|
65
|
+
*,
|
|
66
|
+
include_raw: bool,
|
|
67
|
+
) -> CursorPage[Post]:
|
|
68
|
+
author_id = str(data.get("user_id", username)) if isinstance(data, dict) else username
|
|
69
|
+
items_raw: list[Any]
|
|
70
|
+
if isinstance(data, list):
|
|
71
|
+
items_raw = data
|
|
72
|
+
else:
|
|
73
|
+
items_raw = data.get("threads") or data.get("items") or []
|
|
74
|
+
posts: list[Post] = []
|
|
75
|
+
for thread in items_raw:
|
|
76
|
+
thread_items = thread.get("thread_items") if isinstance(thread, dict) else None
|
|
77
|
+
if thread_items:
|
|
78
|
+
for item in thread_items:
|
|
79
|
+
posts.append(_post_from_thread_item(item, author_id, include_raw=include_raw))
|
|
80
|
+
elif isinstance(thread, dict):
|
|
81
|
+
posts.append(_post_from_thread_item(thread, author_id, include_raw=include_raw))
|
|
82
|
+
return CursorPage(items=posts, has_more=False, next_cursor=None)
|
rrss/normalize/tiktok.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from rrss.models import MediaItem, Post, User
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def user_from_tiktok(raw: dict[str, Any], *, include_raw: bool) -> User:
|
|
10
|
+
stats = raw.get("stats") or {}
|
|
11
|
+
return User(
|
|
12
|
+
id=str(raw.get("id") or raw.get("uid") or raw.get("user_id", "")),
|
|
13
|
+
username=raw.get("uniqueId") or raw.get("username"),
|
|
14
|
+
display_name=raw.get("nickname"),
|
|
15
|
+
bio=raw.get("signature"),
|
|
16
|
+
profile_url=None,
|
|
17
|
+
avatar_url=raw.get("avatarLarger") or raw.get("avatarThumb"),
|
|
18
|
+
follower_count=stats.get("followerCount") or raw.get("follower_count"),
|
|
19
|
+
following_count=stats.get("followingCount") or raw.get("following_count"),
|
|
20
|
+
is_verified=raw.get("verified"),
|
|
21
|
+
platform="tiktok",
|
|
22
|
+
raw=raw if include_raw else None,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def post_from_tiktok(raw: dict[str, Any], *, include_raw: bool) -> Post:
|
|
27
|
+
create_time = raw.get("createTime") or raw.get("create_time")
|
|
28
|
+
created = datetime.fromtimestamp(int(create_time), tz=UTC) if create_time else None
|
|
29
|
+
author = raw.get("author") or {}
|
|
30
|
+
author_id = str(author.get("id") or raw.get("author_id", ""))
|
|
31
|
+
video = raw.get("video") or {}
|
|
32
|
+
cover = video.get("cover") or video.get("originCover")
|
|
33
|
+
media: list[MediaItem] = []
|
|
34
|
+
if cover:
|
|
35
|
+
media.append(MediaItem(type="image", url=str(cover)))
|
|
36
|
+
play = video.get("playAddr") or video.get("downloadAddr")
|
|
37
|
+
if play:
|
|
38
|
+
media.append(MediaItem(type="video", url=str(play)))
|
|
39
|
+
stats = raw.get("stats") or {}
|
|
40
|
+
return Post(
|
|
41
|
+
id=str(raw.get("id") or raw.get("aweme_id", "")),
|
|
42
|
+
author_id=author_id,
|
|
43
|
+
text=raw.get("desc"),
|
|
44
|
+
created_at=created,
|
|
45
|
+
url=None,
|
|
46
|
+
media=media,
|
|
47
|
+
like_count=stats.get("diggCount") or raw.get("digg_count"),
|
|
48
|
+
comment_count=stats.get("commentCount") or raw.get("comment_count"),
|
|
49
|
+
platform="tiktok",
|
|
50
|
+
raw=raw if include_raw else None,
|
|
51
|
+
)
|
rrss/normalize/x.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Any, Literal
|
|
5
|
+
|
|
6
|
+
from rrss.errors import NotFound
|
|
7
|
+
from rrss.models import Comment, CursorPage, MediaItem, Post, User
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _get(obj: Any, key: str, default: Any = None) -> Any:
|
|
11
|
+
if obj is None:
|
|
12
|
+
return default
|
|
13
|
+
if isinstance(obj, dict):
|
|
14
|
+
return obj.get(key, default)
|
|
15
|
+
return getattr(obj, key, default)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _is_response(raw: Any) -> bool:
|
|
19
|
+
if isinstance(raw, dict):
|
|
20
|
+
return "data" in raw and any(key in raw for key in ("includes", "meta", "errors"))
|
|
21
|
+
return hasattr(raw, "data") and any(hasattr(raw, key) for key in ("includes", "meta", "errors"))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _parts(raw: Any) -> tuple[Any, Any, Any]:
|
|
25
|
+
if raw is None:
|
|
26
|
+
return None, {}, {}
|
|
27
|
+
if _is_response(raw):
|
|
28
|
+
return _get(raw, "data"), _get(raw, "includes") or {}, _get(raw, "meta") or {}
|
|
29
|
+
return raw, {}, {}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _metric(metrics: Any, key: str) -> int | None:
|
|
33
|
+
if metrics is None:
|
|
34
|
+
return None
|
|
35
|
+
value = metrics.get(key) if isinstance(metrics, dict) else getattr(metrics, key, None)
|
|
36
|
+
if value is None:
|
|
37
|
+
return None
|
|
38
|
+
return int(value)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _parse_time(value: Any) -> datetime | None:
|
|
42
|
+
if value is None:
|
|
43
|
+
return None
|
|
44
|
+
if isinstance(value, datetime):
|
|
45
|
+
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
|
46
|
+
text = str(value).replace("Z", "+00:00")
|
|
47
|
+
return datetime.fromisoformat(text)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _raw_dict(obj: Any) -> dict[str, object]:
|
|
51
|
+
if isinstance(obj, dict):
|
|
52
|
+
return obj
|
|
53
|
+
data = getattr(obj, "data", None)
|
|
54
|
+
if isinstance(data, dict):
|
|
55
|
+
return data
|
|
56
|
+
return {}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _entity(raw: Any, *, what: str) -> Any:
|
|
60
|
+
data, _includes, _meta = _parts(raw)
|
|
61
|
+
if isinstance(data, list):
|
|
62
|
+
data = data[0] if data else None
|
|
63
|
+
if data is None:
|
|
64
|
+
raise NotFound(f"X {what} payload was empty")
|
|
65
|
+
return data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def user_from_x(raw: Any, *, include_raw: bool) -> User:
|
|
69
|
+
data = _entity(raw, what="user")
|
|
70
|
+
metrics = _get(data, "public_metrics") or {}
|
|
71
|
+
username = _get(data, "username")
|
|
72
|
+
return User(
|
|
73
|
+
id=str(_get(data, "id")),
|
|
74
|
+
username=username,
|
|
75
|
+
display_name=_get(data, "name"),
|
|
76
|
+
bio=_get(data, "description"),
|
|
77
|
+
profile_url=f"https://x.com/{username}" if username else None,
|
|
78
|
+
avatar_url=_get(data, "profile_image_url"),
|
|
79
|
+
follower_count=_metric(metrics, "followers_count"),
|
|
80
|
+
following_count=_metric(metrics, "following_count"),
|
|
81
|
+
is_verified=_get(data, "verified"),
|
|
82
|
+
platform="x",
|
|
83
|
+
raw=_raw_dict(data) if include_raw else None,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _author_username(includes: Any, author_id: str) -> str | None:
|
|
88
|
+
users = includes.get("users") if isinstance(includes, dict) else _get(includes, "users")
|
|
89
|
+
for user in users or []:
|
|
90
|
+
if str(_get(user, "id")) == author_id and _get(user, "username"):
|
|
91
|
+
return str(_get(user, "username"))
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _media(data: Any, includes: Any) -> list[MediaItem]:
|
|
96
|
+
attachments = _get(data, "attachments") or {}
|
|
97
|
+
keys = attachments.get("media_keys") if isinstance(attachments, dict) else []
|
|
98
|
+
library = includes.get("media") if isinstance(includes, dict) else _get(includes, "media")
|
|
99
|
+
by_key: dict[str, Any] = {}
|
|
100
|
+
for item in library or []:
|
|
101
|
+
by_key[str(_get(item, "media_key"))] = item
|
|
102
|
+
media: list[MediaItem] = []
|
|
103
|
+
for key in keys or []:
|
|
104
|
+
item = by_key.get(str(key))
|
|
105
|
+
if item is None:
|
|
106
|
+
continue
|
|
107
|
+
url = _get(item, "url") or _get(item, "preview_image_url")
|
|
108
|
+
if not url:
|
|
109
|
+
continue
|
|
110
|
+
kind = str(_get(item, "type") or "photo")
|
|
111
|
+
media_type: Literal["image", "video"] = (
|
|
112
|
+
"video" if kind in {"video", "animated_gif"} else "image"
|
|
113
|
+
)
|
|
114
|
+
media.append(MediaItem(type=media_type, url=str(url)))
|
|
115
|
+
return media
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def post_from_x(raw: Any, *, include_raw: bool, includes: Any | None = None) -> Post:
|
|
119
|
+
data, response_includes, _meta = _parts(raw)
|
|
120
|
+
if isinstance(data, list):
|
|
121
|
+
data = data[0] if data else None
|
|
122
|
+
if data is None:
|
|
123
|
+
raise NotFound("X post payload was empty")
|
|
124
|
+
merged_includes = includes if includes is not None else response_includes
|
|
125
|
+
author_id = str(_get(data, "author_id") or "")
|
|
126
|
+
tweet_id = str(_get(data, "id"))
|
|
127
|
+
username = _author_username(merged_includes, author_id)
|
|
128
|
+
url = (
|
|
129
|
+
f"https://x.com/{username}/status/{tweet_id}"
|
|
130
|
+
if username
|
|
131
|
+
else f"https://x.com/i/web/status/{tweet_id}"
|
|
132
|
+
)
|
|
133
|
+
metrics = _get(data, "public_metrics") or {}
|
|
134
|
+
return Post(
|
|
135
|
+
id=tweet_id,
|
|
136
|
+
author_id=author_id,
|
|
137
|
+
text=_get(data, "text"),
|
|
138
|
+
created_at=_parse_time(_get(data, "created_at")),
|
|
139
|
+
url=url,
|
|
140
|
+
media=_media(data, merged_includes),
|
|
141
|
+
like_count=_metric(metrics, "like_count"),
|
|
142
|
+
comment_count=_metric(metrics, "reply_count"),
|
|
143
|
+
platform="x",
|
|
144
|
+
raw=_raw_dict(data) if include_raw else None,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _next_token(meta: Any) -> str | None:
|
|
149
|
+
token = meta.get("next_token") if isinstance(meta, dict) else _get(meta, "next_token")
|
|
150
|
+
if isinstance(token, str) and token:
|
|
151
|
+
return token
|
|
152
|
+
return None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def posts_page_from_x(raw: Any, *, include_raw: bool) -> CursorPage[Post]:
|
|
156
|
+
if raw is None:
|
|
157
|
+
return CursorPage(items=[], has_more=False, next_cursor=None)
|
|
158
|
+
data, includes, meta = _parts(raw)
|
|
159
|
+
rows = data or []
|
|
160
|
+
if not isinstance(rows, list):
|
|
161
|
+
rows = [rows]
|
|
162
|
+
posts = [post_from_x(row, include_raw=include_raw, includes=includes) for row in rows]
|
|
163
|
+
token = _next_token(meta)
|
|
164
|
+
return CursorPage(items=posts, has_more=token is not None, next_cursor=token)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def comments_page_from_x(raw: Any, *, post_id: str, include_raw: bool) -> CursorPage[Comment]:
|
|
168
|
+
del include_raw
|
|
169
|
+
if raw is None:
|
|
170
|
+
return CursorPage(items=[], has_more=False, next_cursor=None)
|
|
171
|
+
data, _includes, meta = _parts(raw)
|
|
172
|
+
rows = data or []
|
|
173
|
+
if not isinstance(rows, list):
|
|
174
|
+
rows = [rows]
|
|
175
|
+
items: list[Comment] = []
|
|
176
|
+
for row in rows:
|
|
177
|
+
comment_id = str(_get(row, "id") or "")
|
|
178
|
+
if not comment_id or comment_id == post_id:
|
|
179
|
+
continue
|
|
180
|
+
author = _get(row, "author_id")
|
|
181
|
+
items.append(
|
|
182
|
+
Comment(
|
|
183
|
+
id=comment_id,
|
|
184
|
+
post_id=str(_get(row, "conversation_id") or post_id),
|
|
185
|
+
author_id=str(author) if author is not None else None,
|
|
186
|
+
text=str(_get(row, "text") or ""),
|
|
187
|
+
created_at=_parse_time(_get(row, "created_at")),
|
|
188
|
+
platform="x",
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
token = _next_token(meta)
|
|
192
|
+
return CursorPage(items=items, has_more=token is not None, next_cursor=token)
|
rrss/ratelimit.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import TypeVar
|
|
7
|
+
|
|
8
|
+
from rrss.errors import RateLimited
|
|
9
|
+
|
|
10
|
+
T = TypeVar("T")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def with_retries(
|
|
14
|
+
fn: Callable[[], T],
|
|
15
|
+
*,
|
|
16
|
+
max_attempts: int = 3,
|
|
17
|
+
base_delay: float = 0.5,
|
|
18
|
+
) -> T:
|
|
19
|
+
last_exc: Exception | None = None
|
|
20
|
+
for attempt in range(max_attempts):
|
|
21
|
+
try:
|
|
22
|
+
return fn()
|
|
23
|
+
except RateLimited as exc:
|
|
24
|
+
last_exc = exc
|
|
25
|
+
if attempt + 1 >= max_attempts:
|
|
26
|
+
raise
|
|
27
|
+
jitter = random.uniform(0, 0.25)
|
|
28
|
+
time.sleep(base_delay * (2**attempt) + jitter)
|
|
29
|
+
assert last_exc is not None
|
|
30
|
+
raise last_exc
|