rrss-sdk-wrapper 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rrss/models.py ADDED
@@ -0,0 +1,61 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass, field
4
+ from datetime import datetime
5
+ from typing import Generic, Literal, TypeVar
6
+
7
+ Platform = Literal["instagram", "threads", "tiktok", "facebook", "x"]
8
+
9
+ T = TypeVar("T")
10
+
11
+
12
+ @dataclass
13
+ class MediaItem:
14
+ type: Literal["image", "video"]
15
+ url: str
16
+
17
+
18
+ @dataclass
19
+ class User:
20
+ id: str
21
+ platform: Platform
22
+ username: str | None = None
23
+ display_name: str | None = None
24
+ bio: str | None = None
25
+ profile_url: str | None = None
26
+ avatar_url: str | None = None
27
+ follower_count: int | None = None
28
+ following_count: int | None = None
29
+ is_verified: bool | None = None
30
+ raw: dict[str, object] | None = None
31
+
32
+
33
+ @dataclass
34
+ class Post:
35
+ id: str
36
+ author_id: str
37
+ platform: Platform
38
+ text: str | None = None
39
+ created_at: datetime | None = None
40
+ url: str | None = None
41
+ media: list[MediaItem] = field(default_factory=list)
42
+ like_count: int | None = None
43
+ comment_count: int | None = None
44
+ raw: dict[str, object] | None = None
45
+
46
+
47
+ @dataclass
48
+ class Comment:
49
+ id: str
50
+ post_id: str
51
+ text: str
52
+ platform: Platform
53
+ author_id: str | None = None
54
+ created_at: datetime | None = None
55
+
56
+
57
+ @dataclass
58
+ class CursorPage(Generic[T]):
59
+ items: list[T]
60
+ has_more: bool
61
+ next_cursor: str | None = None
@@ -0,0 +1 @@
1
+ """Platform-specific normalization helpers."""
@@ -0,0 +1,82 @@
1
+ from __future__ import annotations
2
+
3
+ from datetime import UTC, datetime
4
+ from typing import Any
5
+
6
+ from rrss.models import Comment, MediaItem, Post, User
7
+
8
+
9
+ def _get(obj: Any, key: str, default: Any = None) -> Any:
10
+ if isinstance(obj, dict):
11
+ return obj.get(key, default)
12
+ return getattr(obj, key, default)
13
+
14
+
15
+ def user_from_instagrapi(raw: Any, *, include_raw: bool) -> User:
16
+ return User(
17
+ id=str(_get(raw, "pk") or _get(raw, "id")),
18
+ username=_get(raw, "username"),
19
+ display_name=_get(raw, "full_name"),
20
+ bio=_get(raw, "biography"),
21
+ profile_url=None,
22
+ avatar_url=_get(raw, "profile_pic_url_hd") or _get(raw, "profile_pic_url"),
23
+ follower_count=_get(raw, "follower_count"),
24
+ following_count=_get(raw, "following_count"),
25
+ is_verified=_get(raw, "is_verified"),
26
+ platform="instagram",
27
+ raw=raw if include_raw else None,
28
+ )
29
+
30
+
31
+ def post_from_instagrapi(raw: Any, *, include_raw: bool) -> Post:
32
+ caption = _get(raw, "caption_text")
33
+ if caption is None:
34
+ cap_obj = _get(raw, "caption")
35
+ if cap_obj is not None:
36
+ caption = _get(cap_obj, "text")
37
+ taken = _get(raw, "taken_at")
38
+ created: datetime | None = None
39
+ if taken is not None:
40
+ created = datetime.fromtimestamp(int(taken), tz=UTC)
41
+ code = _get(raw, "code")
42
+ url = f"https://www.instagram.com/p/{code}/" if code else None
43
+ media: list[MediaItem] = []
44
+ video_url = _get(raw, "video_url")
45
+ thumb = _get(raw, "thumbnail_url")
46
+ if video_url:
47
+ media.append(MediaItem(type="video", url=str(video_url)))
48
+ elif thumb:
49
+ media.append(MediaItem(type="image", url=str(thumb)))
50
+ user_field = _get(raw, "user")
51
+ if isinstance(user_field, dict):
52
+ author_id = str(user_field.get("pk", ""))
53
+ else:
54
+ author_id = str(_get(raw, "user_id", ""))
55
+ return Post(
56
+ id=str(_get(raw, "pk") or _get(raw, "id")),
57
+ author_id=author_id,
58
+ text=caption,
59
+ created_at=created,
60
+ url=url,
61
+ media=media,
62
+ like_count=_get(raw, "like_count"),
63
+ comment_count=_get(raw, "comment_count"),
64
+ platform="instagram",
65
+ raw=raw if include_raw else None,
66
+ )
67
+
68
+
69
+ def comment_from_instagrapi(raw: Any, *, post_id: str, include_raw: bool) -> Comment:
70
+ del include_raw
71
+ created: datetime | None = None
72
+ ts = _get(raw, "created_at_utc") or _get(raw, "created_at")
73
+ if ts is not None:
74
+ created = datetime.fromtimestamp(int(ts), tz=UTC)
75
+ return Comment(
76
+ id=str(_get(raw, "pk") or _get(raw, "id")),
77
+ post_id=post_id,
78
+ author_id=str(_get(raw, "user_id")) if _get(raw, "user_id") is not None else None,
79
+ text=str(_get(raw, "text", "")),
80
+ created_at=created,
81
+ platform="instagram",
82
+ )
@@ -0,0 +1,82 @@
1
+ from __future__ import annotations
2
+
3
+ from datetime import UTC, datetime
4
+ from typing import Any
5
+
6
+ from rrss.models import CursorPage, MediaItem, Post, User
7
+
8
+
9
+ def user_from_sidecar(username: str, data: dict[str, Any], *, include_raw: bool) -> User:
10
+ return User(
11
+ id=str(data.get("user_id", data.get("id", username))),
12
+ username=username,
13
+ display_name=data.get("full_name"),
14
+ bio=data.get("biography"),
15
+ profile_url=(
16
+ (data.get("bio_links") or [{}])[0].get("url") if data.get("bio_links") else None
17
+ ),
18
+ avatar_url=data.get("profile_pic_url"),
19
+ follower_count=data.get("follower_count"),
20
+ following_count=data.get("following_count"),
21
+ is_verified=data.get("is_verified"),
22
+ platform="threads",
23
+ raw=data if include_raw else None,
24
+ )
25
+
26
+
27
+ def _post_from_thread_item(item: dict[str, Any], author_id: str, *, include_raw: bool) -> Post:
28
+ post = item.get("post") or item
29
+ pid = str(post.get("pk") or post.get("code") or post.get("id", ""))
30
+ caption = post.get("caption") or {}
31
+ text = caption.get("text") if isinstance(caption, dict) else caption
32
+ media: list[MediaItem] = []
33
+ for key in ("carousel_media", "image_versions2"):
34
+ block = post.get(key)
35
+ if isinstance(block, list):
36
+ for entry in block:
37
+ if not isinstance(entry, dict):
38
+ continue
39
+ url = entry.get("url") or entry.get("download_url")
40
+ if url:
41
+ media.append(MediaItem(type="image", url=str(url)))
42
+ elif isinstance(block, dict):
43
+ candidates = block.get("candidates") or []
44
+ if candidates:
45
+ media.append(MediaItem(type="image", url=str(candidates[0].get("url", ""))))
46
+ taken = post.get("taken_at")
47
+ created = datetime.fromtimestamp(int(taken), tz=UTC) if taken else None
48
+ return Post(
49
+ id=pid,
50
+ author_id=author_id,
51
+ text=str(text) if text is not None else None,
52
+ created_at=created,
53
+ url=None,
54
+ media=media,
55
+ like_count=post.get("like_count"),
56
+ comment_count=post.get("reply_count"),
57
+ platform="threads",
58
+ raw=post if include_raw else None,
59
+ )
60
+
61
+
62
+ def threads_page_from_sidecar(
63
+ username: str,
64
+ data: dict[str, Any] | list[Any],
65
+ *,
66
+ include_raw: bool,
67
+ ) -> CursorPage[Post]:
68
+ author_id = str(data.get("user_id", username)) if isinstance(data, dict) else username
69
+ items_raw: list[Any]
70
+ if isinstance(data, list):
71
+ items_raw = data
72
+ else:
73
+ items_raw = data.get("threads") or data.get("items") or []
74
+ posts: list[Post] = []
75
+ for thread in items_raw:
76
+ thread_items = thread.get("thread_items") if isinstance(thread, dict) else None
77
+ if thread_items:
78
+ for item in thread_items:
79
+ posts.append(_post_from_thread_item(item, author_id, include_raw=include_raw))
80
+ elif isinstance(thread, dict):
81
+ posts.append(_post_from_thread_item(thread, author_id, include_raw=include_raw))
82
+ return CursorPage(items=posts, has_more=False, next_cursor=None)
@@ -0,0 +1,51 @@
1
+ from __future__ import annotations
2
+
3
+ from datetime import UTC, datetime
4
+ from typing import Any
5
+
6
+ from rrss.models import MediaItem, Post, User
7
+
8
+
9
+ def user_from_tiktok(raw: dict[str, Any], *, include_raw: bool) -> User:
10
+ stats = raw.get("stats") or {}
11
+ return User(
12
+ id=str(raw.get("id") or raw.get("uid") or raw.get("user_id", "")),
13
+ username=raw.get("uniqueId") or raw.get("username"),
14
+ display_name=raw.get("nickname"),
15
+ bio=raw.get("signature"),
16
+ profile_url=None,
17
+ avatar_url=raw.get("avatarLarger") or raw.get("avatarThumb"),
18
+ follower_count=stats.get("followerCount") or raw.get("follower_count"),
19
+ following_count=stats.get("followingCount") or raw.get("following_count"),
20
+ is_verified=raw.get("verified"),
21
+ platform="tiktok",
22
+ raw=raw if include_raw else None,
23
+ )
24
+
25
+
26
+ def post_from_tiktok(raw: dict[str, Any], *, include_raw: bool) -> Post:
27
+ create_time = raw.get("createTime") or raw.get("create_time")
28
+ created = datetime.fromtimestamp(int(create_time), tz=UTC) if create_time else None
29
+ author = raw.get("author") or {}
30
+ author_id = str(author.get("id") or raw.get("author_id", ""))
31
+ video = raw.get("video") or {}
32
+ cover = video.get("cover") or video.get("originCover")
33
+ media: list[MediaItem] = []
34
+ if cover:
35
+ media.append(MediaItem(type="image", url=str(cover)))
36
+ play = video.get("playAddr") or video.get("downloadAddr")
37
+ if play:
38
+ media.append(MediaItem(type="video", url=str(play)))
39
+ stats = raw.get("stats") or {}
40
+ return Post(
41
+ id=str(raw.get("id") or raw.get("aweme_id", "")),
42
+ author_id=author_id,
43
+ text=raw.get("desc"),
44
+ created_at=created,
45
+ url=None,
46
+ media=media,
47
+ like_count=stats.get("diggCount") or raw.get("digg_count"),
48
+ comment_count=stats.get("commentCount") or raw.get("comment_count"),
49
+ platform="tiktok",
50
+ raw=raw if include_raw else None,
51
+ )
rrss/normalize/x.py ADDED
@@ -0,0 +1,192 @@
1
+ from __future__ import annotations
2
+
3
+ from datetime import UTC, datetime
4
+ from typing import Any, Literal
5
+
6
+ from rrss.errors import NotFound
7
+ from rrss.models import Comment, CursorPage, MediaItem, Post, User
8
+
9
+
10
+ def _get(obj: Any, key: str, default: Any = None) -> Any:
11
+ if obj is None:
12
+ return default
13
+ if isinstance(obj, dict):
14
+ return obj.get(key, default)
15
+ return getattr(obj, key, default)
16
+
17
+
18
+ def _is_response(raw: Any) -> bool:
19
+ if isinstance(raw, dict):
20
+ return "data" in raw and any(key in raw for key in ("includes", "meta", "errors"))
21
+ return hasattr(raw, "data") and any(hasattr(raw, key) for key in ("includes", "meta", "errors"))
22
+
23
+
24
+ def _parts(raw: Any) -> tuple[Any, Any, Any]:
25
+ if raw is None:
26
+ return None, {}, {}
27
+ if _is_response(raw):
28
+ return _get(raw, "data"), _get(raw, "includes") or {}, _get(raw, "meta") or {}
29
+ return raw, {}, {}
30
+
31
+
32
+ def _metric(metrics: Any, key: str) -> int | None:
33
+ if metrics is None:
34
+ return None
35
+ value = metrics.get(key) if isinstance(metrics, dict) else getattr(metrics, key, None)
36
+ if value is None:
37
+ return None
38
+ return int(value)
39
+
40
+
41
+ def _parse_time(value: Any) -> datetime | None:
42
+ if value is None:
43
+ return None
44
+ if isinstance(value, datetime):
45
+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
46
+ text = str(value).replace("Z", "+00:00")
47
+ return datetime.fromisoformat(text)
48
+
49
+
50
+ def _raw_dict(obj: Any) -> dict[str, object]:
51
+ if isinstance(obj, dict):
52
+ return obj
53
+ data = getattr(obj, "data", None)
54
+ if isinstance(data, dict):
55
+ return data
56
+ return {}
57
+
58
+
59
+ def _entity(raw: Any, *, what: str) -> Any:
60
+ data, _includes, _meta = _parts(raw)
61
+ if isinstance(data, list):
62
+ data = data[0] if data else None
63
+ if data is None:
64
+ raise NotFound(f"X {what} payload was empty")
65
+ return data
66
+
67
+
68
+ def user_from_x(raw: Any, *, include_raw: bool) -> User:
69
+ data = _entity(raw, what="user")
70
+ metrics = _get(data, "public_metrics") or {}
71
+ username = _get(data, "username")
72
+ return User(
73
+ id=str(_get(data, "id")),
74
+ username=username,
75
+ display_name=_get(data, "name"),
76
+ bio=_get(data, "description"),
77
+ profile_url=f"https://x.com/{username}" if username else None,
78
+ avatar_url=_get(data, "profile_image_url"),
79
+ follower_count=_metric(metrics, "followers_count"),
80
+ following_count=_metric(metrics, "following_count"),
81
+ is_verified=_get(data, "verified"),
82
+ platform="x",
83
+ raw=_raw_dict(data) if include_raw else None,
84
+ )
85
+
86
+
87
+ def _author_username(includes: Any, author_id: str) -> str | None:
88
+ users = includes.get("users") if isinstance(includes, dict) else _get(includes, "users")
89
+ for user in users or []:
90
+ if str(_get(user, "id")) == author_id and _get(user, "username"):
91
+ return str(_get(user, "username"))
92
+ return None
93
+
94
+
95
+ def _media(data: Any, includes: Any) -> list[MediaItem]:
96
+ attachments = _get(data, "attachments") or {}
97
+ keys = attachments.get("media_keys") if isinstance(attachments, dict) else []
98
+ library = includes.get("media") if isinstance(includes, dict) else _get(includes, "media")
99
+ by_key: dict[str, Any] = {}
100
+ for item in library or []:
101
+ by_key[str(_get(item, "media_key"))] = item
102
+ media: list[MediaItem] = []
103
+ for key in keys or []:
104
+ item = by_key.get(str(key))
105
+ if item is None:
106
+ continue
107
+ url = _get(item, "url") or _get(item, "preview_image_url")
108
+ if not url:
109
+ continue
110
+ kind = str(_get(item, "type") or "photo")
111
+ media_type: Literal["image", "video"] = (
112
+ "video" if kind in {"video", "animated_gif"} else "image"
113
+ )
114
+ media.append(MediaItem(type=media_type, url=str(url)))
115
+ return media
116
+
117
+
118
+ def post_from_x(raw: Any, *, include_raw: bool, includes: Any | None = None) -> Post:
119
+ data, response_includes, _meta = _parts(raw)
120
+ if isinstance(data, list):
121
+ data = data[0] if data else None
122
+ if data is None:
123
+ raise NotFound("X post payload was empty")
124
+ merged_includes = includes if includes is not None else response_includes
125
+ author_id = str(_get(data, "author_id") or "")
126
+ tweet_id = str(_get(data, "id"))
127
+ username = _author_username(merged_includes, author_id)
128
+ url = (
129
+ f"https://x.com/{username}/status/{tweet_id}"
130
+ if username
131
+ else f"https://x.com/i/web/status/{tweet_id}"
132
+ )
133
+ metrics = _get(data, "public_metrics") or {}
134
+ return Post(
135
+ id=tweet_id,
136
+ author_id=author_id,
137
+ text=_get(data, "text"),
138
+ created_at=_parse_time(_get(data, "created_at")),
139
+ url=url,
140
+ media=_media(data, merged_includes),
141
+ like_count=_metric(metrics, "like_count"),
142
+ comment_count=_metric(metrics, "reply_count"),
143
+ platform="x",
144
+ raw=_raw_dict(data) if include_raw else None,
145
+ )
146
+
147
+
148
+ def _next_token(meta: Any) -> str | None:
149
+ token = meta.get("next_token") if isinstance(meta, dict) else _get(meta, "next_token")
150
+ if isinstance(token, str) and token:
151
+ return token
152
+ return None
153
+
154
+
155
+ def posts_page_from_x(raw: Any, *, include_raw: bool) -> CursorPage[Post]:
156
+ if raw is None:
157
+ return CursorPage(items=[], has_more=False, next_cursor=None)
158
+ data, includes, meta = _parts(raw)
159
+ rows = data or []
160
+ if not isinstance(rows, list):
161
+ rows = [rows]
162
+ posts = [post_from_x(row, include_raw=include_raw, includes=includes) for row in rows]
163
+ token = _next_token(meta)
164
+ return CursorPage(items=posts, has_more=token is not None, next_cursor=token)
165
+
166
+
167
+ def comments_page_from_x(raw: Any, *, post_id: str, include_raw: bool) -> CursorPage[Comment]:
168
+ del include_raw
169
+ if raw is None:
170
+ return CursorPage(items=[], has_more=False, next_cursor=None)
171
+ data, _includes, meta = _parts(raw)
172
+ rows = data or []
173
+ if not isinstance(rows, list):
174
+ rows = [rows]
175
+ items: list[Comment] = []
176
+ for row in rows:
177
+ comment_id = str(_get(row, "id") or "")
178
+ if not comment_id or comment_id == post_id:
179
+ continue
180
+ author = _get(row, "author_id")
181
+ items.append(
182
+ Comment(
183
+ id=comment_id,
184
+ post_id=str(_get(row, "conversation_id") or post_id),
185
+ author_id=str(author) if author is not None else None,
186
+ text=str(_get(row, "text") or ""),
187
+ created_at=_parse_time(_get(row, "created_at")),
188
+ platform="x",
189
+ )
190
+ )
191
+ token = _next_token(meta)
192
+ return CursorPage(items=items, has_more=token is not None, next_cursor=token)
rrss/ratelimit.py ADDED
@@ -0,0 +1,30 @@
1
+ from __future__ import annotations
2
+
3
+ import random
4
+ import time
5
+ from collections.abc import Callable
6
+ from typing import TypeVar
7
+
8
+ from rrss.errors import RateLimited
9
+
10
+ T = TypeVar("T")
11
+
12
+
13
+ def with_retries(
14
+ fn: Callable[[], T],
15
+ *,
16
+ max_attempts: int = 3,
17
+ base_delay: float = 0.5,
18
+ ) -> T:
19
+ last_exc: Exception | None = None
20
+ for attempt in range(max_attempts):
21
+ try:
22
+ return fn()
23
+ except RateLimited as exc:
24
+ last_exc = exc
25
+ if attempt + 1 >= max_attempts:
26
+ raise
27
+ jitter = random.uniform(0, 0.25)
28
+ time.sleep(base_delay * (2**attempt) + jitter)
29
+ assert last_exc is not None
30
+ raise last_exc