reddrix 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
reddrix/__init__.py ADDED
@@ -0,0 +1,2 @@
1
+ from .reddrix import Reddrix
2
+ __version__ = "0.1.0"
reddrix/cli.py ADDED
@@ -0,0 +1,25 @@
1
+ from camoufox.pkgman import CamoufoxFetcher
2
+ import argparse
3
+
4
+
5
+ def install_binary():
6
+ CamoufoxFetcher().install(replace=True)
7
+
8
+ def main():
9
+ parser = argparse.ArgumentParser(prog="reddrix", description="Reddrix --- all-in-one Reddit scraper")
10
+ subparsers = parser.add_subparsers(dest="command")
11
+ subparsers.add_parser(
12
+ name="install",
13
+ help="Installs the Camoufox binary dependency."
14
+ )
15
+
16
+ args = parser.parse_args()
17
+
18
+ if args.command == "install":
19
+ install_binary()
20
+ else:
21
+ print(f"[reddrix] `{args.command}` is not a valid command.", flush=True)
22
+ parser.print_help()
23
+
24
+ if __name__ == "__main__":
25
+ main()
reddrix/exceptions.py ADDED
@@ -0,0 +1,8 @@
1
+ class FatalAntiBotBlockError(Exception):
2
+ """Raised when the scraper is blocked after exhausting all attempts."""
3
+ pass
4
+
5
+
6
+ class AntiBotBlockError(Exception):
7
+ """Raised when the scraper is blocked once."""
8
+ pass
reddrix/reddrix.py ADDED
@@ -0,0 +1,772 @@
1
+ from playwright.async_api import Browser, BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, Error
2
+ from curl_cffi.requests.exceptions import SessionClosed, HTTPError, JSONDecodeError
3
+ from .exceptions import AntiBotBlockError, FatalAntiBotBlockError
4
+ from playwright._impl._errors import TargetClosedError
5
+ from camoufox.exceptions import UnknownIPLocation
6
+ from camoufox.async_api import AsyncCamoufox
7
+ from urllib.parse import urlencode, urlparse
8
+ from collections.abc import AsyncGenerator
9
+ from datetime import datetime, timezone
10
+ from whiskeysour import WhiskeySour
11
+ from curl_cffi import AsyncSession
12
+ from functools import wraps
13
+ from platform import system
14
+ from typing import Literal
15
+ import asyncio
16
+ import random
17
+ import time
18
+
19
+
20
+ COOKIE_URL = "https://www.reddit.com/r/kasralainy" # Lightweight sub page to reduce network bandwidth if using a proxy.
21
+ LOGIN_BTN_SELECTOR = "#login-button" # Used as a success signal for session cookies.
22
+ MAX_RETRIES = 5
23
+ PLATFORMS = {
24
+ "windows": "windows",
25
+ "linux": "linux",
26
+ "darwin": "macos"
27
+ }
28
+ TIME_FRAMES = {
29
+ "all_time": "all",
30
+ "past_hour": "hour",
31
+ "today": "day",
32
+ "past_week": "week",
33
+ "past_month": "month",
34
+ "past_year": "year"
35
+ }
36
+ COOKIE_LOCK = asyncio.Lock()
37
+
38
+ def auto_retry(max_attempts: int):
39
+ def decorator(func):
40
+ @wraps(func)
41
+ async def wrapper(*args, **kwargs):
42
+ url = kwargs.get("url") or args[1] if func.__name__ == "fetch_page_data" else None
43
+ scraper: Reddrix = args[0]
44
+
45
+ for attempt in range(max_attempts + 1):
46
+ current_time = time.time()
47
+ try:
48
+ result = await func(*args, **kwargs)
49
+ except AntiBotBlockError:
50
+ if attempt == max_attempts:
51
+ if url:
52
+ raise FatalAntiBotBlockError(f"Error fetching URL: {url}\n\nError Message: Cookies expired, and attempts have been exhausted.")
53
+ else:
54
+ raise FatalAntiBotBlockError(f"Error executing function: {func.__name__}\n\nError Message: Cookies expired, and attempts have been exhausted.")
55
+
56
+ async with COOKIE_LOCK:
57
+ if current_time - scraper.cookies_last_renewed_at >= 8:
58
+ await scraper.refresh_cookies()
59
+
60
+ await asyncio.sleep(min(2 ** attempt + random.uniform(0, 5), 60))
61
+ else:
62
+ return result
63
+
64
+ return wrapper
65
+
66
+ return decorator
67
+
68
+ class Reddrix:
69
+ def __init__(
70
+ self,
71
+ mode: Literal["standard", "keyword_search", "domain_search"]="standard",
72
+ http_client: Literal["curl_cffi", "camoufox"]="curl_cffi",
73
+ max_results: int=999999,
74
+ max_posts_per_collection: int=999999,
75
+ sort: Literal["default", "new", "hot"]="default",
76
+ time_frame: Literal["all_time", "past_hour", "today", "past_week", "past_month", "past_year"]="all_time",
77
+ subreddit_urls: list[str]=[],
78
+ post_urls: list[str]=[],
79
+ profile_urls: list[str]=[],
80
+ domains: list[str]=[],
81
+ query: str=None,
82
+ include_comments: bool=False,
83
+ max_comments_depth: int=10,
84
+ max_comments_per_post: int=999999,
85
+ max_top_comments_per_post: int=999999,
86
+ proxies: list[str]=[],
87
+ verbose: bool=False
88
+ ):
89
+ self.mode = mode
90
+ self.max_results = max_results
91
+ self.max_posts_per_collection = max_posts_per_collection
92
+ self.sort = sort
93
+ self.time_frame = TIME_FRAMES[time_frame]
94
+ self.subreddit_urls = subreddit_urls
95
+ self.post_urls = post_urls
96
+ self.profile_urls = profile_urls
97
+ self.domains = domains
98
+ self.query = query
99
+ self.include_comments = include_comments
100
+ self.max_comments_depth = max_comments_depth
101
+ self.max_comments_per_post = max_comments_per_post
102
+ self.max_top_comments_per_post = max_top_comments_per_post
103
+ self.proxies = proxies
104
+ self.is_verbose = verbose
105
+ self.used_proxies = set()
106
+
107
+ self.http_client = http_client
108
+ self.browser: Browser | None = None
109
+ self.context: BrowserContext | None = None
110
+ self.page: Page | None = None
111
+ self.session: AsyncSession | None = None
112
+ self.started = False
113
+ self.cookie_renewal_in_progress = False
114
+ self.cookies_last_accessed_at = 0.0
115
+ self.request_semaphore = asyncio.Semaphore(10)
116
+ self.items_queue = asyncio.Queue(maxsize=1000)
117
+
118
+ async def start(self):
119
+ return await self.__aenter__()
120
+
121
+ async def stop(self):
122
+ await self.__aexit__(None, None, None)
123
+
124
+ def new_proxy_url(self) -> str | None:
125
+ if not self.proxies:
126
+ return None
127
+
128
+ proxy_to_be_used = None
129
+ for proxy in self.proxies:
130
+ if proxy not in self.used_proxies:
131
+ self.used_proxies.add(proxy)
132
+ proxy_to_be_used = proxy
133
+ break
134
+
135
+ if not proxy_to_be_used:
136
+ self.used_proxies.clear()
137
+ proxy_to_be_used = random.choice(self.proxies)
138
+ self.used_proxies.add(proxy_to_be_used)
139
+
140
+ return proxy_to_be_used
141
+
142
+ async def __aenter__(self):
143
+ initialization_successful = False
144
+ self.log("INITIALIZING SCRAPER")
145
+ for attempt in range(1, MAX_RETRIES + 1):
146
+ try:
147
+ if self.http_client == "camoufox":
148
+ proxy_url = self.new_proxy_url()
149
+ parsed_url = urlparse(proxy_url)
150
+ self.camoufox = AsyncCamoufox(
151
+ os=PLATFORMS[system().lower()],
152
+ headless=True,
153
+ fingerprint_preset=True,
154
+ proxy={
155
+ "server": f"{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port}",
156
+ "username": parsed_url.username,
157
+ "password": parsed_url.password
158
+ } if proxy_url else None,
159
+ geoip=True if proxy_url else False
160
+ )
161
+ self.browser: Browser = await self.camoufox.start()
162
+ self.context = await self.browser.new_context()
163
+ self.page = await self.context.new_page()
164
+ await self.page.goto(COOKIE_URL)
165
+ await self.page.wait_for_selector(LOGIN_BTN_SELECTOR, timeout=15000)
166
+ else:
167
+ self.session = AsyncSession(proxy=self.new_proxy_url(), allow_redirects=True)
168
+ self.session.headers["User-Agent"] = f"Reddrix{random.randint(0, 999999)}/v0.1.0"
169
+ res = await self.session.get("https://old.reddit.com/login")
170
+ res.raise_for_status()
171
+
172
+ self.started = True
173
+ except (ValueError, PlaywrightTimeoutError, TimeoutError, Error, HTTPError, UnknownIPLocation):
174
+ self.log(f"INITIALIZATION ATTEMPT {attempt}/{MAX_RETRIES} FAILED. RETRYING...", level="error")
175
+ if self.http_client == "camoufox":
176
+ try:
177
+ await self.context.close()
178
+ await self.browser.__aexit__(None, None, None)
179
+ await self.session.__aexit__(None, None, None)
180
+ except AttributeError:
181
+ pass
182
+ else:
183
+ await self.session.close()
184
+
185
+ self.started = False
186
+ else:
187
+ initialization_successful = True
188
+ self.started = True
189
+ self.cookies_last_accessed_at = time.time()
190
+ break
191
+
192
+ if not initialization_successful:
193
+ raise FatalAntiBotBlockError(f"FAILED TO CONNECT TO REDDIT AFTER {MAX_RETRIES} ATTEMPTS.")
194
+
195
+ return self
196
+
197
+ async def __aexit__(self, exc_type, exc, tb):
198
+ if self.http_client == "camoufox":
199
+ try:
200
+ await self.context.close()
201
+ await self.browser.__aexit__(None, None, None)
202
+ await self.camoufox.__aexit__(None, None, None)
203
+ except AttributeError:
204
+ pass
205
+ else:
206
+ await self.session.close()
207
+
208
+ self.started = False
209
+
210
+ async def refresh_cookies(self):
211
+ if self.cookie_renewal_in_progress:
212
+ return
213
+
214
+ self.cookie_renewal_in_progress = True
215
+ cookie_refresh_successful = False
216
+ self.log("REFRESHING COOKIES...")
217
+
218
+ if self.http_client == "camoufox":
219
+ try:
220
+ await self.context.close()
221
+ await self.browser.__aexit__(None, None, None)
222
+ await self.session.__aexit__(None, None, None)
223
+ except AttributeError:
224
+ pass
225
+ else:
226
+ await self.session.close()
227
+
228
+
229
+ for attempt in range(1, MAX_RETRIES + 1):
230
+ try:
231
+ if self.http_client == "camoufox":
232
+ proxy_url = self.new_proxy_url()
233
+ parsed_url = urlparse(proxy_url)
234
+ self.camoufox = AsyncCamoufox(
235
+ os=PLATFORMS[system().lower()],
236
+ headless=True,
237
+ fingerprint_preset=True,
238
+ proxy={
239
+ "server": f"{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port}",
240
+ "username": parsed_url.username,
241
+ "password": parsed_url.password
242
+ } if proxy_url else None,
243
+ geoip=True if proxy_url else False
244
+ )
245
+ self.browser: Browser = await self.camoufox.start()
246
+ self.context = await self.browser.new_context()
247
+ self.page = await self.context.new_page()
248
+ await self.page.goto(COOKIE_URL)
249
+ await self.page.wait_for_selector(LOGIN_BTN_SELECTOR, timeout=15000)
250
+ else:
251
+ self.session = AsyncSession(proxy=self.new_proxy_url(), allow_redirects=True)
252
+ self.session.headers["User-Agent"] = f"Reddrix{random.randint(0, 999999)}/v0.1.0"
253
+ res = await self.session.get("https://old.reddit.com/login")
254
+ res.raise_for_status()
255
+ except (ValueError, PlaywrightTimeoutError, TimeoutError, Error, HTTPError, UnknownIPLocation):
256
+ self.log(f"COOKIE REFRESH ATTEMPT {attempt}/{MAX_RETRIES} FAILED. RETRYING...", level="error")
257
+
258
+ if self.http_client == "camoufox":
259
+ try:
260
+ await self.context.close()
261
+ await self.browser.__aexit__(None, None, None)
262
+ await self.session.__aexit__(None, None, None)
263
+ except AttributeError:
264
+ pass
265
+ else:
266
+ await self.session.close()
267
+
268
+ self.cookies_last_accessed_at = time.time()
269
+ else:
270
+ cookie_refresh_successful = True
271
+ self.cookies_last_accessed_at = time.time()
272
+ break
273
+
274
+ if not cookie_refresh_successful:
275
+ raise FatalAntiBotBlockError(f"FAILED TO REFRESH COOKIES AFTER {MAX_RETRIES} ATTEMPTS.")
276
+
277
+ self.cookie_renewal_in_progress = False
278
+
279
+ def log(self, msg: str=None, level: Literal["info", "warning", "error"]="info"):
280
+ if msg and self.is_verbose:
281
+ now = datetime.fromtimestamp(time.time(), tz=timezone.utc).isoformat()
282
+ print(f"{now} [reddrix] [{level.upper()}] {msg}", flush=True)
283
+
284
+ def parse_item(self, raw: dict) -> dict:
285
+ """Parses each raw post or comment."""
286
+
287
+ data: dict = raw["data"]
288
+ if raw["kind"] == "t3": # Posts
289
+ submission_id = data["id"]
290
+ submission_id_prefixed = f"t3_{submission_id}"
291
+ submission_type = "post"
292
+ submission_url = "https://www.reddit.com" + data["permalink"]
293
+ submitted_at = datetime.fromtimestamp(data["created_utc"], tz=timezone.utc).isoformat()
294
+ parent_id = None
295
+ parent_post_id = None
296
+ parent_post_url = None
297
+ post_type = None
298
+ is_crosspost = "crosspost_parent" in data
299
+ crosspost_parent_id = data.get("crosspost_parent")
300
+
301
+ if is_crosspost:
302
+ crosspost_parent_url = "https://www.reddit.com" + data.get("url_overridden_by_dest")
303
+ else:
304
+ crosspost_parent_url = None
305
+
306
+ if (data.get("is_self", False) or False) and "poll_data" in data:
307
+ post_type = "poll"
308
+ elif (data.get("is_self", False) or False) and "poll_data" not in data:
309
+ post_type = "text"
310
+ elif data.get("post_hint", "") == "link":
311
+ post_type = "link"
312
+ elif data["is_video"] or data.get("domain") == "v.redd.it" or ((data.get("media", {}) or {}).get("oembed", {}) or {}).get("type") == "video":
313
+ post_type = "video"
314
+ elif data.get("post_hint", "") == "image" or data.get("domain") == "i.redd.it":
315
+ post_type = "image"
316
+ elif data.get("is_gallery", False):
317
+ post_type = "gallery"
318
+ elif "self." not in (data.get("domain", "") or ""):
319
+ post_type = "link"
320
+
321
+ title = data["title"]
322
+ body_text = data["selftext"]
323
+ is_edited = False if not data["edited"] else True
324
+ author = data["author"]
325
+ author_flair_text = data.get("author_flair_text")
326
+ author_flair_image_url = (data.get("author_flair_richtext", [{}]) or [{}])[0].get("u")
327
+ depth = None
328
+ post_flair = data.get("link_flair_text") or None
329
+ is_nsfw = data.get("over_18", False) or False
330
+ subreddit = data.get("subreddit_name_prefixed") or None
331
+ score = data["score"]
332
+ upvote_ratio = data.get("upvote_ratio") or None
333
+ num_comments = data["num_comments"]
334
+ num_crossposts = data.get("num_crossposts", 0) or 0
335
+
336
+ if is_crosspost:
337
+ external_url = "https://www.reddit.com" + data.get("url_overridden_by_dest")
338
+ else:
339
+ external_url = data.get("url_overridden_by_dest")
340
+
341
+ if "preview.redd.it" in (data.get("thumbnail", "") or ""):
342
+ if not ((data.get("secure_media", {}) or {}).get("oembed", {}) or {}).get("thumbnail_url"):
343
+ thumbnail_url = f'https://i.redd.it/{data.get("thumbnail").split(".it/")[-1].split("?")[0]}'
344
+ else:
345
+ thumbnail_url = ((data.get("secure_media", {}) or {}).get("oembed", {}) or {}).get("thumbnail_url")
346
+ else:
347
+ thumbnail_url = None
348
+
349
+ if post_type == "video":
350
+ video_url = (((data.get("media", {}) or {}).get("reddit_video", {}) or {}).get("fallback_url", "") or "").split("?")[0]
351
+ if not video_url:
352
+ video_url = external_url
353
+ else:
354
+ video_url = None
355
+
356
+ if post_type == "image":
357
+ image_url = thumbnail_url if isinstance(thumbnail_url, str) and "preview.reddit" in thumbnail_url else external_url
358
+ else:
359
+ image_url = None
360
+
361
+ if post_type == "gallery":
362
+ img_extension = data["thumbnail"].split("?")[0].split(".")[-1]
363
+ gallery_images_urls = [f'https://i.redd.it/{id_}.{img_extension}' for id_ in [item["media_id"] for item in data["gallery_data"]["items"]]]
364
+
365
+ else:
366
+ gallery_images_urls = []
367
+
368
+ return {
369
+ "submissionId": submission_id,
370
+ "submissionIdPrefixed": submission_id_prefixed,
371
+ "submissionType": submission_type,
372
+ "submissionUrl": submission_url,
373
+ "submittedAt": submitted_at,
374
+ "parentId": parent_id,
375
+ "parentPostId": parent_post_id,
376
+ "parentPostUrl": parent_post_url,
377
+ "postType": post_type,
378
+ "isCrosspost": is_crosspost,
379
+ "crosspostParentId": crosspost_parent_id,
380
+ "crosspostParentUrl": crosspost_parent_url,
381
+ "title": title,
382
+ "bodyText": body_text,
383
+ "isEdited": is_edited,
384
+ "author": author,
385
+ "authorFlairText": author_flair_text,
386
+ "authorFlairImageUrl": author_flair_image_url,
387
+ "depth": depth,
388
+ "postFlair": post_flair,
389
+ "isNSFW": is_nsfw,
390
+ "subreddit": subreddit,
391
+ "score": score,
392
+ "upvoteRatio": upvote_ratio,
393
+ "numComments": num_comments,
394
+ "numCrossposts": num_crossposts,
395
+ "externalUrl": external_url,
396
+ "thumbnailUrl": thumbnail_url,
397
+ "videoUrl": video_url,
398
+ "imageUrl": image_url,
399
+ "galleryImagesUrls": gallery_images_urls
400
+ }
401
+
402
+ else: # Comments
403
+ submission_id = data["id"]
404
+ submission_id_prefixed = f"t1_{submission_id}"
405
+ submission_type = "comment"
406
+ submission_url: str = "https://www.reddit.com" + data["permalink"]
407
+ submitted_at = datetime.fromtimestamp(data["created_utc"], tz=timezone.utc).isoformat()
408
+ parent_id = data["parent_id"]
409
+ parent_post_id = data["link_id"]
410
+ parent_post_url = submission_url.split("/comments/")[0] + "/comments/" + submission_url.split("/comments/")[-1].split("/")[0]
411
+ post_type = None
412
+ is_crosspost = False
413
+ crosspost_parent_id = None
414
+ crosspost_parent_url = None
415
+ title = None
416
+ body_text = data["body"]
417
+ is_edited = False if not data["edited"] else True
418
+ author = data["author"]
419
+ author_flair_text = data.get("author_flair_text")
420
+ author_flair_image_url = (data.get("author_flair_richtext", [{}]) or [{}])[0].get("u")
421
+ depth = data["depth"]
422
+ post_flair = None
423
+ is_nsfw = False
424
+ subreddit = data.get("subreddit_name_prefixed")
425
+ score = data["score"]
426
+ upvote_ratio = None
427
+ num_comments = None
428
+ num_crossposts = None
429
+ external_url = None
430
+ thumbnail_url = None
431
+ image_url = None
432
+ video_url = None
433
+ gallery_images_urls = []
434
+
435
+ return {
436
+ "submissionId": submission_id,
437
+ "submissionIdPrefixed": submission_id_prefixed,
438
+ "submissionType": submission_type,
439
+ "submissionUrl": submission_url,
440
+ "submittedAt": submitted_at,
441
+ "parentId": parent_id,
442
+ "parentPostId": parent_post_id,
443
+ "parentPostUrl": parent_post_url,
444
+ "postType": post_type,
445
+ "isCrosspost": is_crosspost,
446
+ "crosspostParentId": crosspost_parent_id,
447
+ "crosspostParentUrl": crosspost_parent_url,
448
+ "title": title,
449
+ "bodyText": body_text,
450
+ "isEdited": is_edited,
451
+ "author": author,
452
+ "authorFlairText": author_flair_text,
453
+ "authorFlairImageUrl": author_flair_image_url,
454
+ "depth": depth,
455
+ "postFlair": post_flair,
456
+ "isNSFW": is_nsfw,
457
+ "subreddit": subreddit,
458
+ "score": score,
459
+ "upvoteRatio": upvote_ratio,
460
+ "numComments": num_comments,
461
+ "numCrossposts": num_crossposts,
462
+ "externalUrl": external_url,
463
+ "thumbnailUrl": thumbnail_url,
464
+ "videoUrl": video_url,
465
+ "imageUrl": image_url,
466
+ "galleryImagesUrls": gallery_images_urls
467
+ }
468
+
469
+ @auto_retry(max_attempts=3)
470
+ async def get(self, url: str, params: dict={}, expected_data_type: Literal["JSON", "HTML/XML"]="JSON") -> dict | list | str:
471
+ if params:
472
+ url += f"{'?' if '?' not in url else ''}{urlencode(params) if '?' not in url else f'&{urlencode(params)}'}"
473
+
474
+ while self.cookie_renewal_in_progress:
475
+ await asyncio.sleep(0.07)
476
+
477
+ if self.http_client == "camoufox":
478
+ try:
479
+ if expected_data_type == "JSON":
480
+ data: list | dict = await self.page.evaluate(r"async (url) => {const res = await fetch(url); return await res.json();}", url)
481
+ else:
482
+ data: str = await self.page.evaluate(r"async (url) => {const res = await fetch(url); return await res.text();}", url)
483
+
484
+ if isinstance(data, dict) and data.get("error", 200) in [403, 404] and ("/user/" in url or "/u/" in url) and "/comments/" not in url:
485
+ return {} if expected_data_type == "JSON" else ""
486
+ elif isinstance(data, dict) and data.get("error", 200) >= 400:
487
+ raise AntiBotBlockError
488
+ except Error:
489
+ raise AntiBotBlockError
490
+ except AttributeError:
491
+ data = {} if expected_data_type == "JSON" else ""
492
+ pass
493
+ else:
494
+ try:
495
+ res = await self.session.get(url)
496
+ res.raise_for_status()
497
+
498
+ if expected_data_type == "JSON":
499
+ data: list | dict = res.json()
500
+ else:
501
+ data: str = res.text
502
+ except HTTPError:
503
+ if "/user/" in url or "/u/" in url:
504
+ return {} if expected_data_type == "JSON" else ""
505
+ else:
506
+ raise AntiBotBlockError
507
+ except JSONDecodeError:
508
+ raise AntiBotBlockError
509
+
510
+ return data
511
+
512
+ @auto_retry(max_attempts=3)
513
+ async def fetch_children_batch(self, post_id: str, id_batch: list) -> tuple[list[dict]]:
514
+ api_url = f"https://www.reddit.com/api/morechildren.json?api_type=json&link_id={post_id}&children={','.join(id_batch)}"
515
+ async with self.request_semaphore:
516
+ await asyncio.sleep(random.uniform(0.5, 4)) # Random jitter
517
+ page_data = await self.get(api_url)
518
+ if page_data["json"]["errors"]:
519
+ raise AntiBotBlockError
520
+
521
+ return [item for item in page_data["json"]["data"]["things"] if item["kind"] == "t1" and item["data"]["id"] != "_" and item["data"]["depth"] <= self.max_comments_depth], [item for item in page_data["json"]["data"]["things"] if item["kind"] == "more" and item["data"]["id"] != "_" and item["data"]["depth"] <= self.max_comments_depth]
522
+
523
+ @auto_retry(max_attempts=3)
524
+ async def fetch_more_comments(self, post_id: str, more_items: list, fetched_comments_count: int, fetched_top_comments_count: int) -> tuple[dict, int, int]:
525
+ more_ids_batches = {}
526
+ more = []
527
+
528
+ for item in more_items:
529
+ item_id = item["data"]["id"]
530
+ more_ids_batches.update({item_id: [[]]})
531
+
532
+ for child in item["data"]["children"]:
533
+ more_ids_batches[item_id][-1].append(child)
534
+ if len(more_ids_batches[item_id][-1]) % 100 == 0:
535
+ more_ids_batches[item_id].append([])
536
+
537
+ if not more_ids_batches[item_id][-1]:
538
+ more_ids_batches[item_id] = more_ids_batches[item_id][:-1]
539
+
540
+ all_batches = [batch for batches in more_ids_batches.values() for batch in batches]
541
+ batch_tasks = [self.fetch_children_batch(post_id, batch) for batch in all_batches]
542
+
543
+ for future in asyncio.as_completed(batch_tasks):
544
+ comments, new_more = await future
545
+
546
+ if fetched_comments_count >= self.max_comments_per_post:
547
+ break
548
+
549
+ more.extend(new_more)
550
+
551
+ for comment in comments:
552
+ if fetched_comments_count >= self.max_comments_per_post:
553
+ break
554
+
555
+ if comment["kind"] == "t1" and comment["data"]["depth"] <= self.max_comments_depth:
556
+ if comment["data"]["depth"] == 0:
557
+ if fetched_comments_count < self.max_comments_per_post and fetched_top_comments_count < self.max_top_comments_per_post:
558
+ await self.items_queue.put(comment)
559
+ fetched_comments_count += 1
560
+ fetched_top_comments_count += 1
561
+ else:
562
+ if fetched_comments_count < self.max_comments_per_post:
563
+ await self.items_queue.put(comment)
564
+ fetched_comments_count += 1
565
+
566
+ return more, fetched_comments_count, fetched_top_comments_count
567
+
568
+ async def fetch_post(self, url: str) -> None:
569
+ fetched_comments_count = 0
570
+ fetched_top_comments_count = 0
571
+ more = []
572
+
573
+ async with self.request_semaphore:
574
+ await asyncio.sleep(random.uniform(0.5, 4)) # Random jitter
575
+ page_data = await self.get(url)
576
+
577
+ post = page_data[0]["data"]["children"][0]
578
+ comments: list = page_data[1]["data"]["children"] or []
579
+
580
+ await self.items_queue.put(post)
581
+
582
+ if not self.include_comments:
583
+ return
584
+
585
+ for comment in comments:
586
+ if fetched_comments_count >= self.max_comments_per_post:
587
+ break
588
+
589
+ if comment["kind"] == "t1" and comment["data"]["depth"] <= self.max_comments_depth:
590
+ if comment["data"]["depth"] == 0:
591
+ if fetched_comments_count < self.max_comments_per_post and fetched_top_comments_count < self.max_top_comments_per_post:
592
+ await self.items_queue.put(comment)
593
+ fetched_comments_count += 1
594
+ fetched_top_comments_count += 1
595
+ else:
596
+ if fetched_comments_count < self.max_comments_per_post:
597
+ await self.items_queue.put(comment)
598
+ fetched_comments_count += 1
599
+
600
+ elif comment["kind"] == "more" and comment["data"]["id"] != "_":
601
+ more.append(comment)
602
+
603
+ while more and fetched_comments_count < self.max_comments_per_post:
604
+ more, fetched_comments_count, fetched_top_comments_count = await self.fetch_more_comments(post_id=post["data"]["name"], more_items=more, fetched_comments_count=fetched_comments_count, fetched_top_comments_count=fetched_top_comments_count)
605
+
606
+ async def fetch_collection(self, url: str) -> None:
607
+ after = ""
608
+ next_page_url = ""
609
+ fetched_posts_count = 0
610
+ post_tasks = []
611
+ if self.mode != "keyword_search":
612
+ while True:
613
+ async with self.request_semaphore:
614
+ await asyncio.sleep(random.uniform(0.5, 4)) # Random jitter
615
+ page_data = await self.get(url, params={"after": after})
616
+
617
+ if not page_data:
618
+ break
619
+
620
+ after = page_data["data"]["after"] or ""
621
+ posts = page_data["data"]["children"]
622
+
623
+ for post in posts:
624
+ if fetched_posts_count >= self.max_posts_per_collection:
625
+ break
626
+
627
+ if post["kind"] == "t3" and fetched_posts_count < self.max_posts_per_collection:
628
+ post_url = "https://www.reddit.com" + post["data"]["permalink"] + ".json?threaded=false"
629
+
630
+ if self.include_comments and post["data"]["num_comments"] > 0:
631
+ post_tasks.append(asyncio.create_task(self.fetch_post(post_url)))
632
+ else:
633
+ await self.items_queue.put(post)
634
+
635
+ fetched_posts_count += 1
636
+
637
+ if not after or fetched_posts_count >= self.max_posts_per_collection:
638
+ break
639
+
640
+ if post_tasks:
641
+ await asyncio.gather(*post_tasks)
642
+ else:
643
+ while True:
644
+ async with self.request_semaphore:
645
+ await asyncio.sleep(random.uniform(0.5, 4)) # Random jitter
646
+ page_html: str = await self.get(url if not next_page_url else next_page_url, expected_data_type="HTML/XML")
647
+
648
+ if not page_html:
649
+ break
650
+
651
+ soup = WhiskeySour(page_html)
652
+ next_page_el = soup.select_one("faceplate-partial[loading=lazy]")
653
+ next_page_url = "https://www.reddit.com" + next_page_el.get("src") if next_page_el else ""
654
+
655
+ posts = soup.select('a[data-testid="post-title"]')
656
+ for post in posts:
657
+ if fetched_posts_count >= self.max_posts_per_collection:
658
+ break
659
+
660
+ if fetched_posts_count < self.max_posts_per_collection:
661
+ post_url = "https://www.reddit.com" + post.get("href") + ".json?threaded=false"
662
+ post_tasks.append(asyncio.create_task(self.fetch_post(post_url)))
663
+
664
+ fetched_posts_count += 1
665
+
666
+ if not next_page_url or fetched_posts_count >= self.max_posts_per_collection:
667
+ break
668
+
669
+ if post_tasks:
670
+ await asyncio.gather(*post_tasks)
671
+
672
+ async def get_items(self) -> AsyncGenerator[dict, None]:
673
+ tasks = []
674
+
675
+ async def run_all_tasks():
676
+ try:
677
+ await asyncio.gather(*tasks)
678
+ except Exception as e:
679
+ await self.items_queue.put(e)
680
+ else:
681
+ await self.items_queue.put("done")
682
+
683
+ if self.mode == "standard":
684
+ # Subreddits
685
+ for url in self.subreddit_urls:
686
+ initial_url = url
687
+ if "reddit.com" not in initial_url or "/r/" not in initial_url:
688
+ continue
689
+
690
+ params = {"limit": 100, "t": self.time_frame}
691
+
692
+ raw_sub_name = initial_url.split("/r/")[-1].split("/")[0]
693
+ final_url = f"https://www.reddit.com/r/{raw_sub_name}"
694
+
695
+ if self.sort != "default":
696
+ final_url += f"/{self.sort}/"
697
+
698
+ final_url += f".json?{urlencode(params)}"
699
+
700
+ tasks.append(self.fetch_collection(final_url))
701
+
702
+ # Posts
703
+ for url in self.post_urls:
704
+ initial_url = url
705
+ if "reddit.com" not in initial_url or "/comments/" not in initial_url:
706
+ continue
707
+
708
+ final_url = initial_url.split("/comments/")[0] + "/comments/" + initial_url.split("/comments/")[-1].split("/")[0] + ".json?threaded=false"
709
+ tasks.append(self.fetch_post(final_url))
710
+
711
+ # Profiles
712
+ for url in self.profile_urls:
713
+ initial_url = url
714
+ if "reddit.com" not in initial_url or ("/u/" not in initial_url and "/user/" not in initial_url):
715
+ continue
716
+
717
+ params = {
718
+ "limit": 100,
719
+ "sort": self.sort if self.sort != "default" else "",
720
+ "t": self.time_frame
721
+ }
722
+
723
+ if "/u/" in initial_url:
724
+ final_url = initial_url.split("/u/")[0] + "/u/" + initial_url.split("/u/")[-1].split("/")[0] + f"/submitted/.json?{urlencode(params)}"
725
+ elif "/user/" in initial_url:
726
+ final_url = initial_url.split("/user/")[0] + "/user/" + initial_url.split("/user/")[-1].split("/")[0] + f"/submitted/.json?{urlencode(params)}"
727
+
728
+ tasks.append(self.fetch_collection(final_url))
729
+ elif self.mode == "keyword_search":
730
+ final_url = ""
731
+
732
+ if self.query:
733
+ params = {
734
+ "q": self.query,
735
+ "type": "posts",
736
+ "sort": "relevance"
737
+ }
738
+
739
+ final_url = f"https://www.reddit.com/search/?{urlencode(params)}"
740
+ tasks.append(self.fetch_collection(final_url))
741
+ else: # domain_search
742
+ for domain in self.domains:
743
+ final_url = f"https://www.reddit.com/domain/{domain}/"
744
+ if self.sort != "default":
745
+ final_url += f"{self.sort}/"
746
+
747
+ final_url += ".json?limit=100"
748
+ tasks.append(self.fetch_collection(final_url))
749
+
750
+
751
+
752
+ runner = asyncio.create_task(run_all_tasks())
753
+ results_count = 0
754
+
755
+ try:
756
+ while True:
757
+ item = await self.items_queue.get()
758
+
759
+ if isinstance(item, Exception):
760
+ raise item
761
+
762
+ if item != "done":
763
+ results_count += 1
764
+
765
+ if results_count > self.max_results:
766
+ break
767
+ else:
768
+ yield self.parse_item(item)
769
+ else:
770
+ break
771
+ finally:
772
+ runner.cancel()
@@ -0,0 +1,341 @@
1
+ Metadata-Version: 2.4
2
+ Name: reddrix
3
+ Version: 0.1.0
4
+ Summary: Reddrix is an open-source, blazing fast Reddit Scraper for Python (No API Key)
5
+ Description-Content-Type: text/markdown
6
+ License-File: LICENSE.md
7
+ Requires-Dist: camoufox[geoip]
8
+ Requires-Dist: whiskeysour
9
+ Requires-Dist: curl-cffi
10
+ Dynamic: description
11
+ Dynamic: description-content-type
12
+ Dynamic: license-file
13
+ Dynamic: requires-dist
14
+ Dynamic: summary
15
+
16
+ <p align="center">
17
+ <img src="https://raw.githubusercontent.com/Coding-Doctor-Omar/reddrix/main/images/reddrix.png" width="350" alt="reddrix logo">
18
+ </p>
19
+
20
+ <p align="center">
21
+ <a href="https://ko-fi.com/CodingDoctorOmar">
22
+ <img src="https://ko-fi.com/img/githubbutton_sm.svg" alt="Support Me on Ko-fi">
23
+ </a>
24
+ </p>
25
+
26
+ # Reddrix &mdash; An open-source, blazing fast Reddit Scraper for Python (No API Key)
27
+
28
+ <p align="center"><h3>A powerful, async-first Reddit scraper (with built-in proxy support) for posts and comments with detailed metrics. No API key required. Scrape subreddits, profiles, and specific posts, or search Reddit site-wide by keywords or domains. Collect rich, structured data including authors, scores, timestamps, flairs, comment threads, and more (30+ fields).</h3></p>
29
+
30
+ > If this repo is useful to you or saved your time, consider starring ⭐ it to support me and help other find it and have an easier time scraping Reddit ❤️.
31
+
32
+ > Need a hosted version, check [Reddit Scraper Pro](https://apify.com/coding-doctor-omar/Reddit-Scraper-Pro) on Apify.
33
+
34
+ <p align="center">
35
+ <a href="#installation">Installation</a> • <a href="#quick-start">Quick Start</a> • <a href="#output-dataset">Output Dataset</a> •
36
+ <a href="#features">Features</a> • <a href="#scraper-options">Scraper Options</a>
37
+ </p>
38
+
39
+ ## Installation
40
+
41
+ ```bash
42
+ pip install reddrix
43
+
44
+ reddrix install # Optional for installing Camoufox if you want to use Reddrix with Camoufox
45
+ ```
46
+
47
+ ## Quick start
48
+
49
+ **Using Async Context Manager:**
50
+
51
+ ```python
52
+ from reddrix import Reddrix
53
+ import asyncio
54
+
55
+ async def main():
56
+ async with Reddrix(
57
+ mode="standard",
58
+ subreddit_urls=["https://www.reddit.com/r/webscraping"],
59
+ sort="new",
60
+ include_comments=True,
61
+ max_results=500,
62
+ max_posts_per_collection=500,
63
+ verbose=True
64
+ ) as reddrix:
65
+ async for item in reddrix.get_items():
66
+ print(item["submissionType"], item["submissionUrl"])
67
+
68
+ if __name__ == "__main__":
69
+ asyncio.run(main())
70
+ ```
71
+
72
+ **Without the context manager:**
73
+
74
+ ```python
75
+ from reddrix import Reddrix
76
+ import asyncio
77
+
78
+ async def main():
79
+ reddrix = Reddrix(
80
+ mode="standard",
81
+ subreddit_urls=["https://www.reddit.com/r/webscraping"],
82
+ sort="new",
83
+ include_comments=True,
84
+ max_results=500,
85
+ max_posts_per_collection=500,
86
+ proxies=["http://user:pass@hostname:port", "http://user2:pass2@hostname2:port2"], # Built-in automatic proxy rotation
87
+ verbose=True
88
+ )
89
+ await reddrix.start()
90
+
91
+ async for item in reddrix.get_items():
92
+ print(item["submissionType"], item["submissionUrl"])
93
+
94
+ await reddrix.stop()
95
+
96
+
97
+ if __name__ == "__main__":
98
+ asyncio.run(main())
99
+ ```
100
+
101
+ **Scraping Posts and Comments Based on Keywords:**
102
+
103
+ ```python
104
+ from reddrix import Reddrix
105
+ import asyncio
106
+
107
+ async def main():
108
+ async with Reddrix(
109
+ mode="keyword_search",
110
+ sort="new", # The sort option is ignored for this mode
111
+ query="how to scrape reddit",
112
+ include_comments=True,
113
+ max_results=500,
114
+ max_posts_per_collection=500,
115
+ verbose=True
116
+ ) as reddrix:
117
+ async for item in reddrix.get_items():
118
+ print(item["submissionType"], item["submissionUrl"])
119
+
120
+ if __name__ == "__main__":
121
+ asyncio.run(main())
122
+ ```
123
+
124
+ **Scraping Posts and Comments Based on Domains:**
125
+
126
+ ```python
127
+ from reddrix import Reddrix
128
+ import asyncio
129
+
130
+ async def main():
131
+ async with Reddrix(
132
+ mode="domain_search",
133
+ sort="new",
134
+ domains=["github.com", "apify.com"],
135
+ include_comments=True,
136
+ max_results=500,
137
+ max_posts_per_collection=500,
138
+ verbose=True
139
+ ) as reddrix:
140
+ async for item in reddrix.get_items():
141
+ print(item["submissionType"], item["submissionUrl"])
142
+
143
+ if __name__ == "__main__":
144
+ asyncio.run(main())
145
+ ```
146
+
147
+
148
+
149
+ **Using Reddrix as a standalone Reddit http client for your custom scraping setup:**
150
+
151
+ ```python
152
+ from reddrix import Reddrix
153
+ import asyncio
154
+
155
+ async def main():
156
+ async with Reddrix() as reddrix:
157
+ json_data = await reddrix.get("https://www.reddit.com/r/webscraping.json")
158
+ html_content = await reddrix.get(
159
+ url="https://www.reddit.com/search?q=how+to+scrape+reddit",
160
+ expected_data_type="HTML/XML" # You must specify the data type if you want to fetch html or xml.
161
+ )
162
+
163
+ if __name__ == "__main__":
164
+ asyncio.run(main())
165
+ ```
166
+
167
+ **Using Reddrix with Camoufox:**
168
+
169
+ ```bash
170
+ pip install reddrix
171
+ reddrix install # Installs the camoufox browser
172
+ ```
173
+
174
+ ```python
175
+ from reddrix import Reddrix
176
+ import asyncio
177
+
178
+ async def main():
179
+ async with Reddrix(
180
+ mode="standard",
181
+ http_client="camoufox", # By default, the http_client is "curl_cffi", but you can choose Camoufox.
182
+ subreddit_urls=["https://www.reddit.com/r/webscraping"],
183
+ sort="new",
184
+ include_comments=True,
185
+ max_results=500,
186
+ max_posts_per_collection=500,
187
+ verbose=True
188
+ ) as reddrix:
189
+ async for item in reddrix.get_items():
190
+ print(item["submissionType"], item["submissionUrl"])
191
+
192
+ if __name__ == "__main__":
193
+ asyncio.run(main())
194
+ ```
195
+
196
+ This option is included in anticipation of future aggressive anti-scraping measures from Reddit. Even when Camoufox is used as the http client, the scraper uses it intelligently. It still utilizes the same JSON endpoints that curl_cffi uses via `page.evaluate()` with JS fetch. This results in nearly the same scraping speed, but with more stealth. The only caveat is more resource usage and slower scraper initialization speed.
197
+
198
+ ## Output Dataset
199
+
200
+ If you use the preset scraping setup from `reddrix.get_items()`, the output dataset is a Python dictionary.
201
+
202
+ **Example Reddit Post Output:**
203
+
204
+ ```python
205
+ {
206
+ "submissionId": "1w1gc8p",
207
+ "submissionIdPrefixed": "t3_1w1gc8p",
208
+ "submissionType": "post",
209
+ "submissionUrl": "https://www.reddit.com/r/Sat/comments/1w1gc8p/is_this_good_improvement_for_10_hours_of/",
210
+ "submittedAt": "2026-08-29T07:53:42+00:00",
211
+ "parentId": None,
212
+ "parentPostId": None,
213
+ "parentPostUrl": None,
214
+ "postType": "image",
215
+ "isCrosspost": False,
216
+ "crosspostParentId": None,
217
+ "crosspostParentUrl": None,
218
+ "title": "Is this good improvement for 10 hours of dedicated study + need help for rw section",
219
+ "bodyText": "math significantly improved but reading writing is too tough for me (international student)\n\nplease help, i have to give the sat in two months, will reading novels improve my rw score?",
220
+ "isEdited": False,
221
+ "author": "stcer",
222
+ "authorFlairText": None,
223
+ "authorFlairImageUrl": None,
224
+ "depth": None,
225
+ "postFlair": None,
226
+ "isNSFW": False,
227
+ "subreddit": "r/Sat",
228
+ "score": 32,
229
+ "upvoteRatio": 0.98,
230
+ "numComments": 29,
231
+ "numCrossposts": 0,
232
+ "externalUrl": "https://i.redd.it/cjhezz5bs9mh1.jpeg",
233
+ "thumbnailUrl": None,
234
+ "videoUrl": None,
235
+ "imageUrl": "https://i.redd.it/cjhezz5bs9mh1.jpeg",
236
+ "galleryImagesUrls": None
237
+ }
238
+ ```
239
+
240
+ **Example Comment Output:**
241
+
242
+ ```python
243
+ {
244
+ "submissionId": "p6lwobb",
245
+ "submissionIdPrefixed": "t1_p6lwobb",
246
+ "submissionType": "comment",
247
+ "submissionUrl": "https://www.reddit.com/r/Sat/comments/1w1gc8p/is_this_good_improvement_for_10_hours_of/p6lwobb/",
248
+ "submittedAt": "2026-08-29T13:41:52+00:00",
249
+ "parentId": "t3_1w1gc8p",
250
+ "parentPostId": "t3_1w1gc8p",
251
+ "parentPostUrl": "https://www.reddit.com/r/Sat/comments/1w1gc8p",
252
+ "postType": None,
253
+ "isCrosspost": False,
254
+ "crosspostParentId": None,
255
+ "crosspostParentUrl": None,
256
+ "title": None,
257
+ "bodyText": "How did you do that in 10 hours? Can you please give me the sources that you used? I also need help in rw section and I’ll take the sat in 2 weeks",
258
+ "isEdited": False,
259
+ "author": "AnalysisStill4932",
260
+ "authorFlairText": None,
261
+ "authorFlairImageUrl": None,
262
+ "depth": 0,
263
+ "postFlair": None,
264
+ "isNSFW": False,
265
+ "subreddit": "r/Sat",
266
+ "score": 9,
267
+ "upvoteRatio": None,
268
+ "numComments": None,
269
+ "numCrossposts": None,
270
+ "externalUrl": None,
271
+ "thumbnailUrl": None,
272
+ "videoUrl": None,
273
+ "imageUrl": None,
274
+ "galleryImagesUrls": None
275
+ }
276
+ ```
277
+ **Output Fields Explanation:**
278
+
279
+ 1. `submissionId` &mdash; [`string`] The ID of the post or comment.
280
+ 2. `submissionIdPrefixed` &mdash; [`string`] The prefixed ID of the post or comment.
281
+ 3. `submissionType` &mdash; [`string`] Whether the submission is a `post` or `comment`.
282
+ 4. `submissionUrl` &mdash; [`string`] The URL for the submission.
283
+ 5. `submittedAt` &mdash; [`string`] The date and time the post or comment was submitted in UTC timezone and ISO format.
284
+ 6. `parentId` &mdash; [`string`, `None`] The ID of the direct parent post or comment.
285
+ 7. `parentPostId` &mdash; [`string`, `None`] The ID of the parent post. For posts, this field will always be `None` since posts have no parent.
286
+ 8. `parentPostUrl` &mdash; [`string`, `None`] The URL of the parent post. For posts, this field will always be `None` since posts have no parent.
287
+ 9. `postType` &mdash; [`string`, `None`] The post type. Can be any of `text`, `poll`, `link`, `image`, `gallery`, or `video`. For comments, this field will always be `None`.
288
+ 10. `isCrosspost` &mdash; [`boolean`] Whether the post is a crosspost.
289
+ 11. `crosspostParentId` &mdash; [`string`, `None`] The ID of the parent of the crosspost (for posts that are not crossposts, will be `None`).
290
+ 12. `crosspostParentUrl` &mdash; [`string`, `None`] The URL of the parent of the crosspost (for posts that are not crossposts, will be `None`).
291
+ 13. `title` &mdash; [`string`, `None`] The post title (will always be `None` for comments).
292
+ 14. `bodyText` &mdash; [`string`, `None`] The main text content of a post or comment.
293
+ 15. `isEdited` &mdash; [`boolean`] Whether the post or comment is edited.
294
+ 16. `author` &mdash; [`string`] The author of the post or comment.
295
+ 17. `authorFlairText` &mdash; [`string`, `None`] The text content of the author flair or a text version of it.
296
+ 18. `authorFlairImageUrl` &mdash; [`string`, `None`] The URL for the image in an author flair.
297
+ 19. `depth` &mdash; [`integer`, `None`] The comment depth (top-level comments have a depth of `0` and posts have a depth of `None`).
298
+ 20. `postFlair` &mdash; [`string`, `None`] The post flair (will be `None` for comments).
299
+ 21. `isNSFW` &mdash; [`boolean`] Whether the post or comment is NSFW.
300
+ 22. `subreddit` &mdash; [`string`, `None`] The subreddit in which the post or comment was submitted.
301
+ 23. `score` &mdash; [`integer`, `None`] The vote score of the post or comment.
302
+ 24. `upvoteRatio` &mdash; [`float`, `None`] The ratio of upvotes to total number of votes of a post (always `None` for comments).
303
+ 25. `numComments` &mdash; [`integer`, `None`] The number of comments in a post (always `None` for comments).
304
+ 26. `numCrossposts` &mdash; [`integer`, `None`] The number of crossposts for a post (always `None` for comments).
305
+ 27. `externalUrl` &mdash; [`string`, `None`] The external URL in a post. For comments, this field will be `None` because external URLs will be included in the `bodyText` anyway.
306
+ 28. `thumbnailUrl` &mdash; [`string`, `None`] The thumbnail image URL for `video`, `image`, and `gallery` post types (always `None` for any other submission).
307
+ 29. `videoUrl` &mdash; [`string`, `None`] The URL for the video in a post (always `None` for any submission other than `video` posts).
308
+ 30. `imageUrl` &mdash; [`string`, `None`] The URL for the image in a post (always `None` for any submission other than `image` posts).
309
+ 31. `galleryImagesUrls` &mdash; [`list`] A list of URLs for all images in a gallery (always empty for any submission other than `gallery` posts).
310
+
311
+ ## Features
312
+
313
+ - 🚀 **Async-First:** High-performance asynchronous scraping with `asyncio` and an async iterator for streaming results.
314
+ - 🔑 **No API Key Required:** Scrape Reddit without requiring an API key.
315
+ - 🔍 **Multiple Search Modes:** Scrape subreddits, user profiles, and specific posts, or search Reddit by keywords and domains.
316
+ - 💬 **Comprehensive Data Extraction:** Collect posts and comments with 30+ structured fields, including authors, scores, timestamps, flairs, media URLs, and comment thread metadata.
317
+ - 🌐 **Flexible HTTP Clients:** Choose between `curl_cffi` for lightweight HTTP requests and `camoufox` for intelligent, browser-based fetching.
318
+ - 🛡️ **Resilient Scraping:** Built-in retries, cookie renewal, and configurable proxies.
319
+ - ⚙️ **Fine-Grained Control:** Configure sorting, time frames, result limits, comment depth, and comment counts.
320
+ - 🧩 **Standalone HTTP Client:** Use Reddrix's asynchronous `get()` method in your own custom scraping workflows.
321
+ - 🐍 **Pythonic API:** Manage scraper resources with an async context manager or explicit `start()` and `stop()` methods.
322
+
323
+ ## Scraper Options
324
+
325
+ 1. `mode` (string) &mdash; One of `standard`, `keyword_search`, and `domain_search`. Default is `standard`.
326
+ 2. `http_client` (string) &mdash; One of `curl_cffi` and `camoufox`. Default is `curl_cffi`.
327
+ 3. `max_results` (integer) &mdash; The maximum total number of items (posts and comments combined) to return across all sources. Default is `999999`.
328
+ 4. `max_posts_per_collection` (integer) &mdash; The maximum number of posts to scrape from each collection. A collection is a single subreddit, profile, keyword search, or domain. Default is `999999`.
329
+ 5. `sort` (string) &mdash; One of `default`, `new`, and `hot`. Applies to subreddits, profiles, and `domain_search`. This option is ignored in `keyword_search` mode, which always uses Reddit's relevance sorting. Default is `default`.
330
+ 6. `time_frame` (string) &mdash; One of `all_time`, `past_hour`, `today`, `past_week`, `past_month`, and `past_year`. Used in `standard` mode for subreddits and profiles, and ignored in `keyword_search` and `domain_search` modes. Default is `all_time`.
331
+ 7. `subreddit_urls` (list of strings) &mdash; Subreddit URLs to scrape, e.g. `["https://www.reddit.com/r/webscraping"]`. Used in `standard` mode only. URLs that are not valid subreddit URLs are skipped. Default is `[]`.
332
+ 8. `post_urls` (list of strings) &mdash; Specific post URLs to scrape, e.g. `["https://www.reddit.com/r/webscraping/comments/abc123/post_title/"]`. Used in `standard` mode only. URLs that are not valid post URLs are skipped. Default is `[]`.
333
+ 9. `profile_urls` (list of strings) &mdash; User profile URLs to scrape the submitted posts of, e.g. `["https://www.reddit.com/user/username"]`. Both `/user/` and `/u/` URLs are supported. Used in `standard` mode only. URLs that are not valid profile URLs are skipped. Default is `[]`.
334
+ 10. `domains` (list of strings) &mdash; Domains to search for in Reddit posts that link to them, e.g. `["github.com", "apify.com"]`. Used in `domain_search` mode only. Default is `[]`.
335
+ 11. `query` (string) &mdash; The search keywords to look up site-wide on Reddit. Used in `keyword_search` mode only. Default is `None`.
336
+ 12. `include_comments` (boolean) &mdash; Whether to scrape the comments of each post in addition to the post itself. Comments are returned as separate items with `submissionType` set to `comment`. Default is `False`.
337
+ 13. `max_comments_depth` (integer) &mdash; The maximum comment depth to scrape. Top-level comments have a depth of `0`, their replies have a depth of `1`, and so on. Only applies if `include_comments` is `True`. Default is `10`.
338
+ 14. `max_comments_per_post` (integer) &mdash; The maximum total number of comments (top-level comments and replies) to scrape per post. Only applies if `include_comments` is `True`. Default is `999999`.
339
+ 15. `max_top_comments_per_post` (integer) &mdash; The maximum number of top-level comments (depth `0`) to scrape per post. Only applies if `include_comments` is `True`. Default is `999999`.
340
+ 16. `proxies` (list of strings) &mdash; A list of proxy URLs in the format `scheme://user:pass@hostname:port`. Reddrix rotates through them automatically, picking an unused proxy first and reusing proxies only after the whole list has been used. Default is `[]` (no proxy).
341
+ 17. `verbose` (boolean) &mdash; Whether to print timestamped log messages (initialization, retries, etc.) to the console. Default is `False`.
@@ -0,0 +1,10 @@
1
+ reddrix/__init__.py,sha256=ijAbs5HKh-89Vbmo8j9TJn2yvMyGKFSntgyXquNJ8Kk,53
2
+ reddrix/cli.py,sha256=g3lzM-9CBPb3vd5odmPKpISODJQX574cLSHVn_Vq7zk,698
3
+ reddrix/exceptions.py,sha256=ie91Rj3GoGsodWlQ8om2JC2IuzE3qlLJBG6XhQtl8_I,230
4
+ reddrix/reddrix.py,sha256=SH5KCWxSyHU1SvA_Thcz1LQ8kEsy6FpyChtga2Wpl3A,34219
5
+ reddrix-0.1.0.dist-info/licenses/LICENSE.md,sha256=jyoYoLT3j1Gzbj0bo0OaGZjLKpjdEJ_01R31byfJQDw,1091
6
+ reddrix-0.1.0.dist-info/METADATA,sha256=BNv-6a4Zjkwc38BPPyPkFfmGgq8KX--19LM37HzGeq4,16800
7
+ reddrix-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
8
+ reddrix-0.1.0.dist-info/entry_points.txt,sha256=JlLGCBJqrnLvveS11NF6ReiS56FwIkRWLuBP3XaAAQw,45
9
+ reddrix-0.1.0.dist-info/top_level.txt,sha256=XOi5znNdB6-QZgZZY6y5NQ4EfmF7pgXTkfAY733iZP4,8
10
+ reddrix-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ reddrix = reddrix.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Omar Abdelhamid
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ reddrix