deepsift 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deepsift/scraper.py ADDED
@@ -0,0 +1,397 @@
1
+ """
2
+ Generalized Reddit Scraper Library.
3
+ Coordinates parallel shard post scanning and async comment extraction.
4
+ """
5
+ import os
6
+ import json
7
+ import asyncio
8
+ import threading
9
+ from datetime import datetime, date, timedelta, timezone
10
+ from concurrent.futures import ThreadPoolExecutor, as_completed
11
+ from typing import List, Union, Optional, Tuple, Type
12
+ import requests
13
+ import httpx
14
+ from tqdm import tqdm
15
+ from tqdm.asyncio import tqdm as async_tqdm
16
+
17
+ from deepsift.models import Post, Comment, UnretrievedFieldError
18
+ from deepsift.filters import Filters
19
+ from deepsift.sources import Source, ArcticShiftSource
20
+ from deepsift.storage import Storage, ParquetStorage
21
+
22
+
23
+ def to_utc_datetime(val: Union[datetime, date, str, int, float, None]) -> Optional[datetime]:
24
+ """Normalizes input to timezone-aware UTC datetime."""
25
+ if val is None:
26
+ return None
27
+ if isinstance(val, (int, float)):
28
+ return datetime.fromtimestamp(val, tz=timezone.utc)
29
+ if isinstance(val, datetime):
30
+ return val.astimezone(timezone.utc) if val.tzinfo else val.replace(tzinfo=timezone.utc)
31
+ if isinstance(val, date):
32
+ return datetime(val.year, val.month, val.day, tzinfo=timezone.utc)
33
+ if isinstance(val, str):
34
+ v = val.strip().strip("'\"")
35
+ try:
36
+ num = float(v)
37
+ return datetime.fromtimestamp(num, tz=timezone.utc)
38
+ except ValueError:
39
+ pass
40
+ v = v.replace("Z", "+00:00")
41
+ try:
42
+ val = datetime.fromisoformat(v)
43
+ except ValueError:
44
+ val = datetime.strptime(v, "%Y-%m-%d")
45
+ return val.astimezone(timezone.utc) if val.tzinfo else val.replace(tzinfo=timezone.utc)
46
+ raise TypeError(f"Expected datetime, date, numeric timestamp, or str date, got {type(val).__name__}")
47
+
48
+
49
+ class RedditScraper:
50
+ """Generalized orchestrator for historical Reddit data collection."""
51
+
52
+ def __init__(
53
+ self,
54
+ filters: Optional[Filters] = None,
55
+ source: Optional[Source] = None,
56
+ storage_cls: Type[Storage] = ParquetStorage,
57
+ post_workers: int = 4,
58
+ comment_workers: int = 5,
59
+ shard_days: int = 180,
60
+ ):
61
+ if post_workers <= 0:
62
+ raise ValueError(f"post_workers must be positive integer >= 1, got {post_workers}")
63
+ if comment_workers <= 0:
64
+ raise ValueError(f"comment_workers must be positive integer >= 1, got {comment_workers}")
65
+ if shard_days <= 0:
66
+ raise ValueError(f"shard_days must be positive integer >= 1, got {shard_days}")
67
+
68
+ self.filters = filters or Filters()
69
+ self.source = source or ArcticShiftSource()
70
+ self.storage_cls = storage_cls
71
+ self.post_workers = post_workers
72
+ self.comment_workers = comment_workers
73
+ self.shard_days = shard_days
74
+ self.errors: List[dict] = []
75
+ self._errors_lock = threading.Lock()
76
+ self.succeeded_posts: List[str] = []
77
+ self.succeeded_comments: int = 0
78
+ self.targeted_subreddits: List[str] = []
79
+ self.started_at: Optional[datetime] = None
80
+ self.completed_at: Optional[datetime] = None
81
+
82
+ def save_errors_log(
83
+ self, errors_path: str = "data/errors.json", fatal_error: Optional[BaseException] = None
84
+ ) -> Optional[str]:
85
+ """Saves a structured JSON log of all failed requests, successes, and timestamps."""
86
+ if fatal_error:
87
+ fatal_entry = {
88
+ "stage": "fatal_pipeline_error",
89
+ "error_type": type(fatal_error).__name__,
90
+ "error_message": str(fatal_error) or "Execution interrupted by user or system signal",
91
+ "timestamp": datetime.now(timezone.utc).isoformat(),
92
+ }
93
+ with self._errors_lock:
94
+ self.errors.append(fatal_entry)
95
+
96
+ if not self.errors:
97
+ return None
98
+
99
+ dir_path = os.path.dirname(os.path.abspath(errors_path))
100
+ if dir_path:
101
+ os.makedirs(dir_path, exist_ok=True)
102
+
103
+ end_dt = self.completed_at or datetime.now(timezone.utc)
104
+ duration = (end_dt - self.started_at).total_seconds() if self.started_at else None
105
+
106
+ payload = {
107
+ "timestamp": datetime.now(timezone.utc).isoformat(),
108
+ "status": "failed" if fatal_error else "completed_with_errors",
109
+ "execution": {
110
+ "started_at": self.started_at.isoformat() if self.started_at else None,
111
+ "completed_at": end_dt.isoformat(),
112
+ "duration_seconds": round(duration, 2) if duration is not None else None,
113
+ "source": getattr(self.source, "name", "unknown"),
114
+ },
115
+ "summary": {
116
+ "total_errors": len(self.errors),
117
+ "total_posts_succeeded": len(self.succeeded_posts),
118
+ "total_comments_succeeded": self.succeeded_comments,
119
+ },
120
+ "succeeded": {
121
+ "posts_count": len(self.succeeded_posts),
122
+ "comments_count": self.succeeded_comments,
123
+ "subreddits": list(self.targeted_subreddits),
124
+ "post_ids": self.succeeded_posts[:1000],
125
+ },
126
+ "failures": list(self.errors),
127
+ }
128
+ with open(errors_path, "w", encoding="utf-8") as f:
129
+ json.dump(payload, f, indent=2)
130
+
131
+ print(f"\n[!] {len(self.errors)} error(s) occurred during execution.")
132
+ print(f" Refer to '{errors_path}' to see what failed, what succeeded, and full timestamps.")
133
+ return errors_path
134
+
135
+ def _generate_shards(self, from_dt: Union[datetime, str], to_dt: Optional[Union[datetime, str]]) -> Tuple[list, datetime, datetime]:
136
+ end = to_utc_datetime(to_dt) or datetime.now(timezone.utc)
137
+ start = to_utc_datetime(from_dt)
138
+ if not start or start >= end:
139
+ raise ValueError(f"Invalid range: from_dt ({start}) must precede to_dt ({end})")
140
+
141
+ shards, cur = [], end
142
+ while cur > start:
143
+ prev = max(cur - timedelta(days=self.shard_days), start)
144
+ shards.append((int(prev.timestamp()), int(cur.timestamp())))
145
+ cur = prev
146
+ return shards, start, end
147
+
148
+ def _scan_shard(self, subreddit: str, after_ts: int, before_ts: int, streamer: Storage, seen: set, lock: threading.Lock) -> list[str]:
149
+ matched = []
150
+ reached_end = False
151
+ try:
152
+ with requests.Session() as session:
153
+ for batch in self.source.fetch_posts_shard(subreddit, after_ts, before_ts, fields=self.filters.required_post_fields, session=session):
154
+ rows = []
155
+ for raw in batch:
156
+ try:
157
+ if not isinstance(raw, dict):
158
+ continue
159
+ if int(float(raw.get("created_utc", 0))) < after_ts:
160
+ reached_end = True
161
+ break
162
+ post = Post(raw)
163
+ if "subreddit" not in post._data and subreddit:
164
+ post._data["subreddit"] = subreddit
165
+ if post.id not in seen and self.filters.is_valid_post(post):
166
+ with lock:
167
+ if post.id not in seen:
168
+ seen.add(post.id)
169
+ rows.append(post.to_dict())
170
+ matched.append(post.id)
171
+ except UnretrievedFieldError:
172
+ raise
173
+ except Exception:
174
+ continue
175
+ if rows:
176
+ with lock:
177
+ streamer.write_batch(rows)
178
+ if reached_end:
179
+ break
180
+ except Exception as e:
181
+ error_entry = {
182
+ "stage": "post_shard",
183
+ "subreddit": subreddit,
184
+ "time_window": {
185
+ "after_ts": after_ts,
186
+ "before_ts": before_ts,
187
+ "after_utc": datetime.fromtimestamp(after_ts, tz=timezone.utc).isoformat(),
188
+ "before_utc": datetime.fromtimestamp(before_ts, tz=timezone.utc).isoformat(),
189
+ },
190
+ "error_type": type(e).__name__,
191
+ "error_message": str(e),
192
+ "timestamp": datetime.now(timezone.utc).isoformat(),
193
+ }
194
+ with self._errors_lock:
195
+ self.errors.append(error_entry)
196
+ print(f"[!] Warning: Shard [{after_ts}, {before_ts}] for r/{subreddit} encountered an error: {e}")
197
+ return matched
198
+
199
+ def scrape_posts(
200
+ self,
201
+ subreddits: Union[str, List[str]],
202
+ from_dt: Union[datetime, str, int, float],
203
+ to_dt: Optional[Union[datetime, str, int, float]] = None,
204
+ output_path: str = "data/curated_posts.parquet",
205
+ storage_cls: Optional[Type[Storage]] = None,
206
+ errors_output: str = "data/errors.json",
207
+ _in_pipeline: bool = False,
208
+ ) -> list[str]:
209
+ subs = [subreddits] if isinstance(subreddits, str) else list(subreddits)
210
+ if not subs:
211
+ raise ValueError("At least one subreddit must be specified.")
212
+ if not _in_pipeline:
213
+ self.started_at = datetime.now(timezone.utc)
214
+ self.targeted_subreddits = subs
215
+ self.succeeded_posts = []
216
+
217
+ if os.path.exists(output_path):
218
+ os.remove(output_path)
219
+
220
+ fatal_exc = None
221
+ unique_ids = []
222
+ try:
223
+ shards, start_dt, end_dt = self._generate_shards(from_dt, to_dt)
224
+ seen, lock, all_matched = set(), threading.Lock(), []
225
+
226
+ print(f"[*] Post scan using [{self.source.name}] across {len(subs)} subreddits ({start_dt:%Y-%m-%d} to {end_dt:%Y-%m-%d})")
227
+ with (storage_cls or self.storage_cls)(output_path) as streamer:
228
+ for sub in subs:
229
+ executor = ThreadPoolExecutor(max_workers=self.post_workers)
230
+ try:
231
+ futures = [executor.submit(self._scan_shard, sub, a, b, streamer, seen, lock) for a, b in shards]
232
+ for f in tqdm(as_completed(futures), total=len(futures), desc=f"r/{sub}"):
233
+ all_matched.extend(f.result())
234
+ finally:
235
+ executor.shutdown(wait=False, cancel_futures=True)
236
+
237
+ unique_ids = list(dict.fromkeys(all_matched))
238
+ self.succeeded_posts = unique_ids
239
+ if unique_ids:
240
+ print(f"[✓] Saved {len(unique_ids):,} posts to {output_path}")
241
+ return unique_ids
242
+ except BaseException as e:
243
+ fatal_exc = e
244
+ raise
245
+ finally:
246
+ if not _in_pipeline:
247
+ self.completed_at = datetime.now(timezone.utc)
248
+ self.save_errors_log(errors_path=errors_output, fatal_error=fatal_exc)
249
+
250
+ async def _scrape_comments_async(
251
+ self,
252
+ post_ids: list[str],
253
+ output_path: str,
254
+ chunk_size: int = 500,
255
+ storage_cls: Optional[Type[Storage]] = None,
256
+ errors_output: str = "data/errors.json",
257
+ _in_pipeline: bool = False,
258
+ ) -> int:
259
+ if not _in_pipeline and self.started_at is None:
260
+ self.started_at = datetime.now(timezone.utc)
261
+
262
+ if os.path.exists(output_path):
263
+ os.remove(output_path)
264
+
265
+ sem = asyncio.Semaphore(self.comment_workers)
266
+ total_saved, buffer = 0, []
267
+ fatal_exc = None
268
+
269
+ async def _fetch(client: httpx.AsyncClient, pid: str):
270
+ try:
271
+ async with sem:
272
+ cid, raw, success = await self.source.fetch_comments_for_post(client, pid, fields=self.filters.required_comment_fields)
273
+ if not success:
274
+ error_entry = {
275
+ "stage": "comment_fetch",
276
+ "post_id": pid,
277
+ "error_type": "FetchFailed",
278
+ "error_message": "Data source reported failure (all retries exhausted)",
279
+ "timestamp": datetime.now(timezone.utc).isoformat(),
280
+ }
281
+ with self._errors_lock:
282
+ self.errors.append(error_entry)
283
+ return [c.to_dict() for raw_c in (raw or []) if self.filters.is_valid_comment(c := Comment(raw_c, cid))]
284
+ except UnretrievedFieldError:
285
+ raise
286
+ except Exception as e:
287
+ error_entry = {
288
+ "stage": "comment_fetch",
289
+ "post_id": pid,
290
+ "error_type": type(e).__name__,
291
+ "error_message": str(e),
292
+ "timestamp": datetime.now(timezone.utc).isoformat(),
293
+ }
294
+ with self._errors_lock:
295
+ self.errors.append(error_entry)
296
+ return []
297
+
298
+ try:
299
+ print(f"[*] Fetching comments using [{self.source.name}] for {len(post_ids):,} posts...")
300
+ max_conn = max(self.comment_workers * 2, 20)
301
+ with (storage_cls or self.storage_cls)(output_path) as streamer:
302
+ async with httpx.AsyncClient(limits=httpx.Limits(max_keepalive_connections=max_conn, max_connections=max_conn)) as client:
303
+ tasks = [_fetch(client, pid) for pid in post_ids]
304
+ for fut in async_tqdm.as_completed(tasks, total=len(tasks), desc="Comments"):
305
+ comments = await fut
306
+ if comments:
307
+ buffer.extend(comments)
308
+ if len(buffer) >= chunk_size:
309
+ await asyncio.to_thread(streamer.write_batch, list(buffer))
310
+ total_saved += len(buffer)
311
+ buffer.clear()
312
+ if buffer:
313
+ await asyncio.to_thread(streamer.write_batch, list(buffer))
314
+ total_saved += len(buffer)
315
+
316
+ self.succeeded_comments += total_saved
317
+ if total_saved > 0:
318
+ print(f"[✓] Saved {total_saved:,} comments to {output_path}")
319
+ return total_saved
320
+ except BaseException as e:
321
+ fatal_exc = e
322
+ raise
323
+ finally:
324
+ if not _in_pipeline:
325
+ self.completed_at = datetime.now(timezone.utc)
326
+ self.save_errors_log(errors_path=errors_output, fatal_error=fatal_exc)
327
+
328
+ def scrape_comments(
329
+ self,
330
+ post_ids: Union[str, list[str]],
331
+ output_path: str = "data/curated_comments.parquet",
332
+ chunk_size: int = 500,
333
+ storage_cls: Optional[Type[Storage]] = None,
334
+ errors_output: str = "data/errors.json",
335
+ _in_pipeline: bool = False,
336
+ ) -> int:
337
+ pids = [post_ids] if isinstance(post_ids, str) else list(dict.fromkeys(post_ids))
338
+ if not pids:
339
+ return 0
340
+ try:
341
+ loop = asyncio.get_running_loop()
342
+ except RuntimeError:
343
+ loop = None
344
+
345
+ coro = self._scrape_comments_async(
346
+ pids, output_path, chunk_size, storage_cls, errors_output=errors_output, _in_pipeline=_in_pipeline
347
+ )
348
+ if loop and loop.is_running():
349
+ import nest_asyncio
350
+ nest_asyncio.apply()
351
+ return loop.run_until_complete(coro)
352
+ return asyncio.run(coro)
353
+
354
+ def run(
355
+ self,
356
+ subreddits: Union[str, List[str]],
357
+ from_dt: Union[datetime, str],
358
+ to_dt: Optional[Union[datetime, str]] = None,
359
+ posts_output: str = "data/curated_posts.parquet",
360
+ comments_output: str = "data/curated_comments.parquet",
361
+ errors_output: str = "data/errors.json",
362
+ storage_cls: Optional[Type[Storage]] = None,
363
+ ) -> tuple[list[str], int]:
364
+ subs = [subreddits] if isinstance(subreddits, str) else list(subreddits)
365
+ self.started_at = datetime.now(timezone.utc)
366
+ self.targeted_subreddits = subs
367
+ self.succeeded_posts = []
368
+ self.succeeded_comments = 0
369
+ fatal_exc = None
370
+ posts = []
371
+ comments = 0
372
+
373
+ try:
374
+ posts = self.scrape_posts(
375
+ subreddits=subs,
376
+ from_dt=from_dt,
377
+ to_dt=to_dt,
378
+ output_path=posts_output,
379
+ storage_cls=storage_cls,
380
+ errors_output=errors_output,
381
+ _in_pipeline=True,
382
+ )
383
+ if posts:
384
+ comments = self.scrape_comments(
385
+ post_ids=posts,
386
+ output_path=comments_output,
387
+ storage_cls=storage_cls,
388
+ errors_output=errors_output,
389
+ _in_pipeline=True,
390
+ )
391
+ return posts, comments
392
+ except BaseException as e:
393
+ fatal_exc = e
394
+ raise
395
+ finally:
396
+ self.completed_at = datetime.now(timezone.utc)
397
+ self.save_errors_log(errors_path=errors_output, fatal_error=fatal_exc)
@@ -0,0 +1,14 @@
1
+ """
2
+ Data sources subpackage for Reddit scraping.
3
+ """
4
+ from deepsift.sources.base import Source
5
+ from deepsift.sources.arctic_shift import ArcticShiftSource
6
+ from deepsift.sources.registry import SOURCES, get_source, register_source
7
+
8
+ __all__ = [
9
+ "Source",
10
+ "ArcticShiftSource",
11
+ "SOURCES",
12
+ "get_source",
13
+ "register_source",
14
+ ]
@@ -0,0 +1,263 @@
1
+ """
2
+ Arctic Shift API data source implementation.
3
+ """
4
+ import time
5
+ import asyncio
6
+ from typing import Iterator, List, Dict, Any, Optional, Tuple, Iterable
7
+ import requests
8
+ import httpx
9
+ from deepsift.sources.base import Source
10
+
11
+
12
+ class ArcticShiftSource(Source):
13
+ """
14
+ Arctic Shift API data source implementation (https://arctic-shift.photon-reddit.com).
15
+ Supports server-side field projection via the `fields=` query parameter.
16
+ """
17
+
18
+ BASE_URL = "https://arctic-shift.photon-reddit.com"
19
+ DEFAULT_USER_AGENT = "reddit-research-scraper/0.2.0"
20
+
21
+ def __init__(
22
+ self,
23
+ base_url: str = BASE_URL,
24
+ timeout: float = 15.0,
25
+ rate_limit_delay: float = 0.3,
26
+ batch_size: int = 100,
27
+ user_agent: str = DEFAULT_USER_AGENT,
28
+ ):
29
+ self.base_url = base_url.rstrip("/")
30
+ self.timeout = timeout
31
+ self.rate_limit_delay = rate_limit_delay
32
+ self.batch_size = batch_size
33
+ self.user_agent = user_agent
34
+ self.headers = {"User-Agent": self.user_agent}
35
+
36
+ @property
37
+ def name(self) -> str:
38
+ return "arctic_shift"
39
+
40
+ def fetch_posts_shard(
41
+ self,
42
+ subreddit: str,
43
+ after_ts: int,
44
+ before_ts: int,
45
+ fields: Optional[Iterable[str]] = None,
46
+ session: Optional[requests.Session] = None,
47
+ ) -> Iterator[List[Dict[str, Any]]]:
48
+ """
49
+ Paginates backwards through Arctic Shift posts in time window [after_ts, before_ts].
50
+ If fields is specified, requests only those fields from the server.
51
+ """
52
+ close_session = session is None
53
+ s = session or requests.Session()
54
+ current_before = before_ts
55
+ projected = self.resolve_post_fields(fields)
56
+ fields_str = ",".join(sorted(projected)) if projected else None
57
+
58
+ try:
59
+ while current_before > after_ts:
60
+ params = {
61
+ "subreddit": subreddit,
62
+ "limit": self.batch_size,
63
+ "before": current_before,
64
+ "after": after_ts,
65
+ }
66
+ if fields_str:
67
+ params["fields"] = fields_str
68
+
69
+ data = None
70
+ backoff = 2.0
71
+ for _ in range(5):
72
+ try:
73
+ res = s.get(
74
+ f"{self.base_url}/api/posts/search",
75
+ params=params,
76
+ headers=self.headers,
77
+ timeout=self.timeout,
78
+ )
79
+ if res.status_code == 200:
80
+ data = res.json().get("data", [])
81
+ break
82
+ if res.status_code == 429:
83
+ retry_after = float(res.headers.get("Retry-After", backoff))
84
+ time.sleep(min(retry_after, 30.0))
85
+ else:
86
+ time.sleep(backoff)
87
+ backoff *= 1.8
88
+ except requests.RequestException:
89
+ time.sleep(backoff)
90
+ backoff *= 1.8
91
+
92
+ if data is None:
93
+ raise ConnectionError(
94
+ f"Failed to fetch posts shard for r/{subreddit} in window [{after_ts}, {current_before}] "
95
+ f"after 5 retries (timeout or server error)."
96
+ )
97
+
98
+ if not data:
99
+ break
100
+
101
+ yield data
102
+
103
+ oldest = min(int(float(p.get("created_utc", current_before))) for p in data)
104
+ if oldest <= after_ts:
105
+ break
106
+ current_before = current_before - 1 if oldest >= current_before else oldest
107
+ time.sleep(self.rate_limit_delay)
108
+ finally:
109
+ if close_session:
110
+ s.close()
111
+
112
+ async def fetch_comments_for_post(
113
+ self,
114
+ client: httpx.AsyncClient,
115
+ post_id: str,
116
+ fields: Optional[Iterable[str]] = None,
117
+ max_retries: int = 5,
118
+ ) -> Tuple[str, List[Dict[str, Any]], bool]:
119
+ """
120
+ Fetches comments from Arctic Shift.
121
+ Tree endpoint is attempted first; falls back to search endpoint with fields projection if needed.
122
+ """
123
+ clean_id = str(post_id).replace("t3_", "")
124
+ tree_url = f"{self.base_url}/api/comments/tree"
125
+ params = {"link_id": f"t3_{clean_id}"}
126
+ backoff = 2.0
127
+
128
+ for _ in range(max_retries):
129
+ try:
130
+ resp = await client.get(
131
+ tree_url,
132
+ params=params,
133
+ headers=self.headers,
134
+ timeout=self.timeout,
135
+ )
136
+ if resp.status_code == 200:
137
+ json_data = resp.json()
138
+ raw = json_data.get("data")
139
+ if raw is None and "comments" in json_data:
140
+ raw = json_data["comments"]
141
+ cleaned = self._unpack_nodes(raw if raw is not None else json_data, clean_id)
142
+ if not cleaned and raw is None:
143
+ cleaned, _ = await self._fetch_search_fallback(client, clean_id, fields=fields)
144
+ return clean_id, cleaned, True
145
+
146
+ if resp.status_code == 404:
147
+ break
148
+
149
+ await asyncio.sleep(backoff)
150
+ backoff *= 1.6
151
+ except Exception:
152
+ await asyncio.sleep(backoff)
153
+ backoff *= 1.6
154
+
155
+ fallback, success = await self._fetch_search_fallback(client, clean_id, fields=fields)
156
+ return clean_id, fallback, success
157
+
158
+ def _unpack_nodes(self, nodes: Any, post_id: str) -> List[Dict[str, Any]]:
159
+ """
160
+ Flattens nested comment tree into raw dictionaries using a stack.
161
+ Avoids recursion limits for deeply nested reply threads.
162
+ """
163
+ flat = []
164
+ if not isinstance(nodes, list):
165
+ nodes = [nodes] if isinstance(nodes, dict) else []
166
+
167
+ stack = list(reversed(nodes))
168
+ while stack:
169
+ node = stack.pop()
170
+ if not isinstance(node, dict):
171
+ continue
172
+
173
+ raw_comment = None
174
+ if "comment" in node and isinstance(node["comment"], dict):
175
+ raw_comment = node["comment"]
176
+ elif "data" in node and isinstance(node["data"], dict):
177
+ raw_comment = node["data"]
178
+ elif "body" in node or "id" in node or "author" in node:
179
+ raw_comment = node
180
+
181
+ replies = node.get("replies") or node.get("children") or []
182
+ if raw_comment and not replies:
183
+ replies = raw_comment.get("replies") or raw_comment.get("children") or []
184
+
185
+ if isinstance(replies, dict):
186
+ replies = replies.get("data", {}).get("children", []) if "data" in replies else list(replies.values())
187
+ if isinstance(replies, list) and replies:
188
+ for child in reversed(replies):
189
+ stack.append(child)
190
+
191
+ if not raw_comment:
192
+ continue
193
+
194
+ comment_dict = dict(raw_comment)
195
+ if "post_id" not in comment_dict:
196
+ comment_dict["post_id"] = post_id
197
+ flat.append(comment_dict)
198
+
199
+ return flat
200
+
201
+ async def _fetch_search_fallback(
202
+ self, client: httpx.AsyncClient, clean_id: str, fields: Optional[Iterable[str]] = None
203
+ ) -> Tuple[List[Dict[str, Any]], bool]:
204
+ """Fallback to /api/comments/search with field projection and backward pagination."""
205
+ all_comments = []
206
+ seen_ids = set()
207
+ projected = self.resolve_comment_fields(fields)
208
+ query_fields = (set(projected) | {"id", "created_utc"}) if projected else None
209
+ fields_str = ",".join(sorted(query_fields)) if query_fields else None
210
+ current_before = None
211
+ ever_succeeded = False
212
+
213
+ for _ in range(25):
214
+ params = {"link_id": f"t3_{clean_id}", "limit": self.batch_size}
215
+ if fields_str:
216
+ params["fields"] = fields_str
217
+ if current_before is not None:
218
+ params["before"] = current_before
219
+
220
+ raw = None
221
+ for attempt in range(2):
222
+ try:
223
+ resp = await client.get(
224
+ f"{self.base_url}/api/comments/search",
225
+ params=params,
226
+ headers=self.headers,
227
+ timeout=self.timeout,
228
+ )
229
+ if resp.status_code == 200:
230
+ raw = resp.json().get("data", [])
231
+ ever_succeeded = True
232
+ break
233
+ if resp.status_code == 429:
234
+ await asyncio.sleep(1.0)
235
+ except Exception:
236
+ if attempt == 0:
237
+ await asyncio.sleep(0.5)
238
+
239
+ if raw is None or not raw:
240
+ break
241
+
242
+ for item in raw:
243
+ cid = str(item.get("id", ""))
244
+ if cid and cid in seen_ids:
245
+ continue
246
+ if cid:
247
+ seen_ids.add(cid)
248
+ if "post_id" not in item:
249
+ item["post_id"] = clean_id
250
+ all_comments.append(item)
251
+ if len(raw) < self.batch_size:
252
+ break
253
+ oldest_ts = raw[-1].get("created_utc")
254
+ if oldest_ts is not None:
255
+ try:
256
+ oldest_int = int(float(oldest_ts))
257
+ current_before = oldest_int - 1 if (current_before is not None and oldest_int >= current_before) else oldest_int
258
+ except (ValueError, TypeError):
259
+ break
260
+ else:
261
+ break
262
+
263
+ return all_comments, ever_succeeded