deepsift 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepsift/__init__.py +39 -0
- deepsift/__main__.py +8 -0
- deepsift/cli.py +171 -0
- deepsift/filters.py +111 -0
- deepsift/models/__init__.py +17 -0
- deepsift/models/base.py +100 -0
- deepsift/models/comment.py +38 -0
- deepsift/models/enums.py +243 -0
- deepsift/models/post.py +29 -0
- deepsift/scraper.py +397 -0
- deepsift/sources/__init__.py +14 -0
- deepsift/sources/arctic_shift.py +263 -0
- deepsift/sources/base.py +102 -0
- deepsift/sources/dummy.py +207 -0
- deepsift/sources/registry.py +24 -0
- deepsift/storage/__init__.py +10 -0
- deepsift/storage/base.py +37 -0
- deepsift/storage/parquet.py +86 -0
- deepsift-0.2.0.dist-info/METADATA +115 -0
- deepsift-0.2.0.dist-info/RECORD +24 -0
- deepsift-0.2.0.dist-info/WHEEL +5 -0
- deepsift-0.2.0.dist-info/entry_points.txt +3 -0
- deepsift-0.2.0.dist-info/licenses/LICENSE +21 -0
- deepsift-0.2.0.dist-info/top_level.txt +1 -0
deepsift/scraper.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Generalized Reddit Scraper Library.
|
|
3
|
+
Coordinates parallel shard post scanning and async comment extraction.
|
|
4
|
+
"""
|
|
5
|
+
import os
|
|
6
|
+
import json
|
|
7
|
+
import asyncio
|
|
8
|
+
import threading
|
|
9
|
+
from datetime import datetime, date, timedelta, timezone
|
|
10
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
11
|
+
from typing import List, Union, Optional, Tuple, Type
|
|
12
|
+
import requests
|
|
13
|
+
import httpx
|
|
14
|
+
from tqdm import tqdm
|
|
15
|
+
from tqdm.asyncio import tqdm as async_tqdm
|
|
16
|
+
|
|
17
|
+
from deepsift.models import Post, Comment, UnretrievedFieldError
|
|
18
|
+
from deepsift.filters import Filters
|
|
19
|
+
from deepsift.sources import Source, ArcticShiftSource
|
|
20
|
+
from deepsift.storage import Storage, ParquetStorage
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def to_utc_datetime(val: Union[datetime, date, str, int, float, None]) -> Optional[datetime]:
|
|
24
|
+
"""Normalizes input to timezone-aware UTC datetime."""
|
|
25
|
+
if val is None:
|
|
26
|
+
return None
|
|
27
|
+
if isinstance(val, (int, float)):
|
|
28
|
+
return datetime.fromtimestamp(val, tz=timezone.utc)
|
|
29
|
+
if isinstance(val, datetime):
|
|
30
|
+
return val.astimezone(timezone.utc) if val.tzinfo else val.replace(tzinfo=timezone.utc)
|
|
31
|
+
if isinstance(val, date):
|
|
32
|
+
return datetime(val.year, val.month, val.day, tzinfo=timezone.utc)
|
|
33
|
+
if isinstance(val, str):
|
|
34
|
+
v = val.strip().strip("'\"")
|
|
35
|
+
try:
|
|
36
|
+
num = float(v)
|
|
37
|
+
return datetime.fromtimestamp(num, tz=timezone.utc)
|
|
38
|
+
except ValueError:
|
|
39
|
+
pass
|
|
40
|
+
v = v.replace("Z", "+00:00")
|
|
41
|
+
try:
|
|
42
|
+
val = datetime.fromisoformat(v)
|
|
43
|
+
except ValueError:
|
|
44
|
+
val = datetime.strptime(v, "%Y-%m-%d")
|
|
45
|
+
return val.astimezone(timezone.utc) if val.tzinfo else val.replace(tzinfo=timezone.utc)
|
|
46
|
+
raise TypeError(f"Expected datetime, date, numeric timestamp, or str date, got {type(val).__name__}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class RedditScraper:
|
|
50
|
+
"""Generalized orchestrator for historical Reddit data collection."""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
filters: Optional[Filters] = None,
|
|
55
|
+
source: Optional[Source] = None,
|
|
56
|
+
storage_cls: Type[Storage] = ParquetStorage,
|
|
57
|
+
post_workers: int = 4,
|
|
58
|
+
comment_workers: int = 5,
|
|
59
|
+
shard_days: int = 180,
|
|
60
|
+
):
|
|
61
|
+
if post_workers <= 0:
|
|
62
|
+
raise ValueError(f"post_workers must be positive integer >= 1, got {post_workers}")
|
|
63
|
+
if comment_workers <= 0:
|
|
64
|
+
raise ValueError(f"comment_workers must be positive integer >= 1, got {comment_workers}")
|
|
65
|
+
if shard_days <= 0:
|
|
66
|
+
raise ValueError(f"shard_days must be positive integer >= 1, got {shard_days}")
|
|
67
|
+
|
|
68
|
+
self.filters = filters or Filters()
|
|
69
|
+
self.source = source or ArcticShiftSource()
|
|
70
|
+
self.storage_cls = storage_cls
|
|
71
|
+
self.post_workers = post_workers
|
|
72
|
+
self.comment_workers = comment_workers
|
|
73
|
+
self.shard_days = shard_days
|
|
74
|
+
self.errors: List[dict] = []
|
|
75
|
+
self._errors_lock = threading.Lock()
|
|
76
|
+
self.succeeded_posts: List[str] = []
|
|
77
|
+
self.succeeded_comments: int = 0
|
|
78
|
+
self.targeted_subreddits: List[str] = []
|
|
79
|
+
self.started_at: Optional[datetime] = None
|
|
80
|
+
self.completed_at: Optional[datetime] = None
|
|
81
|
+
|
|
82
|
+
def save_errors_log(
|
|
83
|
+
self, errors_path: str = "data/errors.json", fatal_error: Optional[BaseException] = None
|
|
84
|
+
) -> Optional[str]:
|
|
85
|
+
"""Saves a structured JSON log of all failed requests, successes, and timestamps."""
|
|
86
|
+
if fatal_error:
|
|
87
|
+
fatal_entry = {
|
|
88
|
+
"stage": "fatal_pipeline_error",
|
|
89
|
+
"error_type": type(fatal_error).__name__,
|
|
90
|
+
"error_message": str(fatal_error) or "Execution interrupted by user or system signal",
|
|
91
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
92
|
+
}
|
|
93
|
+
with self._errors_lock:
|
|
94
|
+
self.errors.append(fatal_entry)
|
|
95
|
+
|
|
96
|
+
if not self.errors:
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
dir_path = os.path.dirname(os.path.abspath(errors_path))
|
|
100
|
+
if dir_path:
|
|
101
|
+
os.makedirs(dir_path, exist_ok=True)
|
|
102
|
+
|
|
103
|
+
end_dt = self.completed_at or datetime.now(timezone.utc)
|
|
104
|
+
duration = (end_dt - self.started_at).total_seconds() if self.started_at else None
|
|
105
|
+
|
|
106
|
+
payload = {
|
|
107
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
108
|
+
"status": "failed" if fatal_error else "completed_with_errors",
|
|
109
|
+
"execution": {
|
|
110
|
+
"started_at": self.started_at.isoformat() if self.started_at else None,
|
|
111
|
+
"completed_at": end_dt.isoformat(),
|
|
112
|
+
"duration_seconds": round(duration, 2) if duration is not None else None,
|
|
113
|
+
"source": getattr(self.source, "name", "unknown"),
|
|
114
|
+
},
|
|
115
|
+
"summary": {
|
|
116
|
+
"total_errors": len(self.errors),
|
|
117
|
+
"total_posts_succeeded": len(self.succeeded_posts),
|
|
118
|
+
"total_comments_succeeded": self.succeeded_comments,
|
|
119
|
+
},
|
|
120
|
+
"succeeded": {
|
|
121
|
+
"posts_count": len(self.succeeded_posts),
|
|
122
|
+
"comments_count": self.succeeded_comments,
|
|
123
|
+
"subreddits": list(self.targeted_subreddits),
|
|
124
|
+
"post_ids": self.succeeded_posts[:1000],
|
|
125
|
+
},
|
|
126
|
+
"failures": list(self.errors),
|
|
127
|
+
}
|
|
128
|
+
with open(errors_path, "w", encoding="utf-8") as f:
|
|
129
|
+
json.dump(payload, f, indent=2)
|
|
130
|
+
|
|
131
|
+
print(f"\n[!] {len(self.errors)} error(s) occurred during execution.")
|
|
132
|
+
print(f" Refer to '{errors_path}' to see what failed, what succeeded, and full timestamps.")
|
|
133
|
+
return errors_path
|
|
134
|
+
|
|
135
|
+
def _generate_shards(self, from_dt: Union[datetime, str], to_dt: Optional[Union[datetime, str]]) -> Tuple[list, datetime, datetime]:
|
|
136
|
+
end = to_utc_datetime(to_dt) or datetime.now(timezone.utc)
|
|
137
|
+
start = to_utc_datetime(from_dt)
|
|
138
|
+
if not start or start >= end:
|
|
139
|
+
raise ValueError(f"Invalid range: from_dt ({start}) must precede to_dt ({end})")
|
|
140
|
+
|
|
141
|
+
shards, cur = [], end
|
|
142
|
+
while cur > start:
|
|
143
|
+
prev = max(cur - timedelta(days=self.shard_days), start)
|
|
144
|
+
shards.append((int(prev.timestamp()), int(cur.timestamp())))
|
|
145
|
+
cur = prev
|
|
146
|
+
return shards, start, end
|
|
147
|
+
|
|
148
|
+
def _scan_shard(self, subreddit: str, after_ts: int, before_ts: int, streamer: Storage, seen: set, lock: threading.Lock) -> list[str]:
|
|
149
|
+
matched = []
|
|
150
|
+
reached_end = False
|
|
151
|
+
try:
|
|
152
|
+
with requests.Session() as session:
|
|
153
|
+
for batch in self.source.fetch_posts_shard(subreddit, after_ts, before_ts, fields=self.filters.required_post_fields, session=session):
|
|
154
|
+
rows = []
|
|
155
|
+
for raw in batch:
|
|
156
|
+
try:
|
|
157
|
+
if not isinstance(raw, dict):
|
|
158
|
+
continue
|
|
159
|
+
if int(float(raw.get("created_utc", 0))) < after_ts:
|
|
160
|
+
reached_end = True
|
|
161
|
+
break
|
|
162
|
+
post = Post(raw)
|
|
163
|
+
if "subreddit" not in post._data and subreddit:
|
|
164
|
+
post._data["subreddit"] = subreddit
|
|
165
|
+
if post.id not in seen and self.filters.is_valid_post(post):
|
|
166
|
+
with lock:
|
|
167
|
+
if post.id not in seen:
|
|
168
|
+
seen.add(post.id)
|
|
169
|
+
rows.append(post.to_dict())
|
|
170
|
+
matched.append(post.id)
|
|
171
|
+
except UnretrievedFieldError:
|
|
172
|
+
raise
|
|
173
|
+
except Exception:
|
|
174
|
+
continue
|
|
175
|
+
if rows:
|
|
176
|
+
with lock:
|
|
177
|
+
streamer.write_batch(rows)
|
|
178
|
+
if reached_end:
|
|
179
|
+
break
|
|
180
|
+
except Exception as e:
|
|
181
|
+
error_entry = {
|
|
182
|
+
"stage": "post_shard",
|
|
183
|
+
"subreddit": subreddit,
|
|
184
|
+
"time_window": {
|
|
185
|
+
"after_ts": after_ts,
|
|
186
|
+
"before_ts": before_ts,
|
|
187
|
+
"after_utc": datetime.fromtimestamp(after_ts, tz=timezone.utc).isoformat(),
|
|
188
|
+
"before_utc": datetime.fromtimestamp(before_ts, tz=timezone.utc).isoformat(),
|
|
189
|
+
},
|
|
190
|
+
"error_type": type(e).__name__,
|
|
191
|
+
"error_message": str(e),
|
|
192
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
193
|
+
}
|
|
194
|
+
with self._errors_lock:
|
|
195
|
+
self.errors.append(error_entry)
|
|
196
|
+
print(f"[!] Warning: Shard [{after_ts}, {before_ts}] for r/{subreddit} encountered an error: {e}")
|
|
197
|
+
return matched
|
|
198
|
+
|
|
199
|
+
def scrape_posts(
|
|
200
|
+
self,
|
|
201
|
+
subreddits: Union[str, List[str]],
|
|
202
|
+
from_dt: Union[datetime, str, int, float],
|
|
203
|
+
to_dt: Optional[Union[datetime, str, int, float]] = None,
|
|
204
|
+
output_path: str = "data/curated_posts.parquet",
|
|
205
|
+
storage_cls: Optional[Type[Storage]] = None,
|
|
206
|
+
errors_output: str = "data/errors.json",
|
|
207
|
+
_in_pipeline: bool = False,
|
|
208
|
+
) -> list[str]:
|
|
209
|
+
subs = [subreddits] if isinstance(subreddits, str) else list(subreddits)
|
|
210
|
+
if not subs:
|
|
211
|
+
raise ValueError("At least one subreddit must be specified.")
|
|
212
|
+
if not _in_pipeline:
|
|
213
|
+
self.started_at = datetime.now(timezone.utc)
|
|
214
|
+
self.targeted_subreddits = subs
|
|
215
|
+
self.succeeded_posts = []
|
|
216
|
+
|
|
217
|
+
if os.path.exists(output_path):
|
|
218
|
+
os.remove(output_path)
|
|
219
|
+
|
|
220
|
+
fatal_exc = None
|
|
221
|
+
unique_ids = []
|
|
222
|
+
try:
|
|
223
|
+
shards, start_dt, end_dt = self._generate_shards(from_dt, to_dt)
|
|
224
|
+
seen, lock, all_matched = set(), threading.Lock(), []
|
|
225
|
+
|
|
226
|
+
print(f"[*] Post scan using [{self.source.name}] across {len(subs)} subreddits ({start_dt:%Y-%m-%d} to {end_dt:%Y-%m-%d})")
|
|
227
|
+
with (storage_cls or self.storage_cls)(output_path) as streamer:
|
|
228
|
+
for sub in subs:
|
|
229
|
+
executor = ThreadPoolExecutor(max_workers=self.post_workers)
|
|
230
|
+
try:
|
|
231
|
+
futures = [executor.submit(self._scan_shard, sub, a, b, streamer, seen, lock) for a, b in shards]
|
|
232
|
+
for f in tqdm(as_completed(futures), total=len(futures), desc=f"r/{sub}"):
|
|
233
|
+
all_matched.extend(f.result())
|
|
234
|
+
finally:
|
|
235
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
236
|
+
|
|
237
|
+
unique_ids = list(dict.fromkeys(all_matched))
|
|
238
|
+
self.succeeded_posts = unique_ids
|
|
239
|
+
if unique_ids:
|
|
240
|
+
print(f"[✓] Saved {len(unique_ids):,} posts to {output_path}")
|
|
241
|
+
return unique_ids
|
|
242
|
+
except BaseException as e:
|
|
243
|
+
fatal_exc = e
|
|
244
|
+
raise
|
|
245
|
+
finally:
|
|
246
|
+
if not _in_pipeline:
|
|
247
|
+
self.completed_at = datetime.now(timezone.utc)
|
|
248
|
+
self.save_errors_log(errors_path=errors_output, fatal_error=fatal_exc)
|
|
249
|
+
|
|
250
|
+
async def _scrape_comments_async(
|
|
251
|
+
self,
|
|
252
|
+
post_ids: list[str],
|
|
253
|
+
output_path: str,
|
|
254
|
+
chunk_size: int = 500,
|
|
255
|
+
storage_cls: Optional[Type[Storage]] = None,
|
|
256
|
+
errors_output: str = "data/errors.json",
|
|
257
|
+
_in_pipeline: bool = False,
|
|
258
|
+
) -> int:
|
|
259
|
+
if not _in_pipeline and self.started_at is None:
|
|
260
|
+
self.started_at = datetime.now(timezone.utc)
|
|
261
|
+
|
|
262
|
+
if os.path.exists(output_path):
|
|
263
|
+
os.remove(output_path)
|
|
264
|
+
|
|
265
|
+
sem = asyncio.Semaphore(self.comment_workers)
|
|
266
|
+
total_saved, buffer = 0, []
|
|
267
|
+
fatal_exc = None
|
|
268
|
+
|
|
269
|
+
async def _fetch(client: httpx.AsyncClient, pid: str):
|
|
270
|
+
try:
|
|
271
|
+
async with sem:
|
|
272
|
+
cid, raw, success = await self.source.fetch_comments_for_post(client, pid, fields=self.filters.required_comment_fields)
|
|
273
|
+
if not success:
|
|
274
|
+
error_entry = {
|
|
275
|
+
"stage": "comment_fetch",
|
|
276
|
+
"post_id": pid,
|
|
277
|
+
"error_type": "FetchFailed",
|
|
278
|
+
"error_message": "Data source reported failure (all retries exhausted)",
|
|
279
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
280
|
+
}
|
|
281
|
+
with self._errors_lock:
|
|
282
|
+
self.errors.append(error_entry)
|
|
283
|
+
return [c.to_dict() for raw_c in (raw or []) if self.filters.is_valid_comment(c := Comment(raw_c, cid))]
|
|
284
|
+
except UnretrievedFieldError:
|
|
285
|
+
raise
|
|
286
|
+
except Exception as e:
|
|
287
|
+
error_entry = {
|
|
288
|
+
"stage": "comment_fetch",
|
|
289
|
+
"post_id": pid,
|
|
290
|
+
"error_type": type(e).__name__,
|
|
291
|
+
"error_message": str(e),
|
|
292
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
293
|
+
}
|
|
294
|
+
with self._errors_lock:
|
|
295
|
+
self.errors.append(error_entry)
|
|
296
|
+
return []
|
|
297
|
+
|
|
298
|
+
try:
|
|
299
|
+
print(f"[*] Fetching comments using [{self.source.name}] for {len(post_ids):,} posts...")
|
|
300
|
+
max_conn = max(self.comment_workers * 2, 20)
|
|
301
|
+
with (storage_cls or self.storage_cls)(output_path) as streamer:
|
|
302
|
+
async with httpx.AsyncClient(limits=httpx.Limits(max_keepalive_connections=max_conn, max_connections=max_conn)) as client:
|
|
303
|
+
tasks = [_fetch(client, pid) for pid in post_ids]
|
|
304
|
+
for fut in async_tqdm.as_completed(tasks, total=len(tasks), desc="Comments"):
|
|
305
|
+
comments = await fut
|
|
306
|
+
if comments:
|
|
307
|
+
buffer.extend(comments)
|
|
308
|
+
if len(buffer) >= chunk_size:
|
|
309
|
+
await asyncio.to_thread(streamer.write_batch, list(buffer))
|
|
310
|
+
total_saved += len(buffer)
|
|
311
|
+
buffer.clear()
|
|
312
|
+
if buffer:
|
|
313
|
+
await asyncio.to_thread(streamer.write_batch, list(buffer))
|
|
314
|
+
total_saved += len(buffer)
|
|
315
|
+
|
|
316
|
+
self.succeeded_comments += total_saved
|
|
317
|
+
if total_saved > 0:
|
|
318
|
+
print(f"[✓] Saved {total_saved:,} comments to {output_path}")
|
|
319
|
+
return total_saved
|
|
320
|
+
except BaseException as e:
|
|
321
|
+
fatal_exc = e
|
|
322
|
+
raise
|
|
323
|
+
finally:
|
|
324
|
+
if not _in_pipeline:
|
|
325
|
+
self.completed_at = datetime.now(timezone.utc)
|
|
326
|
+
self.save_errors_log(errors_path=errors_output, fatal_error=fatal_exc)
|
|
327
|
+
|
|
328
|
+
def scrape_comments(
|
|
329
|
+
self,
|
|
330
|
+
post_ids: Union[str, list[str]],
|
|
331
|
+
output_path: str = "data/curated_comments.parquet",
|
|
332
|
+
chunk_size: int = 500,
|
|
333
|
+
storage_cls: Optional[Type[Storage]] = None,
|
|
334
|
+
errors_output: str = "data/errors.json",
|
|
335
|
+
_in_pipeline: bool = False,
|
|
336
|
+
) -> int:
|
|
337
|
+
pids = [post_ids] if isinstance(post_ids, str) else list(dict.fromkeys(post_ids))
|
|
338
|
+
if not pids:
|
|
339
|
+
return 0
|
|
340
|
+
try:
|
|
341
|
+
loop = asyncio.get_running_loop()
|
|
342
|
+
except RuntimeError:
|
|
343
|
+
loop = None
|
|
344
|
+
|
|
345
|
+
coro = self._scrape_comments_async(
|
|
346
|
+
pids, output_path, chunk_size, storage_cls, errors_output=errors_output, _in_pipeline=_in_pipeline
|
|
347
|
+
)
|
|
348
|
+
if loop and loop.is_running():
|
|
349
|
+
import nest_asyncio
|
|
350
|
+
nest_asyncio.apply()
|
|
351
|
+
return loop.run_until_complete(coro)
|
|
352
|
+
return asyncio.run(coro)
|
|
353
|
+
|
|
354
|
+
def run(
|
|
355
|
+
self,
|
|
356
|
+
subreddits: Union[str, List[str]],
|
|
357
|
+
from_dt: Union[datetime, str],
|
|
358
|
+
to_dt: Optional[Union[datetime, str]] = None,
|
|
359
|
+
posts_output: str = "data/curated_posts.parquet",
|
|
360
|
+
comments_output: str = "data/curated_comments.parquet",
|
|
361
|
+
errors_output: str = "data/errors.json",
|
|
362
|
+
storage_cls: Optional[Type[Storage]] = None,
|
|
363
|
+
) -> tuple[list[str], int]:
|
|
364
|
+
subs = [subreddits] if isinstance(subreddits, str) else list(subreddits)
|
|
365
|
+
self.started_at = datetime.now(timezone.utc)
|
|
366
|
+
self.targeted_subreddits = subs
|
|
367
|
+
self.succeeded_posts = []
|
|
368
|
+
self.succeeded_comments = 0
|
|
369
|
+
fatal_exc = None
|
|
370
|
+
posts = []
|
|
371
|
+
comments = 0
|
|
372
|
+
|
|
373
|
+
try:
|
|
374
|
+
posts = self.scrape_posts(
|
|
375
|
+
subreddits=subs,
|
|
376
|
+
from_dt=from_dt,
|
|
377
|
+
to_dt=to_dt,
|
|
378
|
+
output_path=posts_output,
|
|
379
|
+
storage_cls=storage_cls,
|
|
380
|
+
errors_output=errors_output,
|
|
381
|
+
_in_pipeline=True,
|
|
382
|
+
)
|
|
383
|
+
if posts:
|
|
384
|
+
comments = self.scrape_comments(
|
|
385
|
+
post_ids=posts,
|
|
386
|
+
output_path=comments_output,
|
|
387
|
+
storage_cls=storage_cls,
|
|
388
|
+
errors_output=errors_output,
|
|
389
|
+
_in_pipeline=True,
|
|
390
|
+
)
|
|
391
|
+
return posts, comments
|
|
392
|
+
except BaseException as e:
|
|
393
|
+
fatal_exc = e
|
|
394
|
+
raise
|
|
395
|
+
finally:
|
|
396
|
+
self.completed_at = datetime.now(timezone.utc)
|
|
397
|
+
self.save_errors_log(errors_path=errors_output, fatal_error=fatal_exc)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Data sources subpackage for Reddit scraping.
|
|
3
|
+
"""
|
|
4
|
+
from deepsift.sources.base import Source
|
|
5
|
+
from deepsift.sources.arctic_shift import ArcticShiftSource
|
|
6
|
+
from deepsift.sources.registry import SOURCES, get_source, register_source
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"Source",
|
|
10
|
+
"ArcticShiftSource",
|
|
11
|
+
"SOURCES",
|
|
12
|
+
"get_source",
|
|
13
|
+
"register_source",
|
|
14
|
+
]
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Arctic Shift API data source implementation.
|
|
3
|
+
"""
|
|
4
|
+
import time
|
|
5
|
+
import asyncio
|
|
6
|
+
from typing import Iterator, List, Dict, Any, Optional, Tuple, Iterable
|
|
7
|
+
import requests
|
|
8
|
+
import httpx
|
|
9
|
+
from deepsift.sources.base import Source
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ArcticShiftSource(Source):
|
|
13
|
+
"""
|
|
14
|
+
Arctic Shift API data source implementation (https://arctic-shift.photon-reddit.com).
|
|
15
|
+
Supports server-side field projection via the `fields=` query parameter.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
BASE_URL = "https://arctic-shift.photon-reddit.com"
|
|
19
|
+
DEFAULT_USER_AGENT = "reddit-research-scraper/0.2.0"
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
base_url: str = BASE_URL,
|
|
24
|
+
timeout: float = 15.0,
|
|
25
|
+
rate_limit_delay: float = 0.3,
|
|
26
|
+
batch_size: int = 100,
|
|
27
|
+
user_agent: str = DEFAULT_USER_AGENT,
|
|
28
|
+
):
|
|
29
|
+
self.base_url = base_url.rstrip("/")
|
|
30
|
+
self.timeout = timeout
|
|
31
|
+
self.rate_limit_delay = rate_limit_delay
|
|
32
|
+
self.batch_size = batch_size
|
|
33
|
+
self.user_agent = user_agent
|
|
34
|
+
self.headers = {"User-Agent": self.user_agent}
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def name(self) -> str:
|
|
38
|
+
return "arctic_shift"
|
|
39
|
+
|
|
40
|
+
def fetch_posts_shard(
|
|
41
|
+
self,
|
|
42
|
+
subreddit: str,
|
|
43
|
+
after_ts: int,
|
|
44
|
+
before_ts: int,
|
|
45
|
+
fields: Optional[Iterable[str]] = None,
|
|
46
|
+
session: Optional[requests.Session] = None,
|
|
47
|
+
) -> Iterator[List[Dict[str, Any]]]:
|
|
48
|
+
"""
|
|
49
|
+
Paginates backwards through Arctic Shift posts in time window [after_ts, before_ts].
|
|
50
|
+
If fields is specified, requests only those fields from the server.
|
|
51
|
+
"""
|
|
52
|
+
close_session = session is None
|
|
53
|
+
s = session or requests.Session()
|
|
54
|
+
current_before = before_ts
|
|
55
|
+
projected = self.resolve_post_fields(fields)
|
|
56
|
+
fields_str = ",".join(sorted(projected)) if projected else None
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
while current_before > after_ts:
|
|
60
|
+
params = {
|
|
61
|
+
"subreddit": subreddit,
|
|
62
|
+
"limit": self.batch_size,
|
|
63
|
+
"before": current_before,
|
|
64
|
+
"after": after_ts,
|
|
65
|
+
}
|
|
66
|
+
if fields_str:
|
|
67
|
+
params["fields"] = fields_str
|
|
68
|
+
|
|
69
|
+
data = None
|
|
70
|
+
backoff = 2.0
|
|
71
|
+
for _ in range(5):
|
|
72
|
+
try:
|
|
73
|
+
res = s.get(
|
|
74
|
+
f"{self.base_url}/api/posts/search",
|
|
75
|
+
params=params,
|
|
76
|
+
headers=self.headers,
|
|
77
|
+
timeout=self.timeout,
|
|
78
|
+
)
|
|
79
|
+
if res.status_code == 200:
|
|
80
|
+
data = res.json().get("data", [])
|
|
81
|
+
break
|
|
82
|
+
if res.status_code == 429:
|
|
83
|
+
retry_after = float(res.headers.get("Retry-After", backoff))
|
|
84
|
+
time.sleep(min(retry_after, 30.0))
|
|
85
|
+
else:
|
|
86
|
+
time.sleep(backoff)
|
|
87
|
+
backoff *= 1.8
|
|
88
|
+
except requests.RequestException:
|
|
89
|
+
time.sleep(backoff)
|
|
90
|
+
backoff *= 1.8
|
|
91
|
+
|
|
92
|
+
if data is None:
|
|
93
|
+
raise ConnectionError(
|
|
94
|
+
f"Failed to fetch posts shard for r/{subreddit} in window [{after_ts}, {current_before}] "
|
|
95
|
+
f"after 5 retries (timeout or server error)."
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
if not data:
|
|
99
|
+
break
|
|
100
|
+
|
|
101
|
+
yield data
|
|
102
|
+
|
|
103
|
+
oldest = min(int(float(p.get("created_utc", current_before))) for p in data)
|
|
104
|
+
if oldest <= after_ts:
|
|
105
|
+
break
|
|
106
|
+
current_before = current_before - 1 if oldest >= current_before else oldest
|
|
107
|
+
time.sleep(self.rate_limit_delay)
|
|
108
|
+
finally:
|
|
109
|
+
if close_session:
|
|
110
|
+
s.close()
|
|
111
|
+
|
|
112
|
+
async def fetch_comments_for_post(
|
|
113
|
+
self,
|
|
114
|
+
client: httpx.AsyncClient,
|
|
115
|
+
post_id: str,
|
|
116
|
+
fields: Optional[Iterable[str]] = None,
|
|
117
|
+
max_retries: int = 5,
|
|
118
|
+
) -> Tuple[str, List[Dict[str, Any]], bool]:
|
|
119
|
+
"""
|
|
120
|
+
Fetches comments from Arctic Shift.
|
|
121
|
+
Tree endpoint is attempted first; falls back to search endpoint with fields projection if needed.
|
|
122
|
+
"""
|
|
123
|
+
clean_id = str(post_id).replace("t3_", "")
|
|
124
|
+
tree_url = f"{self.base_url}/api/comments/tree"
|
|
125
|
+
params = {"link_id": f"t3_{clean_id}"}
|
|
126
|
+
backoff = 2.0
|
|
127
|
+
|
|
128
|
+
for _ in range(max_retries):
|
|
129
|
+
try:
|
|
130
|
+
resp = await client.get(
|
|
131
|
+
tree_url,
|
|
132
|
+
params=params,
|
|
133
|
+
headers=self.headers,
|
|
134
|
+
timeout=self.timeout,
|
|
135
|
+
)
|
|
136
|
+
if resp.status_code == 200:
|
|
137
|
+
json_data = resp.json()
|
|
138
|
+
raw = json_data.get("data")
|
|
139
|
+
if raw is None and "comments" in json_data:
|
|
140
|
+
raw = json_data["comments"]
|
|
141
|
+
cleaned = self._unpack_nodes(raw if raw is not None else json_data, clean_id)
|
|
142
|
+
if not cleaned and raw is None:
|
|
143
|
+
cleaned, _ = await self._fetch_search_fallback(client, clean_id, fields=fields)
|
|
144
|
+
return clean_id, cleaned, True
|
|
145
|
+
|
|
146
|
+
if resp.status_code == 404:
|
|
147
|
+
break
|
|
148
|
+
|
|
149
|
+
await asyncio.sleep(backoff)
|
|
150
|
+
backoff *= 1.6
|
|
151
|
+
except Exception:
|
|
152
|
+
await asyncio.sleep(backoff)
|
|
153
|
+
backoff *= 1.6
|
|
154
|
+
|
|
155
|
+
fallback, success = await self._fetch_search_fallback(client, clean_id, fields=fields)
|
|
156
|
+
return clean_id, fallback, success
|
|
157
|
+
|
|
158
|
+
def _unpack_nodes(self, nodes: Any, post_id: str) -> List[Dict[str, Any]]:
|
|
159
|
+
"""
|
|
160
|
+
Flattens nested comment tree into raw dictionaries using a stack.
|
|
161
|
+
Avoids recursion limits for deeply nested reply threads.
|
|
162
|
+
"""
|
|
163
|
+
flat = []
|
|
164
|
+
if not isinstance(nodes, list):
|
|
165
|
+
nodes = [nodes] if isinstance(nodes, dict) else []
|
|
166
|
+
|
|
167
|
+
stack = list(reversed(nodes))
|
|
168
|
+
while stack:
|
|
169
|
+
node = stack.pop()
|
|
170
|
+
if not isinstance(node, dict):
|
|
171
|
+
continue
|
|
172
|
+
|
|
173
|
+
raw_comment = None
|
|
174
|
+
if "comment" in node and isinstance(node["comment"], dict):
|
|
175
|
+
raw_comment = node["comment"]
|
|
176
|
+
elif "data" in node and isinstance(node["data"], dict):
|
|
177
|
+
raw_comment = node["data"]
|
|
178
|
+
elif "body" in node or "id" in node or "author" in node:
|
|
179
|
+
raw_comment = node
|
|
180
|
+
|
|
181
|
+
replies = node.get("replies") or node.get("children") or []
|
|
182
|
+
if raw_comment and not replies:
|
|
183
|
+
replies = raw_comment.get("replies") or raw_comment.get("children") or []
|
|
184
|
+
|
|
185
|
+
if isinstance(replies, dict):
|
|
186
|
+
replies = replies.get("data", {}).get("children", []) if "data" in replies else list(replies.values())
|
|
187
|
+
if isinstance(replies, list) and replies:
|
|
188
|
+
for child in reversed(replies):
|
|
189
|
+
stack.append(child)
|
|
190
|
+
|
|
191
|
+
if not raw_comment:
|
|
192
|
+
continue
|
|
193
|
+
|
|
194
|
+
comment_dict = dict(raw_comment)
|
|
195
|
+
if "post_id" not in comment_dict:
|
|
196
|
+
comment_dict["post_id"] = post_id
|
|
197
|
+
flat.append(comment_dict)
|
|
198
|
+
|
|
199
|
+
return flat
|
|
200
|
+
|
|
201
|
+
async def _fetch_search_fallback(
|
|
202
|
+
self, client: httpx.AsyncClient, clean_id: str, fields: Optional[Iterable[str]] = None
|
|
203
|
+
) -> Tuple[List[Dict[str, Any]], bool]:
|
|
204
|
+
"""Fallback to /api/comments/search with field projection and backward pagination."""
|
|
205
|
+
all_comments = []
|
|
206
|
+
seen_ids = set()
|
|
207
|
+
projected = self.resolve_comment_fields(fields)
|
|
208
|
+
query_fields = (set(projected) | {"id", "created_utc"}) if projected else None
|
|
209
|
+
fields_str = ",".join(sorted(query_fields)) if query_fields else None
|
|
210
|
+
current_before = None
|
|
211
|
+
ever_succeeded = False
|
|
212
|
+
|
|
213
|
+
for _ in range(25):
|
|
214
|
+
params = {"link_id": f"t3_{clean_id}", "limit": self.batch_size}
|
|
215
|
+
if fields_str:
|
|
216
|
+
params["fields"] = fields_str
|
|
217
|
+
if current_before is not None:
|
|
218
|
+
params["before"] = current_before
|
|
219
|
+
|
|
220
|
+
raw = None
|
|
221
|
+
for attempt in range(2):
|
|
222
|
+
try:
|
|
223
|
+
resp = await client.get(
|
|
224
|
+
f"{self.base_url}/api/comments/search",
|
|
225
|
+
params=params,
|
|
226
|
+
headers=self.headers,
|
|
227
|
+
timeout=self.timeout,
|
|
228
|
+
)
|
|
229
|
+
if resp.status_code == 200:
|
|
230
|
+
raw = resp.json().get("data", [])
|
|
231
|
+
ever_succeeded = True
|
|
232
|
+
break
|
|
233
|
+
if resp.status_code == 429:
|
|
234
|
+
await asyncio.sleep(1.0)
|
|
235
|
+
except Exception:
|
|
236
|
+
if attempt == 0:
|
|
237
|
+
await asyncio.sleep(0.5)
|
|
238
|
+
|
|
239
|
+
if raw is None or not raw:
|
|
240
|
+
break
|
|
241
|
+
|
|
242
|
+
for item in raw:
|
|
243
|
+
cid = str(item.get("id", ""))
|
|
244
|
+
if cid and cid in seen_ids:
|
|
245
|
+
continue
|
|
246
|
+
if cid:
|
|
247
|
+
seen_ids.add(cid)
|
|
248
|
+
if "post_id" not in item:
|
|
249
|
+
item["post_id"] = clean_id
|
|
250
|
+
all_comments.append(item)
|
|
251
|
+
if len(raw) < self.batch_size:
|
|
252
|
+
break
|
|
253
|
+
oldest_ts = raw[-1].get("created_utc")
|
|
254
|
+
if oldest_ts is not None:
|
|
255
|
+
try:
|
|
256
|
+
oldest_int = int(float(oldest_ts))
|
|
257
|
+
current_before = oldest_int - 1 if (current_before is not None and oldest_int >= current_before) else oldest_int
|
|
258
|
+
except (ValueError, TypeError):
|
|
259
|
+
break
|
|
260
|
+
else:
|
|
261
|
+
break
|
|
262
|
+
|
|
263
|
+
return all_comments, ever_succeeded
|