deepsift 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {deepsift-0.2.0 → deepsift-0.2.1}/PKG-INFO +1 -1
  2. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/scraper.py +10 -1
  3. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/arctic_shift.py +11 -4
  4. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/PKG-INFO +1 -1
  5. {deepsift-0.2.0 → deepsift-0.2.1}/pyproject.toml +1 -1
  6. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_scraper_dummy.py +51 -0
  7. {deepsift-0.2.0 → deepsift-0.2.1}/LICENSE +0 -0
  8. {deepsift-0.2.0 → deepsift-0.2.1}/README.md +0 -0
  9. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/__init__.py +0 -0
  10. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/__main__.py +0 -0
  11. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/cli.py +0 -0
  12. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/filters.py +0 -0
  13. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/__init__.py +0 -0
  14. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/base.py +0 -0
  15. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/comment.py +0 -0
  16. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/enums.py +0 -0
  17. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/post.py +0 -0
  18. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/__init__.py +0 -0
  19. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/base.py +0 -0
  20. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/dummy.py +0 -0
  21. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/registry.py +0 -0
  22. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/storage/__init__.py +0 -0
  23. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/storage/base.py +0 -0
  24. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/storage/parquet.py +0 -0
  25. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/SOURCES.txt +0 -0
  26. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/dependency_links.txt +0 -0
  27. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/entry_points.txt +0 -0
  28. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/requires.txt +0 -0
  29. {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/top_level.txt +0 -0
  30. {deepsift-0.2.0 → deepsift-0.2.1}/setup.cfg +0 -0
  31. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_cli.py +0 -0
  32. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_evaluation.py +0 -0
  33. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_filters.py +0 -0
  34. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_models.py +0 -0
  35. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_scraper_live.py +0 -0
  36. {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_storage.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: deepsift
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
5
  Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
6
  License-Expression: MIT
@@ -280,7 +280,16 @@ class RedditScraper:
280
280
  }
281
281
  with self._errors_lock:
282
282
  self.errors.append(error_entry)
283
- return [c.to_dict() for raw_c in (raw or []) if self.filters.is_valid_comment(c := Comment(raw_c, cid))]
283
+ valid_comments = []
284
+ for raw_c in (raw or []):
285
+ if not isinstance(raw_c, dict) or raw_c.get("kind") == "more":
286
+ continue
287
+ if "count" in raw_c and "children" in raw_c and "body" not in raw_c:
288
+ continue
289
+ c = Comment(raw_c, cid)
290
+ if self.filters.is_valid_comment(c):
291
+ valid_comments.append(c.to_dict())
292
+ return valid_comments
284
293
  except UnretrievedFieldError:
285
294
  raise
286
295
  except Exception as e:
@@ -170,6 +170,9 @@ class ArcticShiftSource(Source):
170
170
  if not isinstance(node, dict):
171
171
  continue
172
172
 
173
+ if node.get("kind") == "more":
174
+ continue
175
+
173
176
  raw_comment = None
174
177
  if "comment" in node and isinstance(node["comment"], dict):
175
178
  raw_comment = node["comment"]
@@ -178,6 +181,12 @@ class ArcticShiftSource(Source):
178
181
  elif "body" in node or "id" in node or "author" in node:
179
182
  raw_comment = node
180
183
 
184
+ if not raw_comment or raw_comment.get("kind") == "more":
185
+ continue
186
+
187
+ if "count" in raw_comment and "children" in raw_comment and "body" not in raw_comment:
188
+ continue
189
+
181
190
  replies = node.get("replies") or node.get("children") or []
182
191
  if raw_comment and not replies:
183
192
  replies = raw_comment.get("replies") or raw_comment.get("children") or []
@@ -186,10 +195,8 @@ class ArcticShiftSource(Source):
186
195
  replies = replies.get("data", {}).get("children", []) if "data" in replies else list(replies.values())
187
196
  if isinstance(replies, list) and replies:
188
197
  for child in reversed(replies):
189
- stack.append(child)
190
-
191
- if not raw_comment:
192
- continue
198
+ if isinstance(child, dict):
199
+ stack.append(child)
193
200
 
194
201
  comment_dict = dict(raw_comment)
195
202
  if "post_id" not in comment_dict:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: deepsift
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
5
  Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "deepsift"
7
- version = "0.2.0"
7
+ version = "0.2.1"
8
8
  description = "High-performance historical Reddit research toolkit and scraper for Arctic Shift."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -272,3 +272,54 @@ def test_dummy_source_custom_shard_counts(tmp_path):
272
272
  )
273
273
  assert len(posts) == 14 + 28
274
274
 
275
+
276
+ def test_unpack_nodes_filters_more_stubs():
277
+ from deepsift.sources.arctic_shift import ArcticShiftSource
278
+ source = ArcticShiftSource()
279
+
280
+ mock_tree = [
281
+ {
282
+ "kind": "t1",
283
+ "data": {
284
+ "id": "c101",
285
+ "parent_id": "t3_post1",
286
+ "body": "This is a real comment",
287
+ "score": 10,
288
+ "replies": {
289
+ "data": {
290
+ "children": [
291
+ {
292
+ "kind": "more",
293
+ "data": {
294
+ "count": 5,
295
+ "name": "t1_c102",
296
+ "id": "c102",
297
+ "parent_id": "t1_c101",
298
+ "depth": 2,
299
+ "children": ["c103", "c104"],
300
+ },
301
+ }
302
+ ]
303
+ }
304
+ },
305
+ },
306
+ },
307
+ {
308
+ "kind": "more",
309
+ "data": {
310
+ "count": 3,
311
+ "name": "t1_top_more",
312
+ "id": "top_more",
313
+ "parent_id": "t3_post1",
314
+ "children": ["c105"],
315
+ },
316
+ },
317
+ ]
318
+
319
+ unpacked = source._unpack_nodes(mock_tree, "post1")
320
+ assert len(unpacked) == 1
321
+ assert unpacked[0]["id"] == "c101"
322
+ assert unpacked[0]["body"] == "This is a real comment"
323
+ assert unpacked[0]["post_id"] == "post1"
324
+
325
+
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes