deepsift 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deepsift-0.2.0 → deepsift-0.2.1}/PKG-INFO +1 -1
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/scraper.py +10 -1
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/arctic_shift.py +11 -4
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/PKG-INFO +1 -1
- {deepsift-0.2.0 → deepsift-0.2.1}/pyproject.toml +1 -1
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_scraper_dummy.py +51 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/LICENSE +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/README.md +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/__init__.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/__main__.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/cli.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/filters.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/__init__.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/base.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/comment.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/enums.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/models/post.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/__init__.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/base.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/dummy.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/sources/registry.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/storage/__init__.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/storage/base.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift/storage/parquet.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/SOURCES.txt +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/dependency_links.txt +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/entry_points.txt +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/requires.txt +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/deepsift.egg-info/top_level.txt +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/setup.cfg +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_cli.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_evaluation.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_filters.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_models.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_scraper_live.py +0 -0
- {deepsift-0.2.0 → deepsift-0.2.1}/tests/test_storage.py +0 -0
|
@@ -280,7 +280,16 @@ class RedditScraper:
|
|
|
280
280
|
}
|
|
281
281
|
with self._errors_lock:
|
|
282
282
|
self.errors.append(error_entry)
|
|
283
|
-
|
|
283
|
+
valid_comments = []
|
|
284
|
+
for raw_c in (raw or []):
|
|
285
|
+
if not isinstance(raw_c, dict) or raw_c.get("kind") == "more":
|
|
286
|
+
continue
|
|
287
|
+
if "count" in raw_c and "children" in raw_c and "body" not in raw_c:
|
|
288
|
+
continue
|
|
289
|
+
c = Comment(raw_c, cid)
|
|
290
|
+
if self.filters.is_valid_comment(c):
|
|
291
|
+
valid_comments.append(c.to_dict())
|
|
292
|
+
return valid_comments
|
|
284
293
|
except UnretrievedFieldError:
|
|
285
294
|
raise
|
|
286
295
|
except Exception as e:
|
|
@@ -170,6 +170,9 @@ class ArcticShiftSource(Source):
|
|
|
170
170
|
if not isinstance(node, dict):
|
|
171
171
|
continue
|
|
172
172
|
|
|
173
|
+
if node.get("kind") == "more":
|
|
174
|
+
continue
|
|
175
|
+
|
|
173
176
|
raw_comment = None
|
|
174
177
|
if "comment" in node and isinstance(node["comment"], dict):
|
|
175
178
|
raw_comment = node["comment"]
|
|
@@ -178,6 +181,12 @@ class ArcticShiftSource(Source):
|
|
|
178
181
|
elif "body" in node or "id" in node or "author" in node:
|
|
179
182
|
raw_comment = node
|
|
180
183
|
|
|
184
|
+
if not raw_comment or raw_comment.get("kind") == "more":
|
|
185
|
+
continue
|
|
186
|
+
|
|
187
|
+
if "count" in raw_comment and "children" in raw_comment and "body" not in raw_comment:
|
|
188
|
+
continue
|
|
189
|
+
|
|
181
190
|
replies = node.get("replies") or node.get("children") or []
|
|
182
191
|
if raw_comment and not replies:
|
|
183
192
|
replies = raw_comment.get("replies") or raw_comment.get("children") or []
|
|
@@ -186,10 +195,8 @@ class ArcticShiftSource(Source):
|
|
|
186
195
|
replies = replies.get("data", {}).get("children", []) if "data" in replies else list(replies.values())
|
|
187
196
|
if isinstance(replies, list) and replies:
|
|
188
197
|
for child in reversed(replies):
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
if not raw_comment:
|
|
192
|
-
continue
|
|
198
|
+
if isinstance(child, dict):
|
|
199
|
+
stack.append(child)
|
|
193
200
|
|
|
194
201
|
comment_dict = dict(raw_comment)
|
|
195
202
|
if "post_id" not in comment_dict:
|
|
@@ -272,3 +272,54 @@ def test_dummy_source_custom_shard_counts(tmp_path):
|
|
|
272
272
|
)
|
|
273
273
|
assert len(posts) == 14 + 28
|
|
274
274
|
|
|
275
|
+
|
|
276
|
+
def test_unpack_nodes_filters_more_stubs():
|
|
277
|
+
from deepsift.sources.arctic_shift import ArcticShiftSource
|
|
278
|
+
source = ArcticShiftSource()
|
|
279
|
+
|
|
280
|
+
mock_tree = [
|
|
281
|
+
{
|
|
282
|
+
"kind": "t1",
|
|
283
|
+
"data": {
|
|
284
|
+
"id": "c101",
|
|
285
|
+
"parent_id": "t3_post1",
|
|
286
|
+
"body": "This is a real comment",
|
|
287
|
+
"score": 10,
|
|
288
|
+
"replies": {
|
|
289
|
+
"data": {
|
|
290
|
+
"children": [
|
|
291
|
+
{
|
|
292
|
+
"kind": "more",
|
|
293
|
+
"data": {
|
|
294
|
+
"count": 5,
|
|
295
|
+
"name": "t1_c102",
|
|
296
|
+
"id": "c102",
|
|
297
|
+
"parent_id": "t1_c101",
|
|
298
|
+
"depth": 2,
|
|
299
|
+
"children": ["c103", "c104"],
|
|
300
|
+
},
|
|
301
|
+
}
|
|
302
|
+
]
|
|
303
|
+
}
|
|
304
|
+
},
|
|
305
|
+
},
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
"kind": "more",
|
|
309
|
+
"data": {
|
|
310
|
+
"count": 3,
|
|
311
|
+
"name": "t1_top_more",
|
|
312
|
+
"id": "top_more",
|
|
313
|
+
"parent_id": "t3_post1",
|
|
314
|
+
"children": ["c105"],
|
|
315
|
+
},
|
|
316
|
+
},
|
|
317
|
+
]
|
|
318
|
+
|
|
319
|
+
unpacked = source._unpack_nodes(mock_tree, "post1")
|
|
320
|
+
assert len(unpacked) == 1
|
|
321
|
+
assert unpacked[0]["id"] == "c101"
|
|
322
|
+
assert unpacked[0]["body"] == "This is a real comment"
|
|
323
|
+
assert unpacked[0]["post_id"] == "post1"
|
|
324
|
+
|
|
325
|
+
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|