deepsift 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepsift/__init__.py +39 -0
- deepsift/__main__.py +8 -0
- deepsift/cli.py +171 -0
- deepsift/filters.py +111 -0
- deepsift/models/__init__.py +17 -0
- deepsift/models/base.py +100 -0
- deepsift/models/comment.py +38 -0
- deepsift/models/enums.py +243 -0
- deepsift/models/post.py +29 -0
- deepsift/scraper.py +397 -0
- deepsift/sources/__init__.py +14 -0
- deepsift/sources/arctic_shift.py +263 -0
- deepsift/sources/base.py +102 -0
- deepsift/sources/dummy.py +207 -0
- deepsift/sources/registry.py +24 -0
- deepsift/storage/__init__.py +10 -0
- deepsift/storage/base.py +37 -0
- deepsift/storage/parquet.py +86 -0
- deepsift-0.2.0.dist-info/METADATA +115 -0
- deepsift-0.2.0.dist-info/RECORD +24 -0
- deepsift-0.2.0.dist-info/WHEEL +5 -0
- deepsift-0.2.0.dist-info/entry_points.txt +3 -0
- deepsift-0.2.0.dist-info/licenses/LICENSE +21 -0
- deepsift-0.2.0.dist-info/top_level.txt +1 -0
deepsift/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DeepSift Library.
|
|
3
|
+
A high-performance research toolkit for historical Reddit data.
|
|
4
|
+
"""
|
|
5
|
+
from deepsift.models import (
|
|
6
|
+
Post,
|
|
7
|
+
Comment,
|
|
8
|
+
PostField,
|
|
9
|
+
CommentField,
|
|
10
|
+
FieldEnum,
|
|
11
|
+
UnretrievedFieldError,
|
|
12
|
+
)
|
|
13
|
+
from deepsift.filters import Filters, RequiredFieldError
|
|
14
|
+
from deepsift.sources import (
|
|
15
|
+
Source,
|
|
16
|
+
ArcticShiftSource,
|
|
17
|
+
get_source,
|
|
18
|
+
register_source,
|
|
19
|
+
)
|
|
20
|
+
from deepsift.storage import Storage, ParquetStorage
|
|
21
|
+
from deepsift.scraper import RedditScraper
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"PostField",
|
|
25
|
+
"CommentField",
|
|
26
|
+
"FieldEnum",
|
|
27
|
+
"Post",
|
|
28
|
+
"Comment",
|
|
29
|
+
"UnretrievedFieldError",
|
|
30
|
+
"RequiredFieldError",
|
|
31
|
+
"Filters",
|
|
32
|
+
"Source",
|
|
33
|
+
"ArcticShiftSource",
|
|
34
|
+
"get_source",
|
|
35
|
+
"register_source",
|
|
36
|
+
"RedditScraper",
|
|
37
|
+
"Storage",
|
|
38
|
+
"ParquetStorage",
|
|
39
|
+
]
|
deepsift/__main__.py
ADDED
deepsift/cli.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command line interface for the Reddit Research Scraper.
|
|
3
|
+
"""
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
import click
|
|
7
|
+
|
|
8
|
+
from deepsift.models import PostField, CommentField
|
|
9
|
+
from deepsift.filters import Filters
|
|
10
|
+
from deepsift.sources import ArcticShiftSource, get_source
|
|
11
|
+
from deepsift.scraper import RedditScraper
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.group()
|
|
15
|
+
def cli():
|
|
16
|
+
"""Reddit Research Scraper: High-performance historical Reddit data extractor."""
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@cli.command()
|
|
21
|
+
@click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
|
|
22
|
+
@click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
|
|
23
|
+
@click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
|
|
24
|
+
@click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
|
|
25
|
+
@click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep (validated against PostField enum).")
|
|
26
|
+
@click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop (validated against PostField enum).")
|
|
27
|
+
@click.option("--comment-include-fields", multiple=True, help="Important comment fields to keep.")
|
|
28
|
+
@click.option("--comment-exclude-fields", multiple=True, help="Unimportant comment fields to drop.")
|
|
29
|
+
@click.option("--posts-out", default="data/curated_posts.parquet", help="Parquet output destination for posts.")
|
|
30
|
+
@click.option("--comments-out", default="data/curated_comments.parquet", help="Parquet output destination for comments.")
|
|
31
|
+
@click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
|
|
32
|
+
@click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
|
|
33
|
+
@click.option("--comment-workers", default=5, type=int, help="Async comment concurrency workers.")
|
|
34
|
+
@click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
|
|
35
|
+
def run(
|
|
36
|
+
subreddits, from_dt, to_dt, shard_days,
|
|
37
|
+
include_fields, exclude_fields, comment_include_fields, comment_exclude_fields,
|
|
38
|
+
posts_out, comments_out, errors_out, post_workers, comment_workers, source
|
|
39
|
+
):
|
|
40
|
+
"""Run full pipeline: scrape posts, filter cohort, and extract comment trees."""
|
|
41
|
+
target_subs = list(subreddits)
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
filters = Filters(
|
|
45
|
+
post_filter=None,
|
|
46
|
+
comment_filter=None,
|
|
47
|
+
post_include_fields=list(include_fields) if include_fields else None,
|
|
48
|
+
post_exclude_fields=list(exclude_fields) if exclude_fields else None,
|
|
49
|
+
comment_include_fields=list(comment_include_fields) if comment_include_fields else None,
|
|
50
|
+
comment_exclude_fields=list(comment_exclude_fields) if comment_exclude_fields else None,
|
|
51
|
+
validate=True,
|
|
52
|
+
)
|
|
53
|
+
except ValueError as e:
|
|
54
|
+
click.secho(f"\n[!] Field Validation Error: {e}", fg="red", err=True)
|
|
55
|
+
sys.exit(1)
|
|
56
|
+
|
|
57
|
+
scraper = RedditScraper(
|
|
58
|
+
filters=filters,
|
|
59
|
+
source=get_source(source),
|
|
60
|
+
post_workers=post_workers,
|
|
61
|
+
comment_workers=comment_workers,
|
|
62
|
+
shard_days=shard_days,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
start_time = time.time()
|
|
66
|
+
click.echo(f"Reddit Research Scraper: {len(target_subs)} subreddits (shard_days={shard_days})")
|
|
67
|
+
click.echo(f"Window: from {from_dt} to {to_dt or 'now'}")
|
|
68
|
+
if filters.post_include_fields:
|
|
69
|
+
click.echo(f"Post Fields Kept: {sorted(list(filters.post_include_fields))}")
|
|
70
|
+
if filters.post_exclude_fields:
|
|
71
|
+
click.echo(f"Post Fields Dropped: {sorted(list(filters.post_exclude_fields))}")
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
posts, comments = scraper.run(
|
|
75
|
+
subreddits=target_subs,
|
|
76
|
+
from_dt=from_dt,
|
|
77
|
+
to_dt=to_dt,
|
|
78
|
+
posts_output=posts_out,
|
|
79
|
+
comments_output=comments_out,
|
|
80
|
+
errors_output=errors_out,
|
|
81
|
+
)
|
|
82
|
+
except Exception as e:
|
|
83
|
+
click.secho(f"\n[!] Pipeline Error: {e}", fg="red", err=True)
|
|
84
|
+
sys.exit(1)
|
|
85
|
+
|
|
86
|
+
elapsed = round(time.time() - start_time, 1)
|
|
87
|
+
click.echo(f"[✓] Pipeline complete in {elapsed}s")
|
|
88
|
+
click.echo(f" Posts: {len(posts):,} -> {posts_out}")
|
|
89
|
+
click.echo(f" Comments: {comments:,} -> {comments_out}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@cli.command()
|
|
93
|
+
@click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
|
|
94
|
+
@click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
|
|
95
|
+
@click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
|
|
96
|
+
@click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
|
|
97
|
+
@click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep.")
|
|
98
|
+
@click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop.")
|
|
99
|
+
@click.option("--output", "-o", default="data/curated_posts.parquet", help="Parquet output destination.")
|
|
100
|
+
@click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
|
|
101
|
+
@click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
|
|
102
|
+
@click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
|
|
103
|
+
def posts(subreddits, from_dt, to_dt, shard_days, include_fields, exclude_fields, output, errors_out, post_workers, source):
|
|
104
|
+
"""Scrape and filter Reddit posts between --from and --to dates."""
|
|
105
|
+
target_subs = list(subreddits)
|
|
106
|
+
try:
|
|
107
|
+
filters = Filters(
|
|
108
|
+
post_filter=None,
|
|
109
|
+
post_include_fields=list(include_fields) if include_fields else None,
|
|
110
|
+
post_exclude_fields=list(exclude_fields) if exclude_fields else None,
|
|
111
|
+
)
|
|
112
|
+
scraper = RedditScraper(
|
|
113
|
+
filters=filters,
|
|
114
|
+
source=get_source(source),
|
|
115
|
+
post_workers=post_workers,
|
|
116
|
+
shard_days=shard_days,
|
|
117
|
+
)
|
|
118
|
+
scraper.scrape_posts(
|
|
119
|
+
subreddits=target_subs,
|
|
120
|
+
from_dt=from_dt,
|
|
121
|
+
to_dt=to_dt,
|
|
122
|
+
output_path=output,
|
|
123
|
+
errors_output=errors_out,
|
|
124
|
+
)
|
|
125
|
+
except Exception as e:
|
|
126
|
+
click.secho(f"\n[!] Error: {e}", fg="red", err=True)
|
|
127
|
+
sys.exit(1)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@cli.command()
|
|
131
|
+
@click.option("--post-id", "-p", "post_ids", multiple=True, required=True, help="Post ID(s) to fetch comments for.")
|
|
132
|
+
@click.option("--include-fields", "-i", multiple=True, help="Important comment fields to keep.")
|
|
133
|
+
@click.option("--exclude-fields", "-e", multiple=True, help="Unimportant comment fields to drop.")
|
|
134
|
+
@click.option("--output", "-o", default="data/curated_comments.parquet", help="Parquet output destination.")
|
|
135
|
+
@click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
|
|
136
|
+
@click.option("--workers", "-w", default=5, type=int, help="Async concurrency limit.")
|
|
137
|
+
@click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
|
|
138
|
+
def comments(post_ids, include_fields, exclude_fields, output, errors_out, workers, source):
|
|
139
|
+
"""Scrape comment trees for given post IDs."""
|
|
140
|
+
try:
|
|
141
|
+
filters = Filters(
|
|
142
|
+
comment_filter=None,
|
|
143
|
+
comment_include_fields=list(include_fields) if include_fields else None,
|
|
144
|
+
comment_exclude_fields=list(exclude_fields) if exclude_fields else None,
|
|
145
|
+
)
|
|
146
|
+
scraper = RedditScraper(
|
|
147
|
+
filters=filters,
|
|
148
|
+
source=get_source(source),
|
|
149
|
+
comment_workers=workers,
|
|
150
|
+
)
|
|
151
|
+
scraper.scrape_comments(post_ids=list(post_ids), output_path=output, errors_output=errors_out)
|
|
152
|
+
except Exception as e:
|
|
153
|
+
click.secho(f"\n[!] Error: {e}", fg="red", err=True)
|
|
154
|
+
sys.exit(1)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@cli.command()
|
|
158
|
+
def fields():
|
|
159
|
+
"""List all valid PostField and CommentField enumerations with descriptions."""
|
|
160
|
+
click.echo(f"\nVALID POST FIELDS ({len(PostField)}):")
|
|
161
|
+
for p in sorted(list(PostField), key=lambda x: x.value):
|
|
162
|
+
desc = f" — {p.description}" if p.description else ""
|
|
163
|
+
click.echo(f" • {p.value:<30}{desc}")
|
|
164
|
+
click.echo(f"\nVALID COMMENT FIELDS ({len(CommentField)}):")
|
|
165
|
+
for c in sorted(list(CommentField), key=lambda x: x.value):
|
|
166
|
+
desc = f" — {c.description}" if c.description else ""
|
|
167
|
+
click.echo(f" • {c.value:<30}{desc}")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
if __name__ == "__main__":
|
|
171
|
+
cli()
|
deepsift/filters.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Generalized Filters class.
|
|
3
|
+
Accepts custom filter functions:
|
|
4
|
+
post_filter: Callable[[Post], bool]
|
|
5
|
+
comment_filter: Callable[[Comment], bool]
|
|
6
|
+
Calculates the exact server projection fields needed, preventing unwanted fields
|
|
7
|
+
from ever being transmitted over the wire.
|
|
8
|
+
"""
|
|
9
|
+
from typing import Callable, Optional, Iterable, Union, Set
|
|
10
|
+
from deepsift.models import Post, Comment, PostField, CommentField
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RequiredFieldError(ValueError):
|
|
14
|
+
"""Raised when attempting to exclude a fundamentally required entity field."""
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Filters:
|
|
19
|
+
"""
|
|
20
|
+
Generalized filter container.
|
|
21
|
+
Manages custom predicate functions and minimal server-side field projections.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
post_filter: Optional[Callable[[Post], bool]] = None,
|
|
27
|
+
comment_filter: Optional[Callable[[Comment], bool]] = None,
|
|
28
|
+
post_include_fields: Optional[Iterable[Union[str, PostField]]] = None,
|
|
29
|
+
comment_include_fields: Optional[Iterable[Union[str, CommentField]]] = None,
|
|
30
|
+
post_exclude_fields: Optional[Iterable[Union[str, PostField]]] = None,
|
|
31
|
+
comment_exclude_fields: Optional[Iterable[Union[str, CommentField]]] = None,
|
|
32
|
+
additional_filter_fields: Optional[Iterable[Union[str, PostField]]] = None,
|
|
33
|
+
validate: bool = True,
|
|
34
|
+
):
|
|
35
|
+
self.post_filter = post_filter or (lambda post: True)
|
|
36
|
+
self.comment_filter = comment_filter or (lambda comment: True)
|
|
37
|
+
|
|
38
|
+
if validate:
|
|
39
|
+
self.post_include_fields = PostField.validate_many(post_include_fields)
|
|
40
|
+
self.comment_include_fields = CommentField.validate_many(comment_include_fields)
|
|
41
|
+
self.post_exclude_fields = PostField.validate_many(post_exclude_fields)
|
|
42
|
+
self.comment_exclude_fields = CommentField.validate_many(comment_exclude_fields)
|
|
43
|
+
self.additional_filter_fields = PostField.validate_many(additional_filter_fields)
|
|
44
|
+
else:
|
|
45
|
+
self.post_include_fields = {str(f).lower().strip() for f in post_include_fields} if post_include_fields else None
|
|
46
|
+
self.comment_include_fields = {str(f).lower().strip() for f in comment_include_fields} if comment_include_fields else None
|
|
47
|
+
self.post_exclude_fields = {str(f).lower().strip() for f in post_exclude_fields} if post_exclude_fields else None
|
|
48
|
+
self.comment_exclude_fields = {str(f).lower().strip() for f in comment_exclude_fields} if comment_exclude_fields else None
|
|
49
|
+
self.additional_filter_fields = {str(f).lower().strip() for f in additional_filter_fields} if additional_filter_fields else None
|
|
50
|
+
|
|
51
|
+
# Always strictly reject exclusion of required schema fields
|
|
52
|
+
self._validate_required(self.post_exclude_fields, Post.REQUIRED_FIELDS, "post_exclude_fields")
|
|
53
|
+
self._validate_required(self.comment_exclude_fields, Comment.REQUIRED_FIELDS, "comment_exclude_fields")
|
|
54
|
+
|
|
55
|
+
self.required_post_fields = self._compute_projection(
|
|
56
|
+
self.post_include_fields, self.post_exclude_fields, Post.REQUIRED_FIELDS,
|
|
57
|
+
PostField.all_fields(), self.additional_filter_fields
|
|
58
|
+
)
|
|
59
|
+
self.required_comment_fields = self._compute_projection(
|
|
60
|
+
self.comment_include_fields, self.comment_exclude_fields, Comment.REQUIRED_FIELDS,
|
|
61
|
+
CommentField.all_fields()
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def _validate_required(exclude_fields: Optional[Set[str]], required_fields: frozenset, param_name: str):
|
|
66
|
+
if exclude_fields:
|
|
67
|
+
for field in required_fields:
|
|
68
|
+
if field in exclude_fields:
|
|
69
|
+
raise RequiredFieldError(
|
|
70
|
+
f"Required field '{field}' cannot be excluded in {param_name}. "
|
|
71
|
+
f"Required fields: {sorted(list(required_fields))}."
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _compute_projection(
|
|
76
|
+
include: Optional[Set[str]],
|
|
77
|
+
exclude: Optional[Set[str]],
|
|
78
|
+
required: frozenset,
|
|
79
|
+
all_fields: Set[str],
|
|
80
|
+
extra: Optional[Set[str]] = None,
|
|
81
|
+
) -> Optional[Set[str]]:
|
|
82
|
+
if include is None and exclude is None:
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
core = set(required)
|
|
86
|
+
if include is not None:
|
|
87
|
+
fields = set(include) | core
|
|
88
|
+
if exclude:
|
|
89
|
+
fields -= (exclude - core)
|
|
90
|
+
else:
|
|
91
|
+
fields = (all_fields - exclude) | core
|
|
92
|
+
|
|
93
|
+
if extra:
|
|
94
|
+
fields |= extra
|
|
95
|
+
return fields
|
|
96
|
+
|
|
97
|
+
def is_valid_post(self, post: Post) -> bool:
|
|
98
|
+
"""Evaluates custom filter against a typed Post object."""
|
|
99
|
+
return bool(self.post_filter(post))
|
|
100
|
+
|
|
101
|
+
def is_valid_comment(self, comment: Comment) -> bool:
|
|
102
|
+
"""Evaluates custom filter against a typed Comment object."""
|
|
103
|
+
return bool(self.comment_filter(comment))
|
|
104
|
+
|
|
105
|
+
def filter_post_record(self, post: Post) -> dict:
|
|
106
|
+
"""Serializes post to dict."""
|
|
107
|
+
return post.to_dict()
|
|
108
|
+
|
|
109
|
+
def filter_comment_record(self, comment: Comment) -> dict:
|
|
110
|
+
"""Serializes comment to dict."""
|
|
111
|
+
return comment.to_dict()
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Models subpackage for Reddit data structures.
|
|
3
|
+
"""
|
|
4
|
+
from deepsift.models.enums import FieldEnum, PostField, CommentField
|
|
5
|
+
from deepsift.models.base import RedditEntity, UnretrievedFieldError
|
|
6
|
+
from deepsift.models.post import Post
|
|
7
|
+
from deepsift.models.comment import Comment
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"FieldEnum",
|
|
11
|
+
"PostField",
|
|
12
|
+
"CommentField",
|
|
13
|
+
"RedditEntity",
|
|
14
|
+
"Post",
|
|
15
|
+
"Comment",
|
|
16
|
+
"UnretrievedFieldError",
|
|
17
|
+
]
|
deepsift/models/base.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Base model class and utilities for Reddit entity data models.
|
|
3
|
+
"""
|
|
4
|
+
from typing import Dict, Any, Optional, Iterable, Union, Type
|
|
5
|
+
from deepsift.models.enums import FieldEnum
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class UnretrievedFieldError(AttributeError, KeyError):
|
|
9
|
+
"""Raised when accessing an API field that was not retrieved in the payload."""
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RedditEntity:
|
|
14
|
+
"""
|
|
15
|
+
Base data container for Reddit API entities.
|
|
16
|
+
Provides dynamic field access, typo validation, and dictionary proxy methods.
|
|
17
|
+
"""
|
|
18
|
+
_field_enum: Type[FieldEnum]
|
|
19
|
+
|
|
20
|
+
def __init__(self, data: Dict[str, Any]):
|
|
21
|
+
self._data = dict(data)
|
|
22
|
+
|
|
23
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
24
|
+
"""Returns a copy of the underlying entity dictionary."""
|
|
25
|
+
return dict(self._data)
|
|
26
|
+
|
|
27
|
+
def __repr__(self) -> str:
|
|
28
|
+
eid = self._data.get("id", "")
|
|
29
|
+
return f"<{type(self).__name__} id={eid!r}>" if eid else f"<{type(self).__name__}>"
|
|
30
|
+
|
|
31
|
+
def __getattr__(self, name: str) -> Any:
|
|
32
|
+
if name.startswith("_"):
|
|
33
|
+
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
|
|
34
|
+
|
|
35
|
+
# Check if field is present in the dictionary
|
|
36
|
+
if name in self._data:
|
|
37
|
+
return self._data[name]
|
|
38
|
+
|
|
39
|
+
# Check if the field is a valid entity field
|
|
40
|
+
enum_cls = getattr(type(self), "_field_enum", None)
|
|
41
|
+
if enum_cls and not enum_cls.has_field(name):
|
|
42
|
+
try:
|
|
43
|
+
enum_cls.validate(name)
|
|
44
|
+
except ValueError as e:
|
|
45
|
+
raise AttributeError(f"'{type(self).__name__}' has no field '{name}'. {e}") from None
|
|
46
|
+
|
|
47
|
+
# Known entity field, but was not retrieved from the server
|
|
48
|
+
raise UnretrievedFieldError(
|
|
49
|
+
f"Field '{name}' was not retrieved from the server for this {type(self).__name__.lower()}. "
|
|
50
|
+
f"Available fields: {sorted(list(self._data.keys()))}."
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def __getitem__(self, key: Union[str, FieldEnum]) -> Any:
|
|
54
|
+
key_str = key.value if isinstance(key, FieldEnum) else str(key)
|
|
55
|
+
if key_str in self._data:
|
|
56
|
+
return self._data[key_str]
|
|
57
|
+
try:
|
|
58
|
+
return getattr(self, key_str)
|
|
59
|
+
except AttributeError as e:
|
|
60
|
+
if isinstance(e, KeyError):
|
|
61
|
+
raise
|
|
62
|
+
raise KeyError(str(e)) from e
|
|
63
|
+
|
|
64
|
+
def get(self, key: Union[str, FieldEnum], default: Any = None) -> Any:
|
|
65
|
+
try:
|
|
66
|
+
val = self[key]
|
|
67
|
+
return default if val is None else val
|
|
68
|
+
except (AttributeError, KeyError):
|
|
69
|
+
return default
|
|
70
|
+
|
|
71
|
+
def __contains__(self, key: Union[str, FieldEnum]) -> bool:
|
|
72
|
+
key_str = key.value if isinstance(key, FieldEnum) else str(key)
|
|
73
|
+
return key_str in self._data
|
|
74
|
+
|
|
75
|
+
def __len__(self) -> int:
|
|
76
|
+
return len(self._data)
|
|
77
|
+
|
|
78
|
+
def __iter__(self):
|
|
79
|
+
return iter(self._data)
|
|
80
|
+
|
|
81
|
+
def __eq__(self, other: Any) -> bool:
|
|
82
|
+
if not isinstance(other, type(self)):
|
|
83
|
+
return False
|
|
84
|
+
if "id" in self._data and "id" in other._data:
|
|
85
|
+
return self._data["id"] == other._data["id"]
|
|
86
|
+
return self._data == other._data
|
|
87
|
+
|
|
88
|
+
def __hash__(self) -> int:
|
|
89
|
+
if "id" in self._data:
|
|
90
|
+
return hash((type(self), self._data["id"]))
|
|
91
|
+
return id(self)
|
|
92
|
+
|
|
93
|
+
def keys(self):
|
|
94
|
+
return self._data.keys()
|
|
95
|
+
|
|
96
|
+
def values(self):
|
|
97
|
+
return self._data.values()
|
|
98
|
+
|
|
99
|
+
def items(self):
|
|
100
|
+
return self._data.items()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Comment data model for Reddit comments.
|
|
3
|
+
"""
|
|
4
|
+
from typing import Dict, Any
|
|
5
|
+
from deepsift.models.enums import CommentField
|
|
6
|
+
from deepsift.models.base import RedditEntity
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Comment(RedditEntity):
|
|
10
|
+
"""
|
|
11
|
+
Representation of a Reddit comment.
|
|
12
|
+
Dynamically supports all CommentField enumerations.
|
|
13
|
+
"""
|
|
14
|
+
_field_enum = CommentField
|
|
15
|
+
REQUIRED_FIELDS = frozenset({"id", "post_id", "parent_id"})
|
|
16
|
+
|
|
17
|
+
def __init__(self, data: Dict[str, Any], post_id: str = ""):
|
|
18
|
+
super().__init__(data)
|
|
19
|
+
if "id" in self._data and isinstance(self._data["id"], str):
|
|
20
|
+
self._data["id"] = self._data["id"].replace("t1_", "")
|
|
21
|
+
if "post_id" in self._data and isinstance(self._data["post_id"], str):
|
|
22
|
+
self._data["post_id"] = self._data["post_id"].replace("t3_", "")
|
|
23
|
+
elif post_id:
|
|
24
|
+
self._data["post_id"] = str(post_id).replace("t3_", "")
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def from_api_dict(cls, data: Dict[str, Any], post_id: str = "") -> "Comment":
|
|
28
|
+
return cls(data, post_id=post_id)
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def is_top_level(self) -> bool:
|
|
32
|
+
"""Helper to check if comment is a direct reply to the post (starts with t3_)."""
|
|
33
|
+
parent = str(self.get("parent_id") or "")
|
|
34
|
+
return parent.startswith("t3_")
|
|
35
|
+
|
|
36
|
+
def __repr__(self) -> str:
|
|
37
|
+
cid = self._data.get("id", "")
|
|
38
|
+
return f"<Comment id={cid!r}>"
|