deepsift 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deepsift/__init__.py ADDED
@@ -0,0 +1,39 @@
1
+ """
2
+ DeepSift Library.
3
+ A high-performance research toolkit for historical Reddit data.
4
+ """
5
+ from deepsift.models import (
6
+ Post,
7
+ Comment,
8
+ PostField,
9
+ CommentField,
10
+ FieldEnum,
11
+ UnretrievedFieldError,
12
+ )
13
+ from deepsift.filters import Filters, RequiredFieldError
14
+ from deepsift.sources import (
15
+ Source,
16
+ ArcticShiftSource,
17
+ get_source,
18
+ register_source,
19
+ )
20
+ from deepsift.storage import Storage, ParquetStorage
21
+ from deepsift.scraper import RedditScraper
22
+
23
+ __all__ = [
24
+ "PostField",
25
+ "CommentField",
26
+ "FieldEnum",
27
+ "Post",
28
+ "Comment",
29
+ "UnretrievedFieldError",
30
+ "RequiredFieldError",
31
+ "Filters",
32
+ "Source",
33
+ "ArcticShiftSource",
34
+ "get_source",
35
+ "register_source",
36
+ "RedditScraper",
37
+ "Storage",
38
+ "ParquetStorage",
39
+ ]
deepsift/__main__.py ADDED
@@ -0,0 +1,8 @@
1
+ """
2
+ Executable package entrypoint for running:
3
+ python -m deepsift [command] [options]
4
+ """
5
+ from deepsift.cli import cli
6
+
7
+ if __name__ == "__main__":
8
+ cli()
deepsift/cli.py ADDED
@@ -0,0 +1,171 @@
1
+ """
2
+ Command line interface for the Reddit Research Scraper.
3
+ """
4
+ import sys
5
+ import time
6
+ import click
7
+
8
+ from deepsift.models import PostField, CommentField
9
+ from deepsift.filters import Filters
10
+ from deepsift.sources import ArcticShiftSource, get_source
11
+ from deepsift.scraper import RedditScraper
12
+
13
+
14
+ @click.group()
15
+ def cli():
16
+ """Reddit Research Scraper: High-performance historical Reddit data extractor."""
17
+ pass
18
+
19
+
20
+ @cli.command()
21
+ @click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
22
+ @click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
23
+ @click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
24
+ @click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
25
+ @click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep (validated against PostField enum).")
26
+ @click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop (validated against PostField enum).")
27
+ @click.option("--comment-include-fields", multiple=True, help="Important comment fields to keep.")
28
+ @click.option("--comment-exclude-fields", multiple=True, help="Unimportant comment fields to drop.")
29
+ @click.option("--posts-out", default="data/curated_posts.parquet", help="Parquet output destination for posts.")
30
+ @click.option("--comments-out", default="data/curated_comments.parquet", help="Parquet output destination for comments.")
31
+ @click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
32
+ @click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
33
+ @click.option("--comment-workers", default=5, type=int, help="Async comment concurrency workers.")
34
+ @click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
35
+ def run(
36
+ subreddits, from_dt, to_dt, shard_days,
37
+ include_fields, exclude_fields, comment_include_fields, comment_exclude_fields,
38
+ posts_out, comments_out, errors_out, post_workers, comment_workers, source
39
+ ):
40
+ """Run full pipeline: scrape posts, filter cohort, and extract comment trees."""
41
+ target_subs = list(subreddits)
42
+
43
+ try:
44
+ filters = Filters(
45
+ post_filter=None,
46
+ comment_filter=None,
47
+ post_include_fields=list(include_fields) if include_fields else None,
48
+ post_exclude_fields=list(exclude_fields) if exclude_fields else None,
49
+ comment_include_fields=list(comment_include_fields) if comment_include_fields else None,
50
+ comment_exclude_fields=list(comment_exclude_fields) if comment_exclude_fields else None,
51
+ validate=True,
52
+ )
53
+ except ValueError as e:
54
+ click.secho(f"\n[!] Field Validation Error: {e}", fg="red", err=True)
55
+ sys.exit(1)
56
+
57
+ scraper = RedditScraper(
58
+ filters=filters,
59
+ source=get_source(source),
60
+ post_workers=post_workers,
61
+ comment_workers=comment_workers,
62
+ shard_days=shard_days,
63
+ )
64
+
65
+ start_time = time.time()
66
+ click.echo(f"Reddit Research Scraper: {len(target_subs)} subreddits (shard_days={shard_days})")
67
+ click.echo(f"Window: from {from_dt} to {to_dt or 'now'}")
68
+ if filters.post_include_fields:
69
+ click.echo(f"Post Fields Kept: {sorted(list(filters.post_include_fields))}")
70
+ if filters.post_exclude_fields:
71
+ click.echo(f"Post Fields Dropped: {sorted(list(filters.post_exclude_fields))}")
72
+
73
+ try:
74
+ posts, comments = scraper.run(
75
+ subreddits=target_subs,
76
+ from_dt=from_dt,
77
+ to_dt=to_dt,
78
+ posts_output=posts_out,
79
+ comments_output=comments_out,
80
+ errors_output=errors_out,
81
+ )
82
+ except Exception as e:
83
+ click.secho(f"\n[!] Pipeline Error: {e}", fg="red", err=True)
84
+ sys.exit(1)
85
+
86
+ elapsed = round(time.time() - start_time, 1)
87
+ click.echo(f"[✓] Pipeline complete in {elapsed}s")
88
+ click.echo(f" Posts: {len(posts):,} -> {posts_out}")
89
+ click.echo(f" Comments: {comments:,} -> {comments_out}")
90
+
91
+
92
+ @cli.command()
93
+ @click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
94
+ @click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
95
+ @click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
96
+ @click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
97
+ @click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep.")
98
+ @click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop.")
99
+ @click.option("--output", "-o", default="data/curated_posts.parquet", help="Parquet output destination.")
100
+ @click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
101
+ @click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
102
+ @click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
103
+ def posts(subreddits, from_dt, to_dt, shard_days, include_fields, exclude_fields, output, errors_out, post_workers, source):
104
+ """Scrape and filter Reddit posts between --from and --to dates."""
105
+ target_subs = list(subreddits)
106
+ try:
107
+ filters = Filters(
108
+ post_filter=None,
109
+ post_include_fields=list(include_fields) if include_fields else None,
110
+ post_exclude_fields=list(exclude_fields) if exclude_fields else None,
111
+ )
112
+ scraper = RedditScraper(
113
+ filters=filters,
114
+ source=get_source(source),
115
+ post_workers=post_workers,
116
+ shard_days=shard_days,
117
+ )
118
+ scraper.scrape_posts(
119
+ subreddits=target_subs,
120
+ from_dt=from_dt,
121
+ to_dt=to_dt,
122
+ output_path=output,
123
+ errors_output=errors_out,
124
+ )
125
+ except Exception as e:
126
+ click.secho(f"\n[!] Error: {e}", fg="red", err=True)
127
+ sys.exit(1)
128
+
129
+
130
+ @cli.command()
131
+ @click.option("--post-id", "-p", "post_ids", multiple=True, required=True, help="Post ID(s) to fetch comments for.")
132
+ @click.option("--include-fields", "-i", multiple=True, help="Important comment fields to keep.")
133
+ @click.option("--exclude-fields", "-e", multiple=True, help="Unimportant comment fields to drop.")
134
+ @click.option("--output", "-o", default="data/curated_comments.parquet", help="Parquet output destination.")
135
+ @click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
136
+ @click.option("--workers", "-w", default=5, type=int, help="Async concurrency limit.")
137
+ @click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
138
+ def comments(post_ids, include_fields, exclude_fields, output, errors_out, workers, source):
139
+ """Scrape comment trees for given post IDs."""
140
+ try:
141
+ filters = Filters(
142
+ comment_filter=None,
143
+ comment_include_fields=list(include_fields) if include_fields else None,
144
+ comment_exclude_fields=list(exclude_fields) if exclude_fields else None,
145
+ )
146
+ scraper = RedditScraper(
147
+ filters=filters,
148
+ source=get_source(source),
149
+ comment_workers=workers,
150
+ )
151
+ scraper.scrape_comments(post_ids=list(post_ids), output_path=output, errors_output=errors_out)
152
+ except Exception as e:
153
+ click.secho(f"\n[!] Error: {e}", fg="red", err=True)
154
+ sys.exit(1)
155
+
156
+
157
+ @cli.command()
158
+ def fields():
159
+ """List all valid PostField and CommentField enumerations with descriptions."""
160
+ click.echo(f"\nVALID POST FIELDS ({len(PostField)}):")
161
+ for p in sorted(list(PostField), key=lambda x: x.value):
162
+ desc = f" — {p.description}" if p.description else ""
163
+ click.echo(f" • {p.value:<30}{desc}")
164
+ click.echo(f"\nVALID COMMENT FIELDS ({len(CommentField)}):")
165
+ for c in sorted(list(CommentField), key=lambda x: x.value):
166
+ desc = f" — {c.description}" if c.description else ""
167
+ click.echo(f" • {c.value:<30}{desc}")
168
+
169
+
170
+ if __name__ == "__main__":
171
+ cli()
deepsift/filters.py ADDED
@@ -0,0 +1,111 @@
1
+ """
2
+ Generalized Filters class.
3
+ Accepts custom filter functions:
4
+ post_filter: Callable[[Post], bool]
5
+ comment_filter: Callable[[Comment], bool]
6
+ Calculates the exact server projection fields needed, preventing unwanted fields
7
+ from ever being transmitted over the wire.
8
+ """
9
+ from typing import Callable, Optional, Iterable, Union, Set
10
+ from deepsift.models import Post, Comment, PostField, CommentField
11
+
12
+
13
+ class RequiredFieldError(ValueError):
14
+ """Raised when attempting to exclude a fundamentally required entity field."""
15
+ pass
16
+
17
+
18
+ class Filters:
19
+ """
20
+ Generalized filter container.
21
+ Manages custom predicate functions and minimal server-side field projections.
22
+ """
23
+
24
+ def __init__(
25
+ self,
26
+ post_filter: Optional[Callable[[Post], bool]] = None,
27
+ comment_filter: Optional[Callable[[Comment], bool]] = None,
28
+ post_include_fields: Optional[Iterable[Union[str, PostField]]] = None,
29
+ comment_include_fields: Optional[Iterable[Union[str, CommentField]]] = None,
30
+ post_exclude_fields: Optional[Iterable[Union[str, PostField]]] = None,
31
+ comment_exclude_fields: Optional[Iterable[Union[str, CommentField]]] = None,
32
+ additional_filter_fields: Optional[Iterable[Union[str, PostField]]] = None,
33
+ validate: bool = True,
34
+ ):
35
+ self.post_filter = post_filter or (lambda post: True)
36
+ self.comment_filter = comment_filter or (lambda comment: True)
37
+
38
+ if validate:
39
+ self.post_include_fields = PostField.validate_many(post_include_fields)
40
+ self.comment_include_fields = CommentField.validate_many(comment_include_fields)
41
+ self.post_exclude_fields = PostField.validate_many(post_exclude_fields)
42
+ self.comment_exclude_fields = CommentField.validate_many(comment_exclude_fields)
43
+ self.additional_filter_fields = PostField.validate_many(additional_filter_fields)
44
+ else:
45
+ self.post_include_fields = {str(f).lower().strip() for f in post_include_fields} if post_include_fields else None
46
+ self.comment_include_fields = {str(f).lower().strip() for f in comment_include_fields} if comment_include_fields else None
47
+ self.post_exclude_fields = {str(f).lower().strip() for f in post_exclude_fields} if post_exclude_fields else None
48
+ self.comment_exclude_fields = {str(f).lower().strip() for f in comment_exclude_fields} if comment_exclude_fields else None
49
+ self.additional_filter_fields = {str(f).lower().strip() for f in additional_filter_fields} if additional_filter_fields else None
50
+
51
+ # Always strictly reject exclusion of required schema fields
52
+ self._validate_required(self.post_exclude_fields, Post.REQUIRED_FIELDS, "post_exclude_fields")
53
+ self._validate_required(self.comment_exclude_fields, Comment.REQUIRED_FIELDS, "comment_exclude_fields")
54
+
55
+ self.required_post_fields = self._compute_projection(
56
+ self.post_include_fields, self.post_exclude_fields, Post.REQUIRED_FIELDS,
57
+ PostField.all_fields(), self.additional_filter_fields
58
+ )
59
+ self.required_comment_fields = self._compute_projection(
60
+ self.comment_include_fields, self.comment_exclude_fields, Comment.REQUIRED_FIELDS,
61
+ CommentField.all_fields()
62
+ )
63
+
64
+ @staticmethod
65
+ def _validate_required(exclude_fields: Optional[Set[str]], required_fields: frozenset, param_name: str):
66
+ if exclude_fields:
67
+ for field in required_fields:
68
+ if field in exclude_fields:
69
+ raise RequiredFieldError(
70
+ f"Required field '{field}' cannot be excluded in {param_name}. "
71
+ f"Required fields: {sorted(list(required_fields))}."
72
+ )
73
+
74
+ @staticmethod
75
+ def _compute_projection(
76
+ include: Optional[Set[str]],
77
+ exclude: Optional[Set[str]],
78
+ required: frozenset,
79
+ all_fields: Set[str],
80
+ extra: Optional[Set[str]] = None,
81
+ ) -> Optional[Set[str]]:
82
+ if include is None and exclude is None:
83
+ return None
84
+
85
+ core = set(required)
86
+ if include is not None:
87
+ fields = set(include) | core
88
+ if exclude:
89
+ fields -= (exclude - core)
90
+ else:
91
+ fields = (all_fields - exclude) | core
92
+
93
+ if extra:
94
+ fields |= extra
95
+ return fields
96
+
97
+ def is_valid_post(self, post: Post) -> bool:
98
+ """Evaluates custom filter against a typed Post object."""
99
+ return bool(self.post_filter(post))
100
+
101
+ def is_valid_comment(self, comment: Comment) -> bool:
102
+ """Evaluates custom filter against a typed Comment object."""
103
+ return bool(self.comment_filter(comment))
104
+
105
+ def filter_post_record(self, post: Post) -> dict:
106
+ """Serializes post to dict."""
107
+ return post.to_dict()
108
+
109
+ def filter_comment_record(self, comment: Comment) -> dict:
110
+ """Serializes comment to dict."""
111
+ return comment.to_dict()
@@ -0,0 +1,17 @@
1
+ """
2
+ Models subpackage for Reddit data structures.
3
+ """
4
+ from deepsift.models.enums import FieldEnum, PostField, CommentField
5
+ from deepsift.models.base import RedditEntity, UnretrievedFieldError
6
+ from deepsift.models.post import Post
7
+ from deepsift.models.comment import Comment
8
+
9
+ __all__ = [
10
+ "FieldEnum",
11
+ "PostField",
12
+ "CommentField",
13
+ "RedditEntity",
14
+ "Post",
15
+ "Comment",
16
+ "UnretrievedFieldError",
17
+ ]
@@ -0,0 +1,100 @@
1
+ """
2
+ Base model class and utilities for Reddit entity data models.
3
+ """
4
+ from typing import Dict, Any, Optional, Iterable, Union, Type
5
+ from deepsift.models.enums import FieldEnum
6
+
7
+
8
+ class UnretrievedFieldError(AttributeError, KeyError):
9
+ """Raised when accessing an API field that was not retrieved in the payload."""
10
+ pass
11
+
12
+
13
+ class RedditEntity:
14
+ """
15
+ Base data container for Reddit API entities.
16
+ Provides dynamic field access, typo validation, and dictionary proxy methods.
17
+ """
18
+ _field_enum: Type[FieldEnum]
19
+
20
+ def __init__(self, data: Dict[str, Any]):
21
+ self._data = dict(data)
22
+
23
+ def to_dict(self) -> Dict[str, Any]:
24
+ """Returns a copy of the underlying entity dictionary."""
25
+ return dict(self._data)
26
+
27
+ def __repr__(self) -> str:
28
+ eid = self._data.get("id", "")
29
+ return f"<{type(self).__name__} id={eid!r}>" if eid else f"<{type(self).__name__}>"
30
+
31
+ def __getattr__(self, name: str) -> Any:
32
+ if name.startswith("_"):
33
+ raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
34
+
35
+ # Check if field is present in the dictionary
36
+ if name in self._data:
37
+ return self._data[name]
38
+
39
+ # Check if the field is a valid entity field
40
+ enum_cls = getattr(type(self), "_field_enum", None)
41
+ if enum_cls and not enum_cls.has_field(name):
42
+ try:
43
+ enum_cls.validate(name)
44
+ except ValueError as e:
45
+ raise AttributeError(f"'{type(self).__name__}' has no field '{name}'. {e}") from None
46
+
47
+ # Known entity field, but was not retrieved from the server
48
+ raise UnretrievedFieldError(
49
+ f"Field '{name}' was not retrieved from the server for this {type(self).__name__.lower()}. "
50
+ f"Available fields: {sorted(list(self._data.keys()))}."
51
+ )
52
+
53
+ def __getitem__(self, key: Union[str, FieldEnum]) -> Any:
54
+ key_str = key.value if isinstance(key, FieldEnum) else str(key)
55
+ if key_str in self._data:
56
+ return self._data[key_str]
57
+ try:
58
+ return getattr(self, key_str)
59
+ except AttributeError as e:
60
+ if isinstance(e, KeyError):
61
+ raise
62
+ raise KeyError(str(e)) from e
63
+
64
+ def get(self, key: Union[str, FieldEnum], default: Any = None) -> Any:
65
+ try:
66
+ val = self[key]
67
+ return default if val is None else val
68
+ except (AttributeError, KeyError):
69
+ return default
70
+
71
+ def __contains__(self, key: Union[str, FieldEnum]) -> bool:
72
+ key_str = key.value if isinstance(key, FieldEnum) else str(key)
73
+ return key_str in self._data
74
+
75
+ def __len__(self) -> int:
76
+ return len(self._data)
77
+
78
+ def __iter__(self):
79
+ return iter(self._data)
80
+
81
+ def __eq__(self, other: Any) -> bool:
82
+ if not isinstance(other, type(self)):
83
+ return False
84
+ if "id" in self._data and "id" in other._data:
85
+ return self._data["id"] == other._data["id"]
86
+ return self._data == other._data
87
+
88
+ def __hash__(self) -> int:
89
+ if "id" in self._data:
90
+ return hash((type(self), self._data["id"]))
91
+ return id(self)
92
+
93
+ def keys(self):
94
+ return self._data.keys()
95
+
96
+ def values(self):
97
+ return self._data.values()
98
+
99
+ def items(self):
100
+ return self._data.items()
@@ -0,0 +1,38 @@
1
+ """
2
+ Comment data model for Reddit comments.
3
+ """
4
+ from typing import Dict, Any
5
+ from deepsift.models.enums import CommentField
6
+ from deepsift.models.base import RedditEntity
7
+
8
+
9
+ class Comment(RedditEntity):
10
+ """
11
+ Representation of a Reddit comment.
12
+ Dynamically supports all CommentField enumerations.
13
+ """
14
+ _field_enum = CommentField
15
+ REQUIRED_FIELDS = frozenset({"id", "post_id", "parent_id"})
16
+
17
+ def __init__(self, data: Dict[str, Any], post_id: str = ""):
18
+ super().__init__(data)
19
+ if "id" in self._data and isinstance(self._data["id"], str):
20
+ self._data["id"] = self._data["id"].replace("t1_", "")
21
+ if "post_id" in self._data and isinstance(self._data["post_id"], str):
22
+ self._data["post_id"] = self._data["post_id"].replace("t3_", "")
23
+ elif post_id:
24
+ self._data["post_id"] = str(post_id).replace("t3_", "")
25
+
26
+ @classmethod
27
+ def from_api_dict(cls, data: Dict[str, Any], post_id: str = "") -> "Comment":
28
+ return cls(data, post_id=post_id)
29
+
30
+ @property
31
+ def is_top_level(self) -> bool:
32
+ """Helper to check if comment is a direct reply to the post (starts with t3_)."""
33
+ parent = str(self.get("parent_id") or "")
34
+ return parent.startswith("t3_")
35
+
36
+ def __repr__(self) -> str:
37
+ cid = self._data.get("id", "")
38
+ return f"<Comment id={cid!r}>"