deepsift 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. deepsift-0.2.0/LICENSE +21 -0
  2. deepsift-0.2.0/PKG-INFO +115 -0
  3. deepsift-0.2.0/README.md +82 -0
  4. deepsift-0.2.0/deepsift/__init__.py +39 -0
  5. deepsift-0.2.0/deepsift/__main__.py +8 -0
  6. deepsift-0.2.0/deepsift/cli.py +171 -0
  7. deepsift-0.2.0/deepsift/filters.py +111 -0
  8. deepsift-0.2.0/deepsift/models/__init__.py +17 -0
  9. deepsift-0.2.0/deepsift/models/base.py +100 -0
  10. deepsift-0.2.0/deepsift/models/comment.py +38 -0
  11. deepsift-0.2.0/deepsift/models/enums.py +243 -0
  12. deepsift-0.2.0/deepsift/models/post.py +29 -0
  13. deepsift-0.2.0/deepsift/scraper.py +397 -0
  14. deepsift-0.2.0/deepsift/sources/__init__.py +14 -0
  15. deepsift-0.2.0/deepsift/sources/arctic_shift.py +263 -0
  16. deepsift-0.2.0/deepsift/sources/base.py +102 -0
  17. deepsift-0.2.0/deepsift/sources/dummy.py +207 -0
  18. deepsift-0.2.0/deepsift/sources/registry.py +24 -0
  19. deepsift-0.2.0/deepsift/storage/__init__.py +10 -0
  20. deepsift-0.2.0/deepsift/storage/base.py +37 -0
  21. deepsift-0.2.0/deepsift/storage/parquet.py +86 -0
  22. deepsift-0.2.0/deepsift.egg-info/PKG-INFO +115 -0
  23. deepsift-0.2.0/deepsift.egg-info/SOURCES.txt +34 -0
  24. deepsift-0.2.0/deepsift.egg-info/dependency_links.txt +1 -0
  25. deepsift-0.2.0/deepsift.egg-info/entry_points.txt +3 -0
  26. deepsift-0.2.0/deepsift.egg-info/requires.txt +12 -0
  27. deepsift-0.2.0/deepsift.egg-info/top_level.txt +1 -0
  28. deepsift-0.2.0/pyproject.toml +59 -0
  29. deepsift-0.2.0/setup.cfg +4 -0
  30. deepsift-0.2.0/tests/test_cli.py +38 -0
  31. deepsift-0.2.0/tests/test_evaluation.py +41 -0
  32. deepsift-0.2.0/tests/test_filters.py +79 -0
  33. deepsift-0.2.0/tests/test_models.py +136 -0
  34. deepsift-0.2.0/tests/test_scraper_dummy.py +274 -0
  35. deepsift-0.2.0/tests/test_scraper_live.py +196 -0
  36. deepsift-0.2.0/tests/test_storage.py +93 -0
deepsift-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Saaketh Kesireddy
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,115 @@
1
+ Metadata-Version: 2.4
2
+ Name: deepsift
3
+ Version: 0.2.0
4
+ Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
+ Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/saakethk/deepsift
8
+ Project-URL: Documentation, https://saakethk.github.io/deepsift/
9
+ Project-URL: Repository, https://github.com/saakethk/deepsift.git
10
+ Project-URL: Issues, https://github.com/saakethk/deepsift/issues
11
+ Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: requests>=2.31.0
22
+ Requires-Dist: httpx>=0.27.0
23
+ Requires-Dist: pandas>=2.0.0
24
+ Requires-Dist: pyarrow>=14.0.0
25
+ Requires-Dist: tqdm>=4.66.0
26
+ Requires-Dist: click>=8.1.0
27
+ Requires-Dist: nest-asyncio>=1.6.0
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
30
+ Requires-Dist: build>=1.0.0; extra == "dev"
31
+ Requires-Dist: twine>=5.0.0; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ # Resume Model-V3
35
+ What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
36
+
37
+ ## Methodology
38
+ To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
39
+
40
+ We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
41
+
42
+ [![Graph of subscribers per relevant subreddit](data/graph_relevant_subreddits_fig_11_11_26.png)](data/relevant_subs_cohort_11_11_2026.csv)
43
+
44
+ We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
45
+
46
+ We determined that the ideal cohort for our use case needed to satisfy the following factors:
47
+ - have >10 comments and >0 score
48
+ - have a associated url corresponding to a picture (.png or jpeg)
49
+
50
+ After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
51
+
52
+
53
+
54
+ # Testing
55
+ To run the static unit and integration test suite offline using the synthetic `DummySource`:
56
+ ```bash
57
+ pytest -m "not live" -v
58
+ ```
59
+
60
+ To run the live integration tests against the actual Arctic Shift API:
61
+ ```bash
62
+ pytest -m live -v
63
+ ```
64
+
65
+ # Evaluation & Benchmarks
66
+ Comprehensive performance evaluations comparing DeepSift against sequential execution:
67
+
68
+ | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
69
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
70
+ | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
71
+ | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
72
+
73
+ To run the evaluations:
74
+ ```bash
75
+ # Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
76
+ python -m evaluation
77
+ ```
78
+
79
+ # Documentation Site
80
+ The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
81
+
82
+ To run the documentation dev server with hot-reloading:
83
+ ```bash
84
+ # From project root:
85
+ npm run docs
86
+
87
+ # Or with force cache reload:
88
+ npm run docs:force
89
+
90
+ # Or directly inside docs/ using npx:
91
+ cd docs
92
+ npx astro dev --force
93
+ ```
94
+ Then visit `http://localhost:4321/` in your browser.
95
+
96
+ To build the static production site:
97
+ ```bash
98
+ npm run docs:build
99
+ # or: cd docs && npm run build
100
+ ```
101
+ The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
102
+
103
+ Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
104
+ - [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
105
+ - [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
106
+ - [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
107
+ - [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
108
+ - [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
109
+ - [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
110
+
111
+ # TODO
112
+ - [x] make static tests using dummy source (runs on github ci/cd)
113
+ - [x] make live tests using actual source on r/resumes
114
+ - [x] make evaluation script
115
+ - [x] writeup and documentation with benchmarks, docs page, and cleaned repo
@@ -0,0 +1,82 @@
1
+ # Resume Model-V3
2
+ What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
3
+
4
+ ## Methodology
5
+ To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
6
+
7
+ We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
8
+
9
+ [![Graph of subscribers per relevant subreddit](data/graph_relevant_subreddits_fig_11_11_26.png)](data/relevant_subs_cohort_11_11_2026.csv)
10
+
11
+ We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
12
+
13
+ We determined that the ideal cohort for our use case needed to satisfy the following factors:
14
+ - have >10 comments and >0 score
15
+ - have a associated url corresponding to a picture (.png or jpeg)
16
+
17
+ After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
18
+
19
+
20
+
21
+ # Testing
22
+ To run the static unit and integration test suite offline using the synthetic `DummySource`:
23
+ ```bash
24
+ pytest -m "not live" -v
25
+ ```
26
+
27
+ To run the live integration tests against the actual Arctic Shift API:
28
+ ```bash
29
+ pytest -m live -v
30
+ ```
31
+
32
+ # Evaluation & Benchmarks
33
+ Comprehensive performance evaluations comparing DeepSift against sequential execution:
34
+
35
+ | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
36
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
37
+ | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
38
+ | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
39
+
40
+ To run the evaluations:
41
+ ```bash
42
+ # Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
43
+ python -m evaluation
44
+ ```
45
+
46
+ # Documentation Site
47
+ The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
48
+
49
+ To run the documentation dev server with hot-reloading:
50
+ ```bash
51
+ # From project root:
52
+ npm run docs
53
+
54
+ # Or with force cache reload:
55
+ npm run docs:force
56
+
57
+ # Or directly inside docs/ using npx:
58
+ cd docs
59
+ npx astro dev --force
60
+ ```
61
+ Then visit `http://localhost:4321/` in your browser.
62
+
63
+ To build the static production site:
64
+ ```bash
65
+ npm run docs:build
66
+ # or: cd docs && npm run build
67
+ ```
68
+ The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
69
+
70
+ Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
71
+ - [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
72
+ - [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
73
+ - [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
74
+ - [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
75
+ - [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
76
+ - [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
77
+
78
+ # TODO
79
+ - [x] make static tests using dummy source (runs on github ci/cd)
80
+ - [x] make live tests using actual source on r/resumes
81
+ - [x] make evaluation script
82
+ - [x] writeup and documentation with benchmarks, docs page, and cleaned repo
@@ -0,0 +1,39 @@
1
+ """
2
+ DeepSift Library.
3
+ A high-performance research toolkit for historical Reddit data.
4
+ """
5
+ from deepsift.models import (
6
+ Post,
7
+ Comment,
8
+ PostField,
9
+ CommentField,
10
+ FieldEnum,
11
+ UnretrievedFieldError,
12
+ )
13
+ from deepsift.filters import Filters, RequiredFieldError
14
+ from deepsift.sources import (
15
+ Source,
16
+ ArcticShiftSource,
17
+ get_source,
18
+ register_source,
19
+ )
20
+ from deepsift.storage import Storage, ParquetStorage
21
+ from deepsift.scraper import RedditScraper
22
+
23
+ __all__ = [
24
+ "PostField",
25
+ "CommentField",
26
+ "FieldEnum",
27
+ "Post",
28
+ "Comment",
29
+ "UnretrievedFieldError",
30
+ "RequiredFieldError",
31
+ "Filters",
32
+ "Source",
33
+ "ArcticShiftSource",
34
+ "get_source",
35
+ "register_source",
36
+ "RedditScraper",
37
+ "Storage",
38
+ "ParquetStorage",
39
+ ]
@@ -0,0 +1,8 @@
1
+ """
2
+ Executable package entrypoint for running:
3
+ python -m deepsift [command] [options]
4
+ """
5
+ from deepsift.cli import cli
6
+
7
+ if __name__ == "__main__":
8
+ cli()
@@ -0,0 +1,171 @@
1
+ """
2
+ Command line interface for the Reddit Research Scraper.
3
+ """
4
+ import sys
5
+ import time
6
+ import click
7
+
8
+ from deepsift.models import PostField, CommentField
9
+ from deepsift.filters import Filters
10
+ from deepsift.sources import ArcticShiftSource, get_source
11
+ from deepsift.scraper import RedditScraper
12
+
13
+
14
+ @click.group()
15
+ def cli():
16
+ """Reddit Research Scraper: High-performance historical Reddit data extractor."""
17
+ pass
18
+
19
+
20
+ @cli.command()
21
+ @click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
22
+ @click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
23
+ @click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
24
+ @click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
25
+ @click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep (validated against PostField enum).")
26
+ @click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop (validated against PostField enum).")
27
+ @click.option("--comment-include-fields", multiple=True, help="Important comment fields to keep.")
28
+ @click.option("--comment-exclude-fields", multiple=True, help="Unimportant comment fields to drop.")
29
+ @click.option("--posts-out", default="data/curated_posts.parquet", help="Parquet output destination for posts.")
30
+ @click.option("--comments-out", default="data/curated_comments.parquet", help="Parquet output destination for comments.")
31
+ @click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
32
+ @click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
33
+ @click.option("--comment-workers", default=5, type=int, help="Async comment concurrency workers.")
34
+ @click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
35
+ def run(
36
+ subreddits, from_dt, to_dt, shard_days,
37
+ include_fields, exclude_fields, comment_include_fields, comment_exclude_fields,
38
+ posts_out, comments_out, errors_out, post_workers, comment_workers, source
39
+ ):
40
+ """Run full pipeline: scrape posts, filter cohort, and extract comment trees."""
41
+ target_subs = list(subreddits)
42
+
43
+ try:
44
+ filters = Filters(
45
+ post_filter=None,
46
+ comment_filter=None,
47
+ post_include_fields=list(include_fields) if include_fields else None,
48
+ post_exclude_fields=list(exclude_fields) if exclude_fields else None,
49
+ comment_include_fields=list(comment_include_fields) if comment_include_fields else None,
50
+ comment_exclude_fields=list(comment_exclude_fields) if comment_exclude_fields else None,
51
+ validate=True,
52
+ )
53
+ except ValueError as e:
54
+ click.secho(f"\n[!] Field Validation Error: {e}", fg="red", err=True)
55
+ sys.exit(1)
56
+
57
+ scraper = RedditScraper(
58
+ filters=filters,
59
+ source=get_source(source),
60
+ post_workers=post_workers,
61
+ comment_workers=comment_workers,
62
+ shard_days=shard_days,
63
+ )
64
+
65
+ start_time = time.time()
66
+ click.echo(f"Reddit Research Scraper: {len(target_subs)} subreddits (shard_days={shard_days})")
67
+ click.echo(f"Window: from {from_dt} to {to_dt or 'now'}")
68
+ if filters.post_include_fields:
69
+ click.echo(f"Post Fields Kept: {sorted(list(filters.post_include_fields))}")
70
+ if filters.post_exclude_fields:
71
+ click.echo(f"Post Fields Dropped: {sorted(list(filters.post_exclude_fields))}")
72
+
73
+ try:
74
+ posts, comments = scraper.run(
75
+ subreddits=target_subs,
76
+ from_dt=from_dt,
77
+ to_dt=to_dt,
78
+ posts_output=posts_out,
79
+ comments_output=comments_out,
80
+ errors_output=errors_out,
81
+ )
82
+ except Exception as e:
83
+ click.secho(f"\n[!] Pipeline Error: {e}", fg="red", err=True)
84
+ sys.exit(1)
85
+
86
+ elapsed = round(time.time() - start_time, 1)
87
+ click.echo(f"[✓] Pipeline complete in {elapsed}s")
88
+ click.echo(f" Posts: {len(posts):,} -> {posts_out}")
89
+ click.echo(f" Comments: {comments:,} -> {comments_out}")
90
+
91
+
92
+ @cli.command()
93
+ @click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
94
+ @click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
95
+ @click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
96
+ @click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
97
+ @click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep.")
98
+ @click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop.")
99
+ @click.option("--output", "-o", default="data/curated_posts.parquet", help="Parquet output destination.")
100
+ @click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
101
+ @click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
102
+ @click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
103
+ def posts(subreddits, from_dt, to_dt, shard_days, include_fields, exclude_fields, output, errors_out, post_workers, source):
104
+ """Scrape and filter Reddit posts between --from and --to dates."""
105
+ target_subs = list(subreddits)
106
+ try:
107
+ filters = Filters(
108
+ post_filter=None,
109
+ post_include_fields=list(include_fields) if include_fields else None,
110
+ post_exclude_fields=list(exclude_fields) if exclude_fields else None,
111
+ )
112
+ scraper = RedditScraper(
113
+ filters=filters,
114
+ source=get_source(source),
115
+ post_workers=post_workers,
116
+ shard_days=shard_days,
117
+ )
118
+ scraper.scrape_posts(
119
+ subreddits=target_subs,
120
+ from_dt=from_dt,
121
+ to_dt=to_dt,
122
+ output_path=output,
123
+ errors_output=errors_out,
124
+ )
125
+ except Exception as e:
126
+ click.secho(f"\n[!] Error: {e}", fg="red", err=True)
127
+ sys.exit(1)
128
+
129
+
130
+ @cli.command()
131
+ @click.option("--post-id", "-p", "post_ids", multiple=True, required=True, help="Post ID(s) to fetch comments for.")
132
+ @click.option("--include-fields", "-i", multiple=True, help="Important comment fields to keep.")
133
+ @click.option("--exclude-fields", "-e", multiple=True, help="Unimportant comment fields to drop.")
134
+ @click.option("--output", "-o", default="data/curated_comments.parquet", help="Parquet output destination.")
135
+ @click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
136
+ @click.option("--workers", "-w", default=5, type=int, help="Async concurrency limit.")
137
+ @click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
138
+ def comments(post_ids, include_fields, exclude_fields, output, errors_out, workers, source):
139
+ """Scrape comment trees for given post IDs."""
140
+ try:
141
+ filters = Filters(
142
+ comment_filter=None,
143
+ comment_include_fields=list(include_fields) if include_fields else None,
144
+ comment_exclude_fields=list(exclude_fields) if exclude_fields else None,
145
+ )
146
+ scraper = RedditScraper(
147
+ filters=filters,
148
+ source=get_source(source),
149
+ comment_workers=workers,
150
+ )
151
+ scraper.scrape_comments(post_ids=list(post_ids), output_path=output, errors_output=errors_out)
152
+ except Exception as e:
153
+ click.secho(f"\n[!] Error: {e}", fg="red", err=True)
154
+ sys.exit(1)
155
+
156
+
157
+ @cli.command()
158
+ def fields():
159
+ """List all valid PostField and CommentField enumerations with descriptions."""
160
+ click.echo(f"\nVALID POST FIELDS ({len(PostField)}):")
161
+ for p in sorted(list(PostField), key=lambda x: x.value):
162
+ desc = f" — {p.description}" if p.description else ""
163
+ click.echo(f" • {p.value:<30}{desc}")
164
+ click.echo(f"\nVALID COMMENT FIELDS ({len(CommentField)}):")
165
+ for c in sorted(list(CommentField), key=lambda x: x.value):
166
+ desc = f" — {c.description}" if c.description else ""
167
+ click.echo(f" • {c.value:<30}{desc}")
168
+
169
+
170
+ if __name__ == "__main__":
171
+ cli()
@@ -0,0 +1,111 @@
1
+ """
2
+ Generalized Filters class.
3
+ Accepts custom filter functions:
4
+ post_filter: Callable[[Post], bool]
5
+ comment_filter: Callable[[Comment], bool]
6
+ Calculates the exact server projection fields needed, preventing unwanted fields
7
+ from ever being transmitted over the wire.
8
+ """
9
+ from typing import Callable, Optional, Iterable, Union, Set
10
+ from deepsift.models import Post, Comment, PostField, CommentField
11
+
12
+
13
+ class RequiredFieldError(ValueError):
14
+ """Raised when attempting to exclude a fundamentally required entity field."""
15
+ pass
16
+
17
+
18
+ class Filters:
19
+ """
20
+ Generalized filter container.
21
+ Manages custom predicate functions and minimal server-side field projections.
22
+ """
23
+
24
+ def __init__(
25
+ self,
26
+ post_filter: Optional[Callable[[Post], bool]] = None,
27
+ comment_filter: Optional[Callable[[Comment], bool]] = None,
28
+ post_include_fields: Optional[Iterable[Union[str, PostField]]] = None,
29
+ comment_include_fields: Optional[Iterable[Union[str, CommentField]]] = None,
30
+ post_exclude_fields: Optional[Iterable[Union[str, PostField]]] = None,
31
+ comment_exclude_fields: Optional[Iterable[Union[str, CommentField]]] = None,
32
+ additional_filter_fields: Optional[Iterable[Union[str, PostField]]] = None,
33
+ validate: bool = True,
34
+ ):
35
+ self.post_filter = post_filter or (lambda post: True)
36
+ self.comment_filter = comment_filter or (lambda comment: True)
37
+
38
+ if validate:
39
+ self.post_include_fields = PostField.validate_many(post_include_fields)
40
+ self.comment_include_fields = CommentField.validate_many(comment_include_fields)
41
+ self.post_exclude_fields = PostField.validate_many(post_exclude_fields)
42
+ self.comment_exclude_fields = CommentField.validate_many(comment_exclude_fields)
43
+ self.additional_filter_fields = PostField.validate_many(additional_filter_fields)
44
+ else:
45
+ self.post_include_fields = {str(f).lower().strip() for f in post_include_fields} if post_include_fields else None
46
+ self.comment_include_fields = {str(f).lower().strip() for f in comment_include_fields} if comment_include_fields else None
47
+ self.post_exclude_fields = {str(f).lower().strip() for f in post_exclude_fields} if post_exclude_fields else None
48
+ self.comment_exclude_fields = {str(f).lower().strip() for f in comment_exclude_fields} if comment_exclude_fields else None
49
+ self.additional_filter_fields = {str(f).lower().strip() for f in additional_filter_fields} if additional_filter_fields else None
50
+
51
+ # Always strictly reject exclusion of required schema fields
52
+ self._validate_required(self.post_exclude_fields, Post.REQUIRED_FIELDS, "post_exclude_fields")
53
+ self._validate_required(self.comment_exclude_fields, Comment.REQUIRED_FIELDS, "comment_exclude_fields")
54
+
55
+ self.required_post_fields = self._compute_projection(
56
+ self.post_include_fields, self.post_exclude_fields, Post.REQUIRED_FIELDS,
57
+ PostField.all_fields(), self.additional_filter_fields
58
+ )
59
+ self.required_comment_fields = self._compute_projection(
60
+ self.comment_include_fields, self.comment_exclude_fields, Comment.REQUIRED_FIELDS,
61
+ CommentField.all_fields()
62
+ )
63
+
64
+ @staticmethod
65
+ def _validate_required(exclude_fields: Optional[Set[str]], required_fields: frozenset, param_name: str):
66
+ if exclude_fields:
67
+ for field in required_fields:
68
+ if field in exclude_fields:
69
+ raise RequiredFieldError(
70
+ f"Required field '{field}' cannot be excluded in {param_name}. "
71
+ f"Required fields: {sorted(list(required_fields))}."
72
+ )
73
+
74
+ @staticmethod
75
+ def _compute_projection(
76
+ include: Optional[Set[str]],
77
+ exclude: Optional[Set[str]],
78
+ required: frozenset,
79
+ all_fields: Set[str],
80
+ extra: Optional[Set[str]] = None,
81
+ ) -> Optional[Set[str]]:
82
+ if include is None and exclude is None:
83
+ return None
84
+
85
+ core = set(required)
86
+ if include is not None:
87
+ fields = set(include) | core
88
+ if exclude:
89
+ fields -= (exclude - core)
90
+ else:
91
+ fields = (all_fields - exclude) | core
92
+
93
+ if extra:
94
+ fields |= extra
95
+ return fields
96
+
97
+ def is_valid_post(self, post: Post) -> bool:
98
+ """Evaluates custom filter against a typed Post object."""
99
+ return bool(self.post_filter(post))
100
+
101
+ def is_valid_comment(self, comment: Comment) -> bool:
102
+ """Evaluates custom filter against a typed Comment object."""
103
+ return bool(self.comment_filter(comment))
104
+
105
+ def filter_post_record(self, post: Post) -> dict:
106
+ """Serializes post to dict."""
107
+ return post.to_dict()
108
+
109
+ def filter_comment_record(self, comment: Comment) -> dict:
110
+ """Serializes comment to dict."""
111
+ return comment.to_dict()
@@ -0,0 +1,17 @@
1
+ """
2
+ Models subpackage for Reddit data structures.
3
+ """
4
+ from deepsift.models.enums import FieldEnum, PostField, CommentField
5
+ from deepsift.models.base import RedditEntity, UnretrievedFieldError
6
+ from deepsift.models.post import Post
7
+ from deepsift.models.comment import Comment
8
+
9
+ __all__ = [
10
+ "FieldEnum",
11
+ "PostField",
12
+ "CommentField",
13
+ "RedditEntity",
14
+ "Post",
15
+ "Comment",
16
+ "UnretrievedFieldError",
17
+ ]