deepsift 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepsift-0.2.0/LICENSE +21 -0
- deepsift-0.2.0/PKG-INFO +115 -0
- deepsift-0.2.0/README.md +82 -0
- deepsift-0.2.0/deepsift/__init__.py +39 -0
- deepsift-0.2.0/deepsift/__main__.py +8 -0
- deepsift-0.2.0/deepsift/cli.py +171 -0
- deepsift-0.2.0/deepsift/filters.py +111 -0
- deepsift-0.2.0/deepsift/models/__init__.py +17 -0
- deepsift-0.2.0/deepsift/models/base.py +100 -0
- deepsift-0.2.0/deepsift/models/comment.py +38 -0
- deepsift-0.2.0/deepsift/models/enums.py +243 -0
- deepsift-0.2.0/deepsift/models/post.py +29 -0
- deepsift-0.2.0/deepsift/scraper.py +397 -0
- deepsift-0.2.0/deepsift/sources/__init__.py +14 -0
- deepsift-0.2.0/deepsift/sources/arctic_shift.py +263 -0
- deepsift-0.2.0/deepsift/sources/base.py +102 -0
- deepsift-0.2.0/deepsift/sources/dummy.py +207 -0
- deepsift-0.2.0/deepsift/sources/registry.py +24 -0
- deepsift-0.2.0/deepsift/storage/__init__.py +10 -0
- deepsift-0.2.0/deepsift/storage/base.py +37 -0
- deepsift-0.2.0/deepsift/storage/parquet.py +86 -0
- deepsift-0.2.0/deepsift.egg-info/PKG-INFO +115 -0
- deepsift-0.2.0/deepsift.egg-info/SOURCES.txt +34 -0
- deepsift-0.2.0/deepsift.egg-info/dependency_links.txt +1 -0
- deepsift-0.2.0/deepsift.egg-info/entry_points.txt +3 -0
- deepsift-0.2.0/deepsift.egg-info/requires.txt +12 -0
- deepsift-0.2.0/deepsift.egg-info/top_level.txt +1 -0
- deepsift-0.2.0/pyproject.toml +59 -0
- deepsift-0.2.0/setup.cfg +4 -0
- deepsift-0.2.0/tests/test_cli.py +38 -0
- deepsift-0.2.0/tests/test_evaluation.py +41 -0
- deepsift-0.2.0/tests/test_filters.py +79 -0
- deepsift-0.2.0/tests/test_models.py +136 -0
- deepsift-0.2.0/tests/test_scraper_dummy.py +274 -0
- deepsift-0.2.0/tests/test_scraper_live.py +196 -0
- deepsift-0.2.0/tests/test_storage.py +93 -0
deepsift-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Saaketh Kesireddy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
deepsift-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deepsift
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
|
|
5
|
+
Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/saakethk/deepsift
|
|
8
|
+
Project-URL: Documentation, https://saakethk.github.io/deepsift/
|
|
9
|
+
Project-URL: Repository, https://github.com/saakethk/deepsift.git
|
|
10
|
+
Project-URL: Issues, https://github.com/saakethk/deepsift/issues
|
|
11
|
+
Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: requests>=2.31.0
|
|
22
|
+
Requires-Dist: httpx>=0.27.0
|
|
23
|
+
Requires-Dist: pandas>=2.0.0
|
|
24
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
25
|
+
Requires-Dist: tqdm>=4.66.0
|
|
26
|
+
Requires-Dist: click>=8.1.0
|
|
27
|
+
Requires-Dist: nest-asyncio>=1.6.0
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# Resume Model-V3
|
|
35
|
+
What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
|
|
36
|
+
|
|
37
|
+
## Methodology
|
|
38
|
+
To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
|
|
39
|
+
|
|
40
|
+
We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
|
|
41
|
+
|
|
42
|
+
[](data/relevant_subs_cohort_11_11_2026.csv)
|
|
43
|
+
|
|
44
|
+
We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
|
|
45
|
+
|
|
46
|
+
We determined that the ideal cohort for our use case needed to satisfy the following factors:
|
|
47
|
+
- have >10 comments and >0 score
|
|
48
|
+
- have a associated url corresponding to a picture (.png or jpeg)
|
|
49
|
+
|
|
50
|
+
After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Testing
|
|
55
|
+
To run the static unit and integration test suite offline using the synthetic `DummySource`:
|
|
56
|
+
```bash
|
|
57
|
+
pytest -m "not live" -v
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
To run the live integration tests against the actual Arctic Shift API:
|
|
61
|
+
```bash
|
|
62
|
+
pytest -m live -v
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
# Evaluation & Benchmarks
|
|
66
|
+
Comprehensive performance evaluations comparing DeepSift against sequential execution:
|
|
67
|
+
|
|
68
|
+
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
69
|
+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
70
|
+
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
71
|
+
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
72
|
+
|
|
73
|
+
To run the evaluations:
|
|
74
|
+
```bash
|
|
75
|
+
# Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
|
|
76
|
+
python -m evaluation
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
# Documentation Site
|
|
80
|
+
The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
|
|
81
|
+
|
|
82
|
+
To run the documentation dev server with hot-reloading:
|
|
83
|
+
```bash
|
|
84
|
+
# From project root:
|
|
85
|
+
npm run docs
|
|
86
|
+
|
|
87
|
+
# Or with force cache reload:
|
|
88
|
+
npm run docs:force
|
|
89
|
+
|
|
90
|
+
# Or directly inside docs/ using npx:
|
|
91
|
+
cd docs
|
|
92
|
+
npx astro dev --force
|
|
93
|
+
```
|
|
94
|
+
Then visit `http://localhost:4321/` in your browser.
|
|
95
|
+
|
|
96
|
+
To build the static production site:
|
|
97
|
+
```bash
|
|
98
|
+
npm run docs:build
|
|
99
|
+
# or: cd docs && npm run build
|
|
100
|
+
```
|
|
101
|
+
The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
|
|
102
|
+
|
|
103
|
+
Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
|
|
104
|
+
- [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
|
|
105
|
+
- [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
|
|
106
|
+
- [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
|
|
107
|
+
- [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
|
|
108
|
+
- [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
|
|
109
|
+
- [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
|
|
110
|
+
|
|
111
|
+
# TODO
|
|
112
|
+
- [x] make static tests using dummy source (runs on github ci/cd)
|
|
113
|
+
- [x] make live tests using actual source on r/resumes
|
|
114
|
+
- [x] make evaluation script
|
|
115
|
+
- [x] writeup and documentation with benchmarks, docs page, and cleaned repo
|
deepsift-0.2.0/README.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Resume Model-V3
|
|
2
|
+
What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
|
|
3
|
+
|
|
4
|
+
## Methodology
|
|
5
|
+
To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
|
|
6
|
+
|
|
7
|
+
We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
|
|
8
|
+
|
|
9
|
+
[](data/relevant_subs_cohort_11_11_2026.csv)
|
|
10
|
+
|
|
11
|
+
We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
|
|
12
|
+
|
|
13
|
+
We determined that the ideal cohort for our use case needed to satisfy the following factors:
|
|
14
|
+
- have >10 comments and >0 score
|
|
15
|
+
- have a associated url corresponding to a picture (.png or jpeg)
|
|
16
|
+
|
|
17
|
+
After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# Testing
|
|
22
|
+
To run the static unit and integration test suite offline using the synthetic `DummySource`:
|
|
23
|
+
```bash
|
|
24
|
+
pytest -m "not live" -v
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
To run the live integration tests against the actual Arctic Shift API:
|
|
28
|
+
```bash
|
|
29
|
+
pytest -m live -v
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
# Evaluation & Benchmarks
|
|
33
|
+
Comprehensive performance evaluations comparing DeepSift against sequential execution:
|
|
34
|
+
|
|
35
|
+
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
36
|
+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
37
|
+
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
38
|
+
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
39
|
+
|
|
40
|
+
To run the evaluations:
|
|
41
|
+
```bash
|
|
42
|
+
# Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
|
|
43
|
+
python -m evaluation
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
# Documentation Site
|
|
47
|
+
The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
|
|
48
|
+
|
|
49
|
+
To run the documentation dev server with hot-reloading:
|
|
50
|
+
```bash
|
|
51
|
+
# From project root:
|
|
52
|
+
npm run docs
|
|
53
|
+
|
|
54
|
+
# Or with force cache reload:
|
|
55
|
+
npm run docs:force
|
|
56
|
+
|
|
57
|
+
# Or directly inside docs/ using npx:
|
|
58
|
+
cd docs
|
|
59
|
+
npx astro dev --force
|
|
60
|
+
```
|
|
61
|
+
Then visit `http://localhost:4321/` in your browser.
|
|
62
|
+
|
|
63
|
+
To build the static production site:
|
|
64
|
+
```bash
|
|
65
|
+
npm run docs:build
|
|
66
|
+
# or: cd docs && npm run build
|
|
67
|
+
```
|
|
68
|
+
The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
|
|
69
|
+
|
|
70
|
+
Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
|
|
71
|
+
- [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
|
|
72
|
+
- [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
|
|
73
|
+
- [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
|
|
74
|
+
- [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
|
|
75
|
+
- [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
|
|
76
|
+
- [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
|
|
77
|
+
|
|
78
|
+
# TODO
|
|
79
|
+
- [x] make static tests using dummy source (runs on github ci/cd)
|
|
80
|
+
- [x] make live tests using actual source on r/resumes
|
|
81
|
+
- [x] make evaluation script
|
|
82
|
+
- [x] writeup and documentation with benchmarks, docs page, and cleaned repo
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DeepSift Library.
|
|
3
|
+
A high-performance research toolkit for historical Reddit data.
|
|
4
|
+
"""
|
|
5
|
+
from deepsift.models import (
|
|
6
|
+
Post,
|
|
7
|
+
Comment,
|
|
8
|
+
PostField,
|
|
9
|
+
CommentField,
|
|
10
|
+
FieldEnum,
|
|
11
|
+
UnretrievedFieldError,
|
|
12
|
+
)
|
|
13
|
+
from deepsift.filters import Filters, RequiredFieldError
|
|
14
|
+
from deepsift.sources import (
|
|
15
|
+
Source,
|
|
16
|
+
ArcticShiftSource,
|
|
17
|
+
get_source,
|
|
18
|
+
register_source,
|
|
19
|
+
)
|
|
20
|
+
from deepsift.storage import Storage, ParquetStorage
|
|
21
|
+
from deepsift.scraper import RedditScraper
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"PostField",
|
|
25
|
+
"CommentField",
|
|
26
|
+
"FieldEnum",
|
|
27
|
+
"Post",
|
|
28
|
+
"Comment",
|
|
29
|
+
"UnretrievedFieldError",
|
|
30
|
+
"RequiredFieldError",
|
|
31
|
+
"Filters",
|
|
32
|
+
"Source",
|
|
33
|
+
"ArcticShiftSource",
|
|
34
|
+
"get_source",
|
|
35
|
+
"register_source",
|
|
36
|
+
"RedditScraper",
|
|
37
|
+
"Storage",
|
|
38
|
+
"ParquetStorage",
|
|
39
|
+
]
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command line interface for the Reddit Research Scraper.
|
|
3
|
+
"""
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
import click
|
|
7
|
+
|
|
8
|
+
from deepsift.models import PostField, CommentField
|
|
9
|
+
from deepsift.filters import Filters
|
|
10
|
+
from deepsift.sources import ArcticShiftSource, get_source
|
|
11
|
+
from deepsift.scraper import RedditScraper
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.group()
|
|
15
|
+
def cli():
|
|
16
|
+
"""Reddit Research Scraper: High-performance historical Reddit data extractor."""
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@cli.command()
|
|
21
|
+
@click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
|
|
22
|
+
@click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
|
|
23
|
+
@click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
|
|
24
|
+
@click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
|
|
25
|
+
@click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep (validated against PostField enum).")
|
|
26
|
+
@click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop (validated against PostField enum).")
|
|
27
|
+
@click.option("--comment-include-fields", multiple=True, help="Important comment fields to keep.")
|
|
28
|
+
@click.option("--comment-exclude-fields", multiple=True, help="Unimportant comment fields to drop.")
|
|
29
|
+
@click.option("--posts-out", default="data/curated_posts.parquet", help="Parquet output destination for posts.")
|
|
30
|
+
@click.option("--comments-out", default="data/curated_comments.parquet", help="Parquet output destination for comments.")
|
|
31
|
+
@click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
|
|
32
|
+
@click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
|
|
33
|
+
@click.option("--comment-workers", default=5, type=int, help="Async comment concurrency workers.")
|
|
34
|
+
@click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
|
|
35
|
+
def run(
|
|
36
|
+
subreddits, from_dt, to_dt, shard_days,
|
|
37
|
+
include_fields, exclude_fields, comment_include_fields, comment_exclude_fields,
|
|
38
|
+
posts_out, comments_out, errors_out, post_workers, comment_workers, source
|
|
39
|
+
):
|
|
40
|
+
"""Run full pipeline: scrape posts, filter cohort, and extract comment trees."""
|
|
41
|
+
target_subs = list(subreddits)
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
filters = Filters(
|
|
45
|
+
post_filter=None,
|
|
46
|
+
comment_filter=None,
|
|
47
|
+
post_include_fields=list(include_fields) if include_fields else None,
|
|
48
|
+
post_exclude_fields=list(exclude_fields) if exclude_fields else None,
|
|
49
|
+
comment_include_fields=list(comment_include_fields) if comment_include_fields else None,
|
|
50
|
+
comment_exclude_fields=list(comment_exclude_fields) if comment_exclude_fields else None,
|
|
51
|
+
validate=True,
|
|
52
|
+
)
|
|
53
|
+
except ValueError as e:
|
|
54
|
+
click.secho(f"\n[!] Field Validation Error: {e}", fg="red", err=True)
|
|
55
|
+
sys.exit(1)
|
|
56
|
+
|
|
57
|
+
scraper = RedditScraper(
|
|
58
|
+
filters=filters,
|
|
59
|
+
source=get_source(source),
|
|
60
|
+
post_workers=post_workers,
|
|
61
|
+
comment_workers=comment_workers,
|
|
62
|
+
shard_days=shard_days,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
start_time = time.time()
|
|
66
|
+
click.echo(f"Reddit Research Scraper: {len(target_subs)} subreddits (shard_days={shard_days})")
|
|
67
|
+
click.echo(f"Window: from {from_dt} to {to_dt or 'now'}")
|
|
68
|
+
if filters.post_include_fields:
|
|
69
|
+
click.echo(f"Post Fields Kept: {sorted(list(filters.post_include_fields))}")
|
|
70
|
+
if filters.post_exclude_fields:
|
|
71
|
+
click.echo(f"Post Fields Dropped: {sorted(list(filters.post_exclude_fields))}")
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
posts, comments = scraper.run(
|
|
75
|
+
subreddits=target_subs,
|
|
76
|
+
from_dt=from_dt,
|
|
77
|
+
to_dt=to_dt,
|
|
78
|
+
posts_output=posts_out,
|
|
79
|
+
comments_output=comments_out,
|
|
80
|
+
errors_output=errors_out,
|
|
81
|
+
)
|
|
82
|
+
except Exception as e:
|
|
83
|
+
click.secho(f"\n[!] Pipeline Error: {e}", fg="red", err=True)
|
|
84
|
+
sys.exit(1)
|
|
85
|
+
|
|
86
|
+
elapsed = round(time.time() - start_time, 1)
|
|
87
|
+
click.echo(f"[✓] Pipeline complete in {elapsed}s")
|
|
88
|
+
click.echo(f" Posts: {len(posts):,} -> {posts_out}")
|
|
89
|
+
click.echo(f" Comments: {comments:,} -> {comments_out}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@cli.command()
|
|
93
|
+
@click.option("--subreddits", "-s", multiple=True, default=["resumes"], help="Subreddit(s) to scan.")
|
|
94
|
+
@click.option("--from", "from_dt", required=True, help="Start date/datetime (YYYY-MM-DD or ISO).")
|
|
95
|
+
@click.option("--to", "to_dt", default=None, help="End date/datetime (YYYY-MM-DD or ISO, defaults to today).")
|
|
96
|
+
@click.option("--shard-days", default=180, type=int, help="Shard window size in days.")
|
|
97
|
+
@click.option("--include-fields", "-i", multiple=True, help="Important post fields to keep.")
|
|
98
|
+
@click.option("--exclude-fields", "-e", multiple=True, help="Unimportant post fields to drop.")
|
|
99
|
+
@click.option("--output", "-o", default="data/curated_posts.parquet", help="Parquet output destination.")
|
|
100
|
+
@click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
|
|
101
|
+
@click.option("--post-workers", default=4, type=int, help="Parallel shard post workers.")
|
|
102
|
+
@click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
|
|
103
|
+
def posts(subreddits, from_dt, to_dt, shard_days, include_fields, exclude_fields, output, errors_out, post_workers, source):
|
|
104
|
+
"""Scrape and filter Reddit posts between --from and --to dates."""
|
|
105
|
+
target_subs = list(subreddits)
|
|
106
|
+
try:
|
|
107
|
+
filters = Filters(
|
|
108
|
+
post_filter=None,
|
|
109
|
+
post_include_fields=list(include_fields) if include_fields else None,
|
|
110
|
+
post_exclude_fields=list(exclude_fields) if exclude_fields else None,
|
|
111
|
+
)
|
|
112
|
+
scraper = RedditScraper(
|
|
113
|
+
filters=filters,
|
|
114
|
+
source=get_source(source),
|
|
115
|
+
post_workers=post_workers,
|
|
116
|
+
shard_days=shard_days,
|
|
117
|
+
)
|
|
118
|
+
scraper.scrape_posts(
|
|
119
|
+
subreddits=target_subs,
|
|
120
|
+
from_dt=from_dt,
|
|
121
|
+
to_dt=to_dt,
|
|
122
|
+
output_path=output,
|
|
123
|
+
errors_output=errors_out,
|
|
124
|
+
)
|
|
125
|
+
except Exception as e:
|
|
126
|
+
click.secho(f"\n[!] Error: {e}", fg="red", err=True)
|
|
127
|
+
sys.exit(1)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@cli.command()
|
|
131
|
+
@click.option("--post-id", "-p", "post_ids", multiple=True, required=True, help="Post ID(s) to fetch comments for.")
|
|
132
|
+
@click.option("--include-fields", "-i", multiple=True, help="Important comment fields to keep.")
|
|
133
|
+
@click.option("--exclude-fields", "-e", multiple=True, help="Unimportant comment fields to drop.")
|
|
134
|
+
@click.option("--output", "-o", default="data/curated_comments.parquet", help="Parquet output destination.")
|
|
135
|
+
@click.option("--errors-out", default="data/errors.json", help="JSON output destination for failure logs.")
|
|
136
|
+
@click.option("--workers", "-w", default=5, type=int, help="Async concurrency limit.")
|
|
137
|
+
@click.option("--source", default="arctic_shift", type=click.Choice(["arctic_shift"]), help="Data source provider.")
|
|
138
|
+
def comments(post_ids, include_fields, exclude_fields, output, errors_out, workers, source):
|
|
139
|
+
"""Scrape comment trees for given post IDs."""
|
|
140
|
+
try:
|
|
141
|
+
filters = Filters(
|
|
142
|
+
comment_filter=None,
|
|
143
|
+
comment_include_fields=list(include_fields) if include_fields else None,
|
|
144
|
+
comment_exclude_fields=list(exclude_fields) if exclude_fields else None,
|
|
145
|
+
)
|
|
146
|
+
scraper = RedditScraper(
|
|
147
|
+
filters=filters,
|
|
148
|
+
source=get_source(source),
|
|
149
|
+
comment_workers=workers,
|
|
150
|
+
)
|
|
151
|
+
scraper.scrape_comments(post_ids=list(post_ids), output_path=output, errors_output=errors_out)
|
|
152
|
+
except Exception as e:
|
|
153
|
+
click.secho(f"\n[!] Error: {e}", fg="red", err=True)
|
|
154
|
+
sys.exit(1)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@cli.command()
|
|
158
|
+
def fields():
|
|
159
|
+
"""List all valid PostField and CommentField enumerations with descriptions."""
|
|
160
|
+
click.echo(f"\nVALID POST FIELDS ({len(PostField)}):")
|
|
161
|
+
for p in sorted(list(PostField), key=lambda x: x.value):
|
|
162
|
+
desc = f" — {p.description}" if p.description else ""
|
|
163
|
+
click.echo(f" • {p.value:<30}{desc}")
|
|
164
|
+
click.echo(f"\nVALID COMMENT FIELDS ({len(CommentField)}):")
|
|
165
|
+
for c in sorted(list(CommentField), key=lambda x: x.value):
|
|
166
|
+
desc = f" — {c.description}" if c.description else ""
|
|
167
|
+
click.echo(f" • {c.value:<30}{desc}")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
if __name__ == "__main__":
|
|
171
|
+
cli()
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Generalized Filters class.
|
|
3
|
+
Accepts custom filter functions:
|
|
4
|
+
post_filter: Callable[[Post], bool]
|
|
5
|
+
comment_filter: Callable[[Comment], bool]
|
|
6
|
+
Calculates the exact server projection fields needed, preventing unwanted fields
|
|
7
|
+
from ever being transmitted over the wire.
|
|
8
|
+
"""
|
|
9
|
+
from typing import Callable, Optional, Iterable, Union, Set
|
|
10
|
+
from deepsift.models import Post, Comment, PostField, CommentField
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RequiredFieldError(ValueError):
|
|
14
|
+
"""Raised when attempting to exclude a fundamentally required entity field."""
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Filters:
|
|
19
|
+
"""
|
|
20
|
+
Generalized filter container.
|
|
21
|
+
Manages custom predicate functions and minimal server-side field projections.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
post_filter: Optional[Callable[[Post], bool]] = None,
|
|
27
|
+
comment_filter: Optional[Callable[[Comment], bool]] = None,
|
|
28
|
+
post_include_fields: Optional[Iterable[Union[str, PostField]]] = None,
|
|
29
|
+
comment_include_fields: Optional[Iterable[Union[str, CommentField]]] = None,
|
|
30
|
+
post_exclude_fields: Optional[Iterable[Union[str, PostField]]] = None,
|
|
31
|
+
comment_exclude_fields: Optional[Iterable[Union[str, CommentField]]] = None,
|
|
32
|
+
additional_filter_fields: Optional[Iterable[Union[str, PostField]]] = None,
|
|
33
|
+
validate: bool = True,
|
|
34
|
+
):
|
|
35
|
+
self.post_filter = post_filter or (lambda post: True)
|
|
36
|
+
self.comment_filter = comment_filter or (lambda comment: True)
|
|
37
|
+
|
|
38
|
+
if validate:
|
|
39
|
+
self.post_include_fields = PostField.validate_many(post_include_fields)
|
|
40
|
+
self.comment_include_fields = CommentField.validate_many(comment_include_fields)
|
|
41
|
+
self.post_exclude_fields = PostField.validate_many(post_exclude_fields)
|
|
42
|
+
self.comment_exclude_fields = CommentField.validate_many(comment_exclude_fields)
|
|
43
|
+
self.additional_filter_fields = PostField.validate_many(additional_filter_fields)
|
|
44
|
+
else:
|
|
45
|
+
self.post_include_fields = {str(f).lower().strip() for f in post_include_fields} if post_include_fields else None
|
|
46
|
+
self.comment_include_fields = {str(f).lower().strip() for f in comment_include_fields} if comment_include_fields else None
|
|
47
|
+
self.post_exclude_fields = {str(f).lower().strip() for f in post_exclude_fields} if post_exclude_fields else None
|
|
48
|
+
self.comment_exclude_fields = {str(f).lower().strip() for f in comment_exclude_fields} if comment_exclude_fields else None
|
|
49
|
+
self.additional_filter_fields = {str(f).lower().strip() for f in additional_filter_fields} if additional_filter_fields else None
|
|
50
|
+
|
|
51
|
+
# Always strictly reject exclusion of required schema fields
|
|
52
|
+
self._validate_required(self.post_exclude_fields, Post.REQUIRED_FIELDS, "post_exclude_fields")
|
|
53
|
+
self._validate_required(self.comment_exclude_fields, Comment.REQUIRED_FIELDS, "comment_exclude_fields")
|
|
54
|
+
|
|
55
|
+
self.required_post_fields = self._compute_projection(
|
|
56
|
+
self.post_include_fields, self.post_exclude_fields, Post.REQUIRED_FIELDS,
|
|
57
|
+
PostField.all_fields(), self.additional_filter_fields
|
|
58
|
+
)
|
|
59
|
+
self.required_comment_fields = self._compute_projection(
|
|
60
|
+
self.comment_include_fields, self.comment_exclude_fields, Comment.REQUIRED_FIELDS,
|
|
61
|
+
CommentField.all_fields()
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def _validate_required(exclude_fields: Optional[Set[str]], required_fields: frozenset, param_name: str):
|
|
66
|
+
if exclude_fields:
|
|
67
|
+
for field in required_fields:
|
|
68
|
+
if field in exclude_fields:
|
|
69
|
+
raise RequiredFieldError(
|
|
70
|
+
f"Required field '{field}' cannot be excluded in {param_name}. "
|
|
71
|
+
f"Required fields: {sorted(list(required_fields))}."
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _compute_projection(
|
|
76
|
+
include: Optional[Set[str]],
|
|
77
|
+
exclude: Optional[Set[str]],
|
|
78
|
+
required: frozenset,
|
|
79
|
+
all_fields: Set[str],
|
|
80
|
+
extra: Optional[Set[str]] = None,
|
|
81
|
+
) -> Optional[Set[str]]:
|
|
82
|
+
if include is None and exclude is None:
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
core = set(required)
|
|
86
|
+
if include is not None:
|
|
87
|
+
fields = set(include) | core
|
|
88
|
+
if exclude:
|
|
89
|
+
fields -= (exclude - core)
|
|
90
|
+
else:
|
|
91
|
+
fields = (all_fields - exclude) | core
|
|
92
|
+
|
|
93
|
+
if extra:
|
|
94
|
+
fields |= extra
|
|
95
|
+
return fields
|
|
96
|
+
|
|
97
|
+
def is_valid_post(self, post: Post) -> bool:
|
|
98
|
+
"""Evaluates custom filter against a typed Post object."""
|
|
99
|
+
return bool(self.post_filter(post))
|
|
100
|
+
|
|
101
|
+
def is_valid_comment(self, comment: Comment) -> bool:
|
|
102
|
+
"""Evaluates custom filter against a typed Comment object."""
|
|
103
|
+
return bool(self.comment_filter(comment))
|
|
104
|
+
|
|
105
|
+
def filter_post_record(self, post: Post) -> dict:
|
|
106
|
+
"""Serializes post to dict."""
|
|
107
|
+
return post.to_dict()
|
|
108
|
+
|
|
109
|
+
def filter_comment_record(self, comment: Comment) -> dict:
|
|
110
|
+
"""Serializes comment to dict."""
|
|
111
|
+
return comment.to_dict()
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Models subpackage for Reddit data structures.
|
|
3
|
+
"""
|
|
4
|
+
from deepsift.models.enums import FieldEnum, PostField, CommentField
|
|
5
|
+
from deepsift.models.base import RedditEntity, UnretrievedFieldError
|
|
6
|
+
from deepsift.models.post import Post
|
|
7
|
+
from deepsift.models.comment import Comment
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"FieldEnum",
|
|
11
|
+
"PostField",
|
|
12
|
+
"CommentField",
|
|
13
|
+
"RedditEntity",
|
|
14
|
+
"Post",
|
|
15
|
+
"Comment",
|
|
16
|
+
"UnretrievedFieldError",
|
|
17
|
+
]
|