deepsift 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepsift-0.2.2/PKG-INFO +212 -0
- deepsift-0.2.2/README.md +179 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/pyproject.toml +10 -2
- deepsift-0.2.2/src/deepsift.egg-info/PKG-INFO +212 -0
- deepsift-0.2.2/src/deepsift.egg-info/SOURCES.txt +33 -0
- deepsift-0.2.1/PKG-INFO +0 -115
- deepsift-0.2.1/README.md +0 -82
- deepsift-0.2.1/deepsift.egg-info/PKG-INFO +0 -115
- deepsift-0.2.1/deepsift.egg-info/SOURCES.txt +0 -34
- deepsift-0.2.1/tests/test_evaluation.py +0 -41
- {deepsift-0.2.1 → deepsift-0.2.2}/LICENSE +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/setup.cfg +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/__init__.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/__main__.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/cli.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/filters.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/__init__.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/base.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/comment.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/enums.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/post.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/scraper.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/__init__.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/arctic_shift.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/base.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/dummy.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/registry.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/storage/__init__.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/storage/base.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/storage/parquet.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/dependency_links.txt +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/entry_points.txt +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/requires.txt +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/top_level.txt +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_cli.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_filters.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_models.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_scraper_dummy.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_scraper_live.py +0 -0
- {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_storage.py +0 -0
deepsift-0.2.2/PKG-INFO
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deepsift
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
|
|
5
|
+
Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/saakethk/deepsift
|
|
8
|
+
Project-URL: Documentation, https://saakethk.github.io/deepsift/
|
|
9
|
+
Project-URL: Repository, https://github.com/saakethk/deepsift.git
|
|
10
|
+
Project-URL: Issues, https://github.com/saakethk/deepsift/issues
|
|
11
|
+
Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: requests>=2.31.0
|
|
22
|
+
Requires-Dist: httpx>=0.27.0
|
|
23
|
+
Requires-Dist: pandas>=2.0.0
|
|
24
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
25
|
+
Requires-Dist: tqdm>=4.66.0
|
|
26
|
+
Requires-Dist: click>=8.1.0
|
|
27
|
+
Requires-Dist: nest-asyncio>=1.6.0
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# DeepSift
|
|
35
|
+
|
|
36
|
+
[](https://pypi.org/project/deepsift/)
|
|
37
|
+
[](https://www.python.org/downloads/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
[](tests/)
|
|
40
|
+
|
|
41
|
+
DeepSift is a high-performance Python library designed for empirical and academic researchers to scrape, filter, and curate massive historical Reddit datasets at scale.
|
|
42
|
+
|
|
43
|
+
By combining **parallel temporal sharding**, **server-side field projection**, and **PyArrow Parquet streaming**, DeepSift overcomes the common bottlenecks of Reddit data collection—slashing bandwidth, keeping memory footprint constant, and achieving over **4.5x faster throughput** than standard sequential scrapers.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Key Performance Advantages
|
|
48
|
+
|
|
49
|
+
* **Multi-Worker Temporal Sharding**: Splits arbitrary multi-month or multi-year windows into discrete temporal segments across parallel thread workers with adaptive backoff and rate-limit recovery.
|
|
50
|
+
* **Server-Side Field Projection**: Mathematically projects predicates to request only the attributes needed by your downstream study over the wire, cutting payload bandwidth by up to **24.0x**.
|
|
51
|
+
* **Constant-Memory Parquet Streaming**: Flushes chunked batches straight to columnar Apache Parquet files via PyArrow, reducing peak RAM consumption by up to **16.8x** compared to in-memory buffering.
|
|
52
|
+
* **Asynchronous Comment Tree Resolution**: Extracts and flattens deeply nested discussion trees using `httpx` async workers with configurable concurrency limits and automatic `"more"` stub pruning.
|
|
53
|
+
* **Polymorphic Schema Coercion**: Handles schema anomalies (such as polymorphic `edited` timestamps, flairs, and variable-typed metadata) without failing batch serialization.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Installation
|
|
58
|
+
|
|
59
|
+
Install the library directly from PyPI:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install deepsift
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
To install local development dependencies:
|
|
66
|
+
```bash
|
|
67
|
+
pip install deepsift[dev]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## Quickstart
|
|
73
|
+
|
|
74
|
+
### 1. Python API
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import pandas as pd
|
|
78
|
+
from deepsift import RedditScraper, Filters, PostField, CommentField
|
|
79
|
+
|
|
80
|
+
# Request only the fields needed for your analysis
|
|
81
|
+
filters = Filters(
|
|
82
|
+
post_include_fields=[
|
|
83
|
+
PostField.ID,
|
|
84
|
+
PostField.TITLE,
|
|
85
|
+
PostField.SELFTEXT,
|
|
86
|
+
PostField.SCORE,
|
|
87
|
+
PostField.NUM_COMMENTS,
|
|
88
|
+
PostField.AUTHOR,
|
|
89
|
+
PostField.CREATED_UTC,
|
|
90
|
+
PostField.SUBREDDIT,
|
|
91
|
+
],
|
|
92
|
+
comment_include_fields=[
|
|
93
|
+
CommentField.ID,
|
|
94
|
+
CommentField.POST_ID,
|
|
95
|
+
CommentField.PARENT_ID,
|
|
96
|
+
CommentField.AUTHOR,
|
|
97
|
+
CommentField.BODY,
|
|
98
|
+
CommentField.SCORE,
|
|
99
|
+
CommentField.CREATED_UTC,
|
|
100
|
+
],
|
|
101
|
+
post_filter=lambda p: (p.score or 0) >= 3 and (p.num_comments or 0) >= 1,
|
|
102
|
+
comment_filter=lambda c: len(c.body or "") >= 20 and c.author != "[deleted]",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Configure parallel workers and temporal shard granularity
|
|
106
|
+
scraper = RedditScraper(
|
|
107
|
+
filters=filters,
|
|
108
|
+
post_workers=4,
|
|
109
|
+
comment_workers=5,
|
|
110
|
+
shard_days=7,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Run full ingestion pipeline (submissions + comment trees)
|
|
114
|
+
post_ids, comment_count = scraper.run(
|
|
115
|
+
subreddits=["MachineLearning"],
|
|
116
|
+
from_dt="2024-01-01",
|
|
117
|
+
to_dt="2024-01-14",
|
|
118
|
+
posts_output="data/posts.parquet",
|
|
119
|
+
comments_output="data/comments.parquet",
|
|
120
|
+
errors_output="data/errors.json",
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
print(f"Scraped {len(post_ids)} posts and {comment_count} comments.")
|
|
124
|
+
|
|
125
|
+
# Open directly into pandas, Polars, or DuckDB
|
|
126
|
+
df_posts = pd.read_parquet("data/posts.parquet")
|
|
127
|
+
df_comments = pd.read_parquet("data/comments.parquet")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### 2. Command Line Interface (CLI)
|
|
131
|
+
|
|
132
|
+
DeepSift includes a CLI for pipeline execution and quick queries:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
# Run complete post + comment pipeline
|
|
136
|
+
deepsift run --subreddit MachineLearning --from 2024-01-01 --to 2024-01-07 --posts data/posts.parquet --comments data/comments.parquet
|
|
137
|
+
|
|
138
|
+
# Scrape posts only
|
|
139
|
+
deepsift posts --subreddit resumes --from 2024-01-01 --to 2024-01-30 --output data/resumes_posts.parquet
|
|
140
|
+
|
|
141
|
+
# Extract comments for specific post IDs
|
|
142
|
+
deepsift comments --post-ids id1,id2,id3 --output data/extracted_comments.parquet
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### 3. Offline Testing with `DummySource`
|
|
146
|
+
|
|
147
|
+
For unit testing, benchmarking, or development without network calls, swap in `DummySource` to generate realistic synthetic posts and comments:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from deepsift import RedditScraper, Filters
|
|
151
|
+
from deepsift.sources.dummy import DummySource
|
|
152
|
+
|
|
153
|
+
dummy = DummySource(posts_per_shard=100, comments_per_post=5, simulated_latency=0.0)
|
|
154
|
+
scraper = RedditScraper(source=dummy, shard_days=1)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## Evaluation & Benchmarks
|
|
160
|
+
|
|
161
|
+
Empirical performance evaluation comparing DeepSift (6 parallel workers) against standard sequential extraction:
|
|
162
|
+
|
|
163
|
+
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
164
|
+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
165
|
+
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
166
|
+
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
167
|
+
|
|
168
|
+
To reproduce the benchmark suite:
|
|
169
|
+
```bash
|
|
170
|
+
python -m evaluation
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## Running Tests
|
|
176
|
+
|
|
177
|
+
Run the test suite offline using synthetic data:
|
|
178
|
+
```bash
|
|
179
|
+
pytest -m "not live" -v
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Run live integration tests against real historical endpoints:
|
|
183
|
+
```bash
|
|
184
|
+
pytest -m live -v
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## Documentation
|
|
190
|
+
|
|
191
|
+
The full documentation website is built with [Astro Starlight](https://starlight.astro.build/) and is located in the [`docs/`](docs/) directory.
|
|
192
|
+
|
|
193
|
+
To start the documentation dev server locally:
|
|
194
|
+
```bash
|
|
195
|
+
cd docs
|
|
196
|
+
npm install
|
|
197
|
+
npm run dev
|
|
198
|
+
```
|
|
199
|
+
Then visit `http://localhost:4321/` in your browser.
|
|
200
|
+
|
|
201
|
+
Documentation sources:
|
|
202
|
+
* [Overview & Architecture](docs/src/content/docs/architecture.md)
|
|
203
|
+
* [Quickstart Guide](docs/src/content/docs/quickstart.md)
|
|
204
|
+
* [Python API Reference](docs/src/content/docs/api.md)
|
|
205
|
+
* [CLI Reference](docs/src/content/docs/cli.md)
|
|
206
|
+
* [Benchmarks & Scaling](docs/src/content/docs/benchmarks.md)
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## License
|
|
211
|
+
|
|
212
|
+
This project is licensed under the [MIT License](LICENSE).
|
deepsift-0.2.2/README.md
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# DeepSift
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/deepsift/)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](tests/)
|
|
7
|
+
|
|
8
|
+
DeepSift is a high-performance Python library designed for empirical and academic researchers to scrape, filter, and curate massive historical Reddit datasets at scale.
|
|
9
|
+
|
|
10
|
+
By combining **parallel temporal sharding**, **server-side field projection**, and **PyArrow Parquet streaming**, DeepSift overcomes the common bottlenecks of Reddit data collection—slashing bandwidth, keeping memory footprint constant, and achieving over **4.5x faster throughput** than standard sequential scrapers.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## Key Performance Advantages
|
|
15
|
+
|
|
16
|
+
* **Multi-Worker Temporal Sharding**: Splits arbitrary multi-month or multi-year windows into discrete temporal segments across parallel thread workers with adaptive backoff and rate-limit recovery.
|
|
17
|
+
* **Server-Side Field Projection**: Mathematically projects predicates to request only the attributes needed by your downstream study over the wire, cutting payload bandwidth by up to **24.0x**.
|
|
18
|
+
* **Constant-Memory Parquet Streaming**: Flushes chunked batches straight to columnar Apache Parquet files via PyArrow, reducing peak RAM consumption by up to **16.8x** compared to in-memory buffering.
|
|
19
|
+
* **Asynchronous Comment Tree Resolution**: Extracts and flattens deeply nested discussion trees using `httpx` async workers with configurable concurrency limits and automatic `"more"` stub pruning.
|
|
20
|
+
* **Polymorphic Schema Coercion**: Handles schema anomalies (such as polymorphic `edited` timestamps, flairs, and variable-typed metadata) without failing batch serialization.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
Install the library directly from PyPI:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install deepsift
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
To install local development dependencies:
|
|
33
|
+
```bash
|
|
34
|
+
pip install deepsift[dev]
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## Quickstart
|
|
40
|
+
|
|
41
|
+
### 1. Python API
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
import pandas as pd
|
|
45
|
+
from deepsift import RedditScraper, Filters, PostField, CommentField
|
|
46
|
+
|
|
47
|
+
# Request only the fields needed for your analysis
|
|
48
|
+
filters = Filters(
|
|
49
|
+
post_include_fields=[
|
|
50
|
+
PostField.ID,
|
|
51
|
+
PostField.TITLE,
|
|
52
|
+
PostField.SELFTEXT,
|
|
53
|
+
PostField.SCORE,
|
|
54
|
+
PostField.NUM_COMMENTS,
|
|
55
|
+
PostField.AUTHOR,
|
|
56
|
+
PostField.CREATED_UTC,
|
|
57
|
+
PostField.SUBREDDIT,
|
|
58
|
+
],
|
|
59
|
+
comment_include_fields=[
|
|
60
|
+
CommentField.ID,
|
|
61
|
+
CommentField.POST_ID,
|
|
62
|
+
CommentField.PARENT_ID,
|
|
63
|
+
CommentField.AUTHOR,
|
|
64
|
+
CommentField.BODY,
|
|
65
|
+
CommentField.SCORE,
|
|
66
|
+
CommentField.CREATED_UTC,
|
|
67
|
+
],
|
|
68
|
+
post_filter=lambda p: (p.score or 0) >= 3 and (p.num_comments or 0) >= 1,
|
|
69
|
+
comment_filter=lambda c: len(c.body or "") >= 20 and c.author != "[deleted]",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Configure parallel workers and temporal shard granularity
|
|
73
|
+
scraper = RedditScraper(
|
|
74
|
+
filters=filters,
|
|
75
|
+
post_workers=4,
|
|
76
|
+
comment_workers=5,
|
|
77
|
+
shard_days=7,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# Run full ingestion pipeline (submissions + comment trees)
|
|
81
|
+
post_ids, comment_count = scraper.run(
|
|
82
|
+
subreddits=["MachineLearning"],
|
|
83
|
+
from_dt="2024-01-01",
|
|
84
|
+
to_dt="2024-01-14",
|
|
85
|
+
posts_output="data/posts.parquet",
|
|
86
|
+
comments_output="data/comments.parquet",
|
|
87
|
+
errors_output="data/errors.json",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
print(f"Scraped {len(post_ids)} posts and {comment_count} comments.")
|
|
91
|
+
|
|
92
|
+
# Open directly into pandas, Polars, or DuckDB
|
|
93
|
+
df_posts = pd.read_parquet("data/posts.parquet")
|
|
94
|
+
df_comments = pd.read_parquet("data/comments.parquet")
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### 2. Command Line Interface (CLI)
|
|
98
|
+
|
|
99
|
+
DeepSift includes a CLI for pipeline execution and quick queries:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
# Run complete post + comment pipeline
|
|
103
|
+
deepsift run --subreddit MachineLearning --from 2024-01-01 --to 2024-01-07 --posts data/posts.parquet --comments data/comments.parquet
|
|
104
|
+
|
|
105
|
+
# Scrape posts only
|
|
106
|
+
deepsift posts --subreddit resumes --from 2024-01-01 --to 2024-01-30 --output data/resumes_posts.parquet
|
|
107
|
+
|
|
108
|
+
# Extract comments for specific post IDs
|
|
109
|
+
deepsift comments --post-ids id1,id2,id3 --output data/extracted_comments.parquet
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### 3. Offline Testing with `DummySource`
|
|
113
|
+
|
|
114
|
+
For unit testing, benchmarking, or development without network calls, swap in `DummySource` to generate realistic synthetic posts and comments:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from deepsift import RedditScraper, Filters
|
|
118
|
+
from deepsift.sources.dummy import DummySource
|
|
119
|
+
|
|
120
|
+
dummy = DummySource(posts_per_shard=100, comments_per_post=5, simulated_latency=0.0)
|
|
121
|
+
scraper = RedditScraper(source=dummy, shard_days=1)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## Evaluation & Benchmarks
|
|
127
|
+
|
|
128
|
+
Empirical performance evaluation comparing DeepSift (6 parallel workers) against standard sequential extraction:
|
|
129
|
+
|
|
130
|
+
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
131
|
+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
132
|
+
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
133
|
+
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
134
|
+
|
|
135
|
+
To reproduce the benchmark suite:
|
|
136
|
+
```bash
|
|
137
|
+
python -m evaluation
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Running Tests
|
|
143
|
+
|
|
144
|
+
Run the test suite offline using synthetic data:
|
|
145
|
+
```bash
|
|
146
|
+
pytest -m "not live" -v
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Run live integration tests against real historical endpoints:
|
|
150
|
+
```bash
|
|
151
|
+
pytest -m live -v
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## Documentation
|
|
157
|
+
|
|
158
|
+
The full documentation website is built with [Astro Starlight](https://starlight.astro.build/) and is located in the [`docs/`](docs/) directory.
|
|
159
|
+
|
|
160
|
+
To start the documentation dev server locally:
|
|
161
|
+
```bash
|
|
162
|
+
cd docs
|
|
163
|
+
npm install
|
|
164
|
+
npm run dev
|
|
165
|
+
```
|
|
166
|
+
Then visit `http://localhost:4321/` in your browser.
|
|
167
|
+
|
|
168
|
+
Documentation sources:
|
|
169
|
+
* [Overview & Architecture](docs/src/content/docs/architecture.md)
|
|
170
|
+
* [Quickstart Guide](docs/src/content/docs/quickstart.md)
|
|
171
|
+
* [Python API Reference](docs/src/content/docs/api.md)
|
|
172
|
+
* [CLI Reference](docs/src/content/docs/cli.md)
|
|
173
|
+
* [Benchmarks & Scaling](docs/src/content/docs/benchmarks.md)
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "deepsift"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "High-performance historical Reddit research toolkit and scraper for Arctic Shift."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -50,10 +50,18 @@ deepsift = "deepsift.cli:cli"
|
|
|
50
50
|
reddit-scraper = "deepsift.cli:cli"
|
|
51
51
|
|
|
52
52
|
[tool.setuptools.packages.find]
|
|
53
|
-
where = ["
|
|
53
|
+
where = ["src"]
|
|
54
54
|
include = ["deepsift*"]
|
|
55
55
|
|
|
56
56
|
[tool.pytest.ini_options]
|
|
57
57
|
markers = [
|
|
58
58
|
"live: marks tests requiring live network connectivity to external APIs (Arctic Shift)",
|
|
59
59
|
]
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = "py311"
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint]
|
|
66
|
+
select = ["E", "F", "I", "UP"]
|
|
67
|
+
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deepsift
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
|
|
5
|
+
Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/saakethk/deepsift
|
|
8
|
+
Project-URL: Documentation, https://saakethk.github.io/deepsift/
|
|
9
|
+
Project-URL: Repository, https://github.com/saakethk/deepsift.git
|
|
10
|
+
Project-URL: Issues, https://github.com/saakethk/deepsift/issues
|
|
11
|
+
Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: requests>=2.31.0
|
|
22
|
+
Requires-Dist: httpx>=0.27.0
|
|
23
|
+
Requires-Dist: pandas>=2.0.0
|
|
24
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
25
|
+
Requires-Dist: tqdm>=4.66.0
|
|
26
|
+
Requires-Dist: click>=8.1.0
|
|
27
|
+
Requires-Dist: nest-asyncio>=1.6.0
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# DeepSift
|
|
35
|
+
|
|
36
|
+
[](https://pypi.org/project/deepsift/)
|
|
37
|
+
[](https://www.python.org/downloads/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
[](tests/)
|
|
40
|
+
|
|
41
|
+
DeepSift is a high-performance Python library designed for empirical and academic researchers to scrape, filter, and curate massive historical Reddit datasets at scale.
|
|
42
|
+
|
|
43
|
+
By combining **parallel temporal sharding**, **server-side field projection**, and **PyArrow Parquet streaming**, DeepSift overcomes the common bottlenecks of Reddit data collection—slashing bandwidth, keeping memory footprint constant, and achieving over **4.5x faster throughput** than standard sequential scrapers.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Key Performance Advantages
|
|
48
|
+
|
|
49
|
+
* **Multi-Worker Temporal Sharding**: Splits arbitrary multi-month or multi-year windows into discrete temporal segments across parallel thread workers with adaptive backoff and rate-limit recovery.
|
|
50
|
+
* **Server-Side Field Projection**: Mathematically projects predicates to request only the attributes needed by your downstream study over the wire, cutting payload bandwidth by up to **24.0x**.
|
|
51
|
+
* **Constant-Memory Parquet Streaming**: Flushes chunked batches straight to columnar Apache Parquet files via PyArrow, reducing peak RAM consumption by up to **16.8x** compared to in-memory buffering.
|
|
52
|
+
* **Asynchronous Comment Tree Resolution**: Extracts and flattens deeply nested discussion trees using `httpx` async workers with configurable concurrency limits and automatic `"more"` stub pruning.
|
|
53
|
+
* **Polymorphic Schema Coercion**: Handles schema anomalies (such as polymorphic `edited` timestamps, flairs, and variable-typed metadata) without failing batch serialization.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Installation
|
|
58
|
+
|
|
59
|
+
Install the library directly from PyPI:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install deepsift
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
To install local development dependencies:
|
|
66
|
+
```bash
|
|
67
|
+
pip install deepsift[dev]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## Quickstart
|
|
73
|
+
|
|
74
|
+
### 1. Python API
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import pandas as pd
|
|
78
|
+
from deepsift import RedditScraper, Filters, PostField, CommentField
|
|
79
|
+
|
|
80
|
+
# Request only the fields needed for your analysis
|
|
81
|
+
filters = Filters(
|
|
82
|
+
post_include_fields=[
|
|
83
|
+
PostField.ID,
|
|
84
|
+
PostField.TITLE,
|
|
85
|
+
PostField.SELFTEXT,
|
|
86
|
+
PostField.SCORE,
|
|
87
|
+
PostField.NUM_COMMENTS,
|
|
88
|
+
PostField.AUTHOR,
|
|
89
|
+
PostField.CREATED_UTC,
|
|
90
|
+
PostField.SUBREDDIT,
|
|
91
|
+
],
|
|
92
|
+
comment_include_fields=[
|
|
93
|
+
CommentField.ID,
|
|
94
|
+
CommentField.POST_ID,
|
|
95
|
+
CommentField.PARENT_ID,
|
|
96
|
+
CommentField.AUTHOR,
|
|
97
|
+
CommentField.BODY,
|
|
98
|
+
CommentField.SCORE,
|
|
99
|
+
CommentField.CREATED_UTC,
|
|
100
|
+
],
|
|
101
|
+
post_filter=lambda p: (p.score or 0) >= 3 and (p.num_comments or 0) >= 1,
|
|
102
|
+
comment_filter=lambda c: len(c.body or "") >= 20 and c.author != "[deleted]",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Configure parallel workers and temporal shard granularity
|
|
106
|
+
scraper = RedditScraper(
|
|
107
|
+
filters=filters,
|
|
108
|
+
post_workers=4,
|
|
109
|
+
comment_workers=5,
|
|
110
|
+
shard_days=7,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Run full ingestion pipeline (submissions + comment trees)
|
|
114
|
+
post_ids, comment_count = scraper.run(
|
|
115
|
+
subreddits=["MachineLearning"],
|
|
116
|
+
from_dt="2024-01-01",
|
|
117
|
+
to_dt="2024-01-14",
|
|
118
|
+
posts_output="data/posts.parquet",
|
|
119
|
+
comments_output="data/comments.parquet",
|
|
120
|
+
errors_output="data/errors.json",
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
print(f"Scraped {len(post_ids)} posts and {comment_count} comments.")
|
|
124
|
+
|
|
125
|
+
# Open directly into pandas, Polars, or DuckDB
|
|
126
|
+
df_posts = pd.read_parquet("data/posts.parquet")
|
|
127
|
+
df_comments = pd.read_parquet("data/comments.parquet")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### 2. Command Line Interface (CLI)
|
|
131
|
+
|
|
132
|
+
DeepSift includes a CLI for pipeline execution and quick queries:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
# Run complete post + comment pipeline
|
|
136
|
+
deepsift run --subreddit MachineLearning --from 2024-01-01 --to 2024-01-07 --posts data/posts.parquet --comments data/comments.parquet
|
|
137
|
+
|
|
138
|
+
# Scrape posts only
|
|
139
|
+
deepsift posts --subreddit resumes --from 2024-01-01 --to 2024-01-30 --output data/resumes_posts.parquet
|
|
140
|
+
|
|
141
|
+
# Extract comments for specific post IDs
|
|
142
|
+
deepsift comments --post-ids id1,id2,id3 --output data/extracted_comments.parquet
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### 3. Offline Testing with `DummySource`
|
|
146
|
+
|
|
147
|
+
For unit testing, benchmarking, or development without network calls, swap in `DummySource` to generate realistic synthetic posts and comments:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from deepsift import RedditScraper, Filters
|
|
151
|
+
from deepsift.sources.dummy import DummySource
|
|
152
|
+
|
|
153
|
+
dummy = DummySource(posts_per_shard=100, comments_per_post=5, simulated_latency=0.0)
|
|
154
|
+
scraper = RedditScraper(source=dummy, shard_days=1)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## Evaluation & Benchmarks
|
|
160
|
+
|
|
161
|
+
Empirical performance evaluation comparing DeepSift (6 parallel workers) against standard sequential extraction:
|
|
162
|
+
|
|
163
|
+
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
164
|
+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
165
|
+
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
166
|
+
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
167
|
+
|
|
168
|
+
To reproduce the benchmark suite:
|
|
169
|
+
```bash
|
|
170
|
+
python -m evaluation
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## Running Tests
|
|
176
|
+
|
|
177
|
+
Run the test suite offline using synthetic data:
|
|
178
|
+
```bash
|
|
179
|
+
pytest -m "not live" -v
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Run live integration tests against real historical endpoints:
|
|
183
|
+
```bash
|
|
184
|
+
pytest -m live -v
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## Documentation
|
|
190
|
+
|
|
191
|
+
The full documentation website is built with [Astro Starlight](https://starlight.astro.build/) and is located in the [`docs/`](docs/) directory.
|
|
192
|
+
|
|
193
|
+
To start the documentation dev server locally:
|
|
194
|
+
```bash
|
|
195
|
+
cd docs
|
|
196
|
+
npm install
|
|
197
|
+
npm run dev
|
|
198
|
+
```
|
|
199
|
+
Then visit `http://localhost:4321/` in your browser.
|
|
200
|
+
|
|
201
|
+
Documentation sources:
|
|
202
|
+
* [Overview & Architecture](docs/src/content/docs/architecture.md)
|
|
203
|
+
* [Quickstart Guide](docs/src/content/docs/quickstart.md)
|
|
204
|
+
* [Python API Reference](docs/src/content/docs/api.md)
|
|
205
|
+
* [CLI Reference](docs/src/content/docs/cli.md)
|
|
206
|
+
* [Benchmarks & Scaling](docs/src/content/docs/benchmarks.md)
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## License
|
|
211
|
+
|
|
212
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/deepsift/__init__.py
|
|
5
|
+
src/deepsift/__main__.py
|
|
6
|
+
src/deepsift/cli.py
|
|
7
|
+
src/deepsift/filters.py
|
|
8
|
+
src/deepsift/scraper.py
|
|
9
|
+
src/deepsift.egg-info/PKG-INFO
|
|
10
|
+
src/deepsift.egg-info/SOURCES.txt
|
|
11
|
+
src/deepsift.egg-info/dependency_links.txt
|
|
12
|
+
src/deepsift.egg-info/entry_points.txt
|
|
13
|
+
src/deepsift.egg-info/requires.txt
|
|
14
|
+
src/deepsift.egg-info/top_level.txt
|
|
15
|
+
src/deepsift/models/__init__.py
|
|
16
|
+
src/deepsift/models/base.py
|
|
17
|
+
src/deepsift/models/comment.py
|
|
18
|
+
src/deepsift/models/enums.py
|
|
19
|
+
src/deepsift/models/post.py
|
|
20
|
+
src/deepsift/sources/__init__.py
|
|
21
|
+
src/deepsift/sources/arctic_shift.py
|
|
22
|
+
src/deepsift/sources/base.py
|
|
23
|
+
src/deepsift/sources/dummy.py
|
|
24
|
+
src/deepsift/sources/registry.py
|
|
25
|
+
src/deepsift/storage/__init__.py
|
|
26
|
+
src/deepsift/storage/base.py
|
|
27
|
+
src/deepsift/storage/parquet.py
|
|
28
|
+
tests/test_cli.py
|
|
29
|
+
tests/test_filters.py
|
|
30
|
+
tests/test_models.py
|
|
31
|
+
tests/test_scraper_dummy.py
|
|
32
|
+
tests/test_scraper_live.py
|
|
33
|
+
tests/test_storage.py
|
deepsift-0.2.1/PKG-INFO
DELETED
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: deepsift
|
|
3
|
-
Version: 0.2.1
|
|
4
|
-
Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
|
|
5
|
-
Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/saakethk/deepsift
|
|
8
|
-
Project-URL: Documentation, https://saakethk.github.io/deepsift/
|
|
9
|
-
Project-URL: Repository, https://github.com/saakethk/deepsift.git
|
|
10
|
-
Project-URL: Issues, https://github.com/saakethk/deepsift/issues
|
|
11
|
-
Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
|
|
12
|
-
Classifier: Development Status :: 4 - Beta
|
|
13
|
-
Classifier: Intended Audience :: Science/Research
|
|
14
|
-
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Requires-Python: >=3.11
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
License-File: LICENSE
|
|
21
|
-
Requires-Dist: requests>=2.31.0
|
|
22
|
-
Requires-Dist: httpx>=0.27.0
|
|
23
|
-
Requires-Dist: pandas>=2.0.0
|
|
24
|
-
Requires-Dist: pyarrow>=14.0.0
|
|
25
|
-
Requires-Dist: tqdm>=4.66.0
|
|
26
|
-
Requires-Dist: click>=8.1.0
|
|
27
|
-
Requires-Dist: nest-asyncio>=1.6.0
|
|
28
|
-
Provides-Extra: dev
|
|
29
|
-
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
30
|
-
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
31
|
-
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
32
|
-
Dynamic: license-file
|
|
33
|
-
|
|
34
|
-
# Resume Model-V3
|
|
35
|
-
What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
|
|
36
|
-
|
|
37
|
-
## Methodology
|
|
38
|
-
To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
|
|
39
|
-
|
|
40
|
-
We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
|
|
41
|
-
|
|
42
|
-
[](data/relevant_subs_cohort_11_11_2026.csv)
|
|
43
|
-
|
|
44
|
-
We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
|
|
45
|
-
|
|
46
|
-
We determined that the ideal cohort for our use case needed to satisfy the following factors:
|
|
47
|
-
- have >10 comments and >0 score
|
|
48
|
-
- have a associated url corresponding to a picture (.png or jpeg)
|
|
49
|
-
|
|
50
|
-
After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
# Testing
|
|
55
|
-
To run the static unit and integration test suite offline using the synthetic `DummySource`:
|
|
56
|
-
```bash
|
|
57
|
-
pytest -m "not live" -v
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
To run the live integration tests against the actual Arctic Shift API:
|
|
61
|
-
```bash
|
|
62
|
-
pytest -m live -v
|
|
63
|
-
```
|
|
64
|
-
|
|
65
|
-
# Evaluation & Benchmarks
|
|
66
|
-
Comprehensive performance evaluations comparing DeepSift against sequential execution:
|
|
67
|
-
|
|
68
|
-
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
69
|
-
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
70
|
-
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
71
|
-
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
72
|
-
|
|
73
|
-
To run the evaluations:
|
|
74
|
-
```bash
|
|
75
|
-
# Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
|
|
76
|
-
python -m evaluation
|
|
77
|
-
```
|
|
78
|
-
|
|
79
|
-
# Documentation Site
|
|
80
|
-
The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
|
|
81
|
-
|
|
82
|
-
To run the documentation dev server with hot-reloading:
|
|
83
|
-
```bash
|
|
84
|
-
# From project root:
|
|
85
|
-
npm run docs
|
|
86
|
-
|
|
87
|
-
# Or with force cache reload:
|
|
88
|
-
npm run docs:force
|
|
89
|
-
|
|
90
|
-
# Or directly inside docs/ using npx:
|
|
91
|
-
cd docs
|
|
92
|
-
npx astro dev --force
|
|
93
|
-
```
|
|
94
|
-
Then visit `http://localhost:4321/` in your browser.
|
|
95
|
-
|
|
96
|
-
To build the static production site:
|
|
97
|
-
```bash
|
|
98
|
-
npm run docs:build
|
|
99
|
-
# or: cd docs && npm run build
|
|
100
|
-
```
|
|
101
|
-
The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
|
|
102
|
-
|
|
103
|
-
Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
|
|
104
|
-
- [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
|
|
105
|
-
- [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
|
|
106
|
-
- [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
|
|
107
|
-
- [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
|
|
108
|
-
- [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
|
|
109
|
-
- [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
|
|
110
|
-
|
|
111
|
-
# TODO
|
|
112
|
-
- [x] make static tests using dummy source (runs on github ci/cd)
|
|
113
|
-
- [x] make live tests using actual source on r/resumes
|
|
114
|
-
- [x] make evaluation script
|
|
115
|
-
- [x] writeup and documentation with benchmarks, docs page, and cleaned repo
|
deepsift-0.2.1/README.md
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
# Resume Model-V3
|
|
2
|
-
What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
|
|
3
|
-
|
|
4
|
-
## Methodology
|
|
5
|
-
To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
|
|
6
|
-
|
|
7
|
-
We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
|
|
8
|
-
|
|
9
|
-
[](data/relevant_subs_cohort_11_11_2026.csv)
|
|
10
|
-
|
|
11
|
-
We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
|
|
12
|
-
|
|
13
|
-
We determined that the ideal cohort for our use case needed to satisfy the following factors:
|
|
14
|
-
- have >10 comments and >0 score
|
|
15
|
-
- have a associated url corresponding to a picture (.png or jpeg)
|
|
16
|
-
|
|
17
|
-
After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# Testing
|
|
22
|
-
To run the static unit and integration test suite offline using the synthetic `DummySource`:
|
|
23
|
-
```bash
|
|
24
|
-
pytest -m "not live" -v
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
To run the live integration tests against the actual Arctic Shift API:
|
|
28
|
-
```bash
|
|
29
|
-
pytest -m live -v
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
# Evaluation & Benchmarks
|
|
33
|
-
Comprehensive performance evaluations comparing DeepSift against sequential execution:
|
|
34
|
-
|
|
35
|
-
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
36
|
-
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
37
|
-
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
38
|
-
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
39
|
-
|
|
40
|
-
To run the evaluations:
|
|
41
|
-
```bash
|
|
42
|
-
# Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
|
|
43
|
-
python -m evaluation
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
# Documentation Site
|
|
47
|
-
The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
|
|
48
|
-
|
|
49
|
-
To run the documentation dev server with hot-reloading:
|
|
50
|
-
```bash
|
|
51
|
-
# From project root:
|
|
52
|
-
npm run docs
|
|
53
|
-
|
|
54
|
-
# Or with force cache reload:
|
|
55
|
-
npm run docs:force
|
|
56
|
-
|
|
57
|
-
# Or directly inside docs/ using npx:
|
|
58
|
-
cd docs
|
|
59
|
-
npx astro dev --force
|
|
60
|
-
```
|
|
61
|
-
Then visit `http://localhost:4321/` in your browser.
|
|
62
|
-
|
|
63
|
-
To build the static production site:
|
|
64
|
-
```bash
|
|
65
|
-
npm run docs:build
|
|
66
|
-
# or: cd docs && npm run build
|
|
67
|
-
```
|
|
68
|
-
The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
|
|
69
|
-
|
|
70
|
-
Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
|
|
71
|
-
- [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
|
|
72
|
-
- [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
|
|
73
|
-
- [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
|
|
74
|
-
- [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
|
|
75
|
-
- [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
|
|
76
|
-
- [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
|
|
77
|
-
|
|
78
|
-
# TODO
|
|
79
|
-
- [x] make static tests using dummy source (runs on github ci/cd)
|
|
80
|
-
- [x] make live tests using actual source on r/resumes
|
|
81
|
-
- [x] make evaluation script
|
|
82
|
-
- [x] writeup and documentation with benchmarks, docs page, and cleaned repo
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: deepsift
|
|
3
|
-
Version: 0.2.1
|
|
4
|
-
Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
|
|
5
|
-
Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/saakethk/deepsift
|
|
8
|
-
Project-URL: Documentation, https://saakethk.github.io/deepsift/
|
|
9
|
-
Project-URL: Repository, https://github.com/saakethk/deepsift.git
|
|
10
|
-
Project-URL: Issues, https://github.com/saakethk/deepsift/issues
|
|
11
|
-
Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
|
|
12
|
-
Classifier: Development Status :: 4 - Beta
|
|
13
|
-
Classifier: Intended Audience :: Science/Research
|
|
14
|
-
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Requires-Python: >=3.11
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
License-File: LICENSE
|
|
21
|
-
Requires-Dist: requests>=2.31.0
|
|
22
|
-
Requires-Dist: httpx>=0.27.0
|
|
23
|
-
Requires-Dist: pandas>=2.0.0
|
|
24
|
-
Requires-Dist: pyarrow>=14.0.0
|
|
25
|
-
Requires-Dist: tqdm>=4.66.0
|
|
26
|
-
Requires-Dist: click>=8.1.0
|
|
27
|
-
Requires-Dist: nest-asyncio>=1.6.0
|
|
28
|
-
Provides-Extra: dev
|
|
29
|
-
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
30
|
-
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
31
|
-
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
32
|
-
Dynamic: license-file
|
|
33
|
-
|
|
34
|
-
# Resume Model-V3
|
|
35
|
-
What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
|
|
36
|
-
|
|
37
|
-
## Methodology
|
|
38
|
-
To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
|
|
39
|
-
|
|
40
|
-
We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
|
|
41
|
-
|
|
42
|
-
[](data/relevant_subs_cohort_11_11_2026.csv)
|
|
43
|
-
|
|
44
|
-
We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
|
|
45
|
-
|
|
46
|
-
We determined that the ideal cohort for our use case needed to satisfy the following factors:
|
|
47
|
-
- have >10 comments and >0 score
|
|
48
|
-
- have a associated url corresponding to a picture (.png or jpeg)
|
|
49
|
-
|
|
50
|
-
After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
# Testing
|
|
55
|
-
To run the static unit and integration test suite offline using the synthetic `DummySource`:
|
|
56
|
-
```bash
|
|
57
|
-
pytest -m "not live" -v
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
To run the live integration tests against the actual Arctic Shift API:
|
|
61
|
-
```bash
|
|
62
|
-
pytest -m live -v
|
|
63
|
-
```
|
|
64
|
-
|
|
65
|
-
# Evaluation & Benchmarks
|
|
66
|
-
Comprehensive performance evaluations comparing DeepSift against sequential execution:
|
|
67
|
-
|
|
68
|
-
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
69
|
-
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
70
|
-
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
71
|
-
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
72
|
-
|
|
73
|
-
To run the evaluations:
|
|
74
|
-
```bash
|
|
75
|
-
# Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
|
|
76
|
-
python -m evaluation
|
|
77
|
-
```
|
|
78
|
-
|
|
79
|
-
# Documentation Site
|
|
80
|
-
The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
|
|
81
|
-
|
|
82
|
-
To run the documentation dev server with hot-reloading:
|
|
83
|
-
```bash
|
|
84
|
-
# From project root:
|
|
85
|
-
npm run docs
|
|
86
|
-
|
|
87
|
-
# Or with force cache reload:
|
|
88
|
-
npm run docs:force
|
|
89
|
-
|
|
90
|
-
# Or directly inside docs/ using npx:
|
|
91
|
-
cd docs
|
|
92
|
-
npx astro dev --force
|
|
93
|
-
```
|
|
94
|
-
Then visit `http://localhost:4321/` in your browser.
|
|
95
|
-
|
|
96
|
-
To build the static production site:
|
|
97
|
-
```bash
|
|
98
|
-
npm run docs:build
|
|
99
|
-
# or: cd docs && npm run build
|
|
100
|
-
```
|
|
101
|
-
The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
|
|
102
|
-
|
|
103
|
-
Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
|
|
104
|
-
- [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
|
|
105
|
-
- [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
|
|
106
|
-
- [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
|
|
107
|
-
- [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
|
|
108
|
-
- [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
|
|
109
|
-
- [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
|
|
110
|
-
|
|
111
|
-
# TODO
|
|
112
|
-
- [x] make static tests using dummy source (runs on github ci/cd)
|
|
113
|
-
- [x] make live tests using actual source on r/resumes
|
|
114
|
-
- [x] make evaluation script
|
|
115
|
-
- [x] writeup and documentation with benchmarks, docs page, and cleaned repo
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.md
|
|
3
|
-
pyproject.toml
|
|
4
|
-
deepsift/__init__.py
|
|
5
|
-
deepsift/__main__.py
|
|
6
|
-
deepsift/cli.py
|
|
7
|
-
deepsift/filters.py
|
|
8
|
-
deepsift/scraper.py
|
|
9
|
-
deepsift.egg-info/PKG-INFO
|
|
10
|
-
deepsift.egg-info/SOURCES.txt
|
|
11
|
-
deepsift.egg-info/dependency_links.txt
|
|
12
|
-
deepsift.egg-info/entry_points.txt
|
|
13
|
-
deepsift.egg-info/requires.txt
|
|
14
|
-
deepsift.egg-info/top_level.txt
|
|
15
|
-
deepsift/models/__init__.py
|
|
16
|
-
deepsift/models/base.py
|
|
17
|
-
deepsift/models/comment.py
|
|
18
|
-
deepsift/models/enums.py
|
|
19
|
-
deepsift/models/post.py
|
|
20
|
-
deepsift/sources/__init__.py
|
|
21
|
-
deepsift/sources/arctic_shift.py
|
|
22
|
-
deepsift/sources/base.py
|
|
23
|
-
deepsift/sources/dummy.py
|
|
24
|
-
deepsift/sources/registry.py
|
|
25
|
-
deepsift/storage/__init__.py
|
|
26
|
-
deepsift/storage/base.py
|
|
27
|
-
deepsift/storage/parquet.py
|
|
28
|
-
tests/test_cli.py
|
|
29
|
-
tests/test_evaluation.py
|
|
30
|
-
tests/test_filters.py
|
|
31
|
-
tests/test_models.py
|
|
32
|
-
tests/test_scraper_dummy.py
|
|
33
|
-
tests/test_scraper_live.py
|
|
34
|
-
tests/test_storage.py
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Unit tests for the evaluation and benchmarking package.
|
|
3
|
-
Verifies metrics generation, data parity checks, and report formatting.
|
|
4
|
-
"""
|
|
5
|
-
from evaluation.__main__ import run_benchmark, format_markdown_report
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def test_evaluation_run_benchmark_dummy():
|
|
9
|
-
metrics = run_benchmark(
|
|
10
|
-
mode="dummy",
|
|
11
|
-
dummy_posts=20,
|
|
12
|
-
dummy_comments_per_post=2,
|
|
13
|
-
dummy_seed=42,
|
|
14
|
-
post_workers=2,
|
|
15
|
-
comment_workers=2,
|
|
16
|
-
)
|
|
17
|
-
|
|
18
|
-
assert "deepsift" in metrics
|
|
19
|
-
assert "sequential" in metrics
|
|
20
|
-
assert "comparison" in metrics
|
|
21
|
-
|
|
22
|
-
ds = metrics["deepsift"]
|
|
23
|
-
sq = metrics["sequential"]
|
|
24
|
-
comp = metrics["comparison"]
|
|
25
|
-
|
|
26
|
-
assert ds["posts_count"] == 20
|
|
27
|
-
assert sq["posts_count"] == 20
|
|
28
|
-
assert comp["posts_count_match"] is True
|
|
29
|
-
|
|
30
|
-
assert ds["comments_count"] == 40
|
|
31
|
-
assert sq["comments_count"] == 40
|
|
32
|
-
assert comp["comments_count_match"] is True
|
|
33
|
-
|
|
34
|
-
assert ds["elapsed_seconds"] > 0
|
|
35
|
-
assert sq["elapsed_seconds"] > 0
|
|
36
|
-
assert comp["speedup_factor"] > 0
|
|
37
|
-
|
|
38
|
-
report = format_markdown_report(metrics)
|
|
39
|
-
assert "# DeepSift vs Sequential Execution Evaluation" in report
|
|
40
|
-
assert "| **Total Elapsed Time** |" in report
|
|
41
|
-
assert "| **Posts Scraped** |" in report
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|