deepsift 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. deepsift-0.2.2/PKG-INFO +212 -0
  2. deepsift-0.2.2/README.md +179 -0
  3. {deepsift-0.2.1 → deepsift-0.2.2}/pyproject.toml +10 -2
  4. deepsift-0.2.2/src/deepsift.egg-info/PKG-INFO +212 -0
  5. deepsift-0.2.2/src/deepsift.egg-info/SOURCES.txt +33 -0
  6. deepsift-0.2.1/PKG-INFO +0 -115
  7. deepsift-0.2.1/README.md +0 -82
  8. deepsift-0.2.1/deepsift.egg-info/PKG-INFO +0 -115
  9. deepsift-0.2.1/deepsift.egg-info/SOURCES.txt +0 -34
  10. deepsift-0.2.1/tests/test_evaluation.py +0 -41
  11. {deepsift-0.2.1 → deepsift-0.2.2}/LICENSE +0 -0
  12. {deepsift-0.2.1 → deepsift-0.2.2}/setup.cfg +0 -0
  13. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/__init__.py +0 -0
  14. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/__main__.py +0 -0
  15. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/cli.py +0 -0
  16. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/filters.py +0 -0
  17. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/__init__.py +0 -0
  18. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/base.py +0 -0
  19. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/comment.py +0 -0
  20. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/enums.py +0 -0
  21. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/models/post.py +0 -0
  22. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/scraper.py +0 -0
  23. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/__init__.py +0 -0
  24. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/arctic_shift.py +0 -0
  25. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/base.py +0 -0
  26. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/dummy.py +0 -0
  27. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/sources/registry.py +0 -0
  28. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/storage/__init__.py +0 -0
  29. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/storage/base.py +0 -0
  30. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift/storage/parquet.py +0 -0
  31. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/dependency_links.txt +0 -0
  32. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/entry_points.txt +0 -0
  33. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/requires.txt +0 -0
  34. {deepsift-0.2.1 → deepsift-0.2.2/src}/deepsift.egg-info/top_level.txt +0 -0
  35. {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_cli.py +0 -0
  36. {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_filters.py +0 -0
  37. {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_models.py +0 -0
  38. {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_scraper_dummy.py +0 -0
  39. {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_scraper_live.py +0 -0
  40. {deepsift-0.2.1 → deepsift-0.2.2}/tests/test_storage.py +0 -0
@@ -0,0 +1,212 @@
1
+ Metadata-Version: 2.4
2
+ Name: deepsift
3
+ Version: 0.2.2
4
+ Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
+ Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/saakethk/deepsift
8
+ Project-URL: Documentation, https://saakethk.github.io/deepsift/
9
+ Project-URL: Repository, https://github.com/saakethk/deepsift.git
10
+ Project-URL: Issues, https://github.com/saakethk/deepsift/issues
11
+ Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: requests>=2.31.0
22
+ Requires-Dist: httpx>=0.27.0
23
+ Requires-Dist: pandas>=2.0.0
24
+ Requires-Dist: pyarrow>=14.0.0
25
+ Requires-Dist: tqdm>=4.66.0
26
+ Requires-Dist: click>=8.1.0
27
+ Requires-Dist: nest-asyncio>=1.6.0
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
30
+ Requires-Dist: build>=1.0.0; extra == "dev"
31
+ Requires-Dist: twine>=5.0.0; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ # DeepSift
35
+
36
+ [![PyPI version](https://img.shields.io/pypi/v/deepsift.svg)](https://pypi.org/project/deepsift/)
37
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
38
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
39
+ [![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](tests/)
40
+
41
+ DeepSift is a high-performance Python library designed for empirical and academic researchers to scrape, filter, and curate massive historical Reddit datasets at scale.
42
+
43
+ By combining **parallel temporal sharding**, **server-side field projection**, and **PyArrow Parquet streaming**, DeepSift overcomes the common bottlenecks of Reddit data collection—slashing bandwidth, keeping memory footprint constant, and achieving over **4.5x faster throughput** than standard sequential scrapers.
44
+
45
+ ---
46
+
47
+ ## Key Performance Advantages
48
+
49
+ * **Multi-Worker Temporal Sharding**: Splits arbitrary multi-month or multi-year windows into discrete temporal segments across parallel thread workers with adaptive backoff and rate-limit recovery.
50
+ * **Server-Side Field Projection**: Mathematically projects predicates to request only the attributes needed by your downstream study over the wire, cutting payload bandwidth by up to **24.0x**.
51
+ * **Constant-Memory Parquet Streaming**: Flushes chunked batches straight to columnar Apache Parquet files via PyArrow, reducing peak RAM consumption by up to **16.8x** compared to in-memory buffering.
52
+ * **Asynchronous Comment Tree Resolution**: Extracts and flattens deeply nested discussion trees using `httpx` async workers with configurable concurrency limits and automatic `"more"` stub pruning.
53
+ * **Polymorphic Schema Coercion**: Handles schema anomalies (such as polymorphic `edited` timestamps, flairs, and variable-typed metadata) without failing batch serialization.
54
+
55
+ ---
56
+
57
+ ## Installation
58
+
59
+ Install the library directly from PyPI:
60
+
61
+ ```bash
62
+ pip install deepsift
63
+ ```
64
+
65
+ To install local development dependencies:
66
+ ```bash
67
+ pip install deepsift[dev]
68
+ ```
69
+
70
+ ---
71
+
72
+ ## Quickstart
73
+
74
+ ### 1. Python API
75
+
76
+ ```python
77
+ import pandas as pd
78
+ from deepsift import RedditScraper, Filters, PostField, CommentField
79
+
80
+ # Request only the fields needed for your analysis
81
+ filters = Filters(
82
+ post_include_fields=[
83
+ PostField.ID,
84
+ PostField.TITLE,
85
+ PostField.SELFTEXT,
86
+ PostField.SCORE,
87
+ PostField.NUM_COMMENTS,
88
+ PostField.AUTHOR,
89
+ PostField.CREATED_UTC,
90
+ PostField.SUBREDDIT,
91
+ ],
92
+ comment_include_fields=[
93
+ CommentField.ID,
94
+ CommentField.POST_ID,
95
+ CommentField.PARENT_ID,
96
+ CommentField.AUTHOR,
97
+ CommentField.BODY,
98
+ CommentField.SCORE,
99
+ CommentField.CREATED_UTC,
100
+ ],
101
+ post_filter=lambda p: (p.score or 0) >= 3 and (p.num_comments or 0) >= 1,
102
+ comment_filter=lambda c: len(c.body or "") >= 20 and c.author != "[deleted]",
103
+ )
104
+
105
+ # Configure parallel workers and temporal shard granularity
106
+ scraper = RedditScraper(
107
+ filters=filters,
108
+ post_workers=4,
109
+ comment_workers=5,
110
+ shard_days=7,
111
+ )
112
+
113
+ # Run full ingestion pipeline (submissions + comment trees)
114
+ post_ids, comment_count = scraper.run(
115
+ subreddits=["MachineLearning"],
116
+ from_dt="2024-01-01",
117
+ to_dt="2024-01-14",
118
+ posts_output="data/posts.parquet",
119
+ comments_output="data/comments.parquet",
120
+ errors_output="data/errors.json",
121
+ )
122
+
123
+ print(f"Scraped {len(post_ids)} posts and {comment_count} comments.")
124
+
125
+ # Open directly into pandas, Polars, or DuckDB
126
+ df_posts = pd.read_parquet("data/posts.parquet")
127
+ df_comments = pd.read_parquet("data/comments.parquet")
128
+ ```
129
+
130
+ ### 2. Command Line Interface (CLI)
131
+
132
+ DeepSift includes a CLI for pipeline execution and quick queries:
133
+
134
+ ```bash
135
+ # Run complete post + comment pipeline
136
+ deepsift run --subreddit MachineLearning --from 2024-01-01 --to 2024-01-07 --posts data/posts.parquet --comments data/comments.parquet
137
+
138
+ # Scrape posts only
139
+ deepsift posts --subreddit resumes --from 2024-01-01 --to 2024-01-30 --output data/resumes_posts.parquet
140
+
141
+ # Extract comments for specific post IDs
142
+ deepsift comments --post-ids id1,id2,id3 --output data/extracted_comments.parquet
143
+ ```
144
+
145
+ ### 3. Offline Testing with `DummySource`
146
+
147
+ For unit testing, benchmarking, or development without network calls, swap in `DummySource` to generate realistic synthetic posts and comments:
148
+
149
+ ```python
150
+ from deepsift import RedditScraper, Filters
151
+ from deepsift.sources.dummy import DummySource
152
+
153
+ dummy = DummySource(posts_per_shard=100, comments_per_post=5, simulated_latency=0.0)
154
+ scraper = RedditScraper(source=dummy, shard_days=1)
155
+ ```
156
+
157
+ ---
158
+
159
+ ## Evaluation & Benchmarks
160
+
161
+ Empirical performance evaluation comparing DeepSift (6 parallel workers) against standard sequential extraction:
162
+
163
+ | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
164
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
165
+ | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
166
+ | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
167
+
168
+ To reproduce the benchmark suite:
169
+ ```bash
170
+ python -m evaluation
171
+ ```
172
+
173
+ ---
174
+
175
+ ## Running Tests
176
+
177
+ Run the test suite offline using synthetic data:
178
+ ```bash
179
+ pytest -m "not live" -v
180
+ ```
181
+
182
+ Run live integration tests against real historical endpoints:
183
+ ```bash
184
+ pytest -m live -v
185
+ ```
186
+
187
+ ---
188
+
189
+ ## Documentation
190
+
191
+ The full documentation website is built with [Astro Starlight](https://starlight.astro.build/) and is located in the [`docs/`](docs/) directory.
192
+
193
+ To start the documentation dev server locally:
194
+ ```bash
195
+ cd docs
196
+ npm install
197
+ npm run dev
198
+ ```
199
+ Then visit `http://localhost:4321/` in your browser.
200
+
201
+ Documentation sources:
202
+ * [Overview & Architecture](docs/src/content/docs/architecture.md)
203
+ * [Quickstart Guide](docs/src/content/docs/quickstart.md)
204
+ * [Python API Reference](docs/src/content/docs/api.md)
205
+ * [CLI Reference](docs/src/content/docs/cli.md)
206
+ * [Benchmarks & Scaling](docs/src/content/docs/benchmarks.md)
207
+
208
+ ---
209
+
210
+ ## License
211
+
212
+ This project is licensed under the [MIT License](LICENSE).
@@ -0,0 +1,179 @@
1
+ # DeepSift
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/deepsift.svg)](https://pypi.org/project/deepsift/)
4
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
6
+ [![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](tests/)
7
+
8
+ DeepSift is a high-performance Python library designed for empirical and academic researchers to scrape, filter, and curate massive historical Reddit datasets at scale.
9
+
10
+ By combining **parallel temporal sharding**, **server-side field projection**, and **PyArrow Parquet streaming**, DeepSift overcomes the common bottlenecks of Reddit data collection—slashing bandwidth, keeping memory footprint constant, and achieving over **4.5x faster throughput** than standard sequential scrapers.
11
+
12
+ ---
13
+
14
+ ## Key Performance Advantages
15
+
16
+ * **Multi-Worker Temporal Sharding**: Splits arbitrary multi-month or multi-year windows into discrete temporal segments across parallel thread workers with adaptive backoff and rate-limit recovery.
17
+ * **Server-Side Field Projection**: Mathematically projects predicates to request only the attributes needed by your downstream study over the wire, cutting payload bandwidth by up to **24.0x**.
18
+ * **Constant-Memory Parquet Streaming**: Flushes chunked batches straight to columnar Apache Parquet files via PyArrow, reducing peak RAM consumption by up to **16.8x** compared to in-memory buffering.
19
+ * **Asynchronous Comment Tree Resolution**: Extracts and flattens deeply nested discussion trees using `httpx` async workers with configurable concurrency limits and automatic `"more"` stub pruning.
20
+ * **Polymorphic Schema Coercion**: Handles schema anomalies (such as polymorphic `edited` timestamps, flairs, and variable-typed metadata) without failing batch serialization.
21
+
22
+ ---
23
+
24
+ ## Installation
25
+
26
+ Install the library directly from PyPI:
27
+
28
+ ```bash
29
+ pip install deepsift
30
+ ```
31
+
32
+ To install local development dependencies:
33
+ ```bash
34
+ pip install deepsift[dev]
35
+ ```
36
+
37
+ ---
38
+
39
+ ## Quickstart
40
+
41
+ ### 1. Python API
42
+
43
+ ```python
44
+ import pandas as pd
45
+ from deepsift import RedditScraper, Filters, PostField, CommentField
46
+
47
+ # Request only the fields needed for your analysis
48
+ filters = Filters(
49
+ post_include_fields=[
50
+ PostField.ID,
51
+ PostField.TITLE,
52
+ PostField.SELFTEXT,
53
+ PostField.SCORE,
54
+ PostField.NUM_COMMENTS,
55
+ PostField.AUTHOR,
56
+ PostField.CREATED_UTC,
57
+ PostField.SUBREDDIT,
58
+ ],
59
+ comment_include_fields=[
60
+ CommentField.ID,
61
+ CommentField.POST_ID,
62
+ CommentField.PARENT_ID,
63
+ CommentField.AUTHOR,
64
+ CommentField.BODY,
65
+ CommentField.SCORE,
66
+ CommentField.CREATED_UTC,
67
+ ],
68
+ post_filter=lambda p: (p.score or 0) >= 3 and (p.num_comments or 0) >= 1,
69
+ comment_filter=lambda c: len(c.body or "") >= 20 and c.author != "[deleted]",
70
+ )
71
+
72
+ # Configure parallel workers and temporal shard granularity
73
+ scraper = RedditScraper(
74
+ filters=filters,
75
+ post_workers=4,
76
+ comment_workers=5,
77
+ shard_days=7,
78
+ )
79
+
80
+ # Run full ingestion pipeline (submissions + comment trees)
81
+ post_ids, comment_count = scraper.run(
82
+ subreddits=["MachineLearning"],
83
+ from_dt="2024-01-01",
84
+ to_dt="2024-01-14",
85
+ posts_output="data/posts.parquet",
86
+ comments_output="data/comments.parquet",
87
+ errors_output="data/errors.json",
88
+ )
89
+
90
+ print(f"Scraped {len(post_ids)} posts and {comment_count} comments.")
91
+
92
+ # Open directly into pandas, Polars, or DuckDB
93
+ df_posts = pd.read_parquet("data/posts.parquet")
94
+ df_comments = pd.read_parquet("data/comments.parquet")
95
+ ```
96
+
97
+ ### 2. Command Line Interface (CLI)
98
+
99
+ DeepSift includes a CLI for pipeline execution and quick queries:
100
+
101
+ ```bash
102
+ # Run complete post + comment pipeline
103
+ deepsift run --subreddit MachineLearning --from 2024-01-01 --to 2024-01-07 --posts data/posts.parquet --comments data/comments.parquet
104
+
105
+ # Scrape posts only
106
+ deepsift posts --subreddit resumes --from 2024-01-01 --to 2024-01-30 --output data/resumes_posts.parquet
107
+
108
+ # Extract comments for specific post IDs
109
+ deepsift comments --post-ids id1,id2,id3 --output data/extracted_comments.parquet
110
+ ```
111
+
112
+ ### 3. Offline Testing with `DummySource`
113
+
114
+ For unit testing, benchmarking, or development without network calls, swap in `DummySource` to generate realistic synthetic posts and comments:
115
+
116
+ ```python
117
+ from deepsift import RedditScraper, Filters
118
+ from deepsift.sources.dummy import DummySource
119
+
120
+ dummy = DummySource(posts_per_shard=100, comments_per_post=5, simulated_latency=0.0)
121
+ scraper = RedditScraper(source=dummy, shard_days=1)
122
+ ```
123
+
124
+ ---
125
+
126
+ ## Evaluation & Benchmarks
127
+
128
+ Empirical performance evaluation comparing DeepSift (6 parallel workers) against standard sequential extraction:
129
+
130
+ | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
131
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
132
+ | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
133
+ | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
134
+
135
+ To reproduce the benchmark suite:
136
+ ```bash
137
+ python -m evaluation
138
+ ```
139
+
140
+ ---
141
+
142
+ ## Running Tests
143
+
144
+ Run the test suite offline using synthetic data:
145
+ ```bash
146
+ pytest -m "not live" -v
147
+ ```
148
+
149
+ Run live integration tests against real historical endpoints:
150
+ ```bash
151
+ pytest -m live -v
152
+ ```
153
+
154
+ ---
155
+
156
+ ## Documentation
157
+
158
+ The full documentation website is built with [Astro Starlight](https://starlight.astro.build/) and is located in the [`docs/`](docs/) directory.
159
+
160
+ To start the documentation dev server locally:
161
+ ```bash
162
+ cd docs
163
+ npm install
164
+ npm run dev
165
+ ```
166
+ Then visit `http://localhost:4321/` in your browser.
167
+
168
+ Documentation sources:
169
+ * [Overview & Architecture](docs/src/content/docs/architecture.md)
170
+ * [Quickstart Guide](docs/src/content/docs/quickstart.md)
171
+ * [Python API Reference](docs/src/content/docs/api.md)
172
+ * [CLI Reference](docs/src/content/docs/cli.md)
173
+ * [Benchmarks & Scaling](docs/src/content/docs/benchmarks.md)
174
+
175
+ ---
176
+
177
+ ## License
178
+
179
+ This project is licensed under the [MIT License](LICENSE).
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "deepsift"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "High-performance historical Reddit research toolkit and scraper for Arctic Shift."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -50,10 +50,18 @@ deepsift = "deepsift.cli:cli"
50
50
  reddit-scraper = "deepsift.cli:cli"
51
51
 
52
52
  [tool.setuptools.packages.find]
53
- where = ["."]
53
+ where = ["src"]
54
54
  include = ["deepsift*"]
55
55
 
56
56
  [tool.pytest.ini_options]
57
57
  markers = [
58
58
  "live: marks tests requiring live network connectivity to external APIs (Arctic Shift)",
59
59
  ]
60
+
61
+ [tool.ruff]
62
+ line-length = 100
63
+ target-version = "py311"
64
+
65
+ [tool.ruff.lint]
66
+ select = ["E", "F", "I", "UP"]
67
+
@@ -0,0 +1,212 @@
1
+ Metadata-Version: 2.4
2
+ Name: deepsift
3
+ Version: 0.2.2
4
+ Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
+ Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/saakethk/deepsift
8
+ Project-URL: Documentation, https://saakethk.github.io/deepsift/
9
+ Project-URL: Repository, https://github.com/saakethk/deepsift.git
10
+ Project-URL: Issues, https://github.com/saakethk/deepsift/issues
11
+ Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: requests>=2.31.0
22
+ Requires-Dist: httpx>=0.27.0
23
+ Requires-Dist: pandas>=2.0.0
24
+ Requires-Dist: pyarrow>=14.0.0
25
+ Requires-Dist: tqdm>=4.66.0
26
+ Requires-Dist: click>=8.1.0
27
+ Requires-Dist: nest-asyncio>=1.6.0
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
30
+ Requires-Dist: build>=1.0.0; extra == "dev"
31
+ Requires-Dist: twine>=5.0.0; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ # DeepSift
35
+
36
+ [![PyPI version](https://img.shields.io/pypi/v/deepsift.svg)](https://pypi.org/project/deepsift/)
37
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
38
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
39
+ [![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](tests/)
40
+
41
+ DeepSift is a high-performance Python library designed for empirical and academic researchers to scrape, filter, and curate massive historical Reddit datasets at scale.
42
+
43
+ By combining **parallel temporal sharding**, **server-side field projection**, and **PyArrow Parquet streaming**, DeepSift overcomes the common bottlenecks of Reddit data collection—slashing bandwidth, keeping memory footprint constant, and achieving over **4.5x faster throughput** than standard sequential scrapers.
44
+
45
+ ---
46
+
47
+ ## Key Performance Advantages
48
+
49
+ * **Multi-Worker Temporal Sharding**: Splits arbitrary multi-month or multi-year windows into discrete temporal segments across parallel thread workers with adaptive backoff and rate-limit recovery.
50
+ * **Server-Side Field Projection**: Mathematically projects predicates to request only the attributes needed by your downstream study over the wire, cutting payload bandwidth by up to **24.0x**.
51
+ * **Constant-Memory Parquet Streaming**: Flushes chunked batches straight to columnar Apache Parquet files via PyArrow, reducing peak RAM consumption by up to **16.8x** compared to in-memory buffering.
52
+ * **Asynchronous Comment Tree Resolution**: Extracts and flattens deeply nested discussion trees using `httpx` async workers with configurable concurrency limits and automatic `"more"` stub pruning.
53
+ * **Polymorphic Schema Coercion**: Handles schema anomalies (such as polymorphic `edited` timestamps, flairs, and variable-typed metadata) without failing batch serialization.
54
+
55
+ ---
56
+
57
+ ## Installation
58
+
59
+ Install the library directly from PyPI:
60
+
61
+ ```bash
62
+ pip install deepsift
63
+ ```
64
+
65
+ To install local development dependencies:
66
+ ```bash
67
+ pip install deepsift[dev]
68
+ ```
69
+
70
+ ---
71
+
72
+ ## Quickstart
73
+
74
+ ### 1. Python API
75
+
76
+ ```python
77
+ import pandas as pd
78
+ from deepsift import RedditScraper, Filters, PostField, CommentField
79
+
80
+ # Request only the fields needed for your analysis
81
+ filters = Filters(
82
+ post_include_fields=[
83
+ PostField.ID,
84
+ PostField.TITLE,
85
+ PostField.SELFTEXT,
86
+ PostField.SCORE,
87
+ PostField.NUM_COMMENTS,
88
+ PostField.AUTHOR,
89
+ PostField.CREATED_UTC,
90
+ PostField.SUBREDDIT,
91
+ ],
92
+ comment_include_fields=[
93
+ CommentField.ID,
94
+ CommentField.POST_ID,
95
+ CommentField.PARENT_ID,
96
+ CommentField.AUTHOR,
97
+ CommentField.BODY,
98
+ CommentField.SCORE,
99
+ CommentField.CREATED_UTC,
100
+ ],
101
+ post_filter=lambda p: (p.score or 0) >= 3 and (p.num_comments or 0) >= 1,
102
+ comment_filter=lambda c: len(c.body or "") >= 20 and c.author != "[deleted]",
103
+ )
104
+
105
+ # Configure parallel workers and temporal shard granularity
106
+ scraper = RedditScraper(
107
+ filters=filters,
108
+ post_workers=4,
109
+ comment_workers=5,
110
+ shard_days=7,
111
+ )
112
+
113
+ # Run full ingestion pipeline (submissions + comment trees)
114
+ post_ids, comment_count = scraper.run(
115
+ subreddits=["MachineLearning"],
116
+ from_dt="2024-01-01",
117
+ to_dt="2024-01-14",
118
+ posts_output="data/posts.parquet",
119
+ comments_output="data/comments.parquet",
120
+ errors_output="data/errors.json",
121
+ )
122
+
123
+ print(f"Scraped {len(post_ids)} posts and {comment_count} comments.")
124
+
125
+ # Open directly into pandas, Polars, or DuckDB
126
+ df_posts = pd.read_parquet("data/posts.parquet")
127
+ df_comments = pd.read_parquet("data/comments.parquet")
128
+ ```
129
+
130
+ ### 2. Command Line Interface (CLI)
131
+
132
+ DeepSift includes a CLI for pipeline execution and quick queries:
133
+
134
+ ```bash
135
+ # Run complete post + comment pipeline
136
+ deepsift run --subreddit MachineLearning --from 2024-01-01 --to 2024-01-07 --posts data/posts.parquet --comments data/comments.parquet
137
+
138
+ # Scrape posts only
139
+ deepsift posts --subreddit resumes --from 2024-01-01 --to 2024-01-30 --output data/resumes_posts.parquet
140
+
141
+ # Extract comments for specific post IDs
142
+ deepsift comments --post-ids id1,id2,id3 --output data/extracted_comments.parquet
143
+ ```
144
+
145
+ ### 3. Offline Testing with `DummySource`
146
+
147
+ For unit testing, benchmarking, or development without network calls, swap in `DummySource` to generate realistic synthetic posts and comments:
148
+
149
+ ```python
150
+ from deepsift import RedditScraper, Filters
151
+ from deepsift.sources.dummy import DummySource
152
+
153
+ dummy = DummySource(posts_per_shard=100, comments_per_post=5, simulated_latency=0.0)
154
+ scraper = RedditScraper(source=dummy, shard_days=1)
155
+ ```
156
+
157
+ ---
158
+
159
+ ## Evaluation & Benchmarks
160
+
161
+ Empirical performance evaluation comparing DeepSift (6 parallel workers) against standard sequential extraction:
162
+
163
+ | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
164
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
165
+ | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
166
+ | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
167
+
168
+ To reproduce the benchmark suite:
169
+ ```bash
170
+ python -m evaluation
171
+ ```
172
+
173
+ ---
174
+
175
+ ## Running Tests
176
+
177
+ Run the test suite offline using synthetic data:
178
+ ```bash
179
+ pytest -m "not live" -v
180
+ ```
181
+
182
+ Run live integration tests against real historical endpoints:
183
+ ```bash
184
+ pytest -m live -v
185
+ ```
186
+
187
+ ---
188
+
189
+ ## Documentation
190
+
191
+ The full documentation website is built with [Astro Starlight](https://starlight.astro.build/) and is located in the [`docs/`](docs/) directory.
192
+
193
+ To start the documentation dev server locally:
194
+ ```bash
195
+ cd docs
196
+ npm install
197
+ npm run dev
198
+ ```
199
+ Then visit `http://localhost:4321/` in your browser.
200
+
201
+ Documentation sources:
202
+ * [Overview & Architecture](docs/src/content/docs/architecture.md)
203
+ * [Quickstart Guide](docs/src/content/docs/quickstart.md)
204
+ * [Python API Reference](docs/src/content/docs/api.md)
205
+ * [CLI Reference](docs/src/content/docs/cli.md)
206
+ * [Benchmarks & Scaling](docs/src/content/docs/benchmarks.md)
207
+
208
+ ---
209
+
210
+ ## License
211
+
212
+ This project is licensed under the [MIT License](LICENSE).
@@ -0,0 +1,33 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ src/deepsift/__init__.py
5
+ src/deepsift/__main__.py
6
+ src/deepsift/cli.py
7
+ src/deepsift/filters.py
8
+ src/deepsift/scraper.py
9
+ src/deepsift.egg-info/PKG-INFO
10
+ src/deepsift.egg-info/SOURCES.txt
11
+ src/deepsift.egg-info/dependency_links.txt
12
+ src/deepsift.egg-info/entry_points.txt
13
+ src/deepsift.egg-info/requires.txt
14
+ src/deepsift.egg-info/top_level.txt
15
+ src/deepsift/models/__init__.py
16
+ src/deepsift/models/base.py
17
+ src/deepsift/models/comment.py
18
+ src/deepsift/models/enums.py
19
+ src/deepsift/models/post.py
20
+ src/deepsift/sources/__init__.py
21
+ src/deepsift/sources/arctic_shift.py
22
+ src/deepsift/sources/base.py
23
+ src/deepsift/sources/dummy.py
24
+ src/deepsift/sources/registry.py
25
+ src/deepsift/storage/__init__.py
26
+ src/deepsift/storage/base.py
27
+ src/deepsift/storage/parquet.py
28
+ tests/test_cli.py
29
+ tests/test_filters.py
30
+ tests/test_models.py
31
+ tests/test_scraper_dummy.py
32
+ tests/test_scraper_live.py
33
+ tests/test_storage.py
deepsift-0.2.1/PKG-INFO DELETED
@@ -1,115 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: deepsift
3
- Version: 0.2.1
4
- Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
- Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
- License-Expression: MIT
7
- Project-URL: Homepage, https://github.com/saakethk/deepsift
8
- Project-URL: Documentation, https://saakethk.github.io/deepsift/
9
- Project-URL: Repository, https://github.com/saakethk/deepsift.git
10
- Project-URL: Issues, https://github.com/saakethk/deepsift/issues
11
- Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
12
- Classifier: Development Status :: 4 - Beta
13
- Classifier: Intended Audience :: Science/Research
14
- Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
- Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.11
17
- Classifier: Programming Language :: Python :: 3.12
18
- Requires-Python: >=3.11
19
- Description-Content-Type: text/markdown
20
- License-File: LICENSE
21
- Requires-Dist: requests>=2.31.0
22
- Requires-Dist: httpx>=0.27.0
23
- Requires-Dist: pandas>=2.0.0
24
- Requires-Dist: pyarrow>=14.0.0
25
- Requires-Dist: tqdm>=4.66.0
26
- Requires-Dist: click>=8.1.0
27
- Requires-Dist: nest-asyncio>=1.6.0
28
- Provides-Extra: dev
29
- Requires-Dist: pytest>=8.0.0; extra == "dev"
30
- Requires-Dist: build>=1.0.0; extra == "dev"
31
- Requires-Dist: twine>=5.0.0; extra == "dev"
32
- Dynamic: license-file
33
-
34
- # Resume Model-V3
35
- What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
36
-
37
- ## Methodology
38
- To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
39
-
40
- We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
41
-
42
- [![Graph of subscribers per relevant subreddit](data/graph_relevant_subreddits_fig_11_11_26.png)](data/relevant_subs_cohort_11_11_2026.csv)
43
-
44
- We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
45
-
46
- We determined that the ideal cohort for our use case needed to satisfy the following factors:
47
- - have >10 comments and >0 score
48
- - have a associated url corresponding to a picture (.png or jpeg)
49
-
50
- After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
51
-
52
-
53
-
54
- # Testing
55
- To run the static unit and integration test suite offline using the synthetic `DummySource`:
56
- ```bash
57
- pytest -m "not live" -v
58
- ```
59
-
60
- To run the live integration tests against the actual Arctic Shift API:
61
- ```bash
62
- pytest -m live -v
63
- ```
64
-
65
- # Evaluation & Benchmarks
66
- Comprehensive performance evaluations comparing DeepSift against sequential execution:
67
-
68
- | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
69
- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
70
- | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
71
- | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
72
-
73
- To run the evaluations:
74
- ```bash
75
- # Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
76
- python -m evaluation
77
- ```
78
-
79
- # Documentation Site
80
- The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
81
-
82
- To run the documentation dev server with hot-reloading:
83
- ```bash
84
- # From project root:
85
- npm run docs
86
-
87
- # Or with force cache reload:
88
- npm run docs:force
89
-
90
- # Or directly inside docs/ using npx:
91
- cd docs
92
- npx astro dev --force
93
- ```
94
- Then visit `http://localhost:4321/` in your browser.
95
-
96
- To build the static production site:
97
- ```bash
98
- npm run docs:build
99
- # or: cd docs && npm run build
100
- ```
101
- The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
102
-
103
- Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
104
- - [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
105
- - [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
106
- - [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
107
- - [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
108
- - [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
109
- - [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
110
-
111
- # TODO
112
- - [x] make static tests using dummy source (runs on github ci/cd)
113
- - [x] make live tests using actual source on r/resumes
114
- - [x] make evaluation script
115
- - [x] writeup and documentation with benchmarks, docs page, and cleaned repo
deepsift-0.2.1/README.md DELETED
@@ -1,82 +0,0 @@
1
- # Resume Model-V3
2
- What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
3
-
4
- ## Methodology
5
- To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
6
-
7
- We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
8
-
9
- [![Graph of subscribers per relevant subreddit](data/graph_relevant_subreddits_fig_11_11_26.png)](data/relevant_subs_cohort_11_11_2026.csv)
10
-
11
- We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
12
-
13
- We determined that the ideal cohort for our use case needed to satisfy the following factors:
14
- - have >10 comments and >0 score
15
- - have a associated url corresponding to a picture (.png or jpeg)
16
-
17
- After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
18
-
19
-
20
-
21
- # Testing
22
- To run the static unit and integration test suite offline using the synthetic `DummySource`:
23
- ```bash
24
- pytest -m "not live" -v
25
- ```
26
-
27
- To run the live integration tests against the actual Arctic Shift API:
28
- ```bash
29
- pytest -m live -v
30
- ```
31
-
32
- # Evaluation & Benchmarks
33
- Comprehensive performance evaluations comparing DeepSift against sequential execution:
34
-
35
- | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
36
- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
37
- | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
38
- | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
39
-
40
- To run the evaluations:
41
- ```bash
42
- # Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
43
- python -m evaluation
44
- ```
45
-
46
- # Documentation Site
47
- The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
48
-
49
- To run the documentation dev server with hot-reloading:
50
- ```bash
51
- # From project root:
52
- npm run docs
53
-
54
- # Or with force cache reload:
55
- npm run docs:force
56
-
57
- # Or directly inside docs/ using npx:
58
- cd docs
59
- npx astro dev --force
60
- ```
61
- Then visit `http://localhost:4321/` in your browser.
62
-
63
- To build the static production site:
64
- ```bash
65
- npm run docs:build
66
- # or: cd docs && npm run build
67
- ```
68
- The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
69
-
70
- Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
71
- - [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
72
- - [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
73
- - [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
74
- - [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
75
- - [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
76
- - [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
77
-
78
- # TODO
79
- - [x] make static tests using dummy source (runs on github ci/cd)
80
- - [x] make live tests using actual source on r/resumes
81
- - [x] make evaluation script
82
- - [x] writeup and documentation with benchmarks, docs page, and cleaned repo
@@ -1,115 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: deepsift
3
- Version: 0.2.1
4
- Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
- Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
- License-Expression: MIT
7
- Project-URL: Homepage, https://github.com/saakethk/deepsift
8
- Project-URL: Documentation, https://saakethk.github.io/deepsift/
9
- Project-URL: Repository, https://github.com/saakethk/deepsift.git
10
- Project-URL: Issues, https://github.com/saakethk/deepsift/issues
11
- Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
12
- Classifier: Development Status :: 4 - Beta
13
- Classifier: Intended Audience :: Science/Research
14
- Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
- Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.11
17
- Classifier: Programming Language :: Python :: 3.12
18
- Requires-Python: >=3.11
19
- Description-Content-Type: text/markdown
20
- License-File: LICENSE
21
- Requires-Dist: requests>=2.31.0
22
- Requires-Dist: httpx>=0.27.0
23
- Requires-Dist: pandas>=2.0.0
24
- Requires-Dist: pyarrow>=14.0.0
25
- Requires-Dist: tqdm>=4.66.0
26
- Requires-Dist: click>=8.1.0
27
- Requires-Dist: nest-asyncio>=1.6.0
28
- Provides-Extra: dev
29
- Requires-Dist: pytest>=8.0.0; extra == "dev"
30
- Requires-Dist: build>=1.0.0; extra == "dev"
31
- Requires-Dist: twine>=5.0.0; extra == "dev"
32
- Dynamic: license-file
33
-
34
- # Resume Model-V3
35
- What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
36
-
37
- ## Methodology
38
- To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
39
-
40
- We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
41
-
42
- [![Graph of subscribers per relevant subreddit](data/graph_relevant_subreddits_fig_11_11_26.png)](data/relevant_subs_cohort_11_11_2026.csv)
43
-
44
- We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
45
-
46
- We determined that the ideal cohort for our use case needed to satisfy the following factors:
47
- - have >10 comments and >0 score
48
- - have a associated url corresponding to a picture (.png or jpeg)
49
-
50
- After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
51
-
52
-
53
-
54
- # Testing
55
- To run the static unit and integration test suite offline using the synthetic `DummySource`:
56
- ```bash
57
- pytest -m "not live" -v
58
- ```
59
-
60
- To run the live integration tests against the actual Arctic Shift API:
61
- ```bash
62
- pytest -m live -v
63
- ```
64
-
65
- # Evaluation & Benchmarks
66
- Comprehensive performance evaluations comparing DeepSift against sequential execution:
67
-
68
- | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
69
- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
70
- | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
71
- | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
72
-
73
- To run the evaluations:
74
- ```bash
75
- # Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
76
- python -m evaluation
77
- ```
78
-
79
- # Documentation Site
80
- The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
81
-
82
- To run the documentation dev server with hot-reloading:
83
- ```bash
84
- # From project root:
85
- npm run docs
86
-
87
- # Or with force cache reload:
88
- npm run docs:force
89
-
90
- # Or directly inside docs/ using npx:
91
- cd docs
92
- npx astro dev --force
93
- ```
94
- Then visit `http://localhost:4321/` in your browser.
95
-
96
- To build the static production site:
97
- ```bash
98
- npm run docs:build
99
- # or: cd docs && npm run build
100
- ```
101
- The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
102
-
103
- Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
104
- - [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
105
- - [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
106
- - [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
107
- - [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
108
- - [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
109
- - [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
110
-
111
- # TODO
112
- - [x] make static tests using dummy source (runs on github ci/cd)
113
- - [x] make live tests using actual source on r/resumes
114
- - [x] make evaluation script
115
- - [x] writeup and documentation with benchmarks, docs page, and cleaned repo
@@ -1,34 +0,0 @@
1
- LICENSE
2
- README.md
3
- pyproject.toml
4
- deepsift/__init__.py
5
- deepsift/__main__.py
6
- deepsift/cli.py
7
- deepsift/filters.py
8
- deepsift/scraper.py
9
- deepsift.egg-info/PKG-INFO
10
- deepsift.egg-info/SOURCES.txt
11
- deepsift.egg-info/dependency_links.txt
12
- deepsift.egg-info/entry_points.txt
13
- deepsift.egg-info/requires.txt
14
- deepsift.egg-info/top_level.txt
15
- deepsift/models/__init__.py
16
- deepsift/models/base.py
17
- deepsift/models/comment.py
18
- deepsift/models/enums.py
19
- deepsift/models/post.py
20
- deepsift/sources/__init__.py
21
- deepsift/sources/arctic_shift.py
22
- deepsift/sources/base.py
23
- deepsift/sources/dummy.py
24
- deepsift/sources/registry.py
25
- deepsift/storage/__init__.py
26
- deepsift/storage/base.py
27
- deepsift/storage/parquet.py
28
- tests/test_cli.py
29
- tests/test_evaluation.py
30
- tests/test_filters.py
31
- tests/test_models.py
32
- tests/test_scraper_dummy.py
33
- tests/test_scraper_live.py
34
- tests/test_storage.py
@@ -1,41 +0,0 @@
1
- """
2
- Unit tests for the evaluation and benchmarking package.
3
- Verifies metrics generation, data parity checks, and report formatting.
4
- """
5
- from evaluation.__main__ import run_benchmark, format_markdown_report
6
-
7
-
8
- def test_evaluation_run_benchmark_dummy():
9
- metrics = run_benchmark(
10
- mode="dummy",
11
- dummy_posts=20,
12
- dummy_comments_per_post=2,
13
- dummy_seed=42,
14
- post_workers=2,
15
- comment_workers=2,
16
- )
17
-
18
- assert "deepsift" in metrics
19
- assert "sequential" in metrics
20
- assert "comparison" in metrics
21
-
22
- ds = metrics["deepsift"]
23
- sq = metrics["sequential"]
24
- comp = metrics["comparison"]
25
-
26
- assert ds["posts_count"] == 20
27
- assert sq["posts_count"] == 20
28
- assert comp["posts_count_match"] is True
29
-
30
- assert ds["comments_count"] == 40
31
- assert sq["comments_count"] == 40
32
- assert comp["comments_count_match"] is True
33
-
34
- assert ds["elapsed_seconds"] > 0
35
- assert sq["elapsed_seconds"] > 0
36
- assert comp["speedup_factor"] > 0
37
-
38
- report = format_markdown_report(metrics)
39
- assert "# DeepSift vs Sequential Execution Evaluation" in report
40
- assert "| **Total Elapsed Time** |" in report
41
- assert "| **Posts Scraped** |" in report
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes