deepsift 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepsift/__init__.py +39 -0
- deepsift/__main__.py +8 -0
- deepsift/cli.py +171 -0
- deepsift/filters.py +111 -0
- deepsift/models/__init__.py +17 -0
- deepsift/models/base.py +100 -0
- deepsift/models/comment.py +38 -0
- deepsift/models/enums.py +243 -0
- deepsift/models/post.py +29 -0
- deepsift/scraper.py +397 -0
- deepsift/sources/__init__.py +14 -0
- deepsift/sources/arctic_shift.py +263 -0
- deepsift/sources/base.py +102 -0
- deepsift/sources/dummy.py +207 -0
- deepsift/sources/registry.py +24 -0
- deepsift/storage/__init__.py +10 -0
- deepsift/storage/base.py +37 -0
- deepsift/storage/parquet.py +86 -0
- deepsift-0.2.0.dist-info/METADATA +115 -0
- deepsift-0.2.0.dist-info/RECORD +24 -0
- deepsift-0.2.0.dist-info/WHEEL +5 -0
- deepsift-0.2.0.dist-info/entry_points.txt +3 -0
- deepsift-0.2.0.dist-info/licenses/LICENSE +21 -0
- deepsift-0.2.0.dist-info/top_level.txt +1 -0
deepsift/sources/base.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Abstract base class definition for Reddit data sources.
|
|
3
|
+
"""
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from typing import Iterator, List, Dict, Any, Optional, Tuple, Iterable, Set
|
|
6
|
+
import requests
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Source(ABC):
|
|
11
|
+
"""
|
|
12
|
+
Abstract Base Class for Reddit data sources.
|
|
13
|
+
Handles communication with specific APIs, archives, or data stores.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
@staticmethod
|
|
17
|
+
def resolve_post_fields(fields: Optional[Iterable[str]]) -> Optional[Set[str]]:
|
|
18
|
+
"""Ensures required post fields (id, created_utc) are present in any projection."""
|
|
19
|
+
if fields is None:
|
|
20
|
+
return None
|
|
21
|
+
from deepsift.models import Post
|
|
22
|
+
return set(fields) | set(Post.REQUIRED_FIELDS)
|
|
23
|
+
|
|
24
|
+
@staticmethod
|
|
25
|
+
def resolve_comment_fields(fields: Optional[Iterable[str]]) -> Optional[Set[str]]:
|
|
26
|
+
"""Ensures required comment fields (id, post_id, parent_id) are present in any projection."""
|
|
27
|
+
if fields is None:
|
|
28
|
+
return None
|
|
29
|
+
from deepsift.models import Comment
|
|
30
|
+
return set(fields) | set(Comment.REQUIRED_FIELDS)
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def name(self) -> str:
|
|
35
|
+
"""Identifying name of the data source."""
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
@abstractmethod
|
|
39
|
+
def fetch_posts_shard(
|
|
40
|
+
self,
|
|
41
|
+
subreddit: str,
|
|
42
|
+
after_ts: int,
|
|
43
|
+
before_ts: int,
|
|
44
|
+
fields: Optional[Iterable[str]] = None,
|
|
45
|
+
session: Optional[requests.Session] = None,
|
|
46
|
+
) -> Iterator[List[Dict[str, Any]]]:
|
|
47
|
+
"""
|
|
48
|
+
Yields batches of raw post dictionaries for a given subreddit within [after_ts, before_ts].
|
|
49
|
+
If fields is provided, requests only those fields from the server.
|
|
50
|
+
"""
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
@abstractmethod
|
|
54
|
+
async def fetch_comments_for_post(
|
|
55
|
+
self,
|
|
56
|
+
client: httpx.AsyncClient,
|
|
57
|
+
post_id: str,
|
|
58
|
+
fields: Optional[Iterable[str]] = None,
|
|
59
|
+
max_retries: int = 5,
|
|
60
|
+
) -> Tuple[str, List[Dict[str, Any]], bool]:
|
|
61
|
+
"""
|
|
62
|
+
Asynchronously fetches comment dictionaries for a given post.
|
|
63
|
+
Returns: (post_id, comments_list, success_bool)
|
|
64
|
+
"""
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
def fetch_posts(
|
|
68
|
+
self,
|
|
69
|
+
subreddit: str,
|
|
70
|
+
after_ts: int,
|
|
71
|
+
before_ts: int,
|
|
72
|
+
fields: Optional[Iterable[str]] = None,
|
|
73
|
+
) -> List[Dict[str, Any]]:
|
|
74
|
+
"""
|
|
75
|
+
Synchronously gathers all post dictionaries within [after_ts, before_ts] into a single list.
|
|
76
|
+
"""
|
|
77
|
+
posts = []
|
|
78
|
+
for batch in self.fetch_posts_shard(subreddit, after_ts, before_ts, fields=fields):
|
|
79
|
+
posts.extend(batch)
|
|
80
|
+
return posts
|
|
81
|
+
|
|
82
|
+
def fetch_comments(self, post_id: str, fields: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]:
|
|
83
|
+
"""
|
|
84
|
+
Synchronously fetches comments for a single post using a transient httpx client.
|
|
85
|
+
Safely supports execution inside active event loops (such as Jupyter notebooks).
|
|
86
|
+
"""
|
|
87
|
+
import asyncio
|
|
88
|
+
async def _run():
|
|
89
|
+
async with httpx.AsyncClient() as client:
|
|
90
|
+
_, comments, _ = await self.fetch_comments_for_post(client, post_id, fields=fields)
|
|
91
|
+
return comments
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
loop = asyncio.get_running_loop()
|
|
95
|
+
except RuntimeError:
|
|
96
|
+
loop = None
|
|
97
|
+
|
|
98
|
+
if loop and loop.is_running():
|
|
99
|
+
import nest_asyncio
|
|
100
|
+
nest_asyncio.apply()
|
|
101
|
+
return loop.run_until_complete(_run())
|
|
102
|
+
return asyncio.run(_run())
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Dummy and mock data source for scalable offline testing and verification.
|
|
3
|
+
Dynamically generates random, realistic posts and comments to stress-test pipelines.
|
|
4
|
+
"""
|
|
5
|
+
import time
|
|
6
|
+
import asyncio
|
|
7
|
+
import random
|
|
8
|
+
import zlib
|
|
9
|
+
from typing import Iterator, List, Dict, Any, Optional, Tuple, Iterable, Union, Callable
|
|
10
|
+
import requests
|
|
11
|
+
import httpx
|
|
12
|
+
from deepsift.sources.base import Source
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DummySource(Source):
|
|
16
|
+
"""
|
|
17
|
+
In-memory synthetic Reddit data source for offline scale testing.
|
|
18
|
+
Dynamically generates randomized, realistic posts and comments without making network calls.
|
|
19
|
+
Supports server-side field projection simulation.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
SAMPLE_ROLES = [
|
|
23
|
+
"Software Engineer", "Frontend Developer", "Backend Engineer",
|
|
24
|
+
"Fullstack Developer", "Data Scientist", "Machine Learning Engineer",
|
|
25
|
+
"Product Manager", "DevOps Engineer", "Cloud Architect",
|
|
26
|
+
"Cybersecurity Analyst", "UI/UX Designer", "New Grad SWE",
|
|
27
|
+
"Senior Staff Engineer", "QA Automation Engineer", "Intern",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
SAMPLE_CRITIQUES = [
|
|
31
|
+
"Quantify your bullet points with measurable business outcomes and metrics.",
|
|
32
|
+
"Keep the resume strictly to one page for early to mid-level roles.",
|
|
33
|
+
"Remove high school education and irrelevant coursework.",
|
|
34
|
+
"Strong experience section, but consider moving the Skills section to the top.",
|
|
35
|
+
"Use active action verbs (e.g., 'Architected', 'Spearheaded') instead of passive phrasing.",
|
|
36
|
+
"Clarify your tech stack under each project bullet.",
|
|
37
|
+
"The formatting is clean, but tailor your summary statement to the target job description.",
|
|
38
|
+
"Explain the latency reduction and scale of the distributed system you worked on.",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
posts_per_shard: Union[int, Dict[Tuple[int, int], int], Callable[[int, int], int]] = 100,
|
|
44
|
+
batch_size: int = 100,
|
|
45
|
+
comments_per_post: int = 5,
|
|
46
|
+
match_ratio: float = 0.5,
|
|
47
|
+
seed: Optional[int] = 42,
|
|
48
|
+
simulated_latency: float = 0.0,
|
|
49
|
+
):
|
|
50
|
+
self.posts_per_shard = posts_per_shard
|
|
51
|
+
self.batch_size = max(1, batch_size)
|
|
52
|
+
self.comments_per_post = max(0, comments_per_post)
|
|
53
|
+
self.match_ratio = max(0.0, min(1.0, match_ratio))
|
|
54
|
+
self.seed = seed
|
|
55
|
+
self.simulated_latency = max(0.0, float(simulated_latency))
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def name(self) -> str:
|
|
59
|
+
return "dummy"
|
|
60
|
+
|
|
61
|
+
def fetch_posts_shard(
|
|
62
|
+
self,
|
|
63
|
+
subreddit: str,
|
|
64
|
+
after_ts: int,
|
|
65
|
+
before_ts: int,
|
|
66
|
+
fields: Optional[Iterable[str]] = None,
|
|
67
|
+
session: Optional[requests.Session] = None,
|
|
68
|
+
) -> Iterator[List[Dict[str, Any]]]:
|
|
69
|
+
"""
|
|
70
|
+
Yields batches of randomly generated synthetic posts within [after_ts, before_ts].
|
|
71
|
+
Paginates in chunks of batch_size and simulates server-side field projection.
|
|
72
|
+
"""
|
|
73
|
+
shard_seed = (self.seed + after_ts) if self.seed is not None else after_ts
|
|
74
|
+
rng = random.Random(shard_seed)
|
|
75
|
+
fields_set = self.resolve_post_fields(fields)
|
|
76
|
+
|
|
77
|
+
if isinstance(self.posts_per_shard, dict):
|
|
78
|
+
total_posts = self.posts_per_shard.get((after_ts, before_ts), 100)
|
|
79
|
+
elif callable(self.posts_per_shard):
|
|
80
|
+
total_posts = self.posts_per_shard(after_ts, before_ts)
|
|
81
|
+
else:
|
|
82
|
+
total_posts = max(0, int(self.posts_per_shard))
|
|
83
|
+
|
|
84
|
+
if total_posts <= 0:
|
|
85
|
+
return
|
|
86
|
+
|
|
87
|
+
time_step = max(1, (before_ts - after_ts) // (total_posts + 1))
|
|
88
|
+
|
|
89
|
+
current_batch = []
|
|
90
|
+
for i in range(total_posts):
|
|
91
|
+
post_ts = before_ts - (i + 1) * time_step
|
|
92
|
+
if post_ts <= after_ts:
|
|
93
|
+
break
|
|
94
|
+
|
|
95
|
+
post_id = f"dummy_{subreddit}_{i+1}_{post_ts}"
|
|
96
|
+
is_match = rng.random() < self.match_ratio
|
|
97
|
+
role = rng.choice(self.SAMPLE_ROLES)
|
|
98
|
+
|
|
99
|
+
if is_match:
|
|
100
|
+
score = rng.randint(5, 350)
|
|
101
|
+
num_comments = rng.randint(12, 95)
|
|
102
|
+
ext = rng.choice(["png", "jpg", "jpeg"])
|
|
103
|
+
url = f"https://i.redd.it/{post_id}.{ext}"
|
|
104
|
+
is_self = False
|
|
105
|
+
else:
|
|
106
|
+
score = rng.choice([0, rng.randint(0, 4)])
|
|
107
|
+
num_comments = rng.randint(0, 8)
|
|
108
|
+
url = f"https://reddit.com/r/{subreddit}/comments/{post_id}"
|
|
109
|
+
is_self = True
|
|
110
|
+
|
|
111
|
+
full_record = {
|
|
112
|
+
"id": post_id,
|
|
113
|
+
"name": f"t3_{post_id}",
|
|
114
|
+
"title": f"[{subreddit.upper()}] {role} Resume Review - Looking for constructive critique",
|
|
115
|
+
"author": f"candidate_{rng.randint(1, 20000)}",
|
|
116
|
+
"score": score,
|
|
117
|
+
"num_comments": num_comments,
|
|
118
|
+
"url": url,
|
|
119
|
+
"selftext": "Seeking advice on how to improve my resume for upcoming tech recruiting cycles.",
|
|
120
|
+
"subreddit": subreddit,
|
|
121
|
+
"created_utc": post_ts,
|
|
122
|
+
"is_self": is_self,
|
|
123
|
+
"over_18": False,
|
|
124
|
+
"stickied": False,
|
|
125
|
+
"locked": False,
|
|
126
|
+
"retrieved_on": post_ts + 100,
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if fields_set:
|
|
130
|
+
projected = {k: v for k, v in full_record.items() if k in fields_set}
|
|
131
|
+
current_batch.append(projected)
|
|
132
|
+
else:
|
|
133
|
+
current_batch.append(full_record)
|
|
134
|
+
|
|
135
|
+
if len(current_batch) >= self.batch_size:
|
|
136
|
+
if self.simulated_latency > 0:
|
|
137
|
+
time.sleep(self.simulated_latency)
|
|
138
|
+
yield current_batch
|
|
139
|
+
current_batch = []
|
|
140
|
+
|
|
141
|
+
if current_batch:
|
|
142
|
+
if self.simulated_latency > 0:
|
|
143
|
+
time.sleep(self.simulated_latency)
|
|
144
|
+
yield current_batch
|
|
145
|
+
|
|
146
|
+
async def fetch_comments_for_post(
|
|
147
|
+
self,
|
|
148
|
+
client: httpx.AsyncClient,
|
|
149
|
+
post_id: str,
|
|
150
|
+
fields: Optional[Iterable[str]] = None,
|
|
151
|
+
max_retries: int = 5,
|
|
152
|
+
) -> Tuple[str, List[Dict[str, Any]], bool]:
|
|
153
|
+
"""
|
|
154
|
+
Dynamically generates random comments with realistic reply hierarchies for a post.
|
|
155
|
+
Simulates server-side field projection if fields are provided.
|
|
156
|
+
"""
|
|
157
|
+
if self.simulated_latency > 0:
|
|
158
|
+
await asyncio.sleep(self.simulated_latency)
|
|
159
|
+
clean_id = str(post_id).replace("t3_", "")
|
|
160
|
+
fields_set = self.resolve_comment_fields(fields)
|
|
161
|
+
base_hash = zlib.crc32(clean_id.encode())
|
|
162
|
+
comment_seed = (base_hash ^ self.seed) if self.seed is not None else base_hash
|
|
163
|
+
rng = random.Random(comment_seed)
|
|
164
|
+
|
|
165
|
+
comments = []
|
|
166
|
+
created_comment_ids = []
|
|
167
|
+
|
|
168
|
+
for c_idx in range(self.comments_per_post):
|
|
169
|
+
cid = f"c_{clean_id}_{c_idx+1}"
|
|
170
|
+
|
|
171
|
+
# Make the first comment top-level; subsequent comments can be top-level or replies
|
|
172
|
+
if not created_comment_ids or rng.random() > 0.4:
|
|
173
|
+
parent_id = f"t3_{clean_id}"
|
|
174
|
+
else:
|
|
175
|
+
parent_id = f"t1_{rng.choice(created_comment_ids)}"
|
|
176
|
+
|
|
177
|
+
created_comment_ids.append(cid)
|
|
178
|
+
|
|
179
|
+
is_automod = (c_idx == 0 and rng.random() < 0.25)
|
|
180
|
+
author = "AutoModerator" if is_automod else f"reviewer_{rng.randint(1, 10000)}"
|
|
181
|
+
body = (
|
|
182
|
+
"Please make sure your resume does not include personal identification info."
|
|
183
|
+
if is_automod
|
|
184
|
+
else rng.choice(self.SAMPLE_CRITIQUES)
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
record = {
|
|
188
|
+
"id": cid,
|
|
189
|
+
"comment_id": cid,
|
|
190
|
+
"name": f"t1_{cid}",
|
|
191
|
+
"post_id": clean_id,
|
|
192
|
+
"link_id": f"t3_{clean_id}",
|
|
193
|
+
"parent_id": parent_id,
|
|
194
|
+
"author": author,
|
|
195
|
+
"body": body,
|
|
196
|
+
"score": rng.randint(1, 50),
|
|
197
|
+
"created_utc": 1704100000 + c_idx * 1800,
|
|
198
|
+
"subreddit": "resumes",
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if fields_set:
|
|
202
|
+
projected = {k: v for k, v in record.items() if k in fields_set}
|
|
203
|
+
comments.append(projected)
|
|
204
|
+
else:
|
|
205
|
+
comments.append(record)
|
|
206
|
+
|
|
207
|
+
return clean_id, comments, True
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Source registry and factory for Reddit data sources.
|
|
3
|
+
"""
|
|
4
|
+
from typing import Dict, Type
|
|
5
|
+
from deepsift.sources.base import Source
|
|
6
|
+
from deepsift.sources.arctic_shift import ArcticShiftSource
|
|
7
|
+
|
|
8
|
+
SOURCES: Dict[str, Type[Source]] = {
|
|
9
|
+
"arctic_shift": ArcticShiftSource,
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def register_source(name: str, source_cls: Type[Source]):
|
|
14
|
+
"""Registers a new custom Source provider."""
|
|
15
|
+
SOURCES[name.lower().strip()] = source_cls
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_source(name: str = "arctic_shift", **kwargs) -> Source:
|
|
19
|
+
"""Instantiates and returns a registered data source by name."""
|
|
20
|
+
key = name.lower().strip()
|
|
21
|
+
if key not in SOURCES:
|
|
22
|
+
available = ", ".join(sorted(SOURCES.keys()))
|
|
23
|
+
raise ValueError(f"Unknown data source '{name}'. Available sources: {available}")
|
|
24
|
+
return SOURCES[key](**kwargs)
|
deepsift/storage/base.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Abstract base class for storage implementations.
|
|
3
|
+
Defines required interface for streaming scraped Reddit records to disk or databases.
|
|
4
|
+
"""
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import List, Dict, Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Storage(ABC):
|
|
10
|
+
"""
|
|
11
|
+
Abstract Base Class for record storage.
|
|
12
|
+
Enforces thread-safe, batch-oriented serialization and resource cleanup.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, filepath: str):
|
|
16
|
+
self.filepath = filepath
|
|
17
|
+
self.records_written: int = 0
|
|
18
|
+
|
|
19
|
+
def __enter__(self):
|
|
20
|
+
return self
|
|
21
|
+
|
|
22
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
23
|
+
"""Guarantees stream cleanup and footer serialization on context block exit, even if errors occur."""
|
|
24
|
+
self.close()
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
def write_batch(self, batch: List[Dict[str, Any]]) -> None:
|
|
28
|
+
"""
|
|
29
|
+
Writes a batch of record dictionaries to the target storage.
|
|
30
|
+
Implementations should be thread-safe across parallel workers.
|
|
31
|
+
"""
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
@abstractmethod
|
|
35
|
+
def close(self) -> None:
|
|
36
|
+
"""Flushes buffers and releases any underlying file or connection handles."""
|
|
37
|
+
pass
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Parquet streaming writer using PyArrow.
|
|
3
|
+
Thread-safe direct-to-disk chunked writer.
|
|
4
|
+
"""
|
|
5
|
+
import os
|
|
6
|
+
import json
|
|
7
|
+
import threading
|
|
8
|
+
from typing import List, Dict, Any
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
import pyarrow.parquet as pq
|
|
11
|
+
|
|
12
|
+
from deepsift.storage.base import Storage
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ParquetStorage(Storage):
|
|
16
|
+
"""
|
|
17
|
+
Direct-to-disk streaming writer for Parquet format.
|
|
18
|
+
Thread-safe across parallel shard workers.
|
|
19
|
+
Handles dynamic column schemas across batches by serializing complex nested values.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
POLYMORPHIC_FIELDS = frozenset({"edited", "author_flair_text", "link_flair_text", "distinguished"})
|
|
23
|
+
|
|
24
|
+
def __init__(self, filepath: str):
|
|
25
|
+
super().__init__(filepath)
|
|
26
|
+
self.writer = None
|
|
27
|
+
self.schema = None
|
|
28
|
+
self._closed = False
|
|
29
|
+
self._lock = threading.Lock()
|
|
30
|
+
|
|
31
|
+
dir_path = os.path.dirname(os.path.abspath(filepath))
|
|
32
|
+
if dir_path:
|
|
33
|
+
os.makedirs(dir_path, exist_ok=True)
|
|
34
|
+
|
|
35
|
+
def _sanitize_schema(self, schema: pa.Schema) -> pa.Schema:
|
|
36
|
+
"""Promotes null and polymorphic Reddit fields to string to avoid cross-batch type conflicts."""
|
|
37
|
+
fields = []
|
|
38
|
+
for f in schema:
|
|
39
|
+
if pa.types.is_null(f.type) or f.name in self.POLYMORPHIC_FIELDS:
|
|
40
|
+
fields.append(pa.field(f.name, pa.string()))
|
|
41
|
+
else:
|
|
42
|
+
fields.append(f)
|
|
43
|
+
return pa.schema(fields)
|
|
44
|
+
|
|
45
|
+
def write_batch(self, batch: List[Dict[str, Any]]):
|
|
46
|
+
if self._closed:
|
|
47
|
+
raise RuntimeError(f"Cannot write to closed ParquetStorage '{self.filepath}'")
|
|
48
|
+
if not batch:
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
cleaned_batch = []
|
|
52
|
+
for row in batch:
|
|
53
|
+
clean_row = {}
|
|
54
|
+
for k, v in row.items():
|
|
55
|
+
if isinstance(v, (dict, list)):
|
|
56
|
+
clean_row[k] = json.dumps(v, default=str)
|
|
57
|
+
elif k in self.POLYMORPHIC_FIELDS and v is not None and not isinstance(v, str):
|
|
58
|
+
clean_row[k] = str(v)
|
|
59
|
+
else:
|
|
60
|
+
clean_row[k] = v
|
|
61
|
+
cleaned_batch.append(clean_row)
|
|
62
|
+
|
|
63
|
+
with self._lock:
|
|
64
|
+
if self.schema is None:
|
|
65
|
+
initial_table = pa.Table.from_pylist(cleaned_batch)
|
|
66
|
+
self.schema = self._sanitize_schema(initial_table.schema)
|
|
67
|
+
self.writer = pq.ParquetWriter(self.filepath, self.schema, compression="snappy")
|
|
68
|
+
|
|
69
|
+
string_cols = {f.name for f in self.schema if pa.types.is_string(f.type)}
|
|
70
|
+
for row in cleaned_batch:
|
|
71
|
+
for col in string_cols:
|
|
72
|
+
if col in row and row[col] is not None and not isinstance(row[col], str):
|
|
73
|
+
row[col] = str(row[col])
|
|
74
|
+
table = pa.Table.from_pylist(cleaned_batch, schema=self.schema)
|
|
75
|
+
|
|
76
|
+
self.writer.write_table(table)
|
|
77
|
+
self.records_written += len(batch)
|
|
78
|
+
|
|
79
|
+
def close(self):
|
|
80
|
+
with self._lock:
|
|
81
|
+
self._closed = True
|
|
82
|
+
if self.writer is not None:
|
|
83
|
+
self.writer.close()
|
|
84
|
+
self.writer = None
|
|
85
|
+
elif self.records_written == 0:
|
|
86
|
+
print(f"[i] Info: Parquet file '{self.filepath}' was not created due to lack of results.")
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deepsift
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
|
|
5
|
+
Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/saakethk/deepsift
|
|
8
|
+
Project-URL: Documentation, https://saakethk.github.io/deepsift/
|
|
9
|
+
Project-URL: Repository, https://github.com/saakethk/deepsift.git
|
|
10
|
+
Project-URL: Issues, https://github.com/saakethk/deepsift/issues
|
|
11
|
+
Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: requests>=2.31.0
|
|
22
|
+
Requires-Dist: httpx>=0.27.0
|
|
23
|
+
Requires-Dist: pandas>=2.0.0
|
|
24
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
25
|
+
Requires-Dist: tqdm>=4.66.0
|
|
26
|
+
Requires-Dist: click>=8.1.0
|
|
27
|
+
Requires-Dist: nest-asyncio>=1.6.0
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# Resume Model-V3
|
|
35
|
+
What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
|
|
36
|
+
|
|
37
|
+
## Methodology
|
|
38
|
+
To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
|
|
39
|
+
|
|
40
|
+
We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
|
|
41
|
+
|
|
42
|
+
[](data/relevant_subs_cohort_11_11_2026.csv)
|
|
43
|
+
|
|
44
|
+
We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
|
|
45
|
+
|
|
46
|
+
We determined that the ideal cohort for our use case needed to satisfy the following factors:
|
|
47
|
+
- have >10 comments and >0 score
|
|
48
|
+
- have a associated url corresponding to a picture (.png or jpeg)
|
|
49
|
+
|
|
50
|
+
After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Testing
|
|
55
|
+
To run the static unit and integration test suite offline using the synthetic `DummySource`:
|
|
56
|
+
```bash
|
|
57
|
+
pytest -m "not live" -v
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
To run the live integration tests against the actual Arctic Shift API:
|
|
61
|
+
```bash
|
|
62
|
+
pytest -m live -v
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
# Evaluation & Benchmarks
|
|
66
|
+
Comprehensive performance evaluations comparing DeepSift against sequential execution:
|
|
67
|
+
|
|
68
|
+
| Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
|
|
69
|
+
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
|
70
|
+
| **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
|
|
71
|
+
| **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
|
|
72
|
+
|
|
73
|
+
To run the evaluations:
|
|
74
|
+
```bash
|
|
75
|
+
# Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
|
|
76
|
+
python -m evaluation
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
# Documentation Site
|
|
80
|
+
The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
|
|
81
|
+
|
|
82
|
+
To run the documentation dev server with hot-reloading:
|
|
83
|
+
```bash
|
|
84
|
+
# From project root:
|
|
85
|
+
npm run docs
|
|
86
|
+
|
|
87
|
+
# Or with force cache reload:
|
|
88
|
+
npm run docs:force
|
|
89
|
+
|
|
90
|
+
# Or directly inside docs/ using npx:
|
|
91
|
+
cd docs
|
|
92
|
+
npx astro dev --force
|
|
93
|
+
```
|
|
94
|
+
Then visit `http://localhost:4321/` in your browser.
|
|
95
|
+
|
|
96
|
+
To build the static production site:
|
|
97
|
+
```bash
|
|
98
|
+
npm run docs:build
|
|
99
|
+
# or: cd docs && npm run build
|
|
100
|
+
```
|
|
101
|
+
The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
|
|
102
|
+
|
|
103
|
+
Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
|
|
104
|
+
- [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
|
|
105
|
+
- [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
|
|
106
|
+
- [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
|
|
107
|
+
- [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
|
|
108
|
+
- [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
|
|
109
|
+
- [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
|
|
110
|
+
|
|
111
|
+
# TODO
|
|
112
|
+
- [x] make static tests using dummy source (runs on github ci/cd)
|
|
113
|
+
- [x] make live tests using actual source on r/resumes
|
|
114
|
+
- [x] make evaluation script
|
|
115
|
+
- [x] writeup and documentation with benchmarks, docs page, and cleaned repo
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
deepsift/__init__.py,sha256=vK8jpeKXHuVHWYkEiHOViomgrJfKtX9w8Lf7YZSPJNM,782
|
|
2
|
+
deepsift/__main__.py,sha256=3qMFrUXdQ0ZeQW3GaTbRykbSm9mSAbUZegq-wYR6KC4,161
|
|
3
|
+
deepsift/cli.py,sha256=bMGdSQQI9N3lSFLpP5tP0dCnM5HgpRwz1_YzLstXLV8,8290
|
|
4
|
+
deepsift/filters.py,sha256=qu96zYiOUKLfd7bEj5oQ9cKzGYqGlNLzoyoIYCuUrok,5006
|
|
5
|
+
deepsift/scraper.py,sha256=0NAVtivTgTKfvUFijOiIIoIA5hBFRobC_jICLJ5bohY,17091
|
|
6
|
+
deepsift/models/__init__.py,sha256=N6K_JAFJOTMH_rcMSLkYHk-TTGoAa0FsDjkn4ZJuPsM,419
|
|
7
|
+
deepsift/models/base.py,sha256=XQWz5S2sKVR1UG4kvJx-QGyDe-TA4GwfvL2tcRmsoyE,3369
|
|
8
|
+
deepsift/models/comment.py,sha256=WyDYE4VdidNxsgmC1eLY7p_FbHAHcasGh5uisA0KTzA,1370
|
|
9
|
+
deepsift/models/enums.py,sha256=ho8xzN1KHUuumJy5K8R1mDK0NxNhK2G23msV7WeOVNo,18800
|
|
10
|
+
deepsift/models/post.py,sha256=a3FJfNhiG_G47ZgF5EHKi30Xq2gElYyrnyxj7ugqDVo,945
|
|
11
|
+
deepsift/sources/__init__.py,sha256=V1yNiRnLmQgIxmCGShbI5EJkSF9ly26LrtToeKumFpY,339
|
|
12
|
+
deepsift/sources/arctic_shift.py,sha256=5Wlozpcy8UoLlTN3HYqqqSSXY3C8gyejq44G1QX9Ais,9891
|
|
13
|
+
deepsift/sources/base.py,sha256=hLzPdKrTMFH5dvw1hyp-yKdUMjQebmEPkR2dxODXqI4,3425
|
|
14
|
+
deepsift/sources/dummy.py,sha256=yqdiEtiqPVZzwnznHgQX9kgGSHrk0QRmLrDwdBVpEiY,8088
|
|
15
|
+
deepsift/sources/registry.py,sha256=NZhegViN04z3huN7UfgmjCg1lEQCXkzs1tiq8ZQorxw,799
|
|
16
|
+
deepsift/storage/__init__.py,sha256=Wy_4jZVgzaN9l_i57ygiKsr-if7vwhxS-HGFgVp6Aak,197
|
|
17
|
+
deepsift/storage/base.py,sha256=-r_OcM8r1Fl7iI8DJujz58afWw_4weYgdCSbghY0xzo,1124
|
|
18
|
+
deepsift/storage/parquet.py,sha256=Jg4bB4nXrtZJ_lUs1M8xHWJYFEYW-PMVWlAAiG3p1oE,3170
|
|
19
|
+
deepsift-0.2.0.dist-info/licenses/LICENSE,sha256=IMFfwW98307ORF7PAt1lZzFP_giCL5YdKLw5l2Umm20,1074
|
|
20
|
+
deepsift-0.2.0.dist-info/METADATA,sha256=rRPK3qIeuD2i4S8oxyRPyCe8SYpyUoNjylJxhz7XerE,6590
|
|
21
|
+
deepsift-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
22
|
+
deepsift-0.2.0.dist-info/entry_points.txt,sha256=kPy8InNH-uJiVhqIejRC7y0saWaqXRXaNkvB5eJSJiE,80
|
|
23
|
+
deepsift-0.2.0.dist-info/top_level.txt,sha256=E2qyWtoKciYlHQb5TMF-4sAq8o1uXMaH9bxdA69pi98,9
|
|
24
|
+
deepsift-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Saaketh Kesireddy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
deepsift
|