deepsift 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,102 @@
1
+ """
2
+ Abstract base class definition for Reddit data sources.
3
+ """
4
+ from abc import ABC, abstractmethod
5
+ from typing import Iterator, List, Dict, Any, Optional, Tuple, Iterable, Set
6
+ import requests
7
+ import httpx
8
+
9
+
10
+ class Source(ABC):
11
+ """
12
+ Abstract Base Class for Reddit data sources.
13
+ Handles communication with specific APIs, archives, or data stores.
14
+ """
15
+
16
+ @staticmethod
17
+ def resolve_post_fields(fields: Optional[Iterable[str]]) -> Optional[Set[str]]:
18
+ """Ensures required post fields (id, created_utc) are present in any projection."""
19
+ if fields is None:
20
+ return None
21
+ from deepsift.models import Post
22
+ return set(fields) | set(Post.REQUIRED_FIELDS)
23
+
24
+ @staticmethod
25
+ def resolve_comment_fields(fields: Optional[Iterable[str]]) -> Optional[Set[str]]:
26
+ """Ensures required comment fields (id, post_id, parent_id) are present in any projection."""
27
+ if fields is None:
28
+ return None
29
+ from deepsift.models import Comment
30
+ return set(fields) | set(Comment.REQUIRED_FIELDS)
31
+
32
+ @property
33
+ @abstractmethod
34
+ def name(self) -> str:
35
+ """Identifying name of the data source."""
36
+ pass
37
+
38
+ @abstractmethod
39
+ def fetch_posts_shard(
40
+ self,
41
+ subreddit: str,
42
+ after_ts: int,
43
+ before_ts: int,
44
+ fields: Optional[Iterable[str]] = None,
45
+ session: Optional[requests.Session] = None,
46
+ ) -> Iterator[List[Dict[str, Any]]]:
47
+ """
48
+ Yields batches of raw post dictionaries for a given subreddit within [after_ts, before_ts].
49
+ If fields is provided, requests only those fields from the server.
50
+ """
51
+ pass
52
+
53
+ @abstractmethod
54
+ async def fetch_comments_for_post(
55
+ self,
56
+ client: httpx.AsyncClient,
57
+ post_id: str,
58
+ fields: Optional[Iterable[str]] = None,
59
+ max_retries: int = 5,
60
+ ) -> Tuple[str, List[Dict[str, Any]], bool]:
61
+ """
62
+ Asynchronously fetches comment dictionaries for a given post.
63
+ Returns: (post_id, comments_list, success_bool)
64
+ """
65
+ pass
66
+
67
+ def fetch_posts(
68
+ self,
69
+ subreddit: str,
70
+ after_ts: int,
71
+ before_ts: int,
72
+ fields: Optional[Iterable[str]] = None,
73
+ ) -> List[Dict[str, Any]]:
74
+ """
75
+ Synchronously gathers all post dictionaries within [after_ts, before_ts] into a single list.
76
+ """
77
+ posts = []
78
+ for batch in self.fetch_posts_shard(subreddit, after_ts, before_ts, fields=fields):
79
+ posts.extend(batch)
80
+ return posts
81
+
82
+ def fetch_comments(self, post_id: str, fields: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]:
83
+ """
84
+ Synchronously fetches comments for a single post using a transient httpx client.
85
+ Safely supports execution inside active event loops (such as Jupyter notebooks).
86
+ """
87
+ import asyncio
88
+ async def _run():
89
+ async with httpx.AsyncClient() as client:
90
+ _, comments, _ = await self.fetch_comments_for_post(client, post_id, fields=fields)
91
+ return comments
92
+
93
+ try:
94
+ loop = asyncio.get_running_loop()
95
+ except RuntimeError:
96
+ loop = None
97
+
98
+ if loop and loop.is_running():
99
+ import nest_asyncio
100
+ nest_asyncio.apply()
101
+ return loop.run_until_complete(_run())
102
+ return asyncio.run(_run())
@@ -0,0 +1,207 @@
1
+ """
2
+ Dummy and mock data source for scalable offline testing and verification.
3
+ Dynamically generates random, realistic posts and comments to stress-test pipelines.
4
+ """
5
+ import time
6
+ import asyncio
7
+ import random
8
+ import zlib
9
+ from typing import Iterator, List, Dict, Any, Optional, Tuple, Iterable, Union, Callable
10
+ import requests
11
+ import httpx
12
+ from deepsift.sources.base import Source
13
+
14
+
15
+ class DummySource(Source):
16
+ """
17
+ In-memory synthetic Reddit data source for offline scale testing.
18
+ Dynamically generates randomized, realistic posts and comments without making network calls.
19
+ Supports server-side field projection simulation.
20
+ """
21
+
22
+ SAMPLE_ROLES = [
23
+ "Software Engineer", "Frontend Developer", "Backend Engineer",
24
+ "Fullstack Developer", "Data Scientist", "Machine Learning Engineer",
25
+ "Product Manager", "DevOps Engineer", "Cloud Architect",
26
+ "Cybersecurity Analyst", "UI/UX Designer", "New Grad SWE",
27
+ "Senior Staff Engineer", "QA Automation Engineer", "Intern",
28
+ ]
29
+
30
+ SAMPLE_CRITIQUES = [
31
+ "Quantify your bullet points with measurable business outcomes and metrics.",
32
+ "Keep the resume strictly to one page for early to mid-level roles.",
33
+ "Remove high school education and irrelevant coursework.",
34
+ "Strong experience section, but consider moving the Skills section to the top.",
35
+ "Use active action verbs (e.g., 'Architected', 'Spearheaded') instead of passive phrasing.",
36
+ "Clarify your tech stack under each project bullet.",
37
+ "The formatting is clean, but tailor your summary statement to the target job description.",
38
+ "Explain the latency reduction and scale of the distributed system you worked on.",
39
+ ]
40
+
41
+ def __init__(
42
+ self,
43
+ posts_per_shard: Union[int, Dict[Tuple[int, int], int], Callable[[int, int], int]] = 100,
44
+ batch_size: int = 100,
45
+ comments_per_post: int = 5,
46
+ match_ratio: float = 0.5,
47
+ seed: Optional[int] = 42,
48
+ simulated_latency: float = 0.0,
49
+ ):
50
+ self.posts_per_shard = posts_per_shard
51
+ self.batch_size = max(1, batch_size)
52
+ self.comments_per_post = max(0, comments_per_post)
53
+ self.match_ratio = max(0.0, min(1.0, match_ratio))
54
+ self.seed = seed
55
+ self.simulated_latency = max(0.0, float(simulated_latency))
56
+
57
+ @property
58
+ def name(self) -> str:
59
+ return "dummy"
60
+
61
+ def fetch_posts_shard(
62
+ self,
63
+ subreddit: str,
64
+ after_ts: int,
65
+ before_ts: int,
66
+ fields: Optional[Iterable[str]] = None,
67
+ session: Optional[requests.Session] = None,
68
+ ) -> Iterator[List[Dict[str, Any]]]:
69
+ """
70
+ Yields batches of randomly generated synthetic posts within [after_ts, before_ts].
71
+ Paginates in chunks of batch_size and simulates server-side field projection.
72
+ """
73
+ shard_seed = (self.seed + after_ts) if self.seed is not None else after_ts
74
+ rng = random.Random(shard_seed)
75
+ fields_set = self.resolve_post_fields(fields)
76
+
77
+ if isinstance(self.posts_per_shard, dict):
78
+ total_posts = self.posts_per_shard.get((after_ts, before_ts), 100)
79
+ elif callable(self.posts_per_shard):
80
+ total_posts = self.posts_per_shard(after_ts, before_ts)
81
+ else:
82
+ total_posts = max(0, int(self.posts_per_shard))
83
+
84
+ if total_posts <= 0:
85
+ return
86
+
87
+ time_step = max(1, (before_ts - after_ts) // (total_posts + 1))
88
+
89
+ current_batch = []
90
+ for i in range(total_posts):
91
+ post_ts = before_ts - (i + 1) * time_step
92
+ if post_ts <= after_ts:
93
+ break
94
+
95
+ post_id = f"dummy_{subreddit}_{i+1}_{post_ts}"
96
+ is_match = rng.random() < self.match_ratio
97
+ role = rng.choice(self.SAMPLE_ROLES)
98
+
99
+ if is_match:
100
+ score = rng.randint(5, 350)
101
+ num_comments = rng.randint(12, 95)
102
+ ext = rng.choice(["png", "jpg", "jpeg"])
103
+ url = f"https://i.redd.it/{post_id}.{ext}"
104
+ is_self = False
105
+ else:
106
+ score = rng.choice([0, rng.randint(0, 4)])
107
+ num_comments = rng.randint(0, 8)
108
+ url = f"https://reddit.com/r/{subreddit}/comments/{post_id}"
109
+ is_self = True
110
+
111
+ full_record = {
112
+ "id": post_id,
113
+ "name": f"t3_{post_id}",
114
+ "title": f"[{subreddit.upper()}] {role} Resume Review - Looking for constructive critique",
115
+ "author": f"candidate_{rng.randint(1, 20000)}",
116
+ "score": score,
117
+ "num_comments": num_comments,
118
+ "url": url,
119
+ "selftext": "Seeking advice on how to improve my resume for upcoming tech recruiting cycles.",
120
+ "subreddit": subreddit,
121
+ "created_utc": post_ts,
122
+ "is_self": is_self,
123
+ "over_18": False,
124
+ "stickied": False,
125
+ "locked": False,
126
+ "retrieved_on": post_ts + 100,
127
+ }
128
+
129
+ if fields_set:
130
+ projected = {k: v for k, v in full_record.items() if k in fields_set}
131
+ current_batch.append(projected)
132
+ else:
133
+ current_batch.append(full_record)
134
+
135
+ if len(current_batch) >= self.batch_size:
136
+ if self.simulated_latency > 0:
137
+ time.sleep(self.simulated_latency)
138
+ yield current_batch
139
+ current_batch = []
140
+
141
+ if current_batch:
142
+ if self.simulated_latency > 0:
143
+ time.sleep(self.simulated_latency)
144
+ yield current_batch
145
+
146
+ async def fetch_comments_for_post(
147
+ self,
148
+ client: httpx.AsyncClient,
149
+ post_id: str,
150
+ fields: Optional[Iterable[str]] = None,
151
+ max_retries: int = 5,
152
+ ) -> Tuple[str, List[Dict[str, Any]], bool]:
153
+ """
154
+ Dynamically generates random comments with realistic reply hierarchies for a post.
155
+ Simulates server-side field projection if fields are provided.
156
+ """
157
+ if self.simulated_latency > 0:
158
+ await asyncio.sleep(self.simulated_latency)
159
+ clean_id = str(post_id).replace("t3_", "")
160
+ fields_set = self.resolve_comment_fields(fields)
161
+ base_hash = zlib.crc32(clean_id.encode())
162
+ comment_seed = (base_hash ^ self.seed) if self.seed is not None else base_hash
163
+ rng = random.Random(comment_seed)
164
+
165
+ comments = []
166
+ created_comment_ids = []
167
+
168
+ for c_idx in range(self.comments_per_post):
169
+ cid = f"c_{clean_id}_{c_idx+1}"
170
+
171
+ # Make the first comment top-level; subsequent comments can be top-level or replies
172
+ if not created_comment_ids or rng.random() > 0.4:
173
+ parent_id = f"t3_{clean_id}"
174
+ else:
175
+ parent_id = f"t1_{rng.choice(created_comment_ids)}"
176
+
177
+ created_comment_ids.append(cid)
178
+
179
+ is_automod = (c_idx == 0 and rng.random() < 0.25)
180
+ author = "AutoModerator" if is_automod else f"reviewer_{rng.randint(1, 10000)}"
181
+ body = (
182
+ "Please make sure your resume does not include personal identification info."
183
+ if is_automod
184
+ else rng.choice(self.SAMPLE_CRITIQUES)
185
+ )
186
+
187
+ record = {
188
+ "id": cid,
189
+ "comment_id": cid,
190
+ "name": f"t1_{cid}",
191
+ "post_id": clean_id,
192
+ "link_id": f"t3_{clean_id}",
193
+ "parent_id": parent_id,
194
+ "author": author,
195
+ "body": body,
196
+ "score": rng.randint(1, 50),
197
+ "created_utc": 1704100000 + c_idx * 1800,
198
+ "subreddit": "resumes",
199
+ }
200
+
201
+ if fields_set:
202
+ projected = {k: v for k, v in record.items() if k in fields_set}
203
+ comments.append(projected)
204
+ else:
205
+ comments.append(record)
206
+
207
+ return clean_id, comments, True
@@ -0,0 +1,24 @@
1
+ """
2
+ Source registry and factory for Reddit data sources.
3
+ """
4
+ from typing import Dict, Type
5
+ from deepsift.sources.base import Source
6
+ from deepsift.sources.arctic_shift import ArcticShiftSource
7
+
8
+ SOURCES: Dict[str, Type[Source]] = {
9
+ "arctic_shift": ArcticShiftSource,
10
+ }
11
+
12
+
13
+ def register_source(name: str, source_cls: Type[Source]):
14
+ """Registers a new custom Source provider."""
15
+ SOURCES[name.lower().strip()] = source_cls
16
+
17
+
18
+ def get_source(name: str = "arctic_shift", **kwargs) -> Source:
19
+ """Instantiates and returns a registered data source by name."""
20
+ key = name.lower().strip()
21
+ if key not in SOURCES:
22
+ available = ", ".join(sorted(SOURCES.keys()))
23
+ raise ValueError(f"Unknown data source '{name}'. Available sources: {available}")
24
+ return SOURCES[key](**kwargs)
@@ -0,0 +1,10 @@
1
+ """
2
+ Storage subpackage for disk serialization.
3
+ """
4
+ from deepsift.storage.base import Storage
5
+ from deepsift.storage.parquet import ParquetStorage
6
+
7
+ __all__ = [
8
+ "Storage",
9
+ "ParquetStorage",
10
+ ]
@@ -0,0 +1,37 @@
1
+ """
2
+ Abstract base class for storage implementations.
3
+ Defines required interface for streaming scraped Reddit records to disk or databases.
4
+ """
5
+ from abc import ABC, abstractmethod
6
+ from typing import List, Dict, Any
7
+
8
+
9
+ class Storage(ABC):
10
+ """
11
+ Abstract Base Class for record storage.
12
+ Enforces thread-safe, batch-oriented serialization and resource cleanup.
13
+ """
14
+
15
+ def __init__(self, filepath: str):
16
+ self.filepath = filepath
17
+ self.records_written: int = 0
18
+
19
+ def __enter__(self):
20
+ return self
21
+
22
+ def __exit__(self, exc_type, exc_val, exc_tb):
23
+ """Guarantees stream cleanup and footer serialization on context block exit, even if errors occur."""
24
+ self.close()
25
+
26
+ @abstractmethod
27
+ def write_batch(self, batch: List[Dict[str, Any]]) -> None:
28
+ """
29
+ Writes a batch of record dictionaries to the target storage.
30
+ Implementations should be thread-safe across parallel workers.
31
+ """
32
+ pass
33
+
34
+ @abstractmethod
35
+ def close(self) -> None:
36
+ """Flushes buffers and releases any underlying file or connection handles."""
37
+ pass
@@ -0,0 +1,86 @@
1
+ """
2
+ Parquet streaming writer using PyArrow.
3
+ Thread-safe direct-to-disk chunked writer.
4
+ """
5
+ import os
6
+ import json
7
+ import threading
8
+ from typing import List, Dict, Any
9
+ import pyarrow as pa
10
+ import pyarrow.parquet as pq
11
+
12
+ from deepsift.storage.base import Storage
13
+
14
+
15
+ class ParquetStorage(Storage):
16
+ """
17
+ Direct-to-disk streaming writer for Parquet format.
18
+ Thread-safe across parallel shard workers.
19
+ Handles dynamic column schemas across batches by serializing complex nested values.
20
+ """
21
+
22
+ POLYMORPHIC_FIELDS = frozenset({"edited", "author_flair_text", "link_flair_text", "distinguished"})
23
+
24
+ def __init__(self, filepath: str):
25
+ super().__init__(filepath)
26
+ self.writer = None
27
+ self.schema = None
28
+ self._closed = False
29
+ self._lock = threading.Lock()
30
+
31
+ dir_path = os.path.dirname(os.path.abspath(filepath))
32
+ if dir_path:
33
+ os.makedirs(dir_path, exist_ok=True)
34
+
35
+ def _sanitize_schema(self, schema: pa.Schema) -> pa.Schema:
36
+ """Promotes null and polymorphic Reddit fields to string to avoid cross-batch type conflicts."""
37
+ fields = []
38
+ for f in schema:
39
+ if pa.types.is_null(f.type) or f.name in self.POLYMORPHIC_FIELDS:
40
+ fields.append(pa.field(f.name, pa.string()))
41
+ else:
42
+ fields.append(f)
43
+ return pa.schema(fields)
44
+
45
+ def write_batch(self, batch: List[Dict[str, Any]]):
46
+ if self._closed:
47
+ raise RuntimeError(f"Cannot write to closed ParquetStorage '{self.filepath}'")
48
+ if not batch:
49
+ return
50
+
51
+ cleaned_batch = []
52
+ for row in batch:
53
+ clean_row = {}
54
+ for k, v in row.items():
55
+ if isinstance(v, (dict, list)):
56
+ clean_row[k] = json.dumps(v, default=str)
57
+ elif k in self.POLYMORPHIC_FIELDS and v is not None and not isinstance(v, str):
58
+ clean_row[k] = str(v)
59
+ else:
60
+ clean_row[k] = v
61
+ cleaned_batch.append(clean_row)
62
+
63
+ with self._lock:
64
+ if self.schema is None:
65
+ initial_table = pa.Table.from_pylist(cleaned_batch)
66
+ self.schema = self._sanitize_schema(initial_table.schema)
67
+ self.writer = pq.ParquetWriter(self.filepath, self.schema, compression="snappy")
68
+
69
+ string_cols = {f.name for f in self.schema if pa.types.is_string(f.type)}
70
+ for row in cleaned_batch:
71
+ for col in string_cols:
72
+ if col in row and row[col] is not None and not isinstance(row[col], str):
73
+ row[col] = str(row[col])
74
+ table = pa.Table.from_pylist(cleaned_batch, schema=self.schema)
75
+
76
+ self.writer.write_table(table)
77
+ self.records_written += len(batch)
78
+
79
+ def close(self):
80
+ with self._lock:
81
+ self._closed = True
82
+ if self.writer is not None:
83
+ self.writer.close()
84
+ self.writer = None
85
+ elif self.records_written == 0:
86
+ print(f"[i] Info: Parquet file '{self.filepath}' was not created due to lack of results.")
@@ -0,0 +1,115 @@
1
+ Metadata-Version: 2.4
2
+ Name: deepsift
3
+ Version: 0.2.0
4
+ Summary: High-performance historical Reddit research toolkit and scraper for Arctic Shift.
5
+ Author-email: Saaketh Kesireddy <saakethr.kesireddy@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/saakethk/deepsift
8
+ Project-URL: Documentation, https://saakethk.github.io/deepsift/
9
+ Project-URL: Repository, https://github.com/saakethk/deepsift.git
10
+ Project-URL: Issues, https://github.com/saakethk/deepsift/issues
11
+ Keywords: reddit,scraper,research,arctic-shift,parquet,dataset,nlp
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: requests>=2.31.0
22
+ Requires-Dist: httpx>=0.27.0
23
+ Requires-Dist: pandas>=2.0.0
24
+ Requires-Dist: pyarrow>=14.0.0
25
+ Requires-Dist: tqdm>=4.66.0
26
+ Requires-Dist: click>=8.1.0
27
+ Requires-Dist: nest-asyncio>=1.6.0
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
30
+ Requires-Dist: build>=1.0.0; extra == "dev"
31
+ Requires-Dist: twine>=5.0.0; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ # Resume Model-V3
35
+ What constitutes an 'effective' resume remains largely subjective and governed by anecdotal heuristics rather than empirical evidence. Furthermore, strict privacy regulations (e.g., GDPR, PII compliance) and corporate liability prevent real-world applicant tracking datasets from being made public. To overcome this data bottleneck, this study leverages crowdsourced data from major resume-focused Reddit communities to empirically analyze community feedback, isolate recurring structural defects, and quantify the presentation features that drive resume efficacy.
36
+
37
+ ## Methodology
38
+ To perform this analysis, we first wanted to identify the largest relevant subreddits related to our topic of resumes. Utilizing the [PRAW](https://praw.readthedocs.io/en/stable/), we pulled the top 100 subreddits for the search term "resume" ([code](find_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)).
39
+
40
+ We found that 47% of the subreddits contained the relevant search term "resume" (ignoring case). This cohort was then refined further to include communities of a significant size (median subscriber count >= **1091**). The cohort of relevant subreddits (n=23) we decided to investigate can be seen below:
41
+
42
+ [![Graph of subscribers per relevant subreddit](data/graph_relevant_subreddits_fig_11_11_26.png)](data/relevant_subs_cohort_11_11_2026.csv)
43
+
44
+ We then got **10,183** posts in the largest relevant subreddit (r/resumes) dating back to May 14, 2026. This was our preliminary cohort consisting of 180 days of data ([code](explore_relevant_posts.ipynb), [data](data/prelim_historical_10000_11_11_26.csv)) which we intended to perform preliminary exploratory data analysis (EDA) on to determine which posts we need to focus on for our larger planned cohort consisting of 5 years of data (1825 days).
45
+
46
+ We determined that the ideal cohort for our use case needed to satisfy the following factors:
47
+ - have >10 comments and >0 score
48
+ - have a associated url corresponding to a picture (.png or jpeg)
49
+
50
+ After evaluating the preliminary cohort of data against these standards, we found that **115** of the posts satisfied the conditions ([code](explore_relevant_posts.ipynb), [data](data/subs_df_snapshot_11_11_2026.csv)). We took these 115 and analyzed them further to determine if their comments provided relevant feedback.
51
+
52
+
53
+
54
+ # Testing
55
+ To run the static unit and integration test suite offline using the synthetic `DummySource`:
56
+ ```bash
57
+ pytest -m "not live" -v
58
+ ```
59
+
60
+ To run the live integration tests against the actual Arctic Shift API:
61
+ ```bash
62
+ pytest -m live -v
63
+ ```
64
+
65
+ # Evaluation & Benchmarks
66
+ Comprehensive performance evaluations comparing DeepSift against sequential execution:
67
+
68
+ | Benchmark Scenario | Cohort Scale | DeepSift (6w) | Sequential (1w) | Speedup | Ingestion Dynamics | Resource Advantage |
69
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
70
+ | **Live API Ingestion** | 930 Posts (10 Days, `r/resumes`) | **3.58 ± 1.29s** | 14.91 ± 2.46s | **4.16x** | **799ms** avg API latency | **24.0x less bandwidth**, **16.8x less RAM** |
71
+ | **Synthetic Scaling** | 1,000 Posts (`DummySource`) | **0.56 ± 0.01s** | 2.58 ± 0.00s | **4.59x** | **250ms** simulated latency | **1.4x less RAM** (streaming Parquet) |
72
+
73
+ To run the evaluations:
74
+ ```bash
75
+ # Unified scaling benchmark (runs both Live API and Synthetic Dummy together)
76
+ python -m evaluation
77
+ ```
78
+
79
+ # Documentation Site
80
+ The documentation site is built with [Starlight](https://starlight.astro.build/) by Astro.
81
+
82
+ To run the documentation dev server with hot-reloading:
83
+ ```bash
84
+ # From project root:
85
+ npm run docs
86
+
87
+ # Or with force cache reload:
88
+ npm run docs:force
89
+
90
+ # Or directly inside docs/ using npx:
91
+ cd docs
92
+ npx astro dev --force
93
+ ```
94
+ Then visit `http://localhost:4321/` in your browser.
95
+
96
+ To build the static production site:
97
+ ```bash
98
+ npm run docs:build
99
+ # or: cd docs && npm run build
100
+ ```
101
+ The compiled output is generated into [docs/dist/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/) (pre-built and viewable at [docs/dist/index.html](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/dist/index.html)).
102
+
103
+ Documentation content sources are structured in [docs/src/content/docs/](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/):
104
+ - [Overview & Hero](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/index.mdx)
105
+ - [Architecture & Pipeline](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/architecture.md)
106
+ - [Quickstart Guide](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/quickstart.md)
107
+ - [Python API Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/api.md)
108
+ - [CLI Reference](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/cli.md)
109
+ - [Benchmarks & Performance](file:///Users/saakethkesireddy/Desktop/job_journal/resume_model/docs/src/content/docs/benchmarks.md)
110
+
111
+ # TODO
112
+ - [x] make static tests using dummy source (runs on github ci/cd)
113
+ - [x] make live tests using actual source on r/resumes
114
+ - [x] make evaluation script
115
+ - [x] writeup and documentation with benchmarks, docs page, and cleaned repo
@@ -0,0 +1,24 @@
1
+ deepsift/__init__.py,sha256=vK8jpeKXHuVHWYkEiHOViomgrJfKtX9w8Lf7YZSPJNM,782
2
+ deepsift/__main__.py,sha256=3qMFrUXdQ0ZeQW3GaTbRykbSm9mSAbUZegq-wYR6KC4,161
3
+ deepsift/cli.py,sha256=bMGdSQQI9N3lSFLpP5tP0dCnM5HgpRwz1_YzLstXLV8,8290
4
+ deepsift/filters.py,sha256=qu96zYiOUKLfd7bEj5oQ9cKzGYqGlNLzoyoIYCuUrok,5006
5
+ deepsift/scraper.py,sha256=0NAVtivTgTKfvUFijOiIIoIA5hBFRobC_jICLJ5bohY,17091
6
+ deepsift/models/__init__.py,sha256=N6K_JAFJOTMH_rcMSLkYHk-TTGoAa0FsDjkn4ZJuPsM,419
7
+ deepsift/models/base.py,sha256=XQWz5S2sKVR1UG4kvJx-QGyDe-TA4GwfvL2tcRmsoyE,3369
8
+ deepsift/models/comment.py,sha256=WyDYE4VdidNxsgmC1eLY7p_FbHAHcasGh5uisA0KTzA,1370
9
+ deepsift/models/enums.py,sha256=ho8xzN1KHUuumJy5K8R1mDK0NxNhK2G23msV7WeOVNo,18800
10
+ deepsift/models/post.py,sha256=a3FJfNhiG_G47ZgF5EHKi30Xq2gElYyrnyxj7ugqDVo,945
11
+ deepsift/sources/__init__.py,sha256=V1yNiRnLmQgIxmCGShbI5EJkSF9ly26LrtToeKumFpY,339
12
+ deepsift/sources/arctic_shift.py,sha256=5Wlozpcy8UoLlTN3HYqqqSSXY3C8gyejq44G1QX9Ais,9891
13
+ deepsift/sources/base.py,sha256=hLzPdKrTMFH5dvw1hyp-yKdUMjQebmEPkR2dxODXqI4,3425
14
+ deepsift/sources/dummy.py,sha256=yqdiEtiqPVZzwnznHgQX9kgGSHrk0QRmLrDwdBVpEiY,8088
15
+ deepsift/sources/registry.py,sha256=NZhegViN04z3huN7UfgmjCg1lEQCXkzs1tiq8ZQorxw,799
16
+ deepsift/storage/__init__.py,sha256=Wy_4jZVgzaN9l_i57ygiKsr-if7vwhxS-HGFgVp6Aak,197
17
+ deepsift/storage/base.py,sha256=-r_OcM8r1Fl7iI8DJujz58afWw_4weYgdCSbghY0xzo,1124
18
+ deepsift/storage/parquet.py,sha256=Jg4bB4nXrtZJ_lUs1M8xHWJYFEYW-PMVWlAAiG3p1oE,3170
19
+ deepsift-0.2.0.dist-info/licenses/LICENSE,sha256=IMFfwW98307ORF7PAt1lZzFP_giCL5YdKLw5l2Umm20,1074
20
+ deepsift-0.2.0.dist-info/METADATA,sha256=rRPK3qIeuD2i4S8oxyRPyCe8SYpyUoNjylJxhz7XerE,6590
21
+ deepsift-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
22
+ deepsift-0.2.0.dist-info/entry_points.txt,sha256=kPy8InNH-uJiVhqIejRC7y0saWaqXRXaNkvB5eJSJiE,80
23
+ deepsift-0.2.0.dist-info/top_level.txt,sha256=E2qyWtoKciYlHQb5TMF-4sAq8o1uXMaH9bxdA69pi98,9
24
+ deepsift-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ deepsift = deepsift.cli:cli
3
+ reddit-scraper = deepsift.cli:cli
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Saaketh Kesireddy
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ deepsift