grad-agent 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- grad_agent-0.1.1/.gitignore +8 -0
- grad_agent-0.1.1/PKG-INFO +192 -0
- grad_agent-0.1.1/README.md +158 -0
- grad_agent-0.1.1/grad_agent/__init__.py +0 -0
- grad_agent-0.1.1/grad_agent/catalog_sync.py +174 -0
- grad_agent-0.1.1/grad_agent/cli.py +184 -0
- grad_agent-0.1.1/grad_agent/config.py +114 -0
- grad_agent-0.1.1/grad_agent/daily_run.py +224 -0
- grad_agent-0.1.1/grad_agent/drafters/__init__.py +0 -0
- grad_agent-0.1.1/grad_agent/drafters/article.py +57 -0
- grad_agent-0.1.1/grad_agent/drafters/cold_email.py +96 -0
- grad_agent-0.1.1/grad_agent/drafters/sop_latex.py +129 -0
- grad_agent-0.1.1/grad_agent/hook.py +136 -0
- grad_agent-0.1.1/grad_agent/lor.py +62 -0
- grad_agent-0.1.1/grad_agent/mark_promoted.py +32 -0
- grad_agent-0.1.1/grad_agent/outputs/__init__.py +0 -0
- grad_agent-0.1.1/grad_agent/outputs/mailer.py +45 -0
- grad_agent-0.1.1/grad_agent/outputs/portfolio_api.py +93 -0
- grad_agent-0.1.1/grad_agent/outreach_log.py +96 -0
- grad_agent-0.1.1/grad_agent/profile_pdf.py +44 -0
- grad_agent-0.1.1/grad_agent/programs.py +59 -0
- grad_agent-0.1.1/grad_agent/projects.py +160 -0
- grad_agent-0.1.1/grad_agent/server.py +284 -0
- grad_agent-0.1.1/grad_agent/sources/__init__.py +0 -0
- grad_agent-0.1.1/grad_agent/sources/arxiv.py +35 -0
- grad_agent-0.1.1/grad_agent/sources/github.py +63 -0
- grad_agent-0.1.1/grad_agent/sources/hf.py +49 -0
- grad_agent-0.1.1/grad_agent/sources/openreview.py +31 -0
- grad_agent-0.1.1/grad_agent/sources/recruiting.py +43 -0
- grad_agent-0.1.1/grad_agent/sources/semantic_scholar.py +75 -0
- grad_agent-0.1.1/grad_agent/sources/websearch.py +30 -0
- grad_agent-0.1.1/grad_agent/templates/env.example +21 -0
- grad_agent-0.1.1/grad_agent/templates/profile.example.yaml +46 -0
- grad_agent-0.1.1/grad_agent/templates/programs.example.yaml +115 -0
- grad_agent-0.1.1/grad_agent/tracker.py +76 -0
- grad_agent-0.1.1/pyproject.toml +53 -0
- grad_agent-0.1.1/tests/__init__.py +0 -0
- grad_agent-0.1.1/tests/test_smoke.py +140 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: grad-agent
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Autonomous MCP agent for graduate school applications: prof discovery + verified cold emails + SOP drafting + application tracking.
|
|
5
|
+
Project-URL: Homepage, https://github.com/i-ninte/grad-agent
|
|
6
|
+
Project-URL: Issues, https://github.com/i-ninte/grad-agent/issues
|
|
7
|
+
Author-email: Kwabena Obeng <slyobeng111@gmail.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: agent,anthropic,claude,cold-email,grad-school,mcp,phd,sop
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Education
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Education
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Requires-Dist: anthropic>=0.40
|
|
19
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
20
|
+
Requires-Dist: fastmcp>=0.2.0
|
|
21
|
+
Requires-Dist: feedparser>=6.0.11
|
|
22
|
+
Requires-Dist: httpx>=0.27.0
|
|
23
|
+
Requires-Dist: markdown>=3.6
|
|
24
|
+
Requires-Dist: openpyxl>=3.1
|
|
25
|
+
Requires-Dist: pandas>=2.0
|
|
26
|
+
Requires-Dist: pymdown-extensions>=10.7
|
|
27
|
+
Requires-Dist: pypdf>=4.0.0
|
|
28
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
29
|
+
Requires-Dist: pyyaml>=6.0
|
|
30
|
+
Provides-Extra: blog
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# grad-agent
|
|
36
|
+
|
|
37
|
+
An autonomous MCP agent that helps you apply to fully funded MS and PhD programs.
|
|
38
|
+
|
|
39
|
+
- Discovers professors on arXiv in your research areas
|
|
40
|
+
- Verifies each candidate is actually faculty (Semantic Scholar, h-index, papers)
|
|
41
|
+
- Scrapes their lab page for a recruiting signal + email address
|
|
42
|
+
- Matches them to your strongest shipped project
|
|
43
|
+
- Drafts a specific, fact-checked cold email (Claude Haiku authors + verifies claims against paper abstracts)
|
|
44
|
+
- Compiles a per-school SOP to PDF (LaTeX)
|
|
45
|
+
- Tracks everything in xlsx: outreach, LOR requests, program deadlines
|
|
46
|
+
- Emails every draft to your inbox for review; nothing is sent to a professor without you
|
|
47
|
+
|
|
48
|
+
Runs as a stdio MCP server for Claude Code / Claude Desktop, or as a plain CLI.
|
|
49
|
+
|
|
50
|
+
## Install
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pipx install grad-agent
|
|
54
|
+
# or, if you prefer pip in a venv
|
|
55
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
56
|
+
pip install grad-agent
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Set up in 5 minutes
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
grad-agent init
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
This writes:
|
|
66
|
+
|
|
67
|
+
- `~/.grad-agent/profile.yaml` — your identity, projects, preferences
|
|
68
|
+
- `~/.grad-agent/programs.yaml` — target programs (seeded)
|
|
69
|
+
- `~/.grad-agent/.env` — secrets template
|
|
70
|
+
|
|
71
|
+
Fill in `~/.grad-agent/.env`:
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
75
|
+
SMTP_SERVER=smtp.gmail.com
|
|
76
|
+
SMTP_PORT=587
|
|
77
|
+
SMTP_USERNAME=you@gmail.com
|
|
78
|
+
SMTP_PASSWORD=<gmail app password>
|
|
79
|
+
SMTP_FROM=you@gmail.com
|
|
80
|
+
# Optional:
|
|
81
|
+
GITHUB_USERNAME=your-gh
|
|
82
|
+
GITHUB_TOKEN=github_pat_...
|
|
83
|
+
HF_USERNAME=your-hf
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Fill in the important bits of `~/.grad-agent/profile.yaml`:
|
|
87
|
+
|
|
88
|
+
- `name`, `identity_line`, `portfolio`
|
|
89
|
+
- `cv_path`, `transcript_path` (absolute paths)
|
|
90
|
+
- `degree_status: bachelors | masters` (drives PhD eligibility gating)
|
|
91
|
+
- `target_term`, `target_degree`
|
|
92
|
+
- `research_areas: [nlp, ai4health, ...]`
|
|
93
|
+
- `seed_projects:` 3 to 10 flagship projects with `name`, `pitch`, `link`, `tags`
|
|
94
|
+
|
|
95
|
+
Then:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
grad-agent sync # scan projects (GitHub + HF + local)
|
|
99
|
+
grad-agent run # one batch, drafts land in your inbox
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Register with Claude Code
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
grad-agent register-claude
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Prints the exact `claude mcp add` command. Once registered, `/mcp` in Claude Code shows `grad-agent` with ~25 tools.
|
|
109
|
+
|
|
110
|
+
## Register with Claude Desktop
|
|
111
|
+
|
|
112
|
+
Edit `~/Library/Application Support/Claude/claude_desktop_config.json`:
|
|
113
|
+
|
|
114
|
+
```json
|
|
115
|
+
{
|
|
116
|
+
"mcpServers": {
|
|
117
|
+
"grad-agent": {
|
|
118
|
+
"command": "grad-agent",
|
|
119
|
+
"args": ["server"]
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Daily autonomous run (macOS)
|
|
126
|
+
|
|
127
|
+
The package ships a launchd plist template. To fire every day at 08:00:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
cp /path/to/grad_agent/templates/com.gradagent.daily.plist ~/Library/LaunchAgents/
|
|
131
|
+
launchctl load ~/Library/LaunchAgents/com.gradagent.daily.plist
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Every morning: 3 verified faculty leads, hooks fact-checked against paper abstracts, in your inbox for review.
|
|
135
|
+
|
|
136
|
+
## What each MCP tool does
|
|
137
|
+
|
|
138
|
+
| Tool | Purpose |
|
|
139
|
+
|---|---|
|
|
140
|
+
| `run_daily_batch(n, area)` | Full pipeline: verify → recruiting → hook + verify → draft → log |
|
|
141
|
+
| `outreach_log_view(limit)` | Show last N rows of the outreach xlsx |
|
|
142
|
+
| `outreach_mark_sent(prof, uni)` | Flag a row as actually sent to the prof |
|
|
143
|
+
| `sync_catalog(source)` | Pull projects from `github`, `hf`, or `local` |
|
|
144
|
+
| `list_projects_in_catalog()` | Show every project the matcher can see |
|
|
145
|
+
| `list_programs()` | Your target programs |
|
|
146
|
+
| `upcoming_deadlines(days)` | Any program deadline in the next N days |
|
|
147
|
+
| `lor_add / lor_outstanding / lor_mark` | Recommendation-letter tracker |
|
|
148
|
+
| `draft_cold_email(...)` | Manual per-prof draft |
|
|
149
|
+
| `draft_sop(...)` | Compile a Columbia-style SOP PDF for one school |
|
|
150
|
+
| `send_draft_to_me(path)` | Ship any draft file to your review inbox |
|
|
151
|
+
| `discover_profs(area)` | arXiv + OpenReview scan (raw candidates, no verification) |
|
|
152
|
+
|
|
153
|
+
Blog publishing tools (`publish_article`, `update_article`, ...) are gated behind `blog.enabled: true` in `profile.yaml` and are specific to the author's Turso-backed Next.js portfolio. Most users can ignore them.
|
|
154
|
+
|
|
155
|
+
## What the agent will not do
|
|
156
|
+
|
|
157
|
+
- Send any email to a professor. Every send is manual, from your Gmail, after you read the draft.
|
|
158
|
+
- Fabricate a paper claim. The hook goes through a second Claude call that rejects any claim not present in the abstracts, and rewrites.
|
|
159
|
+
- Draft for programs you are ineligible for. If your `degree_status` is `bachelors`, PhD programs that require an MSc first are filtered out.
|
|
160
|
+
|
|
161
|
+
## Requirements
|
|
162
|
+
|
|
163
|
+
- Python 3.10+
|
|
164
|
+
- macOS or Linux (Windows untested)
|
|
165
|
+
- `pdflatex` on PATH if you want SOP PDFs (macOS: MacTeX; Ubuntu: `texlive-latex-recommended`)
|
|
166
|
+
- Anthropic API key
|
|
167
|
+
- Gmail (or another SMTP) for the review-mailer
|
|
168
|
+
|
|
169
|
+
## Where your data lives
|
|
170
|
+
|
|
171
|
+
Everything is under `~/.grad-agent/` by default, or `$GRAD_AGENT_HOME` if set:
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
~/.grad-agent/
|
|
175
|
+
profile.yaml identity + preferences
|
|
176
|
+
programs.yaml target programs
|
|
177
|
+
.env secrets (gitignored)
|
|
178
|
+
data/
|
|
179
|
+
outreach_log.xlsx every prof surfaced or drafted
|
|
180
|
+
lor_log.xlsx recommendation-letter tracker
|
|
181
|
+
catalog.json synced projects (GitHub + HF + local)
|
|
182
|
+
db.sqlite drafts + status
|
|
183
|
+
drafts/ per-school SOP + email drafts
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Contributing
|
|
187
|
+
|
|
188
|
+
MIT licensed. PRs welcome for: more program templates, non-Gmail SMTP presets, non-arXiv source adapters, and better prof-verification heuristics.
|
|
189
|
+
|
|
190
|
+
## Author
|
|
191
|
+
|
|
192
|
+
Kwabena Obeng · [i-ninte.github.io/portfolio/](https://i-ninte.github.io/portfolio/)
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# grad-agent
|
|
2
|
+
|
|
3
|
+
An autonomous MCP agent that helps you apply to fully funded MS and PhD programs.
|
|
4
|
+
|
|
5
|
+
- Discovers professors on arXiv in your research areas
|
|
6
|
+
- Verifies each candidate is actually faculty (Semantic Scholar, h-index, papers)
|
|
7
|
+
- Scrapes their lab page for a recruiting signal + email address
|
|
8
|
+
- Matches them to your strongest shipped project
|
|
9
|
+
- Drafts a specific, fact-checked cold email (Claude Haiku authors + verifies claims against paper abstracts)
|
|
10
|
+
- Compiles a per-school SOP to PDF (LaTeX)
|
|
11
|
+
- Tracks everything in xlsx: outreach, LOR requests, program deadlines
|
|
12
|
+
- Emails every draft to your inbox for review; nothing is sent to a professor without you
|
|
13
|
+
|
|
14
|
+
Runs as a stdio MCP server for Claude Code / Claude Desktop, or as a plain CLI.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pipx install grad-agent
|
|
20
|
+
# or, if you prefer pip in a venv
|
|
21
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
22
|
+
pip install grad-agent
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Set up in 5 minutes
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
grad-agent init
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
This writes:
|
|
32
|
+
|
|
33
|
+
- `~/.grad-agent/profile.yaml` — your identity, projects, preferences
|
|
34
|
+
- `~/.grad-agent/programs.yaml` — target programs (seeded)
|
|
35
|
+
- `~/.grad-agent/.env` — secrets template
|
|
36
|
+
|
|
37
|
+
Fill in `~/.grad-agent/.env`:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
41
|
+
SMTP_SERVER=smtp.gmail.com
|
|
42
|
+
SMTP_PORT=587
|
|
43
|
+
SMTP_USERNAME=you@gmail.com
|
|
44
|
+
SMTP_PASSWORD=<gmail app password>
|
|
45
|
+
SMTP_FROM=you@gmail.com
|
|
46
|
+
# Optional:
|
|
47
|
+
GITHUB_USERNAME=your-gh
|
|
48
|
+
GITHUB_TOKEN=github_pat_...
|
|
49
|
+
HF_USERNAME=your-hf
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Fill in the important bits of `~/.grad-agent/profile.yaml`:
|
|
53
|
+
|
|
54
|
+
- `name`, `identity_line`, `portfolio`
|
|
55
|
+
- `cv_path`, `transcript_path` (absolute paths)
|
|
56
|
+
- `degree_status: bachelors | masters` (drives PhD eligibility gating)
|
|
57
|
+
- `target_term`, `target_degree`
|
|
58
|
+
- `research_areas: [nlp, ai4health, ...]`
|
|
59
|
+
- `seed_projects:` 3 to 10 flagship projects with `name`, `pitch`, `link`, `tags`
|
|
60
|
+
|
|
61
|
+
Then:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
grad-agent sync # scan projects (GitHub + HF + local)
|
|
65
|
+
grad-agent run # one batch, drafts land in your inbox
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Register with Claude Code
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
grad-agent register-claude
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Prints the exact `claude mcp add` command. Once registered, `/mcp` in Claude Code shows `grad-agent` with ~25 tools.
|
|
75
|
+
|
|
76
|
+
## Register with Claude Desktop
|
|
77
|
+
|
|
78
|
+
Edit `~/Library/Application Support/Claude/claude_desktop_config.json`:
|
|
79
|
+
|
|
80
|
+
```json
|
|
81
|
+
{
|
|
82
|
+
"mcpServers": {
|
|
83
|
+
"grad-agent": {
|
|
84
|
+
"command": "grad-agent",
|
|
85
|
+
"args": ["server"]
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Daily autonomous run (macOS)
|
|
92
|
+
|
|
93
|
+
The package ships a launchd plist template. To fire every day at 08:00:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
cp /path/to/grad_agent/templates/com.gradagent.daily.plist ~/Library/LaunchAgents/
|
|
97
|
+
launchctl load ~/Library/LaunchAgents/com.gradagent.daily.plist
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Every morning: 3 verified faculty leads, hooks fact-checked against paper abstracts, in your inbox for review.
|
|
101
|
+
|
|
102
|
+
## What each MCP tool does
|
|
103
|
+
|
|
104
|
+
| Tool | Purpose |
|
|
105
|
+
|---|---|
|
|
106
|
+
| `run_daily_batch(n, area)` | Full pipeline: verify → recruiting → hook + verify → draft → log |
|
|
107
|
+
| `outreach_log_view(limit)` | Show last N rows of the outreach xlsx |
|
|
108
|
+
| `outreach_mark_sent(prof, uni)` | Flag a row as actually sent to the prof |
|
|
109
|
+
| `sync_catalog(source)` | Pull projects from `github`, `hf`, or `local` |
|
|
110
|
+
| `list_projects_in_catalog()` | Show every project the matcher can see |
|
|
111
|
+
| `list_programs()` | Your target programs |
|
|
112
|
+
| `upcoming_deadlines(days)` | Any program deadline in the next N days |
|
|
113
|
+
| `lor_add / lor_outstanding / lor_mark` | Recommendation-letter tracker |
|
|
114
|
+
| `draft_cold_email(...)` | Manual per-prof draft |
|
|
115
|
+
| `draft_sop(...)` | Compile a Columbia-style SOP PDF for one school |
|
|
116
|
+
| `send_draft_to_me(path)` | Ship any draft file to your review inbox |
|
|
117
|
+
| `discover_profs(area)` | arXiv + OpenReview scan (raw candidates, no verification) |
|
|
118
|
+
|
|
119
|
+
Blog publishing tools (`publish_article`, `update_article`, ...) are gated behind `blog.enabled: true` in `profile.yaml` and are specific to the author's Turso-backed Next.js portfolio. Most users can ignore them.
|
|
120
|
+
|
|
121
|
+
## What the agent will not do
|
|
122
|
+
|
|
123
|
+
- Send any email to a professor. Every send is manual, from your Gmail, after you read the draft.
|
|
124
|
+
- Fabricate a paper claim. The hook goes through a second Claude call that rejects any claim not present in the abstracts, and rewrites.
|
|
125
|
+
- Draft for programs you are ineligible for. If your `degree_status` is `bachelors`, PhD programs that require an MSc first are filtered out.
|
|
126
|
+
|
|
127
|
+
## Requirements
|
|
128
|
+
|
|
129
|
+
- Python 3.10+
|
|
130
|
+
- macOS or Linux (Windows untested)
|
|
131
|
+
- `pdflatex` on PATH if you want SOP PDFs (macOS: MacTeX; Ubuntu: `texlive-latex-recommended`)
|
|
132
|
+
- Anthropic API key
|
|
133
|
+
- Gmail (or another SMTP) for the review-mailer
|
|
134
|
+
|
|
135
|
+
## Where your data lives
|
|
136
|
+
|
|
137
|
+
Everything is under `~/.grad-agent/` by default, or `$GRAD_AGENT_HOME` if set:
|
|
138
|
+
|
|
139
|
+
```
|
|
140
|
+
~/.grad-agent/
|
|
141
|
+
profile.yaml identity + preferences
|
|
142
|
+
programs.yaml target programs
|
|
143
|
+
.env secrets (gitignored)
|
|
144
|
+
data/
|
|
145
|
+
outreach_log.xlsx every prof surfaced or drafted
|
|
146
|
+
lor_log.xlsx recommendation-letter tracker
|
|
147
|
+
catalog.json synced projects (GitHub + HF + local)
|
|
148
|
+
db.sqlite drafts + status
|
|
149
|
+
drafts/ per-school SOP + email drafts
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Contributing
|
|
153
|
+
|
|
154
|
+
MIT licensed. PRs welcome for: more program templates, non-Gmail SMTP presets, non-arXiv source adapters, and better prof-verification heuristics.
|
|
155
|
+
|
|
156
|
+
## Author
|
|
157
|
+
|
|
158
|
+
Kwabena Obeng · [i-ninte.github.io/portfolio/](https://i-ninte.github.io/portfolio/)
|
|
File without changes
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Extend the project catalog from GitHub + HuggingFace + local NINTE dirs.
|
|
2
|
+
|
|
3
|
+
Writes/updates `catalog.json` in the grad-agent directory. Each entry has the
|
|
4
|
+
same shape as the seed `PROJECTS` list in projects.py:
|
|
5
|
+
{name, pitch, link, tags}
|
|
6
|
+
Claude Haiku distils each source (repo, model, local README) into the pitch
|
|
7
|
+
and tags. Existing entries are updated in place; new ones appended.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
import os, json, re, time
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from . import config as _cfg
|
|
14
|
+
_cfg.load_env_file()
|
|
15
|
+
|
|
16
|
+
from .sources import github as gh
|
|
17
|
+
from .sources import hf
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
import anthropic
|
|
21
|
+
except Exception:
|
|
22
|
+
anthropic = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _catalog_path() -> Path:
|
|
26
|
+
return _cfg.data_dir() / "catalog.json"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _projects_dir() -> Path:
|
|
30
|
+
prof = _cfg.load_profile()
|
|
31
|
+
if prof.projects_dir:
|
|
32
|
+
return Path(prof.projects_dir).expanduser()
|
|
33
|
+
return Path(os.environ.get("NINTE_ROOT", str(Path.home())))
|
|
34
|
+
|
|
35
|
+
# Skip these local dirs when scanning _projects_dir()
|
|
36
|
+
SKIP_LOCAL = {"GRAD", "portfolio", "SNAPSHOT", "Kwabena.zip"}
|
|
37
|
+
|
|
38
|
+
SYSTEM = (
|
|
39
|
+
"You extract a compact JSON record from a project description. Return a "
|
|
40
|
+
"strict JSON object with keys: pitch (one sentence, 25 to 45 words, in "
|
|
41
|
+
"Kwabena's voice, direct and grounded in the actual work described, no "
|
|
42
|
+
"cliches, no dashes), tags (a list of 5 to 12 short lowercase phrases "
|
|
43
|
+
"describing the technical area, methods, and domain, e.g. 'nlp', 'llm', "
|
|
44
|
+
"'african languages', 'yolo', 'time series'). Nothing else. No prose "
|
|
45
|
+
"outside the JSON."
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _distil(name: str, kind: str, description: str, extra: str = "",
|
|
50
|
+
model: str = "claude-haiku-4-5-20251001") -> dict:
|
|
51
|
+
if anthropic is None or not os.environ.get("ANTHROPIC_API_KEY"):
|
|
52
|
+
# Heuristic: pitch = first 40 words of description, tags = topics/tags
|
|
53
|
+
pitch = " ".join((description or f"{kind} named {name}").split()[:40])
|
|
54
|
+
return {"pitch": pitch, "tags": []}
|
|
55
|
+
client = anthropic.Anthropic()
|
|
56
|
+
user = (
|
|
57
|
+
f"KIND: {kind}\nNAME: {name}\nDESCRIPTION: {description}\n\n"
|
|
58
|
+
f"EXTRA CONTEXT (README or metadata):\n{extra[:3000]}\n\n"
|
|
59
|
+
f"Return the JSON now."
|
|
60
|
+
)
|
|
61
|
+
m = client.messages.create(
|
|
62
|
+
model=model, max_tokens=400, system=SYSTEM,
|
|
63
|
+
messages=[{"role": "user", "content": user}],
|
|
64
|
+
)
|
|
65
|
+
raw = m.content[0].text.strip()
|
|
66
|
+
# tolerate ```json fences
|
|
67
|
+
raw = re.sub(r"^```(?:json)?", "", raw).rstrip("`").strip()
|
|
68
|
+
try:
|
|
69
|
+
obj = json.loads(raw)
|
|
70
|
+
obj["pitch"] = obj.get("pitch", "").replace("—", ", ").replace("–", ", ")
|
|
71
|
+
return {"pitch": obj.get("pitch", ""), "tags": obj.get("tags", []) or []}
|
|
72
|
+
except Exception:
|
|
73
|
+
return {"pitch": (description or "")[:200], "tags": []}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _load() -> dict:
|
|
77
|
+
if _catalog_path().exists():
|
|
78
|
+
return json.loads(_catalog_path().read_text())
|
|
79
|
+
return {"projects": {}, "last_synced": None}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _save(cat: dict) -> None:
|
|
83
|
+
cat["last_synced"] = time.strftime("%Y-%m-%d %H:%M")
|
|
84
|
+
_catalog_path().write_text(json.dumps(cat, indent=2))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _upsert(cat: dict, key: str, entry: dict) -> None:
|
|
88
|
+
cat["projects"][key] = entry
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def sync_github() -> int:
|
|
92
|
+
cat = _load(); added = 0
|
|
93
|
+
for r in gh.list_repos():
|
|
94
|
+
if r["fork"] or r["archived"]: continue
|
|
95
|
+
readme = gh.fetch_readme(r["full_name"])
|
|
96
|
+
d = _distil(
|
|
97
|
+
name=r["name"],
|
|
98
|
+
kind="github repo",
|
|
99
|
+
description=r["description"] or "",
|
|
100
|
+
extra=(readme or "") + "\nTopics: " + ", ".join(r["topics"]),
|
|
101
|
+
)
|
|
102
|
+
entry = {
|
|
103
|
+
"name": r["name"],
|
|
104
|
+
"pitch": d["pitch"] or (r["description"] or r["name"]),
|
|
105
|
+
"link": r["html_url"],
|
|
106
|
+
"tags": list(dict.fromkeys(d["tags"] + r["topics"])),
|
|
107
|
+
"source": "github",
|
|
108
|
+
"language": r["language"],
|
|
109
|
+
"stars": r["stars"],
|
|
110
|
+
}
|
|
111
|
+
_upsert(cat, f"gh:{r['full_name']}", entry); added += 1
|
|
112
|
+
_save(cat); return added
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def sync_hf() -> int:
|
|
116
|
+
cat = _load(); added = 0
|
|
117
|
+
for m in hf.list_models() + hf.list_datasets():
|
|
118
|
+
readme = hf.fetch_readme(m["id"], kind=m["kind"] + "s")
|
|
119
|
+
d = _distil(
|
|
120
|
+
name=m["id"],
|
|
121
|
+
kind=f"huggingface {m['kind']}",
|
|
122
|
+
description=(m.get("pipeline_tag") or "") + " " + ", ".join(m["tags"][:10]),
|
|
123
|
+
extra=readme,
|
|
124
|
+
)
|
|
125
|
+
entry = {
|
|
126
|
+
"name": m["id"],
|
|
127
|
+
"pitch": d["pitch"] or f"{m['kind']} at {m['id']}",
|
|
128
|
+
"link": m["url"],
|
|
129
|
+
"tags": list(dict.fromkeys(d["tags"] + m["tags"])),
|
|
130
|
+
"source": "huggingface",
|
|
131
|
+
"downloads": m["downloads"],
|
|
132
|
+
"likes": m["likes"],
|
|
133
|
+
}
|
|
134
|
+
_upsert(cat, f"hf:{m['kind']}:{m['id']}", entry); added += 1
|
|
135
|
+
_save(cat); return added
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def sync_local() -> int:
|
|
139
|
+
cat = _load(); added = 0
|
|
140
|
+
for child in _projects_dir().iterdir():
|
|
141
|
+
if not child.is_dir() or child.name in SKIP_LOCAL: continue
|
|
142
|
+
readme_text = ""
|
|
143
|
+
for cand in ("README.md", "Readme.md", "readme.md", "README.txt"):
|
|
144
|
+
p = child / cand
|
|
145
|
+
if p.exists():
|
|
146
|
+
readme_text = p.read_text(errors="ignore")[:4000]; break
|
|
147
|
+
if not readme_text and not any((child / c).exists() for c in ("package.json", "requirements.txt")):
|
|
148
|
+
continue
|
|
149
|
+
d = _distil(
|
|
150
|
+
name=child.name, kind="local project directory",
|
|
151
|
+
description="", extra=readme_text,
|
|
152
|
+
)
|
|
153
|
+
entry = {
|
|
154
|
+
"name": child.name,
|
|
155
|
+
"pitch": d["pitch"] or child.name,
|
|
156
|
+
"link": str(child),
|
|
157
|
+
"tags": d["tags"],
|
|
158
|
+
"source": "local",
|
|
159
|
+
}
|
|
160
|
+
_upsert(cat, f"local:{child.name}", entry); added += 1
|
|
161
|
+
_save(cat); return added
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def sync_all() -> dict:
|
|
165
|
+
return {
|
|
166
|
+
"github": sync_github(),
|
|
167
|
+
"huggingface": sync_hf(),
|
|
168
|
+
"local": sync_local(),
|
|
169
|
+
"catalog_path": str(_catalog_path()),
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__":
|
|
174
|
+
print(sync_all())
|