pagemint 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pagemint-0.1.0/.github/workflows/ci.yml +28 -0
- pagemint-0.1.0/.github/workflows/release.yml +39 -0
- pagemint-0.1.0/.gitignore +9 -0
- pagemint-0.1.0/CONTRIBUTING.md +53 -0
- pagemint-0.1.0/LICENSE +21 -0
- pagemint-0.1.0/PKG-INFO +365 -0
- pagemint-0.1.0/README.md +335 -0
- pagemint-0.1.0/docs/superpowers/plans/2026-11-18-pagemint-core.md +151 -0
- pagemint-0.1.0/docs/superpowers/plans/2026-11-18-pagemint-jobs.md +137 -0
- pagemint-0.1.0/docs/superpowers/plans/2026-11-19-pagemint-search.md +84 -0
- pagemint-0.1.0/docs/superpowers/specs/2026-11-18-pagemint-core-design.md +362 -0
- pagemint-0.1.0/docs/superpowers/specs/2026-11-18-pagemint-jobs-design.md +274 -0
- pagemint-0.1.0/docs/superpowers/specs/2026-11-19-pagemint-search-design.md +177 -0
- pagemint-0.1.0/pyproject.toml +52 -0
- pagemint-0.1.0/scripts/live_check.py +59 -0
- pagemint-0.1.0/src/pagemint/__init__.py +1 -0
- pagemint-0.1.0/src/pagemint/__main__.py +5 -0
- pagemint-0.1.0/src/pagemint/browser.py +215 -0
- pagemint-0.1.0/src/pagemint/cli.py +226 -0
- pagemint-0.1.0/src/pagemint/client.py +342 -0
- pagemint-0.1.0/src/pagemint/config.py +73 -0
- pagemint-0.1.0/src/pagemint/errors.py +55 -0
- pagemint-0.1.0/src/pagemint/extract.py +204 -0
- pagemint-0.1.0/src/pagemint/fetcher.py +318 -0
- pagemint-0.1.0/src/pagemint/jobs/__init__.py +1 -0
- pagemint-0.1.0/src/pagemint/jobs/detect.py +229 -0
- pagemint-0.1.0/src/pagemint/jobs/jsonld.py +203 -0
- pagemint-0.1.0/src/pagemint/jobs/model.py +105 -0
- pagemint-0.1.0/src/pagemint/jobs/rank.py +157 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/__init__.py +11 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/_common.py +47 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/ashby.py +70 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/greenhouse.py +57 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/lever.py +64 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/recruitee.py +41 -0
- pagemint-0.1.0/src/pagemint/jobs/readers/workday.py +83 -0
- pagemint-0.1.0/src/pagemint/jobs/service.py +320 -0
- pagemint-0.1.0/src/pagemint/jobs/sources/__init__.py +5 -0
- pagemint-0.1.0/src/pagemint/jobs/sources/arbeitnow.py +34 -0
- pagemint-0.1.0/src/pagemint/jobs/sources/himalayas.py +52 -0
- pagemint-0.1.0/src/pagemint/jobs/sources/hn.py +98 -0
- pagemint-0.1.0/src/pagemint/jobs/store.py +185 -0
- pagemint-0.1.0/src/pagemint/links.py +85 -0
- pagemint-0.1.0/src/pagemint/markdown.py +78 -0
- pagemint-0.1.0/src/pagemint/ratelimit.py +41 -0
- pagemint-0.1.0/src/pagemint/robots.py +221 -0
- pagemint-0.1.0/src/pagemint/safety.py +137 -0
- pagemint-0.1.0/src/pagemint/search/__init__.py +1 -0
- pagemint-0.1.0/src/pagemint/search/_http.py +29 -0
- pagemint-0.1.0/src/pagemint/search/brave.py +37 -0
- pagemint-0.1.0/src/pagemint/search/model.py +38 -0
- pagemint-0.1.0/src/pagemint/search/searxng.py +43 -0
- pagemint-0.1.0/src/pagemint/search/secrets.py +53 -0
- pagemint-0.1.0/src/pagemint/search/service.py +129 -0
- pagemint-0.1.0/src/pagemint/search/tavily.py +31 -0
- pagemint-0.1.0/src/pagemint/server.py +171 -0
- pagemint-0.1.0/src/pagemint/sitemap.py +44 -0
- pagemint-0.1.0/tests/conftest.py +150 -0
- pagemint-0.1.0/tests/jobs/fixtures.py +91 -0
- pagemint-0.1.0/tests/jobs/test_detect.py +86 -0
- pagemint-0.1.0/tests/jobs/test_detect_url.py +47 -0
- pagemint-0.1.0/tests/jobs/test_jsonld.py +85 -0
- pagemint-0.1.0/tests/jobs/test_rank.py +118 -0
- pagemint-0.1.0/tests/jobs/test_readers.py +139 -0
- pagemint-0.1.0/tests/jobs/test_service.py +201 -0
- pagemint-0.1.0/tests/jobs/test_sources.py +130 -0
- pagemint-0.1.0/tests/jobs/test_store.py +108 -0
- pagemint-0.1.0/tests/search/test_search_config.py +28 -0
- pagemint-0.1.0/tests/search/test_search_providers.py +129 -0
- pagemint-0.1.0/tests/search/test_search_service.py +185 -0
- pagemint-0.1.0/tests/test_cli.py +114 -0
- pagemint-0.1.0/tests/test_crawl.py +105 -0
- pagemint-0.1.0/tests/test_extract.py +70 -0
- pagemint-0.1.0/tests/test_js.py +252 -0
- pagemint-0.1.0/tests/test_map.py +97 -0
- pagemint-0.1.0/tests/test_robots.py +171 -0
- pagemint-0.1.0/tests/test_safety.py +282 -0
- pagemint-0.1.0/tests/test_scrape.py +94 -0
- pagemint-0.1.0/tests/test_server.py +80 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python-version }}
|
|
22
|
+
cache: pip
|
|
23
|
+
- name: Install PageMint with browser and test extras
|
|
24
|
+
run: python -m pip install -e ".[js,dev]"
|
|
25
|
+
- name: Install Chromium for the JavaScript tests
|
|
26
|
+
run: python -m playwright install --with-deps chromium
|
|
27
|
+
- name: Run tests
|
|
28
|
+
run: python -m pytest -q
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published] # publishing a GitHub release (Releases -> Draft a new release) uploads to PyPI
|
|
6
|
+
workflow_dispatch: # or run it by hand from the Actions tab
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.12"
|
|
19
|
+
- name: Build sdist and wheel
|
|
20
|
+
run: |
|
|
21
|
+
python -m pip install build
|
|
22
|
+
python -m build
|
|
23
|
+
- uses: actions/upload-artifact@v4
|
|
24
|
+
with:
|
|
25
|
+
name: dist
|
|
26
|
+
path: dist/
|
|
27
|
+
|
|
28
|
+
publish:
|
|
29
|
+
needs: build
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
environment: pypi
|
|
32
|
+
permissions:
|
|
33
|
+
id-token: write # PyPI trusted publishing; no API token needed
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/download-artifact@v4
|
|
36
|
+
with:
|
|
37
|
+
name: dist
|
|
38
|
+
path: dist/
|
|
39
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Contributing to PageMint
|
|
2
|
+
|
|
3
|
+
Thanks for helping. Bug reports, fixes and new test cases from real pages are all welcome.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
git clone https://github.com/HarshaVardhan1111/pagemint
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
cd pagemint
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
python -m venv .venv && . .venv/bin/activate
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install -e ".[js,dev]"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
playwright install chromium
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pytest
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The browser tests are skipped automatically if Chromium is not installed. If you already have Chrome or Chromium, point to it with `PAGEMINT_BROWSER_EXECUTABLE=/path/to/chrome`.
|
|
32
|
+
|
|
33
|
+
## Rules for changes
|
|
34
|
+
|
|
35
|
+
1. **Write it yourself.** Do not copy code from Firecrawl or other scrapers, even when their license would allow it. PageMint is written from scratch.
|
|
36
|
+
2. **Permissive dependencies only.** New dependencies must be MIT, BSD or Apache-2.0, and must be added to the license table in the README. Prefer the standard library.
|
|
37
|
+
3. **Safety rules are not optional.** Changes must keep robots.txt handling, per-site rate limits, size and time limits, and the SSRF checks (including on redirects and browser requests). Please do not add options that turn them off for normal users.
|
|
38
|
+
4. **Tests first, no internet.** Add a test that fails without your change. Tests use the local server in `tests/conftest.py`; they must never reach the live internet.
|
|
39
|
+
5. **Never crash the server.** Tools return `{"error": {"type", "message"}}` with a message a person can act on.
|
|
40
|
+
|
|
41
|
+
## Pull requests
|
|
42
|
+
|
|
43
|
+
- Keep each pull request to one change, with a clear description of why.
|
|
44
|
+
- Run `pytest` before pushing; CI runs it on Python 3.11, 3.12 and 3.13.
|
|
45
|
+
- For extraction changes, include the HTML that was handled badly (trimmed) as a test fixture.
|
|
46
|
+
|
|
47
|
+
## Reporting a page that converts badly
|
|
48
|
+
|
|
49
|
+
Open an issue with the URL, what you expected and what you got. If the page is private, attach a trimmed copy of its HTML instead.
|
|
50
|
+
|
|
51
|
+
## Security
|
|
52
|
+
|
|
53
|
+
If you find a way around the SSRF or robots.txt protections, please open a GitHub security advisory (Security tab → Report a vulnerability) instead of a public issue.
|
pagemint-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Harsha
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pagemint-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pagemint
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Free, open-source MCP server that turns web pages into clean Markdown for AI agents.
|
|
5
|
+
Project-URL: Homepage, https://github.com/HarshaVardhan1111/pagemint
|
|
6
|
+
Project-URL: Issues, https://github.com/HarshaVardhan1111/pagemint/issues
|
|
7
|
+
Author: Harsha
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: ai-agents,crawler,markdown,mcp,scraper
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
19
|
+
Requires-Dist: httpcore<2,>=1
|
|
20
|
+
Requires-Dist: httpx<1,>=0.27
|
|
21
|
+
Requires-Dist: lxml>=5
|
|
22
|
+
Requires-Dist: markdownify>=0.13
|
|
23
|
+
Requires-Dist: mcp<3,>=2
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
27
|
+
Provides-Extra: js
|
|
28
|
+
Requires-Dist: playwright>=1.49; extra == 'js'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# PageMint
|
|
32
|
+
|
|
33
|
+
**Free, open-source MCP server that turns web pages into clean Markdown for AI agents.**
|
|
34
|
+
|
|
35
|
+
PageMint gives Claude (and any other MCP client) tools to `scrape` one page, `crawl` a site, `map` a site's URLs, **search open jobs** straight from companies' own hiring systems, and **search the web** with your own SearXNG or your own search API key. It runs on your own machine, so scraping, crawling and job search have no credits, API keys or monthly limits. (Web search needs your own SearXNG or your own search API key; see below.) It is polite by default: it follows robots.txt, rate-limits every site, and refuses to touch private networks.
|
|
36
|
+
|
|
37
|
+
- Clean Markdown: menus, footers, cookie banners and ads are removed; headings, lists, tables, code blocks and links are kept.
|
|
38
|
+
- JavaScript pages: if a page is almost empty without JavaScript, PageMint renders it in headless Chromium (optional install).
|
|
39
|
+
- Written from scratch in Python. MIT license.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
PageMint needs Python 3.11 or newer.
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install pagemint
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or run it without installing, using [uv](https://docs.astral.sh/uv/):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
uvx pagemint --help
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
> **Before the first PyPI release**, install from GitHub instead:
|
|
56
|
+
> `uvx --from git+https://github.com/HarshaVardhan1111/pagemint pagemint --help`
|
|
57
|
+
> (and use the same `--from ...` in the setup commands below).
|
|
58
|
+
|
|
59
|
+
### Optional: JavaScript rendering
|
|
60
|
+
|
|
61
|
+
Plain pages work out of the box. For sites that build their content with JavaScript, add the browser extra and download Chromium once (about 150 MB):
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install "pagemint[js]"
|
|
65
|
+
playwright install chromium
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
With uv:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
uvx --from "pagemint[js]" playwright install chromium
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Without the browser, PageMint still works and tells you when a page needed JavaScript.
|
|
75
|
+
|
|
76
|
+
## Add it to Claude Code
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
claude mcp add pagemint -- uvx pagemint
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
With JavaScript rendering:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
claude mcp add pagemint -- uvx --from "pagemint[js]" pagemint
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Add it to Claude Desktop
|
|
89
|
+
|
|
90
|
+
Open **Settings → Developer → Edit Config** (the file is `claude_desktop_config.json`; on macOS it lives in `~/Library/Application Support/Claude/`, on Windows in `%APPDATA%\Claude\`) and add:
|
|
91
|
+
|
|
92
|
+
```json
|
|
93
|
+
{
|
|
94
|
+
"mcpServers": {
|
|
95
|
+
"pagemint": {
|
|
96
|
+
"command": "uvx",
|
|
97
|
+
"args": ["pagemint"]
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
For JavaScript rendering use `"args": ["--from", "pagemint[js]", "pagemint"]`. Restart Claude Desktop afterwards.
|
|
104
|
+
|
|
105
|
+
## Tools
|
|
106
|
+
|
|
107
|
+
### `scrape(url, format="markdown", render_js=False)`
|
|
108
|
+
|
|
109
|
+
Fetches one page.
|
|
110
|
+
|
|
111
|
+
| Field | Meaning |
|
|
112
|
+
|---|---|
|
|
113
|
+
| `url` | Final URL after redirects |
|
|
114
|
+
| `status_code` | HTTP status code |
|
|
115
|
+
| `title` | Page title (`og:title`, then `<title>`, then the first `<h1>`) |
|
|
116
|
+
| `content` | Markdown (default), cleaned main-content HTML (`format="html"`), or empty (`format="links"`) |
|
|
117
|
+
| `links` | Every http(s) link on the page, absolute and de-duplicated |
|
|
118
|
+
| `rendered_js` | `true` if the page was rendered in the browser |
|
|
119
|
+
| `warnings` | Things worth knowing, e.g. a 404 status or a missing browser |
|
|
120
|
+
|
|
121
|
+
`render_js=True` always uses the browser. Without it, PageMint uses the browser automatically only when the plain page has under 200 characters of text.
|
|
122
|
+
|
|
123
|
+
### `crawl(url, max_pages=20, max_depth=2, same_domain=True, render_js=False)`
|
|
124
|
+
|
|
125
|
+
Follows links breadth-first from `url` and returns each page as Markdown.
|
|
126
|
+
|
|
127
|
+
- `max_depth` counts link hops from the start page (the start page is depth 0).
|
|
128
|
+
- `same_domain=True` stays on the start host (`www.` is treated as the same site).
|
|
129
|
+
- Files such as images, PDFs and archives are not followed.
|
|
130
|
+
- Returns `pages` (url, status_code, title, markdown, depth, truncated, rendered_js), `skipped` (url + reason: robots.txt, blocked, error, limit), `stopped_early` and `warnings`.
|
|
131
|
+
|
|
132
|
+
### `map(url, limit=500)`
|
|
133
|
+
|
|
134
|
+
Lists URLs on a site without fetching every page: it reads the sitemap first (from robots.txt `Sitemap:` lines, otherwise `/sitemap.xml`, including sitemap indexes and `.gz` files), then adds links from the start page if needed. URLs that robots.txt disallows are left out. Returns `urls`, `source` (`sitemap`, `links` or `sitemap+links`) and `warnings`.
|
|
135
|
+
|
|
136
|
+
Errors never crash the server. A failed call returns `{"error": {"type": ..., "message": ...}}` with a plain-English message and is marked as an error to the client.
|
|
137
|
+
|
|
138
|
+
## Job search
|
|
139
|
+
|
|
140
|
+
Four more tools find **open** jobs straight from the systems companies use to post them, so you don't see closed jobs that linger on job boards.
|
|
141
|
+
|
|
142
|
+
| Tool | What it does |
|
|
143
|
+
|---|---|
|
|
144
|
+
| `company_jobs(company, query="", location="", remote=None, limit=50)` | Open jobs at one company. `company` is a name (`"Stripe"`) or a careers-page / job-board URL. The company is added to your saved list. |
|
|
145
|
+
| `search_jobs(query, location="", remote=None, posted_within_days=None, companies=None, sources="all", limit=25)` | Searches all saved companies (or the `companies` you pass) plus the free job sources below, best matches first. `sources="companies"` skips the free sources. Max 100 results. |
|
|
146
|
+
| `check_job(url)` | Is this job still open? Returns `open`, `closed` or `unknown` with the reason, and the full description when open. |
|
|
147
|
+
| `saved_companies(action="list", company=None)` | List, `add` or `remove` saved companies. |
|
|
148
|
+
|
|
149
|
+
Each job has title, company, location, remote / workplace type, employment type, department, posted date, salary (when published), links, a short description, its `source` and a `status`: `open` for jobs read from a company's hiring system, `unverified` for jobs from free sources (run `check_job` on them).
|
|
150
|
+
|
|
151
|
+
### Where jobs come from
|
|
152
|
+
|
|
153
|
+
| Source | How | Notes |
|
|
154
|
+
|---|---|---|
|
|
155
|
+
| Greenhouse | public Job Board API | company name is confirmed from the feed |
|
|
156
|
+
| Lever | public Postings API (global and EU) | |
|
|
157
|
+
| Ashby | public job posting API | |
|
|
158
|
+
| Workday | the career site's own JSON endpoints | needs the careers URL (e.g. `https://acme.wd5.myworkdayjobs.com/External`) |
|
|
159
|
+
| Recruitee | public offers API | experimental |
|
|
160
|
+
| Any other careers page | schema.org `JobPosting` data that job pages carry for search engines | follows up to 30 job links; expired jobs are dropped |
|
|
161
|
+
| [Hacker News "Who is hiring?"](https://news.ycombinator.com/submitted?id=whoishiring) | HN Algolia API, current and previous month | free text, parsed best effort |
|
|
162
|
+
| [Arbeitnow](https://www.arbeitnow.com) | free job board API | mostly Europe; credited as its terms ask |
|
|
163
|
+
| [Himalayas](https://himalayas.app) | free remote jobs API | rate-limits quickly; skipped with a note when it does |
|
|
164
|
+
|
|
165
|
+
**Considered and not used:** LinkedIn, Indeed, Glassdoor and ZipRecruiter (their terms forbid scraping or they block bots), SmartRecruiters (its robots.txt blocks all bots except LinkedIn's), Remotive (robots.txt blocks its API), Workable search (rate-limited, and search URLs are disallowed). Workable careers pages are still read through the general reader when they allow it.
|
|
166
|
+
|
|
167
|
+
### How finding a company works
|
|
168
|
+
|
|
169
|
+
- **By URL** (most reliable): PageMint recognises Greenhouse, Lever, Ashby, Workday, Recruitee and Workable links, also when a company's own careers page embeds one. Otherwise it reads the page's job data.
|
|
170
|
+
- **By name**: PageMint tries a few spellings on Greenhouse, Lever and Ashby. A Greenhouse match whose company name agrees is `confirmed`. A Lever or Ashby match is `guessed`, because those feeds don't include the company name, and the result says so. If it's the wrong company, remove it and pass the careers URL instead.
|
|
171
|
+
|
|
172
|
+
### Matching
|
|
173
|
+
|
|
174
|
+
Results are ranked by keywords (title matters most, then department, then description) and recency, with no AI involved. `location` is a filter that knows common alternative spellings (Bengaluru/Bangalore, Gurugram/Gurgaon, NYC/New York, UK/United Kingdom...). A remote job matches every location only when it isn't limited to a place: "Remote (worldwide)" matches London, "Remote - US" does not.
|
|
175
|
+
|
|
176
|
+
### Your data
|
|
177
|
+
|
|
178
|
+
The saved company list and a cache of job feeds live in a small SQLite file on your computer (`~/.local/share/pagemint` on Linux, `~/Library/Application Support/pagemint` on macOS, `%LOCALAPPDATA%\pagemint` on Windows; change it with `PAGEMINT_DATA_DIR`). Nothing is sent anywhere else.
|
|
179
|
+
|
|
180
|
+
### Job search limits
|
|
181
|
+
|
|
182
|
+
| Setting | Default | Environment variable |
|
|
183
|
+
|---|---|---|
|
|
184
|
+
| Job feed cache | 6 hours (free sources: 1 hour) | `PAGEMINT_JOBS_CACHE_HOURS` |
|
|
185
|
+
| Time budget per search | 60 s (slower feeds are listed under `skipped`) | `PAGEMINT_JOBS_SEARCH_BUDGET` |
|
|
186
|
+
| Saved companies | 500 | |
|
|
187
|
+
| Job feed size | 40 MB (large companies' feeds exceed the 5 MB page limit) | |
|
|
188
|
+
| Workday jobs per company | 200 | |
|
|
189
|
+
|
|
190
|
+
All job requests follow the same robots.txt, rate-limit and safety rules as everything else. Hiring-system hosts are shared by many companies (all Greenhouse boards use one host), so the first search over many saved companies can take a while; later searches use the cache.
|
|
191
|
+
|
|
192
|
+
To try job search against real sites: `python scripts/live_check.py "data engineer" --location London`.
|
|
193
|
+
|
|
194
|
+
## Web search
|
|
195
|
+
|
|
196
|
+
`search(query, limit=10, fetch_pages=0, provider="auto")` returns titles, links and snippets (up to 20). With `fetch_pages=3` it also reads the top 3 result pages and returns them as Markdown, in the same call.
|
|
197
|
+
|
|
198
|
+
There is no free web search that needs no key and allows bots, so you choose one of these. PageMint never uses a paid service unless you set its key.
|
|
199
|
+
|
|
200
|
+
### Option 1: your own SearXNG (free, no key, no monthly limit)
|
|
201
|
+
|
|
202
|
+
[SearXNG](https://docs.searxng.org/) is an open-source search engine you run yourself. With Docker:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
mkdir -p searxng && printf 'use_default_settings: true\nsearch:\n formats: [html, json]\nserver:\n secret_key: "change-me-to-something-random"\n' > searxng/settings.yml
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
docker run -d --name searxng -p 8888:8080 -v "$PWD/searxng:/etc/searxng" searxng/searxng
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Then set `PAGEMINT_SEARXNG_URL=http://localhost:8888`. The `formats: [html, json]` line matters: without it SearXNG refuses PageMint's requests, and PageMint tells you so. SearXNG gets its results from other search engines, so very heavy use can get it rate-limited by them.
|
|
213
|
+
|
|
214
|
+
### Option 2: your own API key
|
|
215
|
+
|
|
216
|
+
Set **one** of these environment variables with a key from your own account:
|
|
217
|
+
|
|
218
|
+
| Variable | Provider |
|
|
219
|
+
|---|---|
|
|
220
|
+
| `PAGEMINT_TAVILY_API_KEY` | [Tavily](https://tavily.com) |
|
|
221
|
+
| `PAGEMINT_BRAVE_API_KEY` | [Brave Search API](https://brave.com/search/api/) |
|
|
222
|
+
|
|
223
|
+
Both have offered free monthly allowances; check their current pricing pages, since these change. Keys are read only from environment variables, never from command-line options or tool arguments, and never appear in PageMint's output or logs.
|
|
224
|
+
|
|
225
|
+
### Adding the settings to Claude
|
|
226
|
+
|
|
227
|
+
Claude Code:
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
claude mcp add pagemint -e PAGEMINT_SEARXNG_URL=http://localhost:8888 -- uvx pagemint
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
Claude Desktop (`claude_desktop_config.json`):
|
|
234
|
+
|
|
235
|
+
```json
|
|
236
|
+
{
|
|
237
|
+
"mcpServers": {
|
|
238
|
+
"pagemint": {
|
|
239
|
+
"command": "uvx",
|
|
240
|
+
"args": ["pagemint"],
|
|
241
|
+
"env": { "PAGEMINT_TAVILY_API_KEY": "your-key" }
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
### Which provider is used
|
|
248
|
+
|
|
249
|
+
- `provider="searxng" | "tavily" | "brave"` uses that one.
|
|
250
|
+
- Otherwise `PAGEMINT_SEARCH_PROVIDER` if set, else SearXNG if configured, else Tavily, else Brave.
|
|
251
|
+
- If it fails, PageMint does **not** switch to another provider on its own (that could move you from free SearXNG to a key that costs money). The error lists your other configured providers. Set `PAGEMINT_SEARCH_FALLBACK=1` to allow trying them automatically when no provider was pinned.
|
|
252
|
+
|
|
253
|
+
### Safety
|
|
254
|
+
|
|
255
|
+
- PageMint never scrapes Google, Bing or DuckDuckGo result pages.
|
|
256
|
+
- Requests that carry your key never follow redirects, so the key can't be sent to another site.
|
|
257
|
+
- Your SearXNG may run on `localhost`: only that exact address is allowed, and only for the search request itself. Result pages read with `fetch_pages` go through the normal checks (robots.txt, private-address block, size and time limits).
|
|
258
|
+
- Calls to the Tavily/Brave APIs with your key, and to your own SearXNG, are not robots.txt-checked: robots.txt is meant for crawlers, not for API calls you have a key for.
|
|
259
|
+
- Search results are text from other websites. Treat them as information, not instructions.
|
|
260
|
+
|
|
261
|
+
## Command line
|
|
262
|
+
|
|
263
|
+
```bash
|
|
264
|
+
pagemint scrape https://example.com
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
pagemint scrape https://example.com --format links
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
pagemint crawl https://example.com/docs --max-pages 50 --out ./docs-md
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
```bash
|
|
276
|
+
pagemint map https://example.com --limit 1000
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
```bash
|
|
280
|
+
pagemint search "python mcp server" --fetch 2
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
- `pagemint` with no command (or `pagemint serve`) runs the MCP server on stdio.
|
|
284
|
+
- `--json` prints the full result.
|
|
285
|
+
- `--js` renders with the browser.
|
|
286
|
+
- `crawl --out DIR` writes one `.md` file per page, with no size limits.
|
|
287
|
+
- Common options: `--rate-limit SECONDS`, `--timeout SECONDS`, `--allow-private`, `--browser PATH`.
|
|
288
|
+
|
|
289
|
+
## Limits and defaults
|
|
290
|
+
|
|
291
|
+
| Setting | Default | Environment variable |
|
|
292
|
+
|---|---|---|
|
|
293
|
+
| Delay between requests to the same site | 1 second | `PAGEMINT_RATE_LIMIT` |
|
|
294
|
+
| Connect / read timeout | 10 s / 20 s | `PAGEMINT_CONNECT_TIMEOUT`, `PAGEMINT_READ_TIMEOUT` |
|
|
295
|
+
| Total time per page (all redirects) | 30 s | `PAGEMINT_PAGE_TIMEOUT` |
|
|
296
|
+
| Maximum page size | 5 MB | `PAGEMINT_MAX_BYTES` |
|
|
297
|
+
| Maximum redirects | 5 | `PAGEMINT_MAX_REDIRECTS` |
|
|
298
|
+
| Browser fallback below | 200 characters of text | `PAGEMINT_JS_FALLBACK_CHARS` (0 turns it off) |
|
|
299
|
+
| Crawl time budget | 120 s | `PAGEMINT_CRAWL_TIME_BUDGET` |
|
|
300
|
+
| Crawl output per page / per call | 20,000 / 300,000 characters | `PAGEMINT_PAGE_CHAR_CAP`, `PAGEMINT_RESULT_CHAR_CAP` |
|
|
301
|
+
| Allow private networks | off | `PAGEMINT_ALLOW_PRIVATE=1` |
|
|
302
|
+
| Chrome/Chromium binary | Playwright's | `PAGEMINT_BROWSER_EXECUTABLE` |
|
|
303
|
+
|
|
304
|
+
Hard limits that cannot be raised: `crawl` at most 100 pages and depth 5; `map` at most 5,000 URLs.
|
|
305
|
+
|
|
306
|
+
When a crawl hits its time or output budget it stops and returns the pages it has, with `stopped_early: true`. Use the CLI with `--out` to save larger crawls to disk.
|
|
307
|
+
|
|
308
|
+
## Safety and politeness
|
|
309
|
+
|
|
310
|
+
- **robots.txt is always respected.** PageMint looks for rules for `PageMint`, then `*`, and supports `Allow`, `Disallow`, `*` and `$` wildcards and `Crawl-delay`. A blocked URL comes back with the rule that blocked it. If robots.txt is missing (4xx), everything is allowed. If it errors (5xx) or times out, the site is treated as off-limits for a minute.
|
|
311
|
+
- **Rate limit per site:** one request per second by default, or slower if `Crawl-delay` asks for it.
|
|
312
|
+
- **User-Agent:** `PageMint/<version> (+https://github.com/HarshaVardhan1111/pagemint)`, so site owners can see who is visiting.
|
|
313
|
+
- **SSRF protection:** only `http` and `https` URLs are fetched. Localhost, private ranges (10/8, 172.16/12, 192.168/16), link-local addresses (including cloud metadata at 169.254.169.254), CGNAT, multicast and IPv6 equivalents are refused. The check runs on every redirect hop, and for direct connections again when the connection opens, which stops DNS-rebinding tricks. In the browser, every request and WebSocket the page opens is checked too, and PageMint follows redirects itself so each hop is checked.
|
|
314
|
+
- **Behind a proxy:** if `HTTPS_PROXY` / `HTTP_PROXY` are set, PageMint uses them. When a name can only be resolved by the proxy, PageMint lets the proxy resolve it. Literal private IPs, localhost, single-word names and internal suffixes (`.internal`, `.local`, `.lan`, `.corp`, `.home.arpa`) are still refused.
|
|
315
|
+
- **Known limits:** DNS-rebinding protection at connect time covers direct connections only. Requests sent through a proxy, and requests made by the browser during JavaScript rendering, are checked before they are sent but not at the moment of connecting. If you scrape untrusted sites from a machine with sensitive internal services, run PageMint without a proxy and without JavaScript rendering, or in an isolated network.
|
|
316
|
+
- **Size and time limits** on every request, so one bad page cannot hang or flood your agent.
|
|
317
|
+
|
|
318
|
+
To scrape your own intranet or a local dev server on purpose, set `PAGEMINT_ALLOW_PRIVATE=1` or pass `--allow-private`.
|
|
319
|
+
|
|
320
|
+
## Libraries and licenses
|
|
321
|
+
|
|
322
|
+
PageMint uses only permissively licensed libraries.
|
|
323
|
+
|
|
324
|
+
| Library | Used for | License |
|
|
325
|
+
|---|---|---|
|
|
326
|
+
| [mcp](https://github.com/modelcontextprotocol/python-sdk) | MCP server (`MCPServer`, formerly FastMCP) | MIT |
|
|
327
|
+
| [httpx](https://github.com/encode/httpx) | HTTP client | BSD-3-Clause |
|
|
328
|
+
| [httpcore](https://github.com/encode/httpcore) | Connection-level SSRF check | BSD-3-Clause |
|
|
329
|
+
| [lxml](https://lxml.de/) | HTML and sitemap parsing | BSD-3-Clause |
|
|
330
|
+
| [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/) | HTML tree for Markdown conversion | MIT |
|
|
331
|
+
| [markdownify](https://github.com/matthewwithanm/python-markdownify) | HTML to Markdown | MIT |
|
|
332
|
+
| [playwright](https://github.com/microsoft/playwright-python) (optional `[js]`) | JavaScript rendering | Apache-2.0 |
|
|
333
|
+
| [pytest](https://pytest.org/) (development) | Tests | MIT |
|
|
334
|
+
| [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) (development) | Async tests | Apache-2.0 |
|
|
335
|
+
|
|
336
|
+
These pull in further dependencies, almost all MIT, BSD or Apache-2.0. Two exceptions:
|
|
337
|
+
|
|
338
|
+
- `certifi` (CA certificates, required by httpx) is **MPL-2.0**.
|
|
339
|
+
- `typing_extensions` (and part of `greenlet`) are **PSF-2.0**.
|
|
340
|
+
|
|
341
|
+
Both are used unmodified, which is compatible with MIT.
|
|
342
|
+
|
|
343
|
+
## Development
|
|
344
|
+
|
|
345
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Tests run entirely against a local HTTP server, with no internet access needed:
|
|
346
|
+
|
|
347
|
+
```bash
|
|
348
|
+
pip install -e ".[js,dev]"
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
```bash
|
|
352
|
+
playwright install chromium
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
```bash
|
|
356
|
+
pytest
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
## Releasing (maintainers)
|
|
360
|
+
|
|
361
|
+
Publishing a GitHub release (Releases → Draft a new release, with a new tag like `v0.1.0` on `main`) builds the package and publishes it to PyPI through [trusted publishing](https://docs.pypi.org/trusted-publishers/), so no API token is stored anywhere. One-time setup on pypi.org: create a free account, then under **Publishing** add a pending publisher with owner `HarshaVardhan1111`, repository `pagemint`, workflow `release.yml` and environment `pypi`.
|
|
362
|
+
|
|
363
|
+
## License
|
|
364
|
+
|
|
365
|
+
MIT. See [LICENSE](LICENSE).
|