pagemint 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. pagemint-0.1.0/.github/workflows/ci.yml +28 -0
  2. pagemint-0.1.0/.github/workflows/release.yml +39 -0
  3. pagemint-0.1.0/.gitignore +9 -0
  4. pagemint-0.1.0/CONTRIBUTING.md +53 -0
  5. pagemint-0.1.0/LICENSE +21 -0
  6. pagemint-0.1.0/PKG-INFO +365 -0
  7. pagemint-0.1.0/README.md +335 -0
  8. pagemint-0.1.0/docs/superpowers/plans/2026-11-18-pagemint-core.md +151 -0
  9. pagemint-0.1.0/docs/superpowers/plans/2026-11-18-pagemint-jobs.md +137 -0
  10. pagemint-0.1.0/docs/superpowers/plans/2026-11-19-pagemint-search.md +84 -0
  11. pagemint-0.1.0/docs/superpowers/specs/2026-11-18-pagemint-core-design.md +362 -0
  12. pagemint-0.1.0/docs/superpowers/specs/2026-11-18-pagemint-jobs-design.md +274 -0
  13. pagemint-0.1.0/docs/superpowers/specs/2026-11-19-pagemint-search-design.md +177 -0
  14. pagemint-0.1.0/pyproject.toml +52 -0
  15. pagemint-0.1.0/scripts/live_check.py +59 -0
  16. pagemint-0.1.0/src/pagemint/__init__.py +1 -0
  17. pagemint-0.1.0/src/pagemint/__main__.py +5 -0
  18. pagemint-0.1.0/src/pagemint/browser.py +215 -0
  19. pagemint-0.1.0/src/pagemint/cli.py +226 -0
  20. pagemint-0.1.0/src/pagemint/client.py +342 -0
  21. pagemint-0.1.0/src/pagemint/config.py +73 -0
  22. pagemint-0.1.0/src/pagemint/errors.py +55 -0
  23. pagemint-0.1.0/src/pagemint/extract.py +204 -0
  24. pagemint-0.1.0/src/pagemint/fetcher.py +318 -0
  25. pagemint-0.1.0/src/pagemint/jobs/__init__.py +1 -0
  26. pagemint-0.1.0/src/pagemint/jobs/detect.py +229 -0
  27. pagemint-0.1.0/src/pagemint/jobs/jsonld.py +203 -0
  28. pagemint-0.1.0/src/pagemint/jobs/model.py +105 -0
  29. pagemint-0.1.0/src/pagemint/jobs/rank.py +157 -0
  30. pagemint-0.1.0/src/pagemint/jobs/readers/__init__.py +11 -0
  31. pagemint-0.1.0/src/pagemint/jobs/readers/_common.py +47 -0
  32. pagemint-0.1.0/src/pagemint/jobs/readers/ashby.py +70 -0
  33. pagemint-0.1.0/src/pagemint/jobs/readers/greenhouse.py +57 -0
  34. pagemint-0.1.0/src/pagemint/jobs/readers/lever.py +64 -0
  35. pagemint-0.1.0/src/pagemint/jobs/readers/recruitee.py +41 -0
  36. pagemint-0.1.0/src/pagemint/jobs/readers/workday.py +83 -0
  37. pagemint-0.1.0/src/pagemint/jobs/service.py +320 -0
  38. pagemint-0.1.0/src/pagemint/jobs/sources/__init__.py +5 -0
  39. pagemint-0.1.0/src/pagemint/jobs/sources/arbeitnow.py +34 -0
  40. pagemint-0.1.0/src/pagemint/jobs/sources/himalayas.py +52 -0
  41. pagemint-0.1.0/src/pagemint/jobs/sources/hn.py +98 -0
  42. pagemint-0.1.0/src/pagemint/jobs/store.py +185 -0
  43. pagemint-0.1.0/src/pagemint/links.py +85 -0
  44. pagemint-0.1.0/src/pagemint/markdown.py +78 -0
  45. pagemint-0.1.0/src/pagemint/ratelimit.py +41 -0
  46. pagemint-0.1.0/src/pagemint/robots.py +221 -0
  47. pagemint-0.1.0/src/pagemint/safety.py +137 -0
  48. pagemint-0.1.0/src/pagemint/search/__init__.py +1 -0
  49. pagemint-0.1.0/src/pagemint/search/_http.py +29 -0
  50. pagemint-0.1.0/src/pagemint/search/brave.py +37 -0
  51. pagemint-0.1.0/src/pagemint/search/model.py +38 -0
  52. pagemint-0.1.0/src/pagemint/search/searxng.py +43 -0
  53. pagemint-0.1.0/src/pagemint/search/secrets.py +53 -0
  54. pagemint-0.1.0/src/pagemint/search/service.py +129 -0
  55. pagemint-0.1.0/src/pagemint/search/tavily.py +31 -0
  56. pagemint-0.1.0/src/pagemint/server.py +171 -0
  57. pagemint-0.1.0/src/pagemint/sitemap.py +44 -0
  58. pagemint-0.1.0/tests/conftest.py +150 -0
  59. pagemint-0.1.0/tests/jobs/fixtures.py +91 -0
  60. pagemint-0.1.0/tests/jobs/test_detect.py +86 -0
  61. pagemint-0.1.0/tests/jobs/test_detect_url.py +47 -0
  62. pagemint-0.1.0/tests/jobs/test_jsonld.py +85 -0
  63. pagemint-0.1.0/tests/jobs/test_rank.py +118 -0
  64. pagemint-0.1.0/tests/jobs/test_readers.py +139 -0
  65. pagemint-0.1.0/tests/jobs/test_service.py +201 -0
  66. pagemint-0.1.0/tests/jobs/test_sources.py +130 -0
  67. pagemint-0.1.0/tests/jobs/test_store.py +108 -0
  68. pagemint-0.1.0/tests/search/test_search_config.py +28 -0
  69. pagemint-0.1.0/tests/search/test_search_providers.py +129 -0
  70. pagemint-0.1.0/tests/search/test_search_service.py +185 -0
  71. pagemint-0.1.0/tests/test_cli.py +114 -0
  72. pagemint-0.1.0/tests/test_crawl.py +105 -0
  73. pagemint-0.1.0/tests/test_extract.py +70 -0
  74. pagemint-0.1.0/tests/test_js.py +252 -0
  75. pagemint-0.1.0/tests/test_map.py +97 -0
  76. pagemint-0.1.0/tests/test_robots.py +171 -0
  77. pagemint-0.1.0/tests/test_safety.py +282 -0
  78. pagemint-0.1.0/tests/test_scrape.py +94 -0
  79. pagemint-0.1.0/tests/test_server.py +80 -0
@@ -0,0 +1,28 @@
1
+ name: tests
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ test:
12
+ runs-on: ubuntu-latest
13
+ strategy:
14
+ fail-fast: false
15
+ matrix:
16
+ python-version: ["3.11", "3.12", "3.13"]
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - uses: actions/setup-python@v5
20
+ with:
21
+ python-version: ${{ matrix.python-version }}
22
+ cache: pip
23
+ - name: Install PageMint with browser and test extras
24
+ run: python -m pip install -e ".[js,dev]"
25
+ - name: Install Chromium for the JavaScript tests
26
+ run: python -m playwright install --with-deps chromium
27
+ - name: Run tests
28
+ run: python -m pytest -q
@@ -0,0 +1,39 @@
1
+ name: release
2
+
3
+ on:
4
+ release:
5
+ types: [published] # publishing a GitHub release (Releases -> Draft a new release) uploads to PyPI
6
+ workflow_dispatch: # or run it by hand from the Actions tab
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ jobs:
12
+ build:
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: actions/setup-python@v5
17
+ with:
18
+ python-version: "3.12"
19
+ - name: Build sdist and wheel
20
+ run: |
21
+ python -m pip install build
22
+ python -m build
23
+ - uses: actions/upload-artifact@v4
24
+ with:
25
+ name: dist
26
+ path: dist/
27
+
28
+ publish:
29
+ needs: build
30
+ runs-on: ubuntu-latest
31
+ environment: pypi
32
+ permissions:
33
+ id-token: write # PyPI trusted publishing; no API token needed
34
+ steps:
35
+ - uses: actions/download-artifact@v4
36
+ with:
37
+ name: dist
38
+ path: dist/
39
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,9 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .venv/
5
+ build/
6
+ dist/
7
+ .pytest_cache/
8
+ .superpowers/
9
+ .coverage
@@ -0,0 +1,53 @@
1
+ # Contributing to PageMint
2
+
3
+ Thanks for helping. Bug reports, fixes and new test cases from real pages are all welcome.
4
+
5
+ ## Setup
6
+
7
+ ```bash
8
+ git clone https://github.com/HarshaVardhan1111/pagemint
9
+ ```
10
+
11
+ ```bash
12
+ cd pagemint
13
+ ```
14
+
15
+ ```bash
16
+ python -m venv .venv && . .venv/bin/activate
17
+ ```
18
+
19
+ ```bash
20
+ pip install -e ".[js,dev]"
21
+ ```
22
+
23
+ ```bash
24
+ playwright install chromium
25
+ ```
26
+
27
+ ```bash
28
+ pytest
29
+ ```
30
+
31
+ The browser tests are skipped automatically if Chromium is not installed. If you already have Chrome or Chromium, point to it with `PAGEMINT_BROWSER_EXECUTABLE=/path/to/chrome`.
32
+
33
+ ## Rules for changes
34
+
35
+ 1. **Write it yourself.** Do not copy code from Firecrawl or other scrapers, even when their license would allow it. PageMint is written from scratch.
36
+ 2. **Permissive dependencies only.** New dependencies must be MIT, BSD or Apache-2.0, and must be added to the license table in the README. Prefer the standard library.
37
+ 3. **Safety rules are not optional.** Changes must keep robots.txt handling, per-site rate limits, size and time limits, and the SSRF checks (including on redirects and browser requests). Please do not add options that turn them off for normal users.
38
+ 4. **Tests first, no internet.** Add a test that fails without your change. Tests use the local server in `tests/conftest.py`; they must never reach the live internet.
39
+ 5. **Never crash the server.** Tools return `{"error": {"type", "message"}}` with a message a person can act on.
40
+
41
+ ## Pull requests
42
+
43
+ - Keep each pull request to one change, with a clear description of why.
44
+ - Run `pytest` before pushing; CI runs it on Python 3.11, 3.12 and 3.13.
45
+ - For extraction changes, include the HTML that was handled badly (trimmed) as a test fixture.
46
+
47
+ ## Reporting a page that converts badly
48
+
49
+ Open an issue with the URL, what you expected and what you got. If the page is private, attach a trimmed copy of its HTML instead.
50
+
51
+ ## Security
52
+
53
+ If you find a way around the SSRF or robots.txt protections, please open a GitHub security advisory (Security tab → Report a vulnerability) instead of a public issue.
pagemint-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Harsha
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,365 @@
1
+ Metadata-Version: 2.5
2
+ Name: pagemint
3
+ Version: 0.1.0
4
+ Summary: Free, open-source MCP server that turns web pages into clean Markdown for AI agents.
5
+ Project-URL: Homepage, https://github.com/HarshaVardhan1111/pagemint
6
+ Project-URL: Issues, https://github.com/HarshaVardhan1111/pagemint/issues
7
+ Author: Harsha
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: ai-agents,crawler,markdown,mcp,scraper
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
17
+ Requires-Python: >=3.11
18
+ Requires-Dist: beautifulsoup4>=4.12
19
+ Requires-Dist: httpcore<2,>=1
20
+ Requires-Dist: httpx<1,>=0.27
21
+ Requires-Dist: lxml>=5
22
+ Requires-Dist: markdownify>=0.13
23
+ Requires-Dist: mcp<3,>=2
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
26
+ Requires-Dist: pytest>=8; extra == 'dev'
27
+ Provides-Extra: js
28
+ Requires-Dist: playwright>=1.49; extra == 'js'
29
+ Description-Content-Type: text/markdown
30
+
31
+ # PageMint
32
+
33
+ **Free, open-source MCP server that turns web pages into clean Markdown for AI agents.**
34
+
35
+ PageMint gives Claude (and any other MCP client) tools to `scrape` one page, `crawl` a site, `map` a site's URLs, **search open jobs** straight from companies' own hiring systems, and **search the web** with your own SearXNG or your own search API key. It runs on your own machine, so scraping, crawling and job search have no credits, API keys or monthly limits. (Web search needs your own SearXNG or your own search API key; see below.) It is polite by default: it follows robots.txt, rate-limits every site, and refuses to touch private networks.
36
+
37
+ - Clean Markdown: menus, footers, cookie banners and ads are removed; headings, lists, tables, code blocks and links are kept.
38
+ - JavaScript pages: if a page is almost empty without JavaScript, PageMint renders it in headless Chromium (optional install).
39
+ - Written from scratch in Python. MIT license.
40
+
41
+ ## Install
42
+
43
+ PageMint needs Python 3.11 or newer.
44
+
45
+ ```bash
46
+ pip install pagemint
47
+ ```
48
+
49
+ Or run it without installing, using [uv](https://docs.astral.sh/uv/):
50
+
51
+ ```bash
52
+ uvx pagemint --help
53
+ ```
54
+
55
+ > **Before the first PyPI release**, install from GitHub instead:
56
+ > `uvx --from git+https://github.com/HarshaVardhan1111/pagemint pagemint --help`
57
+ > (and use the same `--from ...` in the setup commands below).
58
+
59
+ ### Optional: JavaScript rendering
60
+
61
+ Plain pages work out of the box. For sites that build their content with JavaScript, add the browser extra and download Chromium once (about 150 MB):
62
+
63
+ ```bash
64
+ pip install "pagemint[js]"
65
+ playwright install chromium
66
+ ```
67
+
68
+ With uv:
69
+
70
+ ```bash
71
+ uvx --from "pagemint[js]" playwright install chromium
72
+ ```
73
+
74
+ Without the browser, PageMint still works and tells you when a page needed JavaScript.
75
+
76
+ ## Add it to Claude Code
77
+
78
+ ```bash
79
+ claude mcp add pagemint -- uvx pagemint
80
+ ```
81
+
82
+ With JavaScript rendering:
83
+
84
+ ```bash
85
+ claude mcp add pagemint -- uvx --from "pagemint[js]" pagemint
86
+ ```
87
+
88
+ ## Add it to Claude Desktop
89
+
90
+ Open **Settings → Developer → Edit Config** (the file is `claude_desktop_config.json`; on macOS it lives in `~/Library/Application Support/Claude/`, on Windows in `%APPDATA%\Claude\`) and add:
91
+
92
+ ```json
93
+ {
94
+ "mcpServers": {
95
+ "pagemint": {
96
+ "command": "uvx",
97
+ "args": ["pagemint"]
98
+ }
99
+ }
100
+ }
101
+ ```
102
+
103
+ For JavaScript rendering use `"args": ["--from", "pagemint[js]", "pagemint"]`. Restart Claude Desktop afterwards.
104
+
105
+ ## Tools
106
+
107
+ ### `scrape(url, format="markdown", render_js=False)`
108
+
109
+ Fetches one page.
110
+
111
+ | Field | Meaning |
112
+ |---|---|
113
+ | `url` | Final URL after redirects |
114
+ | `status_code` | HTTP status code |
115
+ | `title` | Page title (`og:title`, then `<title>`, then the first `<h1>`) |
116
+ | `content` | Markdown (default), cleaned main-content HTML (`format="html"`), or empty (`format="links"`) |
117
+ | `links` | Every http(s) link on the page, absolute and de-duplicated |
118
+ | `rendered_js` | `true` if the page was rendered in the browser |
119
+ | `warnings` | Things worth knowing, e.g. a 404 status or a missing browser |
120
+
121
+ `render_js=True` always uses the browser. Without it, PageMint uses the browser automatically only when the plain page has under 200 characters of text.
122
+
123
+ ### `crawl(url, max_pages=20, max_depth=2, same_domain=True, render_js=False)`
124
+
125
+ Follows links breadth-first from `url` and returns each page as Markdown.
126
+
127
+ - `max_depth` counts link hops from the start page (the start page is depth 0).
128
+ - `same_domain=True` stays on the start host (`www.` is treated as the same site).
129
+ - Files such as images, PDFs and archives are not followed.
130
+ - Returns `pages` (url, status_code, title, markdown, depth, truncated, rendered_js), `skipped` (url + reason: robots.txt, blocked, error, limit), `stopped_early` and `warnings`.
131
+
132
+ ### `map(url, limit=500)`
133
+
134
+ Lists URLs on a site without fetching every page: it reads the sitemap first (from robots.txt `Sitemap:` lines, otherwise `/sitemap.xml`, including sitemap indexes and `.gz` files), then adds links from the start page if needed. URLs that robots.txt disallows are left out. Returns `urls`, `source` (`sitemap`, `links` or `sitemap+links`) and `warnings`.
135
+
136
+ Errors never crash the server. A failed call returns `{"error": {"type": ..., "message": ...}}` with a plain-English message and is marked as an error to the client.
137
+
138
+ ## Job search
139
+
140
+ Four more tools find **open** jobs straight from the systems companies use to post them, so you don't see closed jobs that linger on job boards.
141
+
142
+ | Tool | What it does |
143
+ |---|---|
144
+ | `company_jobs(company, query="", location="", remote=None, limit=50)` | Open jobs at one company. `company` is a name (`"Stripe"`) or a careers-page / job-board URL. The company is added to your saved list. |
145
+ | `search_jobs(query, location="", remote=None, posted_within_days=None, companies=None, sources="all", limit=25)` | Searches all saved companies (or the `companies` you pass) plus the free job sources below, best matches first. `sources="companies"` skips the free sources. Max 100 results. |
146
+ | `check_job(url)` | Is this job still open? Returns `open`, `closed` or `unknown` with the reason, and the full description when open. |
147
+ | `saved_companies(action="list", company=None)` | List, `add` or `remove` saved companies. |
148
+
149
+ Each job has title, company, location, remote / workplace type, employment type, department, posted date, salary (when published), links, a short description, its `source` and a `status`: `open` for jobs read from a company's hiring system, `unverified` for jobs from free sources (run `check_job` on them).
150
+
151
+ ### Where jobs come from
152
+
153
+ | Source | How | Notes |
154
+ |---|---|---|
155
+ | Greenhouse | public Job Board API | company name is confirmed from the feed |
156
+ | Lever | public Postings API (global and EU) | |
157
+ | Ashby | public job posting API | |
158
+ | Workday | the career site's own JSON endpoints | needs the careers URL (e.g. `https://acme.wd5.myworkdayjobs.com/External`) |
159
+ | Recruitee | public offers API | experimental |
160
+ | Any other careers page | schema.org `JobPosting` data that job pages carry for search engines | follows up to 30 job links; expired jobs are dropped |
161
+ | [Hacker News "Who is hiring?"](https://news.ycombinator.com/submitted?id=whoishiring) | HN Algolia API, current and previous month | free text, parsed best effort |
162
+ | [Arbeitnow](https://www.arbeitnow.com) | free job board API | mostly Europe; credited as its terms ask |
163
+ | [Himalayas](https://himalayas.app) | free remote jobs API | rate-limits quickly; skipped with a note when it does |
164
+
165
+ **Considered and not used:** LinkedIn, Indeed, Glassdoor and ZipRecruiter (their terms forbid scraping or they block bots), SmartRecruiters (its robots.txt blocks all bots except LinkedIn's), Remotive (robots.txt blocks its API), Workable search (rate-limited, and search URLs are disallowed). Workable careers pages are still read through the general reader when they allow it.
166
+
167
+ ### How finding a company works
168
+
169
+ - **By URL** (most reliable): PageMint recognises Greenhouse, Lever, Ashby, Workday, Recruitee and Workable links, also when a company's own careers page embeds one. Otherwise it reads the page's job data.
170
+ - **By name**: PageMint tries a few spellings on Greenhouse, Lever and Ashby. A Greenhouse match whose company name agrees is `confirmed`. A Lever or Ashby match is `guessed`, because those feeds don't include the company name, and the result says so. If it's the wrong company, remove it and pass the careers URL instead.
171
+
172
+ ### Matching
173
+
174
+ Results are ranked by keywords (title matters most, then department, then description) and recency, with no AI involved. `location` is a filter that knows common alternative spellings (Bengaluru/Bangalore, Gurugram/Gurgaon, NYC/New York, UK/United Kingdom...). A remote job matches every location only when it isn't limited to a place: "Remote (worldwide)" matches London, "Remote - US" does not.
175
+
176
+ ### Your data
177
+
178
+ The saved company list and a cache of job feeds live in a small SQLite file on your computer (`~/.local/share/pagemint` on Linux, `~/Library/Application Support/pagemint` on macOS, `%LOCALAPPDATA%\pagemint` on Windows; change it with `PAGEMINT_DATA_DIR`). Nothing is sent anywhere else.
179
+
180
+ ### Job search limits
181
+
182
+ | Setting | Default | Environment variable |
183
+ |---|---|---|
184
+ | Job feed cache | 6 hours (free sources: 1 hour) | `PAGEMINT_JOBS_CACHE_HOURS` |
185
+ | Time budget per search | 60 s (slower feeds are listed under `skipped`) | `PAGEMINT_JOBS_SEARCH_BUDGET` |
186
+ | Saved companies | 500 | |
187
+ | Job feed size | 40 MB (large companies' feeds exceed the 5 MB page limit) | |
188
+ | Workday jobs per company | 200 | |
189
+
190
+ All job requests follow the same robots.txt, rate-limit and safety rules as everything else. Hiring-system hosts are shared by many companies (all Greenhouse boards use one host), so the first search over many saved companies can take a while; later searches use the cache.
191
+
192
+ To try job search against real sites: `python scripts/live_check.py "data engineer" --location London`.
193
+
194
+ ## Web search
195
+
196
+ `search(query, limit=10, fetch_pages=0, provider="auto")` returns titles, links and snippets (up to 20). With `fetch_pages=3` it also reads the top 3 result pages and returns them as Markdown, in the same call.
197
+
198
+ There is no free web search that needs no key and allows bots, so you choose one of these. PageMint never uses a paid service unless you set its key.
199
+
200
+ ### Option 1: your own SearXNG (free, no key, no monthly limit)
201
+
202
+ [SearXNG](https://docs.searxng.org/) is an open-source search engine you run yourself. With Docker:
203
+
204
+ ```bash
205
+ mkdir -p searxng && printf 'use_default_settings: true\nsearch:\n formats: [html, json]\nserver:\n secret_key: "change-me-to-something-random"\n' > searxng/settings.yml
206
+ ```
207
+
208
+ ```bash
209
+ docker run -d --name searxng -p 8888:8080 -v "$PWD/searxng:/etc/searxng" searxng/searxng
210
+ ```
211
+
212
+ Then set `PAGEMINT_SEARXNG_URL=http://localhost:8888`. The `formats: [html, json]` line matters: without it SearXNG refuses PageMint's requests, and PageMint tells you so. SearXNG gets its results from other search engines, so very heavy use can get it rate-limited by them.
213
+
214
+ ### Option 2: your own API key
215
+
216
+ Set **one** of these environment variables with a key from your own account:
217
+
218
+ | Variable | Provider |
219
+ |---|---|
220
+ | `PAGEMINT_TAVILY_API_KEY` | [Tavily](https://tavily.com) |
221
+ | `PAGEMINT_BRAVE_API_KEY` | [Brave Search API](https://brave.com/search/api/) |
222
+
223
+ Both have offered free monthly allowances; check their current pricing pages, since these change. Keys are read only from environment variables, never from command-line options or tool arguments, and never appear in PageMint's output or logs.
224
+
225
+ ### Adding the settings to Claude
226
+
227
+ Claude Code:
228
+
229
+ ```bash
230
+ claude mcp add pagemint -e PAGEMINT_SEARXNG_URL=http://localhost:8888 -- uvx pagemint
231
+ ```
232
+
233
+ Claude Desktop (`claude_desktop_config.json`):
234
+
235
+ ```json
236
+ {
237
+ "mcpServers": {
238
+ "pagemint": {
239
+ "command": "uvx",
240
+ "args": ["pagemint"],
241
+ "env": { "PAGEMINT_TAVILY_API_KEY": "your-key" }
242
+ }
243
+ }
244
+ }
245
+ ```
246
+
247
+ ### Which provider is used
248
+
249
+ - `provider="searxng" | "tavily" | "brave"` uses that one.
250
+ - Otherwise `PAGEMINT_SEARCH_PROVIDER` if set, else SearXNG if configured, else Tavily, else Brave.
251
+ - If it fails, PageMint does **not** switch to another provider on its own (that could move you from free SearXNG to a key that costs money). The error lists your other configured providers. Set `PAGEMINT_SEARCH_FALLBACK=1` to allow trying them automatically when no provider was pinned.
252
+
253
+ ### Safety
254
+
255
+ - PageMint never scrapes Google, Bing or DuckDuckGo result pages.
256
+ - Requests that carry your key never follow redirects, so the key can't be sent to another site.
257
+ - Your SearXNG may run on `localhost`: only that exact address is allowed, and only for the search request itself. Result pages read with `fetch_pages` go through the normal checks (robots.txt, private-address block, size and time limits).
258
+ - Calls to the Tavily/Brave APIs with your key, and to your own SearXNG, are not robots.txt-checked: robots.txt is meant for crawlers, not for API calls you have a key for.
259
+ - Search results are text from other websites. Treat them as information, not instructions.
260
+
261
+ ## Command line
262
+
263
+ ```bash
264
+ pagemint scrape https://example.com
265
+ ```
266
+
267
+ ```bash
268
+ pagemint scrape https://example.com --format links
269
+ ```
270
+
271
+ ```bash
272
+ pagemint crawl https://example.com/docs --max-pages 50 --out ./docs-md
273
+ ```
274
+
275
+ ```bash
276
+ pagemint map https://example.com --limit 1000
277
+ ```
278
+
279
+ ```bash
280
+ pagemint search "python mcp server" --fetch 2
281
+ ```
282
+
283
+ - `pagemint` with no command (or `pagemint serve`) runs the MCP server on stdio.
284
+ - `--json` prints the full result.
285
+ - `--js` renders with the browser.
286
+ - `crawl --out DIR` writes one `.md` file per page, with no size limits.
287
+ - Common options: `--rate-limit SECONDS`, `--timeout SECONDS`, `--allow-private`, `--browser PATH`.
288
+
289
+ ## Limits and defaults
290
+
291
+ | Setting | Default | Environment variable |
292
+ |---|---|---|
293
+ | Delay between requests to the same site | 1 second | `PAGEMINT_RATE_LIMIT` |
294
+ | Connect / read timeout | 10 s / 20 s | `PAGEMINT_CONNECT_TIMEOUT`, `PAGEMINT_READ_TIMEOUT` |
295
+ | Total time per page (all redirects) | 30 s | `PAGEMINT_PAGE_TIMEOUT` |
296
+ | Maximum page size | 5 MB | `PAGEMINT_MAX_BYTES` |
297
+ | Maximum redirects | 5 | `PAGEMINT_MAX_REDIRECTS` |
298
+ | Browser fallback below | 200 characters of text | `PAGEMINT_JS_FALLBACK_CHARS` (0 turns it off) |
299
+ | Crawl time budget | 120 s | `PAGEMINT_CRAWL_TIME_BUDGET` |
300
+ | Crawl output per page / per call | 20,000 / 300,000 characters | `PAGEMINT_PAGE_CHAR_CAP`, `PAGEMINT_RESULT_CHAR_CAP` |
301
+ | Allow private networks | off | `PAGEMINT_ALLOW_PRIVATE=1` |
302
+ | Chrome/Chromium binary | Playwright's | `PAGEMINT_BROWSER_EXECUTABLE` |
303
+
304
+ Hard limits that cannot be raised: `crawl` at most 100 pages and depth 5; `map` at most 5,000 URLs.
305
+
306
+ When a crawl hits its time or output budget it stops and returns the pages it has, with `stopped_early: true`. Use the CLI with `--out` to save larger crawls to disk.
307
+
308
+ ## Safety and politeness
309
+
310
+ - **robots.txt is always respected.** PageMint looks for rules for `PageMint`, then `*`, and supports `Allow`, `Disallow`, `*` and `$` wildcards and `Crawl-delay`. A blocked URL comes back with the rule that blocked it. If robots.txt is missing (4xx), everything is allowed. If it errors (5xx) or times out, the site is treated as off-limits for a minute.
311
+ - **Rate limit per site:** one request per second by default, or slower if `Crawl-delay` asks for it.
312
+ - **User-Agent:** `PageMint/<version> (+https://github.com/HarshaVardhan1111/pagemint)`, so site owners can see who is visiting.
313
+ - **SSRF protection:** only `http` and `https` URLs are fetched. Localhost, private ranges (10/8, 172.16/12, 192.168/16), link-local addresses (including cloud metadata at 169.254.169.254), CGNAT, multicast and IPv6 equivalents are refused. The check runs on every redirect hop, and for direct connections again when the connection opens, which stops DNS-rebinding tricks. In the browser, every request and WebSocket the page opens is checked too, and PageMint follows redirects itself so each hop is checked.
314
+ - **Behind a proxy:** if `HTTPS_PROXY` / `HTTP_PROXY` are set, PageMint uses them. When a name can only be resolved by the proxy, PageMint lets the proxy resolve it. Literal private IPs, localhost, single-word names and internal suffixes (`.internal`, `.local`, `.lan`, `.corp`, `.home.arpa`) are still refused.
315
+ - **Known limits:** DNS-rebinding protection at connect time covers direct connections only. Requests sent through a proxy, and requests made by the browser during JavaScript rendering, are checked before they are sent but not at the moment of connecting. If you scrape untrusted sites from a machine with sensitive internal services, run PageMint without a proxy and without JavaScript rendering, or in an isolated network.
316
+ - **Size and time limits** on every request, so one bad page cannot hang or flood your agent.
317
+
318
+ To scrape your own intranet or a local dev server on purpose, set `PAGEMINT_ALLOW_PRIVATE=1` or pass `--allow-private`.
319
+
320
+ ## Libraries and licenses
321
+
322
+ PageMint uses only permissively licensed libraries.
323
+
324
+ | Library | Used for | License |
325
+ |---|---|---|
326
+ | [mcp](https://github.com/modelcontextprotocol/python-sdk) | MCP server (`MCPServer`, formerly FastMCP) | MIT |
327
+ | [httpx](https://github.com/encode/httpx) | HTTP client | BSD-3-Clause |
328
+ | [httpcore](https://github.com/encode/httpcore) | Connection-level SSRF check | BSD-3-Clause |
329
+ | [lxml](https://lxml.de/) | HTML and sitemap parsing | BSD-3-Clause |
330
+ | [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/) | HTML tree for Markdown conversion | MIT |
331
+ | [markdownify](https://github.com/matthewwithanm/python-markdownify) | HTML to Markdown | MIT |
332
+ | [playwright](https://github.com/microsoft/playwright-python) (optional `[js]`) | JavaScript rendering | Apache-2.0 |
333
+ | [pytest](https://pytest.org/) (development) | Tests | MIT |
334
+ | [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) (development) | Async tests | Apache-2.0 |
335
+
336
+ These pull in further dependencies, almost all MIT, BSD or Apache-2.0. Two exceptions:
337
+
338
+ - `certifi` (CA certificates, required by httpx) is **MPL-2.0**.
339
+ - `typing_extensions` (and part of `greenlet`) are **PSF-2.0**.
340
+
341
+ Both are used unmodified, which is compatible with MIT.
342
+
343
+ ## Development
344
+
345
+ See [CONTRIBUTING.md](CONTRIBUTING.md). Tests run entirely against a local HTTP server, with no internet access needed:
346
+
347
+ ```bash
348
+ pip install -e ".[js,dev]"
349
+ ```
350
+
351
+ ```bash
352
+ playwright install chromium
353
+ ```
354
+
355
+ ```bash
356
+ pytest
357
+ ```
358
+
359
+ ## Releasing (maintainers)
360
+
361
+ Publishing a GitHub release (Releases → Draft a new release, with a new tag like `v0.1.0` on `main`) builds the package and publishes it to PyPI through [trusted publishing](https://docs.pypi.org/trusted-publishers/), so no API token is stored anywhere. One-time setup on pypi.org: create a free account, then under **Publishing** add a pending publisher with owner `HarshaVardhan1111`, repository `pagemint`, workflow `release.yml` and environment `pypi`.
362
+
363
+ ## License
364
+
365
+ MIT. See [LICENSE](LICENSE).