pagemint 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. pagemint/__init__.py +1 -0
  2. pagemint/__main__.py +5 -0
  3. pagemint/browser.py +215 -0
  4. pagemint/cli.py +226 -0
  5. pagemint/client.py +342 -0
  6. pagemint/config.py +73 -0
  7. pagemint/errors.py +55 -0
  8. pagemint/extract.py +204 -0
  9. pagemint/fetcher.py +318 -0
  10. pagemint/jobs/__init__.py +1 -0
  11. pagemint/jobs/detect.py +229 -0
  12. pagemint/jobs/jsonld.py +203 -0
  13. pagemint/jobs/model.py +105 -0
  14. pagemint/jobs/rank.py +157 -0
  15. pagemint/jobs/readers/__init__.py +11 -0
  16. pagemint/jobs/readers/_common.py +47 -0
  17. pagemint/jobs/readers/ashby.py +70 -0
  18. pagemint/jobs/readers/greenhouse.py +57 -0
  19. pagemint/jobs/readers/lever.py +64 -0
  20. pagemint/jobs/readers/recruitee.py +41 -0
  21. pagemint/jobs/readers/workday.py +83 -0
  22. pagemint/jobs/service.py +320 -0
  23. pagemint/jobs/sources/__init__.py +5 -0
  24. pagemint/jobs/sources/arbeitnow.py +34 -0
  25. pagemint/jobs/sources/himalayas.py +52 -0
  26. pagemint/jobs/sources/hn.py +98 -0
  27. pagemint/jobs/store.py +185 -0
  28. pagemint/links.py +85 -0
  29. pagemint/markdown.py +78 -0
  30. pagemint/ratelimit.py +41 -0
  31. pagemint/robots.py +221 -0
  32. pagemint/safety.py +137 -0
  33. pagemint/search/__init__.py +1 -0
  34. pagemint/search/_http.py +29 -0
  35. pagemint/search/brave.py +37 -0
  36. pagemint/search/model.py +38 -0
  37. pagemint/search/searxng.py +43 -0
  38. pagemint/search/secrets.py +53 -0
  39. pagemint/search/service.py +129 -0
  40. pagemint/search/tavily.py +31 -0
  41. pagemint/server.py +171 -0
  42. pagemint/sitemap.py +44 -0
  43. pagemint-0.1.0.dist-info/METADATA +365 -0
  44. pagemint-0.1.0.dist-info/RECORD +47 -0
  45. pagemint-0.1.0.dist-info/WHEEL +4 -0
  46. pagemint-0.1.0.dist-info/entry_points.txt +2 -0
  47. pagemint-0.1.0.dist-info/licenses/LICENSE +21 -0
pagemint/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
pagemint/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from pagemint.cli import main
4
+
5
+ sys.exit(main())
pagemint/browser.py ADDED
@@ -0,0 +1,215 @@
1
+ """JavaScript rendering with Playwright (Chromium). Optional: ``pip install "pagemint[js]"``.
2
+
3
+ Every request the page makes passes through a route handler that applies the
4
+ same SSRF check as plain fetches. Chromium does not route the requests it
5
+ makes when following a redirect, so PageMint never lets the browser follow
6
+ one: the handler fetches with redirects off and follows them itself, checking
7
+ every hop. For the main page a redirect stops the load and PageMint navigates
8
+ to the (checked) target, so the page keeps its real URL. Main-frame
9
+ navigations also respect robots.txt and the per-host rate limit. Images, media
10
+ and fonts are not loaded (they add no text).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import logging
17
+ import os
18
+ from typing import TYPE_CHECKING
19
+ from urllib.parse import urljoin, urlsplit
20
+ from urllib.request import getproxies
21
+
22
+ from pagemint.errors import BrowserUnavailableError, FetchError, FetchTimeoutError, PageMintError, TooLargeError
23
+ from pagemint.fetcher import FetchResult
24
+
25
+ if TYPE_CHECKING:
26
+ from pagemint.config import Settings
27
+ from pagemint.fetcher import Fetcher
28
+
29
+ log = logging.getLogger("pagemint.browser")
30
+
31
+ _SKIP_TYPES = {"image", "media", "font"}
32
+ _REDIRECTS = {301, 302, 303, 307, 308}
33
+ NETWORK_IDLE_WAIT_MS = 2000
34
+
35
+
36
+ class BrowserRenderer:
37
+ def __init__(self, settings: "Settings", fetcher: "Fetcher"):
38
+ self.settings = settings
39
+ self.fetcher = fetcher
40
+ self._pw = None
41
+ self._browser = None
42
+ self._lock = asyncio.Lock()
43
+
44
+ @staticmethod
45
+ def install_hint() -> str:
46
+ return (
47
+ 'JavaScript rendering needs the optional browser install: run pip install "pagemint[js]" '
48
+ "and then playwright install chromium."
49
+ )
50
+
51
+ def _proxy(self) -> dict | None:
52
+ proxies = {k.lower(): v for k, v in getproxies().items()}
53
+ server = proxies.get("https") or proxies.get("http") or proxies.get("all")
54
+ if not server:
55
+ return None
56
+ proxy = {"server": server}
57
+ bypass = proxies.get("no") or os.environ.get("NO_PROXY") or os.environ.get("no_proxy")
58
+ if bypass:
59
+ proxy["bypass"] = bypass
60
+ return proxy
61
+
62
+ async def start(self) -> None:
63
+ """Launch Chromium (once). Raises BrowserUnavailableError if it cannot."""
64
+ async with self._lock:
65
+ if self._browser is not None:
66
+ return
67
+ try:
68
+ from playwright.async_api import async_playwright
69
+ except ImportError as e:
70
+ raise BrowserUnavailableError(self.install_hint()) from e
71
+ try:
72
+ self._pw = await async_playwright().start()
73
+ # No WebRTC traffic outside the proxy (it could reach local addresses).
74
+ kwargs: dict = {"headless": True, "args": ["--force-webrtc-ip-handling-policy=disable_non_proxied_udp"]}
75
+ if self.settings.browser_executable:
76
+ kwargs["executable_path"] = self.settings.browser_executable
77
+ proxy = self._proxy()
78
+ if proxy:
79
+ kwargs["proxy"] = proxy
80
+ self._browser = await self._pw.chromium.launch(**kwargs)
81
+ except Exception as e:
82
+ await self._stop_pw()
83
+ raise BrowserUnavailableError(f"{self.install_hint()} (Chromium could not start: {str(e).splitlines()[0]})") from e
84
+
85
+ async def _stop_pw(self) -> None:
86
+ if self._pw is not None:
87
+ try:
88
+ await self._pw.stop()
89
+ except Exception:
90
+ pass
91
+ self._pw = None
92
+
93
+ async def aclose(self) -> None:
94
+ if self._browser is not None:
95
+ try:
96
+ await self._browser.close()
97
+ except Exception:
98
+ pass
99
+ self._browser = None
100
+ await self._stop_pw()
101
+
102
+ async def render(self, url: str) -> FetchResult:
103
+ """Load ``url`` in Chromium and return the rendered HTML."""
104
+ await self.start()
105
+ s = self.settings
106
+ context = await self._browser.new_context(
107
+ user_agent=s.user_agent, accept_downloads=False, service_workers="block", java_script_enabled=True
108
+ )
109
+ blocked: list[str] = []
110
+ state: dict = {"redirect": None}
111
+ page = await context.new_page()
112
+ timeout_ms = s.read_timeout * 1000
113
+
114
+ async def check(req_url: str, navigation: bool) -> None:
115
+ await self.fetcher.check_host(req_url)
116
+ if navigation:
117
+ await self.fetcher.check_allowed(req_url)
118
+ await self.fetcher.rate.wait(urlsplit(req_url).hostname or "")
119
+
120
+ async def handle(route, request):
121
+ req_url = request.url
122
+ try:
123
+ if urlsplit(req_url).scheme not in ("http", "https"):
124
+ await route.continue_()
125
+ return
126
+ if request.resource_type in _SKIP_TYPES:
127
+ await route.abort("blockedbyclient")
128
+ return
129
+ main_nav = request.is_navigation_request() and request.frame == page.main_frame
130
+ await check(req_url, main_nav)
131
+ current = req_url
132
+ for _ in range(s.max_redirects + 1):
133
+ resp = await route.fetch(url=current, max_redirects=0, timeout=timeout_ms)
134
+ location = resp.headers.get("location")
135
+ if resp.status not in _REDIRECTS or not location:
136
+ break
137
+ nxt = urljoin(current, location)
138
+ if main_nav:
139
+ # Navigate to the target ourselves so the page URL is right.
140
+ state["redirect"] = nxt
141
+ await route.fulfill(status=200, content_type="text/html", body="")
142
+ return
143
+ if urlsplit(nxt).scheme not in ("http", "https"):
144
+ raise FetchError(f"redirect to unsupported URL {nxt}")
145
+ await check(nxt, False)
146
+ current = nxt
147
+ else:
148
+ raise FetchError(f"too many redirects from {req_url}")
149
+ body = await resp.body()
150
+ if len(body) > s.max_bytes:
151
+ raise TooLargeError(f"{req_url} is larger than the {s.max_bytes} byte limit.")
152
+ await route.fulfill(response=resp, body=body)
153
+ except PageMintError as e:
154
+ blocked.append(f"{req_url}: {e.message}")
155
+ await _safe_abort(route)
156
+ except Exception as e: # network errors etc.
157
+ log.debug("browser request failed %s: %s", req_url, e)
158
+ await _safe_abort(route)
159
+
160
+ async def handle_ws(ws_route):
161
+ # page.route does not see WebSockets; check them here.
162
+ try:
163
+ await self.fetcher.check_host(ws_route.url.replace("ws://", "http://", 1).replace("wss://", "https://", 1))
164
+ except Exception as e:
165
+ blocked.append(f"{ws_route.url}: {getattr(e, 'message', e)}")
166
+ try:
167
+ await ws_route.close()
168
+ except Exception:
169
+ pass
170
+ return
171
+ ws_route.connect_to_server()
172
+
173
+ await context.route_web_socket("**/*", handle_ws)
174
+ await page.route("**/*", handle)
175
+ try:
176
+ target = url
177
+ for hop in range(s.max_redirects + 1):
178
+ state["redirect"] = None
179
+ blocked.clear()
180
+ try:
181
+ response = await page.goto(target, wait_until="load", timeout=s.page_timeout * 1000)
182
+ except Exception as e:
183
+ if blocked:
184
+ raise FetchError(f"The browser could not load {target}: {blocked[0]}") from e
185
+ if "Timeout" in type(e).__name__:
186
+ raise FetchTimeoutError(f"Timed out rendering {target} in the browser.") from e
187
+ raise FetchError(f"The browser could not load {target}: {str(e).splitlines()[0]}") from e
188
+ if state["redirect"]:
189
+ target = state["redirect"]
190
+ continue
191
+ break
192
+ else:
193
+ raise FetchError(f"Too many redirects (more than {s.max_redirects}) starting at {url}.")
194
+ try:
195
+ await page.wait_for_load_state("networkidle", timeout=NETWORK_IDLE_WAIT_MS)
196
+ except Exception:
197
+ pass # pages that keep polling never go idle; use what has rendered
198
+ html = await page.content()
199
+ status = response.status if response is not None else 200
200
+ final_url = page.url
201
+ finally:
202
+ await context.close()
203
+ if blocked:
204
+ log.info("browser blocked %d request(s) for %s", len(blocked), url)
205
+ return FetchResult(
206
+ url=final_url, status_code=status, headers={}, content=html.encode("utf-8"),
207
+ content_type="text/html", text=html,
208
+ )
209
+
210
+
211
+ async def _safe_abort(route) -> None:
212
+ try:
213
+ await route.abort("blockedbyclient")
214
+ except Exception:
215
+ pass
pagemint/cli.py ADDED
@@ -0,0 +1,226 @@
1
+ """The ``pagemint`` command.
2
+
3
+ pagemint run the MCP server on stdio (what ``uvx pagemint`` does)
4
+ pagemint serve same, explicit
5
+ pagemint scrape <url> print one page as Markdown
6
+ pagemint crawl <url> print (or save) several pages
7
+ pagemint map <url> print a site's URLs
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import asyncio
14
+ import json
15
+ import logging
16
+ import re
17
+ import sys
18
+ from pathlib import Path
19
+ from urllib.parse import urlsplit
20
+
21
+ from pagemint import __version__
22
+ from pagemint.client import PageMint
23
+ from pagemint.config import Settings
24
+
25
+ _SLUG_RE = re.compile(r"[^A-Za-z0-9.-]+")
26
+ NO_CAP = 10**12
27
+
28
+
29
+ def slug_for(url: str) -> str:
30
+ """A safe file name for a page URL, e.g. ``example.com_docs_intro.md``."""
31
+ p = urlsplit(url)
32
+ path = p.path.strip("/") or "index"
33
+ if p.query:
34
+ path += "_" + p.query
35
+ name = _SLUG_RE.sub("_", f"{p.netloc}_{path}").strip("_")
36
+ return name[:180] + ".md"
37
+
38
+
39
+ def _common(parser: argparse.ArgumentParser, top: bool = False) -> None:
40
+ # Subcommands repeat these options without defaults, so a value given
41
+ # before the subcommand ("pagemint --allow-private scrape URL") is kept.
42
+ d = {} if top else {"default": argparse.SUPPRESS}
43
+ g = parser.add_argument_group("politeness and safety")
44
+ g.add_argument("--rate-limit", type=float, metavar="SECONDS",
45
+ help="minimum seconds between requests to the same site (default 1)", **d)
46
+ g.add_argument("--timeout", type=float, metavar="SECONDS", help="read timeout per request (default 20)", **d)
47
+ g.add_argument("--allow-private", action="store_true",
48
+ help="allow localhost and private network addresses (off by default)", **d)
49
+ g.add_argument("--browser", metavar="PATH", help="path to a Chrome/Chromium binary for JavaScript rendering", **d)
50
+
51
+
52
+ def build_parser() -> argparse.ArgumentParser:
53
+ parser = argparse.ArgumentParser(
54
+ prog="pagemint",
55
+ description="Turn web pages into clean Markdown. With no command, runs the MCP server on stdio.",
56
+ )
57
+ parser.add_argument("--version", action="version", version=f"pagemint {__version__}")
58
+ _common(parser, top=True)
59
+ sub = parser.add_subparsers(dest="command")
60
+
61
+ p = sub.add_parser("serve", help="run the MCP server on stdio")
62
+ _common(p)
63
+
64
+ p = sub.add_parser("scrape", help="fetch one page")
65
+ p.add_argument("url")
66
+ p.add_argument("--format", choices=["markdown", "html", "links"], default="markdown")
67
+ p.add_argument("--js", action="store_true", help="render with a headless browser")
68
+ p.add_argument("--json", action="store_true", help="print the full result as JSON")
69
+ _common(p)
70
+
71
+ p = sub.add_parser("crawl", help="crawl pages starting from a URL")
72
+ p.add_argument("url")
73
+ p.add_argument("--max-pages", type=int, default=20)
74
+ p.add_argument("--max-depth", type=int, default=2)
75
+ p.add_argument("--any-domain", action="store_true", help="also follow links to other sites")
76
+ p.add_argument("--js", action="store_true", help="render every page with a headless browser")
77
+ p.add_argument("--out", metavar="DIR", help="save one .md file per page here (no size limits)")
78
+ p.add_argument("--time-limit", type=float, metavar="SECONDS", help="stop after this long (default 120)")
79
+ p.add_argument("--json", action="store_true", help="print the full result as JSON")
80
+ _common(p)
81
+
82
+ p = sub.add_parser("search", help="search the web with your SearXNG or your own API key")
83
+ p.add_argument("query")
84
+ p.add_argument("--limit", type=int, default=10, help="number of results (max 20)")
85
+ p.add_argument("--fetch", type=int, default=0, metavar="N", help="also read the top N result pages (max 5)")
86
+ p.add_argument("--provider", default="auto", choices=["auto", "searxng", "tavily", "brave"])
87
+ p.add_argument("--searxng-url", metavar="URL", help="your SearXNG, e.g. http://localhost:8888 (keys: env vars only)")
88
+ p.add_argument("--json", action="store_true", help="print the full result as JSON")
89
+ _common(p)
90
+
91
+ p = sub.add_parser("map", help="list URLs on a site")
92
+ p.add_argument("url")
93
+ p.add_argument("--limit", type=int, default=500)
94
+ p.add_argument("--json", action="store_true", help="print the full result as JSON")
95
+ _common(p)
96
+ return parser
97
+
98
+
99
+ def settings_from_args(args: argparse.Namespace) -> Settings:
100
+ overrides: dict = {}
101
+ if args.rate_limit is not None:
102
+ overrides["rate_limit"] = args.rate_limit
103
+ if args.timeout is not None:
104
+ overrides["read_timeout"] = args.timeout
105
+ overrides["connect_timeout"] = min(10.0, args.timeout)
106
+ if args.allow_private:
107
+ overrides["allow_private_networks"] = True
108
+ if args.browser:
109
+ overrides["browser_executable"] = args.browser
110
+ if getattr(args, "out", None):
111
+ overrides["page_char_cap"] = NO_CAP
112
+ overrides["result_char_cap"] = NO_CAP
113
+ if getattr(args, "searxng_url", None):
114
+ overrides["searxng_url"] = args.searxng_url
115
+ if getattr(args, "time_limit", None):
116
+ overrides["crawl_time_budget"] = args.time_limit
117
+ return Settings.from_env(**overrides)
118
+
119
+
120
+ def run_server(settings: Settings) -> None:
121
+ from pagemint.server import build_server
122
+
123
+ build_server(client_factory=lambda: PageMint(settings)).run("stdio")
124
+
125
+
126
+ def _fail(result: dict) -> int:
127
+ err = result["error"]
128
+ print(f"pagemint: {err['type']}: {err['message']}", file=sys.stderr)
129
+ return 1
130
+
131
+
132
+ def _warn(result: dict) -> None:
133
+ for w in result.get("warnings", []):
134
+ print(f"pagemint: warning: {w}", file=sys.stderr)
135
+
136
+
137
+ async def _run(args: argparse.Namespace, settings: Settings) -> int:
138
+ async with PageMint(settings) as client:
139
+ if args.command == "scrape":
140
+ r = await client.scrape(args.url, format=args.format, render_js=args.js)
141
+ if "error" in r:
142
+ return _fail(r)
143
+ if args.json:
144
+ print(json.dumps(r, ensure_ascii=False, indent=2))
145
+ elif args.format == "links":
146
+ print("\n".join(r["links"]))
147
+ else:
148
+ print(r["content"], end="" if r["content"].endswith("\n") else "\n")
149
+ _warn(r)
150
+ return 0
151
+
152
+ if args.command == "crawl":
153
+ r = await client.crawl(
154
+ args.url, max_pages=args.max_pages, max_depth=args.max_depth,
155
+ same_domain=not args.any_domain, render_js=args.js,
156
+ )
157
+ if "error" in r:
158
+ return _fail(r)
159
+ if args.out:
160
+ out = Path(args.out)
161
+ out.mkdir(parents=True, exist_ok=True)
162
+ for page in r["pages"]:
163
+ (out / slug_for(page["url"])).write_text(
164
+ f"<!-- source: {page['url']} -->\n\n{page['markdown']}", encoding="utf-8"
165
+ )
166
+ print(f"pagemint: saved {len(r['pages'])} pages to {out}", file=sys.stderr)
167
+ if args.json:
168
+ print(json.dumps(r, ensure_ascii=False, indent=2))
169
+ elif not args.out:
170
+ for page in r["pages"]:
171
+ print(f"<!-- source: {page['url']} -->\n\n{page['markdown']}\n")
172
+ for s in r["skipped"]:
173
+ print(f"pagemint: skipped {s['url']}: {s['reason']}", file=sys.stderr)
174
+ _warn(r)
175
+ return 0
176
+
177
+ if args.command == "search":
178
+ from pagemint.search.service import run_search
179
+
180
+ r = await run_search(client, args.query, limit=args.limit, fetch_pages=args.fetch, provider=args.provider)
181
+ if "error" in r:
182
+ return _fail(r)
183
+ if args.json:
184
+ print(json.dumps(r, ensure_ascii=False, indent=2))
185
+ else:
186
+ for item in r["results"]:
187
+ print(f"{item['title']} — {item['url']}")
188
+ if item["snippet"]:
189
+ print(f" {item['snippet']}")
190
+ for page in r.get("pages", []):
191
+ if "markdown" in page:
192
+ print(f"\n<!-- source: {page['url']} -->\n\n{page['markdown']}")
193
+ else:
194
+ print(f"pagemint: could not read {page['url']}: {page['error']['message']}", file=sys.stderr)
195
+ _warn(r)
196
+ return 0
197
+
198
+ if args.command == "map":
199
+ r = await client.map_site(args.url, limit=args.limit)
200
+ if "error" in r:
201
+ return _fail(r)
202
+ if args.json:
203
+ print(json.dumps(r, ensure_ascii=False, indent=2))
204
+ else:
205
+ print("\n".join(r["urls"]))
206
+ _warn(r)
207
+ return 0
208
+ return 2
209
+
210
+
211
+ def main(argv: list[str] | None = None) -> int:
212
+ parser = build_parser()
213
+ args = parser.parse_args(argv)
214
+ logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format="pagemint: %(levelname)s: %(message)s")
215
+ settings = settings_from_args(args)
216
+ if args.command in (None, "serve"):
217
+ run_server(settings)
218
+ return 0
219
+ try:
220
+ return asyncio.run(_run(args, settings))
221
+ except KeyboardInterrupt:
222
+ return 130
223
+
224
+
225
+ if __name__ == "__main__": # pragma: no cover
226
+ sys.exit(main())