pagemint 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pagemint/__init__.py +1 -0
- pagemint/__main__.py +5 -0
- pagemint/browser.py +215 -0
- pagemint/cli.py +226 -0
- pagemint/client.py +342 -0
- pagemint/config.py +73 -0
- pagemint/errors.py +55 -0
- pagemint/extract.py +204 -0
- pagemint/fetcher.py +318 -0
- pagemint/jobs/__init__.py +1 -0
- pagemint/jobs/detect.py +229 -0
- pagemint/jobs/jsonld.py +203 -0
- pagemint/jobs/model.py +105 -0
- pagemint/jobs/rank.py +157 -0
- pagemint/jobs/readers/__init__.py +11 -0
- pagemint/jobs/readers/_common.py +47 -0
- pagemint/jobs/readers/ashby.py +70 -0
- pagemint/jobs/readers/greenhouse.py +57 -0
- pagemint/jobs/readers/lever.py +64 -0
- pagemint/jobs/readers/recruitee.py +41 -0
- pagemint/jobs/readers/workday.py +83 -0
- pagemint/jobs/service.py +320 -0
- pagemint/jobs/sources/__init__.py +5 -0
- pagemint/jobs/sources/arbeitnow.py +34 -0
- pagemint/jobs/sources/himalayas.py +52 -0
- pagemint/jobs/sources/hn.py +98 -0
- pagemint/jobs/store.py +185 -0
- pagemint/links.py +85 -0
- pagemint/markdown.py +78 -0
- pagemint/ratelimit.py +41 -0
- pagemint/robots.py +221 -0
- pagemint/safety.py +137 -0
- pagemint/search/__init__.py +1 -0
- pagemint/search/_http.py +29 -0
- pagemint/search/brave.py +37 -0
- pagemint/search/model.py +38 -0
- pagemint/search/searxng.py +43 -0
- pagemint/search/secrets.py +53 -0
- pagemint/search/service.py +129 -0
- pagemint/search/tavily.py +31 -0
- pagemint/server.py +171 -0
- pagemint/sitemap.py +44 -0
- pagemint-0.1.0.dist-info/METADATA +365 -0
- pagemint-0.1.0.dist-info/RECORD +47 -0
- pagemint-0.1.0.dist-info/WHEEL +4 -0
- pagemint-0.1.0.dist-info/entry_points.txt +2 -0
- pagemint-0.1.0.dist-info/licenses/LICENSE +21 -0
pagemint/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
pagemint/__main__.py
ADDED
pagemint/browser.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""JavaScript rendering with Playwright (Chromium). Optional: ``pip install "pagemint[js]"``.
|
|
2
|
+
|
|
3
|
+
Every request the page makes passes through a route handler that applies the
|
|
4
|
+
same SSRF check as plain fetches. Chromium does not route the requests it
|
|
5
|
+
makes when following a redirect, so PageMint never lets the browser follow
|
|
6
|
+
one: the handler fetches with redirects off and follows them itself, checking
|
|
7
|
+
every hop. For the main page a redirect stops the load and PageMint navigates
|
|
8
|
+
to the (checked) target, so the page keeps its real URL. Main-frame
|
|
9
|
+
navigations also respect robots.txt and the per-host rate limit. Images, media
|
|
10
|
+
and fonts are not loaded (they add no text).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import logging
|
|
17
|
+
import os
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
from urllib.parse import urljoin, urlsplit
|
|
20
|
+
from urllib.request import getproxies
|
|
21
|
+
|
|
22
|
+
from pagemint.errors import BrowserUnavailableError, FetchError, FetchTimeoutError, PageMintError, TooLargeError
|
|
23
|
+
from pagemint.fetcher import FetchResult
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from pagemint.config import Settings
|
|
27
|
+
from pagemint.fetcher import Fetcher
|
|
28
|
+
|
|
29
|
+
log = logging.getLogger("pagemint.browser")
|
|
30
|
+
|
|
31
|
+
_SKIP_TYPES = {"image", "media", "font"}
|
|
32
|
+
_REDIRECTS = {301, 302, 303, 307, 308}
|
|
33
|
+
NETWORK_IDLE_WAIT_MS = 2000
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class BrowserRenderer:
|
|
37
|
+
def __init__(self, settings: "Settings", fetcher: "Fetcher"):
|
|
38
|
+
self.settings = settings
|
|
39
|
+
self.fetcher = fetcher
|
|
40
|
+
self._pw = None
|
|
41
|
+
self._browser = None
|
|
42
|
+
self._lock = asyncio.Lock()
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def install_hint() -> str:
|
|
46
|
+
return (
|
|
47
|
+
'JavaScript rendering needs the optional browser install: run pip install "pagemint[js]" '
|
|
48
|
+
"and then playwright install chromium."
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def _proxy(self) -> dict | None:
|
|
52
|
+
proxies = {k.lower(): v for k, v in getproxies().items()}
|
|
53
|
+
server = proxies.get("https") or proxies.get("http") or proxies.get("all")
|
|
54
|
+
if not server:
|
|
55
|
+
return None
|
|
56
|
+
proxy = {"server": server}
|
|
57
|
+
bypass = proxies.get("no") or os.environ.get("NO_PROXY") or os.environ.get("no_proxy")
|
|
58
|
+
if bypass:
|
|
59
|
+
proxy["bypass"] = bypass
|
|
60
|
+
return proxy
|
|
61
|
+
|
|
62
|
+
async def start(self) -> None:
|
|
63
|
+
"""Launch Chromium (once). Raises BrowserUnavailableError if it cannot."""
|
|
64
|
+
async with self._lock:
|
|
65
|
+
if self._browser is not None:
|
|
66
|
+
return
|
|
67
|
+
try:
|
|
68
|
+
from playwright.async_api import async_playwright
|
|
69
|
+
except ImportError as e:
|
|
70
|
+
raise BrowserUnavailableError(self.install_hint()) from e
|
|
71
|
+
try:
|
|
72
|
+
self._pw = await async_playwright().start()
|
|
73
|
+
# No WebRTC traffic outside the proxy (it could reach local addresses).
|
|
74
|
+
kwargs: dict = {"headless": True, "args": ["--force-webrtc-ip-handling-policy=disable_non_proxied_udp"]}
|
|
75
|
+
if self.settings.browser_executable:
|
|
76
|
+
kwargs["executable_path"] = self.settings.browser_executable
|
|
77
|
+
proxy = self._proxy()
|
|
78
|
+
if proxy:
|
|
79
|
+
kwargs["proxy"] = proxy
|
|
80
|
+
self._browser = await self._pw.chromium.launch(**kwargs)
|
|
81
|
+
except Exception as e:
|
|
82
|
+
await self._stop_pw()
|
|
83
|
+
raise BrowserUnavailableError(f"{self.install_hint()} (Chromium could not start: {str(e).splitlines()[0]})") from e
|
|
84
|
+
|
|
85
|
+
async def _stop_pw(self) -> None:
|
|
86
|
+
if self._pw is not None:
|
|
87
|
+
try:
|
|
88
|
+
await self._pw.stop()
|
|
89
|
+
except Exception:
|
|
90
|
+
pass
|
|
91
|
+
self._pw = None
|
|
92
|
+
|
|
93
|
+
async def aclose(self) -> None:
|
|
94
|
+
if self._browser is not None:
|
|
95
|
+
try:
|
|
96
|
+
await self._browser.close()
|
|
97
|
+
except Exception:
|
|
98
|
+
pass
|
|
99
|
+
self._browser = None
|
|
100
|
+
await self._stop_pw()
|
|
101
|
+
|
|
102
|
+
async def render(self, url: str) -> FetchResult:
|
|
103
|
+
"""Load ``url`` in Chromium and return the rendered HTML."""
|
|
104
|
+
await self.start()
|
|
105
|
+
s = self.settings
|
|
106
|
+
context = await self._browser.new_context(
|
|
107
|
+
user_agent=s.user_agent, accept_downloads=False, service_workers="block", java_script_enabled=True
|
|
108
|
+
)
|
|
109
|
+
blocked: list[str] = []
|
|
110
|
+
state: dict = {"redirect": None}
|
|
111
|
+
page = await context.new_page()
|
|
112
|
+
timeout_ms = s.read_timeout * 1000
|
|
113
|
+
|
|
114
|
+
async def check(req_url: str, navigation: bool) -> None:
|
|
115
|
+
await self.fetcher.check_host(req_url)
|
|
116
|
+
if navigation:
|
|
117
|
+
await self.fetcher.check_allowed(req_url)
|
|
118
|
+
await self.fetcher.rate.wait(urlsplit(req_url).hostname or "")
|
|
119
|
+
|
|
120
|
+
async def handle(route, request):
|
|
121
|
+
req_url = request.url
|
|
122
|
+
try:
|
|
123
|
+
if urlsplit(req_url).scheme not in ("http", "https"):
|
|
124
|
+
await route.continue_()
|
|
125
|
+
return
|
|
126
|
+
if request.resource_type in _SKIP_TYPES:
|
|
127
|
+
await route.abort("blockedbyclient")
|
|
128
|
+
return
|
|
129
|
+
main_nav = request.is_navigation_request() and request.frame == page.main_frame
|
|
130
|
+
await check(req_url, main_nav)
|
|
131
|
+
current = req_url
|
|
132
|
+
for _ in range(s.max_redirects + 1):
|
|
133
|
+
resp = await route.fetch(url=current, max_redirects=0, timeout=timeout_ms)
|
|
134
|
+
location = resp.headers.get("location")
|
|
135
|
+
if resp.status not in _REDIRECTS or not location:
|
|
136
|
+
break
|
|
137
|
+
nxt = urljoin(current, location)
|
|
138
|
+
if main_nav:
|
|
139
|
+
# Navigate to the target ourselves so the page URL is right.
|
|
140
|
+
state["redirect"] = nxt
|
|
141
|
+
await route.fulfill(status=200, content_type="text/html", body="")
|
|
142
|
+
return
|
|
143
|
+
if urlsplit(nxt).scheme not in ("http", "https"):
|
|
144
|
+
raise FetchError(f"redirect to unsupported URL {nxt}")
|
|
145
|
+
await check(nxt, False)
|
|
146
|
+
current = nxt
|
|
147
|
+
else:
|
|
148
|
+
raise FetchError(f"too many redirects from {req_url}")
|
|
149
|
+
body = await resp.body()
|
|
150
|
+
if len(body) > s.max_bytes:
|
|
151
|
+
raise TooLargeError(f"{req_url} is larger than the {s.max_bytes} byte limit.")
|
|
152
|
+
await route.fulfill(response=resp, body=body)
|
|
153
|
+
except PageMintError as e:
|
|
154
|
+
blocked.append(f"{req_url}: {e.message}")
|
|
155
|
+
await _safe_abort(route)
|
|
156
|
+
except Exception as e: # network errors etc.
|
|
157
|
+
log.debug("browser request failed %s: %s", req_url, e)
|
|
158
|
+
await _safe_abort(route)
|
|
159
|
+
|
|
160
|
+
async def handle_ws(ws_route):
|
|
161
|
+
# page.route does not see WebSockets; check them here.
|
|
162
|
+
try:
|
|
163
|
+
await self.fetcher.check_host(ws_route.url.replace("ws://", "http://", 1).replace("wss://", "https://", 1))
|
|
164
|
+
except Exception as e:
|
|
165
|
+
blocked.append(f"{ws_route.url}: {getattr(e, 'message', e)}")
|
|
166
|
+
try:
|
|
167
|
+
await ws_route.close()
|
|
168
|
+
except Exception:
|
|
169
|
+
pass
|
|
170
|
+
return
|
|
171
|
+
ws_route.connect_to_server()
|
|
172
|
+
|
|
173
|
+
await context.route_web_socket("**/*", handle_ws)
|
|
174
|
+
await page.route("**/*", handle)
|
|
175
|
+
try:
|
|
176
|
+
target = url
|
|
177
|
+
for hop in range(s.max_redirects + 1):
|
|
178
|
+
state["redirect"] = None
|
|
179
|
+
blocked.clear()
|
|
180
|
+
try:
|
|
181
|
+
response = await page.goto(target, wait_until="load", timeout=s.page_timeout * 1000)
|
|
182
|
+
except Exception as e:
|
|
183
|
+
if blocked:
|
|
184
|
+
raise FetchError(f"The browser could not load {target}: {blocked[0]}") from e
|
|
185
|
+
if "Timeout" in type(e).__name__:
|
|
186
|
+
raise FetchTimeoutError(f"Timed out rendering {target} in the browser.") from e
|
|
187
|
+
raise FetchError(f"The browser could not load {target}: {str(e).splitlines()[0]}") from e
|
|
188
|
+
if state["redirect"]:
|
|
189
|
+
target = state["redirect"]
|
|
190
|
+
continue
|
|
191
|
+
break
|
|
192
|
+
else:
|
|
193
|
+
raise FetchError(f"Too many redirects (more than {s.max_redirects}) starting at {url}.")
|
|
194
|
+
try:
|
|
195
|
+
await page.wait_for_load_state("networkidle", timeout=NETWORK_IDLE_WAIT_MS)
|
|
196
|
+
except Exception:
|
|
197
|
+
pass # pages that keep polling never go idle; use what has rendered
|
|
198
|
+
html = await page.content()
|
|
199
|
+
status = response.status if response is not None else 200
|
|
200
|
+
final_url = page.url
|
|
201
|
+
finally:
|
|
202
|
+
await context.close()
|
|
203
|
+
if blocked:
|
|
204
|
+
log.info("browser blocked %d request(s) for %s", len(blocked), url)
|
|
205
|
+
return FetchResult(
|
|
206
|
+
url=final_url, status_code=status, headers={}, content=html.encode("utf-8"),
|
|
207
|
+
content_type="text/html", text=html,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
async def _safe_abort(route) -> None:
|
|
212
|
+
try:
|
|
213
|
+
await route.abort("blockedbyclient")
|
|
214
|
+
except Exception:
|
|
215
|
+
pass
|
pagemint/cli.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""The ``pagemint`` command.
|
|
2
|
+
|
|
3
|
+
pagemint run the MCP server on stdio (what ``uvx pagemint`` does)
|
|
4
|
+
pagemint serve same, explicit
|
|
5
|
+
pagemint scrape <url> print one page as Markdown
|
|
6
|
+
pagemint crawl <url> print (or save) several pages
|
|
7
|
+
pagemint map <url> print a site's URLs
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from urllib.parse import urlsplit
|
|
20
|
+
|
|
21
|
+
from pagemint import __version__
|
|
22
|
+
from pagemint.client import PageMint
|
|
23
|
+
from pagemint.config import Settings
|
|
24
|
+
|
|
25
|
+
_SLUG_RE = re.compile(r"[^A-Za-z0-9.-]+")
|
|
26
|
+
NO_CAP = 10**12
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def slug_for(url: str) -> str:
|
|
30
|
+
"""A safe file name for a page URL, e.g. ``example.com_docs_intro.md``."""
|
|
31
|
+
p = urlsplit(url)
|
|
32
|
+
path = p.path.strip("/") or "index"
|
|
33
|
+
if p.query:
|
|
34
|
+
path += "_" + p.query
|
|
35
|
+
name = _SLUG_RE.sub("_", f"{p.netloc}_{path}").strip("_")
|
|
36
|
+
return name[:180] + ".md"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _common(parser: argparse.ArgumentParser, top: bool = False) -> None:
|
|
40
|
+
# Subcommands repeat these options without defaults, so a value given
|
|
41
|
+
# before the subcommand ("pagemint --allow-private scrape URL") is kept.
|
|
42
|
+
d = {} if top else {"default": argparse.SUPPRESS}
|
|
43
|
+
g = parser.add_argument_group("politeness and safety")
|
|
44
|
+
g.add_argument("--rate-limit", type=float, metavar="SECONDS",
|
|
45
|
+
help="minimum seconds between requests to the same site (default 1)", **d)
|
|
46
|
+
g.add_argument("--timeout", type=float, metavar="SECONDS", help="read timeout per request (default 20)", **d)
|
|
47
|
+
g.add_argument("--allow-private", action="store_true",
|
|
48
|
+
help="allow localhost and private network addresses (off by default)", **d)
|
|
49
|
+
g.add_argument("--browser", metavar="PATH", help="path to a Chrome/Chromium binary for JavaScript rendering", **d)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
53
|
+
parser = argparse.ArgumentParser(
|
|
54
|
+
prog="pagemint",
|
|
55
|
+
description="Turn web pages into clean Markdown. With no command, runs the MCP server on stdio.",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument("--version", action="version", version=f"pagemint {__version__}")
|
|
58
|
+
_common(parser, top=True)
|
|
59
|
+
sub = parser.add_subparsers(dest="command")
|
|
60
|
+
|
|
61
|
+
p = sub.add_parser("serve", help="run the MCP server on stdio")
|
|
62
|
+
_common(p)
|
|
63
|
+
|
|
64
|
+
p = sub.add_parser("scrape", help="fetch one page")
|
|
65
|
+
p.add_argument("url")
|
|
66
|
+
p.add_argument("--format", choices=["markdown", "html", "links"], default="markdown")
|
|
67
|
+
p.add_argument("--js", action="store_true", help="render with a headless browser")
|
|
68
|
+
p.add_argument("--json", action="store_true", help="print the full result as JSON")
|
|
69
|
+
_common(p)
|
|
70
|
+
|
|
71
|
+
p = sub.add_parser("crawl", help="crawl pages starting from a URL")
|
|
72
|
+
p.add_argument("url")
|
|
73
|
+
p.add_argument("--max-pages", type=int, default=20)
|
|
74
|
+
p.add_argument("--max-depth", type=int, default=2)
|
|
75
|
+
p.add_argument("--any-domain", action="store_true", help="also follow links to other sites")
|
|
76
|
+
p.add_argument("--js", action="store_true", help="render every page with a headless browser")
|
|
77
|
+
p.add_argument("--out", metavar="DIR", help="save one .md file per page here (no size limits)")
|
|
78
|
+
p.add_argument("--time-limit", type=float, metavar="SECONDS", help="stop after this long (default 120)")
|
|
79
|
+
p.add_argument("--json", action="store_true", help="print the full result as JSON")
|
|
80
|
+
_common(p)
|
|
81
|
+
|
|
82
|
+
p = sub.add_parser("search", help="search the web with your SearXNG or your own API key")
|
|
83
|
+
p.add_argument("query")
|
|
84
|
+
p.add_argument("--limit", type=int, default=10, help="number of results (max 20)")
|
|
85
|
+
p.add_argument("--fetch", type=int, default=0, metavar="N", help="also read the top N result pages (max 5)")
|
|
86
|
+
p.add_argument("--provider", default="auto", choices=["auto", "searxng", "tavily", "brave"])
|
|
87
|
+
p.add_argument("--searxng-url", metavar="URL", help="your SearXNG, e.g. http://localhost:8888 (keys: env vars only)")
|
|
88
|
+
p.add_argument("--json", action="store_true", help="print the full result as JSON")
|
|
89
|
+
_common(p)
|
|
90
|
+
|
|
91
|
+
p = sub.add_parser("map", help="list URLs on a site")
|
|
92
|
+
p.add_argument("url")
|
|
93
|
+
p.add_argument("--limit", type=int, default=500)
|
|
94
|
+
p.add_argument("--json", action="store_true", help="print the full result as JSON")
|
|
95
|
+
_common(p)
|
|
96
|
+
return parser
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def settings_from_args(args: argparse.Namespace) -> Settings:
|
|
100
|
+
overrides: dict = {}
|
|
101
|
+
if args.rate_limit is not None:
|
|
102
|
+
overrides["rate_limit"] = args.rate_limit
|
|
103
|
+
if args.timeout is not None:
|
|
104
|
+
overrides["read_timeout"] = args.timeout
|
|
105
|
+
overrides["connect_timeout"] = min(10.0, args.timeout)
|
|
106
|
+
if args.allow_private:
|
|
107
|
+
overrides["allow_private_networks"] = True
|
|
108
|
+
if args.browser:
|
|
109
|
+
overrides["browser_executable"] = args.browser
|
|
110
|
+
if getattr(args, "out", None):
|
|
111
|
+
overrides["page_char_cap"] = NO_CAP
|
|
112
|
+
overrides["result_char_cap"] = NO_CAP
|
|
113
|
+
if getattr(args, "searxng_url", None):
|
|
114
|
+
overrides["searxng_url"] = args.searxng_url
|
|
115
|
+
if getattr(args, "time_limit", None):
|
|
116
|
+
overrides["crawl_time_budget"] = args.time_limit
|
|
117
|
+
return Settings.from_env(**overrides)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def run_server(settings: Settings) -> None:
|
|
121
|
+
from pagemint.server import build_server
|
|
122
|
+
|
|
123
|
+
build_server(client_factory=lambda: PageMint(settings)).run("stdio")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _fail(result: dict) -> int:
|
|
127
|
+
err = result["error"]
|
|
128
|
+
print(f"pagemint: {err['type']}: {err['message']}", file=sys.stderr)
|
|
129
|
+
return 1
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _warn(result: dict) -> None:
|
|
133
|
+
for w in result.get("warnings", []):
|
|
134
|
+
print(f"pagemint: warning: {w}", file=sys.stderr)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
async def _run(args: argparse.Namespace, settings: Settings) -> int:
|
|
138
|
+
async with PageMint(settings) as client:
|
|
139
|
+
if args.command == "scrape":
|
|
140
|
+
r = await client.scrape(args.url, format=args.format, render_js=args.js)
|
|
141
|
+
if "error" in r:
|
|
142
|
+
return _fail(r)
|
|
143
|
+
if args.json:
|
|
144
|
+
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
145
|
+
elif args.format == "links":
|
|
146
|
+
print("\n".join(r["links"]))
|
|
147
|
+
else:
|
|
148
|
+
print(r["content"], end="" if r["content"].endswith("\n") else "\n")
|
|
149
|
+
_warn(r)
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
if args.command == "crawl":
|
|
153
|
+
r = await client.crawl(
|
|
154
|
+
args.url, max_pages=args.max_pages, max_depth=args.max_depth,
|
|
155
|
+
same_domain=not args.any_domain, render_js=args.js,
|
|
156
|
+
)
|
|
157
|
+
if "error" in r:
|
|
158
|
+
return _fail(r)
|
|
159
|
+
if args.out:
|
|
160
|
+
out = Path(args.out)
|
|
161
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
162
|
+
for page in r["pages"]:
|
|
163
|
+
(out / slug_for(page["url"])).write_text(
|
|
164
|
+
f"<!-- source: {page['url']} -->\n\n{page['markdown']}", encoding="utf-8"
|
|
165
|
+
)
|
|
166
|
+
print(f"pagemint: saved {len(r['pages'])} pages to {out}", file=sys.stderr)
|
|
167
|
+
if args.json:
|
|
168
|
+
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
169
|
+
elif not args.out:
|
|
170
|
+
for page in r["pages"]:
|
|
171
|
+
print(f"<!-- source: {page['url']} -->\n\n{page['markdown']}\n")
|
|
172
|
+
for s in r["skipped"]:
|
|
173
|
+
print(f"pagemint: skipped {s['url']}: {s['reason']}", file=sys.stderr)
|
|
174
|
+
_warn(r)
|
|
175
|
+
return 0
|
|
176
|
+
|
|
177
|
+
if args.command == "search":
|
|
178
|
+
from pagemint.search.service import run_search
|
|
179
|
+
|
|
180
|
+
r = await run_search(client, args.query, limit=args.limit, fetch_pages=args.fetch, provider=args.provider)
|
|
181
|
+
if "error" in r:
|
|
182
|
+
return _fail(r)
|
|
183
|
+
if args.json:
|
|
184
|
+
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
185
|
+
else:
|
|
186
|
+
for item in r["results"]:
|
|
187
|
+
print(f"{item['title']} — {item['url']}")
|
|
188
|
+
if item["snippet"]:
|
|
189
|
+
print(f" {item['snippet']}")
|
|
190
|
+
for page in r.get("pages", []):
|
|
191
|
+
if "markdown" in page:
|
|
192
|
+
print(f"\n<!-- source: {page['url']} -->\n\n{page['markdown']}")
|
|
193
|
+
else:
|
|
194
|
+
print(f"pagemint: could not read {page['url']}: {page['error']['message']}", file=sys.stderr)
|
|
195
|
+
_warn(r)
|
|
196
|
+
return 0
|
|
197
|
+
|
|
198
|
+
if args.command == "map":
|
|
199
|
+
r = await client.map_site(args.url, limit=args.limit)
|
|
200
|
+
if "error" in r:
|
|
201
|
+
return _fail(r)
|
|
202
|
+
if args.json:
|
|
203
|
+
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
204
|
+
else:
|
|
205
|
+
print("\n".join(r["urls"]))
|
|
206
|
+
_warn(r)
|
|
207
|
+
return 0
|
|
208
|
+
return 2
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def main(argv: list[str] | None = None) -> int:
|
|
212
|
+
parser = build_parser()
|
|
213
|
+
args = parser.parse_args(argv)
|
|
214
|
+
logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format="pagemint: %(levelname)s: %(message)s")
|
|
215
|
+
settings = settings_from_args(args)
|
|
216
|
+
if args.command in (None, "serve"):
|
|
217
|
+
run_server(settings)
|
|
218
|
+
return 0
|
|
219
|
+
try:
|
|
220
|
+
return asyncio.run(_run(args, settings))
|
|
221
|
+
except KeyboardInterrupt:
|
|
222
|
+
return 130
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
if __name__ == "__main__": # pragma: no cover
|
|
226
|
+
sys.exit(main())
|