mesharc 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mesharc/__init__.py +542 -0
- mesharc/mcp.py +208 -0
- mesharc/py.typed +0 -0
- mesharc-0.1.2.dist-info/METADATA +274 -0
- mesharc-0.1.2.dist-info/RECORD +9 -0
- mesharc-0.1.2.dist-info/WHEEL +5 -0
- mesharc-0.1.2.dist-info/entry_points.txt +2 -0
- mesharc-0.1.2.dist-info/licenses/LICENSE +21 -0
- mesharc-0.1.2.dist-info/top_level.txt +1 -0
mesharc/__init__.py
ADDED
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
"""mesharc: the Python client for the MeshArc API.
|
|
2
|
+
|
|
3
|
+
from mesharc import MeshArc
|
|
4
|
+
arc = MeshArc("mesharc_...") # or MeshArc() with MESHARC_API_KEY set
|
|
5
|
+
|
|
6
|
+
page = arc.scrape("https://example.com/pricing") # one URL, waited for
|
|
7
|
+
batch = arc.scrape(["https://a.com/", "https://b.com/x"]) # many URLs, one row each
|
|
8
|
+
|
|
9
|
+
job = arc.crawl("https://docs.example.com", limit=200) # a site, no project needed
|
|
10
|
+
for page in job.pages(): # pages as they land
|
|
11
|
+
print(page["url"], page["words"])
|
|
12
|
+
project = job.keep(name="Docs", schedule="weekly") # keep it, if it is worth watching
|
|
13
|
+
|
|
14
|
+
for entry in arc.map("https://docs.example.com"): # what the site declares
|
|
15
|
+
print(entry["url"], entry["lastmod"])
|
|
16
|
+
|
|
17
|
+
Every method is one API call (or a poll loop where ``wait`` applies) and
|
|
18
|
+
returns the API's JSON unwrapped, so the API reference at
|
|
19
|
+
https://mesharc.dev/docs/api applies to every return value. Refused
|
|
20
|
+
requests raise ``MeshArcError``; a job the client stopped waiting for
|
|
21
|
+
raises ``MeshArcTimeoutError``.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import os
|
|
27
|
+
import random
|
|
28
|
+
import time
|
|
29
|
+
from typing import Any, Dict, Iterable, Iterator, List, Optional, Union
|
|
30
|
+
from urllib.parse import parse_qs, urlparse
|
|
31
|
+
|
|
32
|
+
import httpx
|
|
33
|
+
|
|
34
|
+
__version__ = "0.1.2"
|
|
35
|
+
__all__ = ["MeshArc", "MeshArcError", "MeshArcTimeoutError", "Crawl"]
|
|
36
|
+
|
|
37
|
+
DEFAULT_BASE = "https://api.mesharc.dev"
|
|
38
|
+
DEFAULT_TIMEOUT = 150.0
|
|
39
|
+
DEFAULT_MAX_RETRIES = 2
|
|
40
|
+
RETRY_STATUSES = frozenset({429, 502, 503, 504})
|
|
41
|
+
|
|
42
|
+
Json = Dict[str, Any]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class MeshArcError(Exception):
|
|
46
|
+
"""A request the API refused, or a job that ended in error.
|
|
47
|
+
|
|
48
|
+
``status`` is the HTTP status (0 for a network failure or a timeout),
|
|
49
|
+
``code`` the machine-readable reason (validation, unauthorized,
|
|
50
|
+
plan_limit, not_found, rate_limited, ...) and ``request_id`` the id
|
|
51
|
+
the API logged the request under -- quote it to support.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def __init__(self, status: int, detail: Any, code: str = "", request_id: str = "") -> None:
|
|
55
|
+
super().__init__(f"{status}: {detail}" + (f" [{request_id}]" if request_id else ""))
|
|
56
|
+
self.status = status
|
|
57
|
+
self.detail = detail
|
|
58
|
+
self.code = code
|
|
59
|
+
self.request_id = request_id
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class MeshArcTimeoutError(MeshArcError, TimeoutError):
|
|
63
|
+
"""A job that was still running when the client stopped waiting.
|
|
64
|
+
|
|
65
|
+
``job_id`` names the job; poll it later with the matching ``get``
|
|
66
|
+
method. Catchable as either ``MeshArcError`` or ``TimeoutError``.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(self, message: str, job_id: str = "") -> None:
|
|
70
|
+
super().__init__(0, message, "timeout")
|
|
71
|
+
self.job_id = job_id
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _running(status: Any) -> bool:
|
|
75
|
+
return status in ("queued", "running")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _cursor_of(next_url: str) -> str:
|
|
79
|
+
return (parse_qs(urlparse(next_url).query).get("cursor") or [""])[0]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _Http:
|
|
83
|
+
"""The transport: one httpx client, bearer auth, JSON errors, retries."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, api_key: str, base_url: str, timeout: float, max_retries: int) -> None:
|
|
86
|
+
self._c = httpx.Client(
|
|
87
|
+
base_url=base_url.rstrip("/") + "/api/v1",
|
|
88
|
+
headers={"Authorization": f"Bearer {api_key}", "User-Agent": f"mesharc-python/{__version__}"},
|
|
89
|
+
timeout=timeout,
|
|
90
|
+
)
|
|
91
|
+
self._max_retries = max(0, int(max_retries))
|
|
92
|
+
|
|
93
|
+
def __call__(self, method: str, path: str, idempotency_key: Optional[str] = None, **kw: Any) -> Any:
|
|
94
|
+
if idempotency_key:
|
|
95
|
+
kw.setdefault("headers", {})["Idempotency-Key"] = str(idempotency_key)
|
|
96
|
+
# A GET or DELETE is safe to repeat; a POST only when it carries an idempotency key.
|
|
97
|
+
repeatable = method in ("GET", "DELETE") or bool(idempotency_key)
|
|
98
|
+
attempt = 0
|
|
99
|
+
while True:
|
|
100
|
+
try:
|
|
101
|
+
r = self._c.request(method, path, **kw)
|
|
102
|
+
except httpx.TimeoutException as exc:
|
|
103
|
+
if repeatable and attempt < self._max_retries:
|
|
104
|
+
attempt += 1
|
|
105
|
+
time.sleep(self._backoff(attempt))
|
|
106
|
+
continue
|
|
107
|
+
raise MeshArcError(0, f"request timed out: {exc}", "timeout") from exc
|
|
108
|
+
except httpx.HTTPError as exc:
|
|
109
|
+
if repeatable and attempt < self._max_retries:
|
|
110
|
+
attempt += 1
|
|
111
|
+
time.sleep(self._backoff(attempt))
|
|
112
|
+
continue
|
|
113
|
+
raise MeshArcError(0, f"network error: {exc}", "network") from exc
|
|
114
|
+
if r.status_code in RETRY_STATUSES and repeatable and attempt < self._max_retries:
|
|
115
|
+
attempt += 1
|
|
116
|
+
time.sleep(self._retry_after(r) or self._backoff(attempt))
|
|
117
|
+
continue
|
|
118
|
+
if r.status_code >= 400:
|
|
119
|
+
raise self._error(r)
|
|
120
|
+
if r.status_code == 204 or not r.content:
|
|
121
|
+
return None
|
|
122
|
+
return r.json()
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def _backoff(attempt: int) -> float:
|
|
126
|
+
return 0.5 * (2 ** (attempt - 1)) + random.random() * 0.25
|
|
127
|
+
|
|
128
|
+
@staticmethod
|
|
129
|
+
def _retry_after(r: httpx.Response) -> Optional[float]:
|
|
130
|
+
header = r.headers.get("Retry-After")
|
|
131
|
+
if not header:
|
|
132
|
+
return None
|
|
133
|
+
try:
|
|
134
|
+
return max(0.0, float(header))
|
|
135
|
+
except ValueError:
|
|
136
|
+
return None
|
|
137
|
+
|
|
138
|
+
@staticmethod
|
|
139
|
+
def _error(r: httpx.Response) -> MeshArcError:
|
|
140
|
+
code, request_id = "", r.headers.get("X-Request-Id", "")
|
|
141
|
+
try:
|
|
142
|
+
body = r.json()
|
|
143
|
+
detail = body.get("error") or body.get("detail") or r.text[:200]
|
|
144
|
+
code = body.get("code", "") or ""
|
|
145
|
+
request_id = body.get("request_id") or request_id
|
|
146
|
+
except Exception: # noqa: BLE001 - a non-JSON body is its own detail
|
|
147
|
+
detail = r.text[:200]
|
|
148
|
+
return MeshArcError(r.status_code, detail, code, request_id)
|
|
149
|
+
|
|
150
|
+
def stream(self, method: str, path: str, **kw: Any) -> Any:
|
|
151
|
+
return self._c.stream(method, path, **kw)
|
|
152
|
+
|
|
153
|
+
def close(self) -> None:
|
|
154
|
+
self._c.close()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class _Projects:
|
|
158
|
+
def __init__(self, http: _Http) -> None:
|
|
159
|
+
self._h = http
|
|
160
|
+
|
|
161
|
+
def list(self) -> List[Json]:
|
|
162
|
+
return self._h("GET", "/projects")
|
|
163
|
+
|
|
164
|
+
def create(self, seed: str, name: Optional[str] = None, schedule: str = "manual", retention: str = "90d",
|
|
165
|
+
config: Optional[Json] = None) -> Json:
|
|
166
|
+
body: Json = {"seed": seed, "schedule": schedule, "retention": retention}
|
|
167
|
+
if name:
|
|
168
|
+
body["name"] = name
|
|
169
|
+
if config:
|
|
170
|
+
body["config"] = config
|
|
171
|
+
return self._h("POST", "/projects", json=body)
|
|
172
|
+
|
|
173
|
+
def get(self, project_id: str) -> Json:
|
|
174
|
+
return self._h("GET", f"/projects/{project_id}")
|
|
175
|
+
|
|
176
|
+
def update(self, project_id: str, **fields: Any) -> Json:
|
|
177
|
+
return self._h("PATCH", f"/projects/{project_id}", json=fields)
|
|
178
|
+
|
|
179
|
+
def delete(self, project_id: str) -> None:
|
|
180
|
+
self._h("DELETE", f"/projects/{project_id}")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class _Runs:
|
|
184
|
+
def __init__(self, http: _Http) -> None:
|
|
185
|
+
self._h = http
|
|
186
|
+
|
|
187
|
+
def list(self, project_id: str, limit: int = 25) -> List[Json]:
|
|
188
|
+
return self._h("GET", f"/projects/{project_id}/runs", params={"limit": limit})
|
|
189
|
+
|
|
190
|
+
def get(self, project_id: str, run_id: str) -> Json:
|
|
191
|
+
return self._h("GET", f"/projects/{project_id}/runs/{run_id}")
|
|
192
|
+
|
|
193
|
+
def start(self, project_id: str, wait: bool = False, poll: float = 3.0, timeout: float = 3600) -> Json:
|
|
194
|
+
run = self._h("POST", f"/projects/{project_id}/runs", json={"trigger": "api"})
|
|
195
|
+
return self.wait(project_id, run["id"], poll, timeout) if wait else run
|
|
196
|
+
|
|
197
|
+
def wait(self, project_id: str, run_id: str, poll: float = 3.0, timeout: float = 3600) -> Json:
|
|
198
|
+
deadline = time.time() + timeout
|
|
199
|
+
while True:
|
|
200
|
+
run = self.get(project_id, run_id)
|
|
201
|
+
if run["status"] != "running" and not run.get("queued"):
|
|
202
|
+
return run
|
|
203
|
+
if time.time() >= deadline:
|
|
204
|
+
raise MeshArcTimeoutError(f"run {run_id} is still {run['status']} after {timeout}s", run_id)
|
|
205
|
+
time.sleep(poll)
|
|
206
|
+
|
|
207
|
+
def cancel(self, project_id: str, run_id: str) -> Json:
|
|
208
|
+
return self._h("POST", f"/projects/{project_id}/runs/{run_id}/cancel")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class Crawl:
|
|
212
|
+
"""A crawl started by ``arc.crawl(url)``: a handle on a running job.
|
|
213
|
+
|
|
214
|
+
``wait()`` blocks until it finishes; ``pages()`` yields pages as they
|
|
215
|
+
land, following the cursor, and ends when the job does; ``keep()``
|
|
216
|
+
turns the one-shot crawl into a project; ``cancel()`` stops it.
|
|
217
|
+
"""
|
|
218
|
+
|
|
219
|
+
def __init__(self, http: _Http, envelope: Json) -> None:
|
|
220
|
+
self._h = http
|
|
221
|
+
self.id: str = envelope["id"]
|
|
222
|
+
self.url: str = envelope.get("url", "")
|
|
223
|
+
self.project_id: str = envelope.get("projectId", "")
|
|
224
|
+
#: Returned once, at creation: the secret the crawl's webhook messages are signed with.
|
|
225
|
+
self.webhook_secret: str = envelope.get("webhookSecret", "")
|
|
226
|
+
self.envelope: Json = envelope
|
|
227
|
+
|
|
228
|
+
def __repr__(self) -> str:
|
|
229
|
+
return f"<Crawl {self.id[:8]} {self.url} {self.envelope.get('status')}>"
|
|
230
|
+
|
|
231
|
+
@property
|
|
232
|
+
def status(self) -> str:
|
|
233
|
+
return self.envelope.get("status", "queued")
|
|
234
|
+
|
|
235
|
+
def refresh(self, formats: str = "markdown") -> Json:
|
|
236
|
+
"""The envelope as it stands now, without its pages."""
|
|
237
|
+
self.envelope = self._h("GET", f"/crawl/{self.id}", params={"limit": 1, "formats": formats})
|
|
238
|
+
return self.envelope
|
|
239
|
+
|
|
240
|
+
def wait(self, poll: float = 3.0, timeout: float = 3600, formats: str = "markdown") -> Json:
|
|
241
|
+
"""Block until the crawl finishes. Returns the envelope."""
|
|
242
|
+
deadline = time.time() + timeout
|
|
243
|
+
while True:
|
|
244
|
+
e = self.refresh(formats)
|
|
245
|
+
if not _running(e["status"]):
|
|
246
|
+
return e
|
|
247
|
+
if time.time() >= deadline:
|
|
248
|
+
raise MeshArcTimeoutError(f"crawl {self.id} is still {e['status']} after {timeout}s", self.id)
|
|
249
|
+
time.sleep(poll)
|
|
250
|
+
|
|
251
|
+
def pages(self, formats: str = "markdown", limit: int = 25, wait: bool = True, poll: float = 3.0,
|
|
252
|
+
timeout: float = 3600) -> Iterator[Json]:
|
|
253
|
+
"""Every page of the crawl, oldest first.
|
|
254
|
+
|
|
255
|
+
While the crawl runs this waits for more pages rather than
|
|
256
|
+
stopping; ``wait=False`` yields what exists and returns.
|
|
257
|
+
"""
|
|
258
|
+
deadline = time.time() + timeout
|
|
259
|
+
cursor: Optional[str] = None
|
|
260
|
+
while True:
|
|
261
|
+
params: Dict[str, Any] = {"formats": formats, "limit": limit}
|
|
262
|
+
if cursor:
|
|
263
|
+
params["cursor"] = cursor
|
|
264
|
+
page = self._h("GET", f"/crawl/{self.id}", params=params)
|
|
265
|
+
self.envelope = {k: v for k, v in page.items() if k != "data"}
|
|
266
|
+
for row in page["data"]:
|
|
267
|
+
yield row
|
|
268
|
+
# The cursor marks where this page ended, so a running crawl is never re-read from the top.
|
|
269
|
+
cursor = page.get("cursor") or cursor
|
|
270
|
+
if page.get("next"):
|
|
271
|
+
cursor = _cursor_of(page["next"]) or cursor
|
|
272
|
+
continue
|
|
273
|
+
if not wait or not _running(page["status"]):
|
|
274
|
+
return
|
|
275
|
+
if time.time() >= deadline:
|
|
276
|
+
raise MeshArcTimeoutError(f"crawl {self.id} is still {page['status']} after {timeout}s", self.id)
|
|
277
|
+
time.sleep(poll)
|
|
278
|
+
|
|
279
|
+
def keep(self, name: Optional[str] = None, schedule: Optional[str] = None,
|
|
280
|
+
retention: Optional[str] = None) -> Json:
|
|
281
|
+
"""Make this one-shot crawl a project. Its run and pages are already in place."""
|
|
282
|
+
body = {k: v for k, v in (("name", name), ("schedule", schedule), ("retention", retention)) if v}
|
|
283
|
+
return self._h("POST", f"/crawl/{self.id}/keep", json=body)
|
|
284
|
+
|
|
285
|
+
def cancel(self) -> Json:
|
|
286
|
+
self._h("DELETE", f"/crawl/{self.id}")
|
|
287
|
+
self.envelope["status"] = "cancelled"
|
|
288
|
+
return self.envelope
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class MeshArc:
|
|
292
|
+
"""One client, one API key, one workspace.
|
|
293
|
+
|
|
294
|
+
``api_key`` falls back to the ``MESHARC_API_KEY`` environment variable.
|
|
295
|
+
``timeout`` is the HTTP timeout per request in seconds; ``max_retries``
|
|
296
|
+
how many times a request that is safe to repeat is retried on 429,
|
|
297
|
+
502, 503, 504 or a network failure. ``base_url`` is for MeshArc's own
|
|
298
|
+
test environments; the hosted API needs nothing there.
|
|
299
|
+
"""
|
|
300
|
+
|
|
301
|
+
def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None,
|
|
302
|
+
timeout: float = DEFAULT_TIMEOUT, max_retries: int = DEFAULT_MAX_RETRIES) -> None:
|
|
303
|
+
key = api_key or os.environ.get("MESHARC_API_KEY") or ""
|
|
304
|
+
if not key:
|
|
305
|
+
raise ValueError("An API key is required: MeshArc('mesharc_...') or set MESHARC_API_KEY.")
|
|
306
|
+
base = base_url or os.environ.get("MESHARC_API_URL") or DEFAULT_BASE
|
|
307
|
+
self._h = _Http(key, base, timeout, max_retries)
|
|
308
|
+
self.projects = _Projects(self._h)
|
|
309
|
+
self.runs = _Runs(self._h)
|
|
310
|
+
|
|
311
|
+
# ------------------------------------------------------------ one or many URLs
|
|
312
|
+
|
|
313
|
+
def extract(self, url: str, config: Optional[Json] = None, wait: bool = True, poll: float = 2.0,
|
|
314
|
+
timeout: float = 300) -> Json:
|
|
315
|
+
"""One URL with every format, as the app's playground reads it."""
|
|
316
|
+
body: Json = {"url": url}
|
|
317
|
+
if config:
|
|
318
|
+
body["config"] = config
|
|
319
|
+
job = self._h("POST", "/playground/extract", json=body)
|
|
320
|
+
if not wait:
|
|
321
|
+
return job
|
|
322
|
+
deadline = time.time() + timeout
|
|
323
|
+
while True:
|
|
324
|
+
r = self._h("GET", f"/playground/{job['id']}")
|
|
325
|
+
if not _running(r["status"]):
|
|
326
|
+
return r
|
|
327
|
+
if time.time() >= deadline:
|
|
328
|
+
raise MeshArcTimeoutError(f"extraction {job['id']} is still {r['status']} after {timeout}s", job["id"])
|
|
329
|
+
time.sleep(poll)
|
|
330
|
+
|
|
331
|
+
def scrape(self, urls: Union[str, Iterable[str]], config: Optional[Json] = None,
|
|
332
|
+
webhook_url: Optional[str] = None, formats: str = "markdown", wait: bool = True,
|
|
333
|
+
poll: float = 3.0, timeout: float = 3600, idempotency_key: Optional[str] = None) -> Json:
|
|
334
|
+
"""URLs in, their content out; no project.
|
|
335
|
+
|
|
336
|
+
One URL (a string) returns the page itself: the API holds the
|
|
337
|
+
request open until the page comes back. A list is a batch and
|
|
338
|
+
returns the finished batch, one row per URL, unless ``wait`` is
|
|
339
|
+
False.
|
|
340
|
+
"""
|
|
341
|
+
if isinstance(urls, str):
|
|
342
|
+
return self.scrape_one(urls, config=config, formats=formats, wait=wait, poll=poll,
|
|
343
|
+
timeout=timeout, idempotency_key=idempotency_key)
|
|
344
|
+
body: Json = {"urls": list(urls)}
|
|
345
|
+
if config:
|
|
346
|
+
body["config"] = config
|
|
347
|
+
if webhook_url:
|
|
348
|
+
body["webhook_url"] = webhook_url
|
|
349
|
+
batch = self._h("POST", "/scrape", json=body, idempotency_key=idempotency_key)
|
|
350
|
+
if not wait:
|
|
351
|
+
return batch
|
|
352
|
+
return self.batch(batch["id"], formats=formats, wait=True, poll=poll, timeout=timeout)
|
|
353
|
+
|
|
354
|
+
def scrape_one(self, url: str, config: Optional[Json] = None, formats: str = "markdown", wait: bool = True,
|
|
355
|
+
timeout_s: int = 60, poll: float = 2.0, timeout: float = 600,
|
|
356
|
+
idempotency_key: Optional[str] = None) -> Json:
|
|
357
|
+
"""One URL, waited for. Returns the page; ``wait=False`` returns the job envelope.
|
|
358
|
+
|
|
359
|
+
``timeout_s`` is how long the API holds the request open for the
|
|
360
|
+
page (60 by default, 120 at most); a slower page comes back as a
|
|
361
|
+
job, which is then polled every ``poll`` seconds for up to
|
|
362
|
+
``timeout`` seconds.
|
|
363
|
+
"""
|
|
364
|
+
body: Json = {"url": url, "formats": formats, "timeout": timeout_s}
|
|
365
|
+
if config:
|
|
366
|
+
body["config"] = config
|
|
367
|
+
out = self._h("POST", "/scrape", json=body, idempotency_key=idempotency_key)
|
|
368
|
+
if not wait:
|
|
369
|
+
return out
|
|
370
|
+
deadline = time.time() + timeout
|
|
371
|
+
while True:
|
|
372
|
+
if out.get("status") == "done":
|
|
373
|
+
return (out.get("data") or [{}])[0]
|
|
374
|
+
if not _running(out.get("status")):
|
|
375
|
+
raise MeshArcError(502, out.get("error") or f"scrape {out.get('status')}", "job_failed")
|
|
376
|
+
if time.time() >= deadline:
|
|
377
|
+
raise MeshArcTimeoutError(f"scrape {out['id']} is still {out['status']} after {timeout}s", out["id"])
|
|
378
|
+
time.sleep(poll)
|
|
379
|
+
out = self._h("GET", f"/scrape/{out['id']}", params={"formats": formats})
|
|
380
|
+
|
|
381
|
+
def batch(self, batch_id: str, formats: str = "markdown", wait: bool = False, poll: float = 3.0,
|
|
382
|
+
timeout: float = 3600) -> Json:
|
|
383
|
+
"""A batch started earlier. With ``wait``, returns once every row is in."""
|
|
384
|
+
deadline = time.time() + timeout
|
|
385
|
+
while True:
|
|
386
|
+
r = self._h("GET", f"/scrape/{batch_id}", params={"formats": formats})
|
|
387
|
+
if not wait or not _running(r["status"]):
|
|
388
|
+
return r
|
|
389
|
+
if time.time() >= deadline:
|
|
390
|
+
raise MeshArcTimeoutError(f"batch {batch_id} is still {r['status']} after {timeout}s", batch_id)
|
|
391
|
+
time.sleep(poll)
|
|
392
|
+
|
|
393
|
+
# ---------------------------------------------------------------- a whole site
|
|
394
|
+
|
|
395
|
+
def crawl(self, url: str, wait: bool = False, poll: float = 3.0, timeout: float = 3600,
|
|
396
|
+
idempotency_key: Optional[str] = None, **opts: Any) -> Crawl:
|
|
397
|
+
"""Crawl a site once, with no project to set up first.
|
|
398
|
+
|
|
399
|
+
Returns a ``Crawl`` handle as soon as the job is queued; ``wait=True``
|
|
400
|
+
blocks until it finishes. ``opts`` are the request's own names --
|
|
401
|
+
``limit``, ``maxDepth``, ``includePaths``, ``excludePaths``,
|
|
402
|
+
``crawlMode``, ``maxAge``, ``maxTier``, ``scrapeOptions``,
|
|
403
|
+
``webhook`` -- and ``config`` takes any project setting directly.
|
|
404
|
+
"""
|
|
405
|
+
job = Crawl(self._h, self._h("POST", "/crawl", json={"url": url, **opts}, idempotency_key=idempotency_key))
|
|
406
|
+
if wait:
|
|
407
|
+
job.wait(poll=poll, timeout=timeout)
|
|
408
|
+
return job
|
|
409
|
+
|
|
410
|
+
def get_crawl(self, crawl_id: str) -> Crawl:
|
|
411
|
+
"""A handle on a crawl started earlier or elsewhere."""
|
|
412
|
+
return Crawl(self._h, self._h("GET", f"/crawl/{crawl_id}", params={"limit": 1}))
|
|
413
|
+
|
|
414
|
+
def map(self, url: str, **opts: Any) -> List[Json]:
|
|
415
|
+
"""Every URL a site declares in its sitemaps, as ``{url, lastmod, changefreq, source, file, section}``."""
|
|
416
|
+
return self.map_details(url, **opts)["data"]
|
|
417
|
+
|
|
418
|
+
def map_details(self, url: str, search: Optional[str] = None, limit: Optional[int] = None,
|
|
419
|
+
timeout_s: int = 10, poll: float = 2.0, timeout: float = 300, **opts: Any) -> Json:
|
|
420
|
+
"""A map with everything the API said about it: how the sitemaps
|
|
421
|
+
were found, the totals, what robots.txt allowed, and the URLs
|
|
422
|
+
under ``data``. ``timeout_s`` is how long the API waits for the
|
|
423
|
+
sitemap tree before answering with a job (10 by default, 60 at
|
|
424
|
+
most), which is then polled.
|
|
425
|
+
"""
|
|
426
|
+
body: Json = {"url": url, "timeout": timeout_s, **opts}
|
|
427
|
+
if search:
|
|
428
|
+
body["search"] = search
|
|
429
|
+
if limit:
|
|
430
|
+
body["limit"] = limit
|
|
431
|
+
out = self._h("POST", "/map", json=body)
|
|
432
|
+
params = {k: v for k, v in (("search", search), ("limit", limit)) if v}
|
|
433
|
+
deadline = time.time() + timeout
|
|
434
|
+
while out.get("status") == "running":
|
|
435
|
+
if time.time() >= deadline:
|
|
436
|
+
raise MeshArcTimeoutError(f"map {out['id']} is still reading {url} after {timeout}s", out["id"])
|
|
437
|
+
time.sleep(poll)
|
|
438
|
+
out = self._h("GET", f"/map/{out['id']}", params=params or None)
|
|
439
|
+
if out.get("status") != "done":
|
|
440
|
+
raise MeshArcError(502, out.get("error") or "no sitemap could be read", "job_failed")
|
|
441
|
+
return out
|
|
442
|
+
|
|
443
|
+
# ---------------------------------------------------------- what a project holds
|
|
444
|
+
|
|
445
|
+
def pages(self, project_id: str, run_id: Optional[str] = None) -> Json:
|
|
446
|
+
"""The pages of a run (the latest finished run by default)."""
|
|
447
|
+
return self._h("GET", f"/projects/{project_id}/pages", params={"run_id": run_id} if run_id else None)
|
|
448
|
+
|
|
449
|
+
def page(self, project_id: str, url: str, run_id: Optional[str] = None) -> Json:
|
|
450
|
+
"""One page in full: bodies, head fields, structured fields, versions."""
|
|
451
|
+
params: Dict[str, Any] = {"url": url}
|
|
452
|
+
if run_id:
|
|
453
|
+
params["run_id"] = run_id
|
|
454
|
+
return self._h("GET", f"/projects/{project_id}/pages/content", params=params)
|
|
455
|
+
|
|
456
|
+
def changes(self, project_id: str, run_id: Optional[str] = None) -> Json:
|
|
457
|
+
"""The change record of a run against the run before it."""
|
|
458
|
+
return self._h("GET", f"/projects/{project_id}/changes", params={"run_id": run_id} if run_id else None)
|
|
459
|
+
|
|
460
|
+
def page_diff(self, project_id: str, url: str, run_id: Optional[str] = None) -> Json:
|
|
461
|
+
"""The word-level diff of one page against the run before."""
|
|
462
|
+
params: Dict[str, Any] = {"url": url}
|
|
463
|
+
if run_id:
|
|
464
|
+
params["run_id"] = run_id
|
|
465
|
+
return self._h("GET", f"/projects/{project_id}/changes/page", params=params)
|
|
466
|
+
|
|
467
|
+
def search(self, project_id: str, q: str, mode: str = "content", run_id: Optional[str] = None) -> Json:
|
|
468
|
+
"""Which pages say this (``content``: words, "phrases") or contain this (``selector``: CSS or XPath)."""
|
|
469
|
+
return self._h("POST", f"/projects/{project_id}/pages/search", json={"mode": mode, "q": q, "run_id": run_id})
|
|
470
|
+
|
|
471
|
+
def recrawl(self, project_id: str, urls: Iterable[str]) -> Json:
|
|
472
|
+
"""Fetch these pages again now, as a scoped run."""
|
|
473
|
+
return self._h("POST", f"/projects/{project_id}/pages/recrawl", json={"urls": list(urls)})
|
|
474
|
+
|
|
475
|
+
def sources(self, project_id: str) -> Json:
|
|
476
|
+
"""The seed, sitemap, URL list, feeds and patterns, with what the last run found through each."""
|
|
477
|
+
return self._h("GET", f"/projects/{project_id}/sources")
|
|
478
|
+
|
|
479
|
+
def export(self, project_id: str, path: str, dataset: str = "pages", fmt: str = "jsonl",
|
|
480
|
+
run_id: Optional[str] = None, urls: Optional[Iterable[str]] = None) -> str:
|
|
481
|
+
"""Stream a dataset (pages, markdown, changes, fields, sitemap) to ``path`` as jsonl or csv.
|
|
482
|
+
|
|
483
|
+
``urls`` restricts the export to those pages. Returns ``path``.
|
|
484
|
+
"""
|
|
485
|
+
if urls:
|
|
486
|
+
ctx = self._h.stream("POST", f"/projects/{project_id}/export",
|
|
487
|
+
json={"dataset": dataset, "format": fmt, "run_id": run_id, "urls": list(urls)})
|
|
488
|
+
else:
|
|
489
|
+
params: Dict[str, Any] = {"dataset": dataset, "format": fmt}
|
|
490
|
+
if run_id:
|
|
491
|
+
params["run_id"] = run_id
|
|
492
|
+
ctx = self._h.stream("GET", f"/projects/{project_id}/export", params=params)
|
|
493
|
+
with ctx as r:
|
|
494
|
+
if r.status_code >= 400:
|
|
495
|
+
r.read()
|
|
496
|
+
raise self._h._error(r)
|
|
497
|
+
with open(path, "wb") as fh:
|
|
498
|
+
for chunk in r.iter_bytes():
|
|
499
|
+
fh.write(chunk)
|
|
500
|
+
return path
|
|
501
|
+
|
|
502
|
+
# --------------------------------------------------------------------- workspace
|
|
503
|
+
|
|
504
|
+
def me(self) -> Json:
|
|
505
|
+
"""The workspace, its plan and limits, and what this key may do."""
|
|
506
|
+
return self._h("GET", "/me")
|
|
507
|
+
|
|
508
|
+
def keys(self) -> List[Json]:
|
|
509
|
+
return self._h("GET", "/me/keys")
|
|
510
|
+
|
|
511
|
+
def create_key(self, name: str = "default", scopes: Optional[List[str]] = None,
|
|
512
|
+
projects: Optional[List[str]] = None, expires_in_days: Optional[int] = None,
|
|
513
|
+
rpm: Optional[int] = None) -> Json:
|
|
514
|
+
"""A new API key. The plaintext is in the response under ``key``, once."""
|
|
515
|
+
body: Json = {"name": name}
|
|
516
|
+
for key, value in (("scopes", scopes), ("projects", projects),
|
|
517
|
+
("expires_in_days", expires_in_days), ("rpm", rpm)):
|
|
518
|
+
if value is not None:
|
|
519
|
+
body[key] = value
|
|
520
|
+
return self._h("POST", "/me/keys", json=body)
|
|
521
|
+
|
|
522
|
+
def revoke_key(self, key_id: str) -> None:
|
|
523
|
+
self._h("DELETE", f"/me/keys/{key_id}")
|
|
524
|
+
|
|
525
|
+
def usage(self) -> Json:
|
|
526
|
+
return self._h("GET", "/me/usage")
|
|
527
|
+
|
|
528
|
+
def monitor(self) -> Json:
|
|
529
|
+
return self._h("GET", "/me/monitor")
|
|
530
|
+
|
|
531
|
+
def meta(self) -> Json:
|
|
532
|
+
"""Verdict meanings, engine costs, the configuration defaults and the ladder."""
|
|
533
|
+
return self._h("GET", "/meta")
|
|
534
|
+
|
|
535
|
+
def close(self) -> None:
|
|
536
|
+
self._h.close()
|
|
537
|
+
|
|
538
|
+
def __enter__(self) -> "MeshArc":
|
|
539
|
+
return self
|
|
540
|
+
|
|
541
|
+
def __exit__(self, *exc: Any) -> None:
|
|
542
|
+
self.close()
|
mesharc/mcp.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""MeshArc as an MCP server: the API's verbs as tools an agent can call.
|
|
2
|
+
|
|
3
|
+
pip install "mesharc[mcp]" # Python 3.10+
|
|
4
|
+
MESHARC_API_KEY=mesharc_... mesharc-mcp # stdio, for Claude Desktop, Claude Code, Cursor
|
|
5
|
+
|
|
6
|
+
claude mcp add mesharc -e MESHARC_API_KEY=mesharc_... -- mesharc-mcp
|
|
7
|
+
|
|
8
|
+
Tools return the API's JSON, trimmed where a body would swamp a context
|
|
9
|
+
window (markdown is capped per page; ask for one page to get all of it).
|
|
10
|
+
Every tool is a call through the Python client, so what the agent gets
|
|
11
|
+
is what the API gives.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
from mcp.server.mcpserver import MCPServer
|
|
18
|
+
except ImportError as exc: # pragma: no cover
|
|
19
|
+
raise SystemExit('The MCP server needs the "mcp" package: pip install "mesharc[mcp]"') from exc
|
|
20
|
+
|
|
21
|
+
from mesharc import MeshArc, MeshArcError
|
|
22
|
+
|
|
23
|
+
MARKDOWN_CAP = 12_000
|
|
24
|
+
PAGES_CAP = 50
|
|
25
|
+
|
|
26
|
+
server = MCPServer(
|
|
27
|
+
"mesharc",
|
|
28
|
+
instructions=(
|
|
29
|
+
"MeshArc turns URLs into clean content and keeps a record of what changed. "
|
|
30
|
+
"Use scrape_urls for a list of pages, extract_url for one page with every format, "
|
|
31
|
+
"map_site to see what URLs a site declares before fetching any of them, and "
|
|
32
|
+
"crawl_site to crawl a whole site once without setting a project up first "
|
|
33
|
+
"(keep_crawl_as_project turns one of those into a watched project afterwards). "
|
|
34
|
+
"The project tools are for a site watched over time: its pages, its change record, "
|
|
35
|
+
"and search inside a run. Blocked pages are reported as blocked, never as missing."
|
|
36
|
+
),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _client():
|
|
41
|
+
key = os.environ.get("MESHARC_API_KEY") or ""
|
|
42
|
+
if not key:
|
|
43
|
+
raise RuntimeError("MESHARC_API_KEY is not set")
|
|
44
|
+
return MeshArc(key, base_url=os.environ.get("MESHARC_API_URL") or None)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _trim_page(p):
|
|
48
|
+
if isinstance(p, dict):
|
|
49
|
+
for k in ("markdown", "text", "cleanHtml", "html"):
|
|
50
|
+
if isinstance(p.get(k), str) and len(p[k]) > MARKDOWN_CAP:
|
|
51
|
+
p[k] = p[k][:MARKDOWN_CAP] + f"\n… [{len(p[k]) - MARKDOWN_CAP} more characters; fetch this page alone for all of it]"
|
|
52
|
+
p.pop("response", None)
|
|
53
|
+
return p
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _safe(fn):
|
|
57
|
+
try:
|
|
58
|
+
return fn()
|
|
59
|
+
except MeshArcError as exc:
|
|
60
|
+
return {"error": exc.detail, "status": exc.status}
|
|
61
|
+
except Exception as exc: # noqa: BLE001
|
|
62
|
+
return {"error": f"{type(exc).__name__}: {exc}"}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@server.tool(description="Scrape a list of URLs (up to 500) into markdown, no project needed. Waits for the batch. "
|
|
66
|
+
"One URL is answered in the same request where the page is quick. "
|
|
67
|
+
"`config` is any subset of a project config, e.g. {\"formats\": [\"markdown\", \"text\"], \"concurrency\": 4, \"render_js\": \"always\"}.")
|
|
68
|
+
def scrape_urls(urls: list[str], config: dict | None = None, formats: str = "markdown") -> dict:
|
|
69
|
+
def go():
|
|
70
|
+
with _client() as s:
|
|
71
|
+
if len(urls) == 1:
|
|
72
|
+
# One URL takes the synchronous path: the API holds the
|
|
73
|
+
# request open for the page rather than making us poll.
|
|
74
|
+
return {"status": "done", "urls": 1,
|
|
75
|
+
"pages": [_trim_page(s.scrape(urls[0], config=config, formats=formats))]}
|
|
76
|
+
b = s.scrape(urls, config=config, formats=formats)
|
|
77
|
+
b["pages"] = [_trim_page(p) for p in b.get("pages", [])[:PAGES_CAP]]
|
|
78
|
+
return b
|
|
79
|
+
return _safe(go)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@server.tool(description="Extract one URL with every format a project can produce (markdown, text, cleanHtml, "
|
|
83
|
+
"json fields, screenshot), through the fetch ladder: plain http first, a browser only "
|
|
84
|
+
"when needed or when render_js is 'always'. Browser `actions` (click, type, select, press, "
|
|
85
|
+
"wait, scroll; a click with repeat 'until_gone' for Load-more buttons; `each` to click "
|
|
86
|
+
"every match of a selector and run nested steps) run before the page is read.")
|
|
87
|
+
def extract_url(url: str, config: dict | None = None) -> dict:
|
|
88
|
+
def go():
|
|
89
|
+
with _client() as s:
|
|
90
|
+
r = s.extract(url, config=config)
|
|
91
|
+
if r.get("page"):
|
|
92
|
+
r["page"] = _trim_page(r["page"])
|
|
93
|
+
r["page"].pop("links", None)
|
|
94
|
+
return r
|
|
95
|
+
return _safe(go)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@server.tool(description="Every URL a site declares in its sitemaps -- robots.txt, the well-known paths, and every "
|
|
99
|
+
"index file walked to its children -- without fetching any of the pages. Cheap, and the right "
|
|
100
|
+
"first step before crawling: it says how big a site is and what sections it has. `search` "
|
|
101
|
+
"narrows to URLs containing a string.")
|
|
102
|
+
def map_site(url: str, search: str | None = None, limit: int = 1000) -> dict:
|
|
103
|
+
def go():
|
|
104
|
+
with _client() as s:
|
|
105
|
+
out = s.map_details(url, search=search, limit=min(limit, 5000))
|
|
106
|
+
return {"url": out["url"], "method": out["method"], "totals": out["totals"],
|
|
107
|
+
"urls": [u["url"] for u in out["data"]],
|
|
108
|
+
"sections": sorted({u.get("section", "") for u in out["data"] if u.get("section")})}
|
|
109
|
+
return _safe(go)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@server.tool(description="Crawl a whole site once and return its pages -- no project needed. Follows links from the "
|
|
113
|
+
"URL given, reads the sitemap, and stops at `limit` pages. Returns when the crawl finishes "
|
|
114
|
+
"(minutes for a large limit); the pages come back with markdown, capped per page. "
|
|
115
|
+
"`crawl_id` in the result can be handed to keep_crawl_as_project.")
|
|
116
|
+
def crawl_site(url: str, limit: int = 50, max_depth: int = 3, include_paths: list[str] | None = None,
|
|
117
|
+
exclude_paths: list[str] | None = None, config: dict | None = None) -> dict:
|
|
118
|
+
def go():
|
|
119
|
+
with _client() as s:
|
|
120
|
+
opts = {"limit": min(limit, 5000), "maxDepth": max_depth}
|
|
121
|
+
if include_paths:
|
|
122
|
+
opts["includePaths"] = include_paths
|
|
123
|
+
if exclude_paths:
|
|
124
|
+
opts["excludePaths"] = exclude_paths
|
|
125
|
+
if config:
|
|
126
|
+
opts["config"] = config
|
|
127
|
+
job = s.crawl(url, **opts)
|
|
128
|
+
pages = [_trim_page(p) for p in job.pages(limit=50)][:PAGES_CAP]
|
|
129
|
+
e = job.envelope
|
|
130
|
+
return {"crawl_id": job.id, "status": e.get("status"), "url": url,
|
|
131
|
+
"pages": pages, "counts": e.get("counts"), "stop": e.get("stop", ""),
|
|
132
|
+
"note": ("this crawl is kept for a day unless keep_crawl_as_project is called"
|
|
133
|
+
if e.get("ephemeral") else "")}
|
|
134
|
+
return _safe(go)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@server.tool(description="Keep a crawl from crawl_site as a project, so the site is watched over time and its changes "
|
|
138
|
+
"are recorded. Nothing is re-fetched: the crawl's pages become the project's first run. "
|
|
139
|
+
"schedule: manual | hourly | daily | weekly.")
|
|
140
|
+
def keep_crawl_as_project(crawl_id: str, name: str | None = None, schedule: str = "manual") -> dict:
|
|
141
|
+
def go():
|
|
142
|
+
with _client() as s:
|
|
143
|
+
return s.get_crawl(crawl_id).keep(name=name, schedule=schedule)
|
|
144
|
+
return _safe(go)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@server.tool(description="The workspace's projects: sites watched over time, with their last run's counts.")
|
|
148
|
+
def list_projects() -> list | dict:
|
|
149
|
+
return _safe(lambda: [{k: p.get(k) for k in ("id", "name", "seed", "host", "schedule", "pages", "coverage", "lastRun", "health")}
|
|
150
|
+
for p in _client().projects.list()])
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@server.tool(description="Create a project for a site (seed URL) so it is crawled on a schedule and its changes recorded. "
|
|
154
|
+
"schedule: manual | hourly | daily | weekly.")
|
|
155
|
+
def create_project(seed: str, name: str | None = None, schedule: str = "manual", config: dict | None = None) -> dict:
|
|
156
|
+
return _safe(lambda: _client().projects.create(seed, name=name, schedule=schedule, config=config))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@server.tool(description="Start a crawl of a project now. With wait=true, returns the finished run (may take minutes).")
|
|
160
|
+
def start_run(project_id: str, wait: bool = False) -> dict:
|
|
161
|
+
return _safe(lambda: _client().runs.start(project_id, wait=wait))
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@server.tool(description="The pages of a project's last finished run (or run_id): url, status, depth, words, when changed.")
|
|
165
|
+
def list_pages(project_id: str, run_id: str | None = None) -> dict:
|
|
166
|
+
def go():
|
|
167
|
+
r = _client().pages(project_id, run_id)
|
|
168
|
+
r["pages"] = r.get("pages", [])[:500]
|
|
169
|
+
return r
|
|
170
|
+
return _safe(go)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@server.tool(description="One stored page in full: markdown, head fields, fields, and its versions across runs.")
|
|
174
|
+
def get_page(project_id: str, url: str, run_id: str | None = None) -> dict:
|
|
175
|
+
return _safe(lambda: _trim_page(_client().page(project_id, url, run_id)))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@server.tool(description="What changed in a project's last run against the run before it: pages added, modified, "
|
|
179
|
+
"removed (withheld when the crawl reached under 90% of the site), and head-field changes.")
|
|
180
|
+
def get_changes(project_id: str, run_id: str | None = None) -> dict:
|
|
181
|
+
def go():
|
|
182
|
+
r = _client().changes(project_id, run_id)
|
|
183
|
+
ch = r.get("change") or {}
|
|
184
|
+
for k in ("feed", "fields", "withheld"):
|
|
185
|
+
if isinstance(ch.get(k), list):
|
|
186
|
+
ch[k] = ch[k][:200]
|
|
187
|
+
return r
|
|
188
|
+
return _safe(go)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@server.tool(description="Search inside a project's run. mode 'content': every word must appear, \"quoted phrases\" as written, "
|
|
192
|
+
"over the extracted markdown. mode 'selector': a CSS selector or XPath (starting with / or () over the stored html.")
|
|
193
|
+
def search_pages(project_id: str, q: str, mode: str = "content", run_id: str | None = None) -> dict:
|
|
194
|
+
return _safe(lambda: _client().search(project_id, q, mode=mode, run_id=run_id))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@server.tool(description="Fetch listed pages of a project again now, as a run of their own compared against the last full run.")
|
|
198
|
+
def recrawl_pages(project_id: str, urls: list[str]) -> dict:
|
|
199
|
+
return _safe(lambda: _client().recrawl(project_id, urls))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def main() -> None:
|
|
203
|
+
"""The `mesharc-mcp` command: serve over stdio."""
|
|
204
|
+
server.run("stdio")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
if __name__ == "__main__":
|
|
208
|
+
main()
|
mesharc/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mesharc
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Python client for the MeshArc API: a URL in, clean content out, and a record of what changed.
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://mesharc.dev
|
|
7
|
+
Project-URL: Documentation, https://mesharc.dev/docs/sdks
|
|
8
|
+
Project-URL: Source, https://github.com/mesharc-org/mesharc-python
|
|
9
|
+
Project-URL: Issues, https://github.com/mesharc-org/mesharc-python/issues
|
|
10
|
+
Keywords: scraping,crawler,web-data,change-detection,sitemap,markdown,llm
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Typing :: Typed
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: httpx>=0.25
|
|
21
|
+
Provides-Extra: mcp
|
|
22
|
+
Requires-Dist: mcp>=1.0; python_version >= "3.10" and extra == "mcp"
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
25
|
+
Requires-Dist: build; extra == "dev"
|
|
26
|
+
Requires-Dist: twine; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# mesharc
|
|
30
|
+
|
|
31
|
+
The Python client for the [MeshArc](https://mesharc.dev) API: a URL in, clean content out, and a record of what changed.
|
|
32
|
+
|
|
33
|
+
- **Scrape** one page or a batch — markdown, text, HTML, links, structured fields, a screenshot.
|
|
34
|
+
- **Crawl** a whole site with no project to set up first, and keep it as one if it turns out to be worth watching.
|
|
35
|
+
- **Map** what a site declares in its sitemaps before fetching any of it.
|
|
36
|
+
- **Watch** a site over time: projects, scheduled runs, and a change record — pages added, removed, modified, field by field.
|
|
37
|
+
|
|
38
|
+
Python 3.9 or newer. One dependency (`httpx`). Fully typed.
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install mesharc
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Authentication
|
|
47
|
+
|
|
48
|
+
Every call needs an API key. Create one in the app under **Settings → API keys** — it is shown once — and give it to the client, or put it in `MESHARC_API_KEY` and construct the client with nothing:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from mesharc import MeshArc
|
|
52
|
+
|
|
53
|
+
arc = MeshArc("mesharc_...")
|
|
54
|
+
# or, with MESHARC_API_KEY in the environment:
|
|
55
|
+
arc = MeshArc()
|
|
56
|
+
# options: MeshArc(api_key, timeout=150.0, max_retries=2)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The key is only ever sent as a bearer header to `api.mesharc.dev`.
|
|
60
|
+
|
|
61
|
+
A key carries the scopes it was made with (`read`, `write`, `admin`), optionally a set of projects it may see, an expiry and a rate limit. A route the key may not use answers `403`; a project it may not see answers `404`.
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from mesharc import MeshArc
|
|
67
|
+
|
|
68
|
+
arc = MeshArc("mesharc_...")
|
|
69
|
+
|
|
70
|
+
page = arc.scrape("https://example.com/pricing")
|
|
71
|
+
print(page["markdown"])
|
|
72
|
+
print(page["verdict"], page["method"], page["credits"]) # ok crawler 1
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`scrape` holds the request open until the page comes back (60 s by default), so there is nothing to poll for an ordinary page.
|
|
76
|
+
|
|
77
|
+
## Reading pages
|
|
78
|
+
|
|
79
|
+
### One page
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
page = arc.scrape("https://quotes.toscrape.com/js/", config={"render_js": "always"})
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`config` is any setting a project takes, by its API name (`render_js`, `only_main_content`, `formats`, `max_tier`, `wait_for_selector`, `actions`, …). The full list, with defaults, is at [mesharc.dev/docs/configuration](https://mesharc.dev/docs/configuration).
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
page = arc.scrape_one(
|
|
89
|
+
"https://example.com/",
|
|
90
|
+
formats="markdown,text,cleanHtml", # which bodies to return
|
|
91
|
+
timeout_s=120, # how long the API holds the request (120 max)
|
|
92
|
+
idempotency_key="pricing-2026-09-18", # the same key returns the first answer for 24 h
|
|
93
|
+
)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Many pages
|
|
97
|
+
|
|
98
|
+
A list of URLs is a batch: grouped by host, fetched in parallel where the config allows, and returned as one row per URL.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
batch = arc.scrape(["https://a.com/", "https://b.com/x"], config={"concurrency": 4})
|
|
102
|
+
for row in batch["pages"]:
|
|
103
|
+
print(row["url"], row["httpStatus"], row["verdict"], row["credits"])
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`arc.scrape(urls, wait=False)` returns the batch id at once; `arc.batch(id, wait=True)` finishes it later. Pass `webhook_url=` to be told instead of polling (`batch.finished`, signed with a secret returned once).
|
|
107
|
+
|
|
108
|
+
### What a page looks like
|
|
109
|
+
|
|
110
|
+
Every page row carries the same fields, whether it came from a scrape, a crawl or a project:
|
|
111
|
+
|
|
112
|
+
| Field | Meaning |
|
|
113
|
+
|---|---|
|
|
114
|
+
| `markdown`, `text`, `cleanHtml`, `html`, `links`, `fields`, `screenshot` | The bodies you asked for |
|
|
115
|
+
| `httpStatus` | The status the site answered with |
|
|
116
|
+
| `verdict` | `ok`, `thin` (short, but a page), `blocked` (refused, or a 404), `skipped` |
|
|
117
|
+
| `errorCode` | `OK`, or what went wrong: `BLOCKED`, `NOT_FOUND`, `TIER_LIMIT`, `CAPTCHA`, `LOGIN_REQUIRED`, `RATE_LIMITED`, `TIMEOUT` … |
|
|
118
|
+
| `shape`, `warnings`, `signals` | `listing` / `table` / `form` for a short page whose markup says what it is; `short`; why the judge decided as it did |
|
|
119
|
+
| `method`, `tier`, `climbedTo` | The engine that read it (`crawler`, `tls`, `minted`, `browser`, `browser-residential`, …), its tier, and how far a refused page climbed |
|
|
120
|
+
| `credits`, `billedAs` | What it cost; the rung it is priced at when not the one that fetched it |
|
|
121
|
+
| `words`, `language`, `head`, `reason`, `crawledAt` | Size, language, the head fields, the judge's sentence, when |
|
|
122
|
+
|
|
123
|
+
A page the site refused costs 0, and so does a 404.
|
|
124
|
+
|
|
125
|
+
## Crawling a site
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
job = arc.crawl(
|
|
129
|
+
"https://docs.example.com",
|
|
130
|
+
limit=200, # page budget
|
|
131
|
+
maxDepth=3, # link hops from the seed
|
|
132
|
+
includePaths=["/docs/*"],
|
|
133
|
+
crawlMode="sitemap_first", # what the sitemap declares first, then links
|
|
134
|
+
maxTier="browser", # how far a refused page may climb
|
|
135
|
+
scrapeOptions={"formats": ["markdown", "links"]},
|
|
136
|
+
config={"crawl_delay_ms": 500}, # any project setting, directly
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
for page in job.pages(): # follows the cursor while the crawl runs
|
|
140
|
+
print(page["url"], page["words"])
|
|
141
|
+
print(job.status, job.envelope["counts"], job.envelope["creditsUsed"])
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`crawl` returns a handle immediately; `job.pages()` yields pages as they land and ends when the crawl does. `wait=True` blocks until it finishes; `job.wait()`, `job.refresh()`, `job.cancel()` do what they say; `arc.get_crawl(id)` reattaches to a crawl started elsewhere.
|
|
145
|
+
|
|
146
|
+
A one-shot crawl expires after 30 days. If the site is worth watching:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
project = job.keep(name="Docs", schedule="weekly")
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
A `webhook=` in the options (`{"url", "events", "metadata"}`) is told about `crawl.started`, `crawl.page` (fifty pages a message) and `crawl.completed`; its signing secret comes back once as `job.webhook_secret`.
|
|
153
|
+
|
|
154
|
+
## Mapping a site
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
for u in arc.map("https://docs.example.com"):
|
|
158
|
+
print(u["url"], u["lastmod"])
|
|
159
|
+
|
|
160
|
+
details = arc.map_details("https://www.gov.uk/", search="visa", limit=500)
|
|
161
|
+
print(details["totals"], details["creditsUsed"]) # {'files': 29, 'urls': 508431, …} 29
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
A map costs one credit per sitemap file read — most sites are one file.
|
|
165
|
+
|
|
166
|
+
## Watching a site: projects and runs
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
project = arc.projects.create(
|
|
170
|
+
"https://docs.example.com",
|
|
171
|
+
name="Docs",
|
|
172
|
+
schedule="weekly", # manual | hourly | daily | weekly
|
|
173
|
+
config={"max_pages": 300, "include_paths": ["/docs/*"]},
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
run = arc.runs.start(project["id"], wait=True) # the first run
|
|
177
|
+
# ...a week later, or arc.runs.start again: the second run produces the change record
|
|
178
|
+
|
|
179
|
+
record = arc.changes(project["id"])
|
|
180
|
+
print(record["change"]["counts"]) # {'added': …, 'removed': …, 'modified': …, 'withheld': …}
|
|
181
|
+
|
|
182
|
+
diff = arc.page_diff(project["id"], "https://docs.example.com/pricing")
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
| Method | What it does |
|
|
186
|
+
|---|---|
|
|
187
|
+
| `projects.list()` · `projects.get(id)` · `projects.update(id, name=, schedule=, retention=, config=)` · `projects.delete(id)` | The projects |
|
|
188
|
+
| `runs.list(project_id)` · `runs.start(project_id, wait=)` · `runs.wait(project_id, run_id)` · `runs.get(project_id, run_id)` · `runs.cancel(project_id, run_id)` | Runs |
|
|
189
|
+
| `pages(project_id, run_id=None)` · `page(project_id, url, run_id=None)` | The pages of a run; one page in full |
|
|
190
|
+
| `changes(project_id, run_id=None)` · `page_diff(project_id, url, run_id=None)` | The change record; one page's word-level diff |
|
|
191
|
+
| `search(project_id, q, mode="content" \| "selector", run_id=None)` | Which pages say this (words, `"phrases"`) or contain this (CSS / XPath) |
|
|
192
|
+
| `recrawl(project_id, urls)` | Fetch these pages again, now |
|
|
193
|
+
| `sources(project_id)` | The seed, sitemap, URL list, feeds and patterns with what the last run found through each |
|
|
194
|
+
| `export(project_id, path, dataset="pages", fmt="jsonl", run_id=None, urls=None)` | Stream a dataset (`pages`, `markdown`, `changes`, `fields`, `sitemap`) as `jsonl` or `csv` to a file |
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
arc.export(project["id"], "pages.csv", dataset="pages", fmt="csv")
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## The workspace
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
me = arc.me() # the workspace, its plan and limits, credits used and remaining, what this key may do
|
|
204
|
+
usage = arc.usage() # pages per day, this month by engine
|
|
205
|
+
monitor = arc.monitor() # what is queued and running
|
|
206
|
+
meta = arc.meta() # verdict meanings, engine costs, the config defaults
|
|
207
|
+
keys = arc.keys()
|
|
208
|
+
key = arc.create_key("ci", scopes=["read", "write"], projects=[project["id"]], expires_in_days=90) # key["key"], once
|
|
209
|
+
arc.revoke_key(key["id"])
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## Errors
|
|
213
|
+
|
|
214
|
+
Every failure raises `MeshArcError`:
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
from mesharc import MeshArc, MeshArcError
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
arc.crawl("https://example.com", limit=1_000_000)
|
|
221
|
+
except MeshArcError as exc:
|
|
222
|
+
print(exc.status, exc.code, exc.detail, exc.request_id)
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
| `code` | Status | Meaning |
|
|
226
|
+
|---|---|---|
|
|
227
|
+
| `validation` | 400 / 422 | Something in the request is wrong; `detail` says what |
|
|
228
|
+
| `unauthorized` | 401 | No key, or a revoked or expired one |
|
|
229
|
+
| `plan_limit` | 402 | The plan does not include this, or the credits are spent |
|
|
230
|
+
| `forbidden` | 403 | The key's scopes do not allow it |
|
|
231
|
+
| `not_found` | 404 | No such thing — or not one this key may see |
|
|
232
|
+
| `conflict` | 409 | The request contradicts current state |
|
|
233
|
+
| `rate_limited` | 429 | Over the key's rate limit; `X-RateLimit-Reset` says when |
|
|
234
|
+
| `internal` | 500 | Quote `request_id` to support |
|
|
235
|
+
|
|
236
|
+
`request_id` is the id the API put on the response and in its own logs, so a support conversation starts from one string.
|
|
237
|
+
|
|
238
|
+
Two more cases: a network failure or a request that hits `timeout` raises `MeshArcError` with `status == 0` and `code` `network` or `timeout`; a job the client stopped waiting for raises `MeshArcTimeoutError` — both a `MeshArcError` and a `TimeoutError` — which carries `job_id` so you can poll it later (`arc.get_crawl(id)`, `arc.batch(id)`).
|
|
239
|
+
|
|
240
|
+
## Idempotency and timeouts
|
|
241
|
+
|
|
242
|
+
- `scrape`, `scrape_one` and `crawl` take `idempotency_key=`: send the same key again within 24 hours and you get the first answer back rather than a second job.
|
|
243
|
+
- Waiting calls take `wait=`, `poll=` (seconds between polls) and `timeout=` (seconds before `TimeoutError`). `wait=False` returns the envelope at once; the default polls every 3 s for up to an hour.
|
|
244
|
+
- `timeout_s` on a single scrape is how long the API itself holds the request open (60 s by default, 120 at most); a slower page comes back as an id and is polled.
|
|
245
|
+
- `MeshArc(..., timeout=150.0)` is the HTTP timeout per request. A request is retried on 429, 502, 503, 504 and network failures when it is safe to repeat — a GET, a DELETE, or a POST with an idempotency key — up to `max_retries` times (2), honouring `Retry-After`.
|
|
246
|
+
|
|
247
|
+
## Credits
|
|
248
|
+
|
|
249
|
+
Every response says what it cost: `credits` on a page, `creditsUsed` on a job envelope, `X-MeshArc-Credits` on the HTTP response. A page costs the engine that read it — a plain fetch 1, a render 4 — and a refused page or a 404 costs nothing. The schedule and the plans are at [mesharc.dev/docs/billing](https://mesharc.dev/docs/billing).
|
|
250
|
+
|
|
251
|
+
## The MCP server
|
|
252
|
+
|
|
253
|
+
The package also ships MeshArc as an MCP server, so Claude Desktop, Claude Code, Cursor and any MCP client can scrape, crawl, map and read change records as tools. Python 3.10+.
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
pip install "mesharc[mcp]"
|
|
257
|
+
MESHARC_API_KEY=mesharc_... mesharc-mcp # serves over stdio
|
|
258
|
+
|
|
259
|
+
# Claude Code
|
|
260
|
+
claude mcp add mesharc -e MESHARC_API_KEY=mesharc_... -- mesharc-mcp
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Tools: `scrape_urls`, `extract_url`, `map_site`, `crawl_site`, `keep_crawl_as_project`, `list_projects`, `create_project`, `start_run`, `list_pages`, `get_page`, `get_changes`, `search_pages`, `recrawl_pages`. Every tool is a call through this client, trimmed where a body would swamp a context window (markdown is capped per page; ask for one page to get all of it).
|
|
264
|
+
|
|
265
|
+
## Anything else
|
|
266
|
+
|
|
267
|
+
The client is a thin wrapper: every method is one API call and returns the API's JSON as a `dict`. The full reference is at [mesharc.dev/docs/api](https://mesharc.dev/docs/api). Call `arc.close()` when you are done, or use the client as a context manager.
|
|
268
|
+
|
|
269
|
+
- Documentation: [mesharc.dev/docs](https://mesharc.dev/docs)
|
|
270
|
+
- Node client: `npm install mesharc` — [mesharc-node](https://github.com/mesharc-org/mesharc-node)
|
|
271
|
+
- Issues and pull requests: [mesharc-python](https://github.com/mesharc-org/mesharc-python)
|
|
272
|
+
- Questions: hello@mesharc.dev
|
|
273
|
+
|
|
274
|
+
MIT.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
mesharc/__init__.py,sha256=hmvqwrTHauN2Tqvg3MjrjMLnONhxcGxYLnkXbOJfJoA,24301
|
|
2
|
+
mesharc/mcp.py,sha256=GTL8mWD1rpO5uKBdg5Wr2fOGR0XMr4Dv9EXWbhcccCM,10464
|
|
3
|
+
mesharc/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
mesharc-0.1.2.dist-info/licenses/LICENSE,sha256=gUvBXupjjMPMIBGEtujNeVFxFKCDu8RAGWpOgfV0Nas,1085
|
|
5
|
+
mesharc-0.1.2.dist-info/METADATA,sha256=jKyPYYy0nE-45pDVuOJj7Vq4q8YvE55v1eJ4VehlSXI,13448
|
|
6
|
+
mesharc-0.1.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
7
|
+
mesharc-0.1.2.dist-info/entry_points.txt,sha256=zqwmWOlh9U__HARPXqxApyRYJQoeHhsRbQyyCh3WNHQ,49
|
|
8
|
+
mesharc-0.1.2.dist-info/top_level.txt,sha256=KXErx_IOhEs5npgSe59IMxFKY68k1jHDYX2bfU7-jKI,8
|
|
9
|
+
mesharc-0.1.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 MeshArc
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mesharc
|