mesharc 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mesharc/__init__.py ADDED
@@ -0,0 +1,542 @@
1
+ """mesharc: the Python client for the MeshArc API.
2
+
3
+ from mesharc import MeshArc
4
+ arc = MeshArc("mesharc_...") # or MeshArc() with MESHARC_API_KEY set
5
+
6
+ page = arc.scrape("https://example.com/pricing") # one URL, waited for
7
+ batch = arc.scrape(["https://a.com/", "https://b.com/x"]) # many URLs, one row each
8
+
9
+ job = arc.crawl("https://docs.example.com", limit=200) # a site, no project needed
10
+ for page in job.pages(): # pages as they land
11
+ print(page["url"], page["words"])
12
+ project = job.keep(name="Docs", schedule="weekly") # keep it, if it is worth watching
13
+
14
+ for entry in arc.map("https://docs.example.com"): # what the site declares
15
+ print(entry["url"], entry["lastmod"])
16
+
17
+ Every method is one API call (or a poll loop where ``wait`` applies) and
18
+ returns the API's JSON unwrapped, so the API reference at
19
+ https://mesharc.dev/docs/api applies to every return value. Refused
20
+ requests raise ``MeshArcError``; a job the client stopped waiting for
21
+ raises ``MeshArcTimeoutError``.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import os
27
+ import random
28
+ import time
29
+ from typing import Any, Dict, Iterable, Iterator, List, Optional, Union
30
+ from urllib.parse import parse_qs, urlparse
31
+
32
+ import httpx
33
+
34
+ __version__ = "0.1.2"
35
+ __all__ = ["MeshArc", "MeshArcError", "MeshArcTimeoutError", "Crawl"]
36
+
37
+ DEFAULT_BASE = "https://api.mesharc.dev"
38
+ DEFAULT_TIMEOUT = 150.0
39
+ DEFAULT_MAX_RETRIES = 2
40
+ RETRY_STATUSES = frozenset({429, 502, 503, 504})
41
+
42
+ Json = Dict[str, Any]
43
+
44
+
45
+ class MeshArcError(Exception):
46
+ """A request the API refused, or a job that ended in error.
47
+
48
+ ``status`` is the HTTP status (0 for a network failure or a timeout),
49
+ ``code`` the machine-readable reason (validation, unauthorized,
50
+ plan_limit, not_found, rate_limited, ...) and ``request_id`` the id
51
+ the API logged the request under -- quote it to support.
52
+ """
53
+
54
+ def __init__(self, status: int, detail: Any, code: str = "", request_id: str = "") -> None:
55
+ super().__init__(f"{status}: {detail}" + (f" [{request_id}]" if request_id else ""))
56
+ self.status = status
57
+ self.detail = detail
58
+ self.code = code
59
+ self.request_id = request_id
60
+
61
+
62
+ class MeshArcTimeoutError(MeshArcError, TimeoutError):
63
+ """A job that was still running when the client stopped waiting.
64
+
65
+ ``job_id`` names the job; poll it later with the matching ``get``
66
+ method. Catchable as either ``MeshArcError`` or ``TimeoutError``.
67
+ """
68
+
69
+ def __init__(self, message: str, job_id: str = "") -> None:
70
+ super().__init__(0, message, "timeout")
71
+ self.job_id = job_id
72
+
73
+
74
+ def _running(status: Any) -> bool:
75
+ return status in ("queued", "running")
76
+
77
+
78
+ def _cursor_of(next_url: str) -> str:
79
+ return (parse_qs(urlparse(next_url).query).get("cursor") or [""])[0]
80
+
81
+
82
+ class _Http:
83
+ """The transport: one httpx client, bearer auth, JSON errors, retries."""
84
+
85
+ def __init__(self, api_key: str, base_url: str, timeout: float, max_retries: int) -> None:
86
+ self._c = httpx.Client(
87
+ base_url=base_url.rstrip("/") + "/api/v1",
88
+ headers={"Authorization": f"Bearer {api_key}", "User-Agent": f"mesharc-python/{__version__}"},
89
+ timeout=timeout,
90
+ )
91
+ self._max_retries = max(0, int(max_retries))
92
+
93
+ def __call__(self, method: str, path: str, idempotency_key: Optional[str] = None, **kw: Any) -> Any:
94
+ if idempotency_key:
95
+ kw.setdefault("headers", {})["Idempotency-Key"] = str(idempotency_key)
96
+ # A GET or DELETE is safe to repeat; a POST only when it carries an idempotency key.
97
+ repeatable = method in ("GET", "DELETE") or bool(idempotency_key)
98
+ attempt = 0
99
+ while True:
100
+ try:
101
+ r = self._c.request(method, path, **kw)
102
+ except httpx.TimeoutException as exc:
103
+ if repeatable and attempt < self._max_retries:
104
+ attempt += 1
105
+ time.sleep(self._backoff(attempt))
106
+ continue
107
+ raise MeshArcError(0, f"request timed out: {exc}", "timeout") from exc
108
+ except httpx.HTTPError as exc:
109
+ if repeatable and attempt < self._max_retries:
110
+ attempt += 1
111
+ time.sleep(self._backoff(attempt))
112
+ continue
113
+ raise MeshArcError(0, f"network error: {exc}", "network") from exc
114
+ if r.status_code in RETRY_STATUSES and repeatable and attempt < self._max_retries:
115
+ attempt += 1
116
+ time.sleep(self._retry_after(r) or self._backoff(attempt))
117
+ continue
118
+ if r.status_code >= 400:
119
+ raise self._error(r)
120
+ if r.status_code == 204 or not r.content:
121
+ return None
122
+ return r.json()
123
+
124
+ @staticmethod
125
+ def _backoff(attempt: int) -> float:
126
+ return 0.5 * (2 ** (attempt - 1)) + random.random() * 0.25
127
+
128
+ @staticmethod
129
+ def _retry_after(r: httpx.Response) -> Optional[float]:
130
+ header = r.headers.get("Retry-After")
131
+ if not header:
132
+ return None
133
+ try:
134
+ return max(0.0, float(header))
135
+ except ValueError:
136
+ return None
137
+
138
+ @staticmethod
139
+ def _error(r: httpx.Response) -> MeshArcError:
140
+ code, request_id = "", r.headers.get("X-Request-Id", "")
141
+ try:
142
+ body = r.json()
143
+ detail = body.get("error") or body.get("detail") or r.text[:200]
144
+ code = body.get("code", "") or ""
145
+ request_id = body.get("request_id") or request_id
146
+ except Exception: # noqa: BLE001 - a non-JSON body is its own detail
147
+ detail = r.text[:200]
148
+ return MeshArcError(r.status_code, detail, code, request_id)
149
+
150
+ def stream(self, method: str, path: str, **kw: Any) -> Any:
151
+ return self._c.stream(method, path, **kw)
152
+
153
+ def close(self) -> None:
154
+ self._c.close()
155
+
156
+
157
+ class _Projects:
158
+ def __init__(self, http: _Http) -> None:
159
+ self._h = http
160
+
161
+ def list(self) -> List[Json]:
162
+ return self._h("GET", "/projects")
163
+
164
+ def create(self, seed: str, name: Optional[str] = None, schedule: str = "manual", retention: str = "90d",
165
+ config: Optional[Json] = None) -> Json:
166
+ body: Json = {"seed": seed, "schedule": schedule, "retention": retention}
167
+ if name:
168
+ body["name"] = name
169
+ if config:
170
+ body["config"] = config
171
+ return self._h("POST", "/projects", json=body)
172
+
173
+ def get(self, project_id: str) -> Json:
174
+ return self._h("GET", f"/projects/{project_id}")
175
+
176
+ def update(self, project_id: str, **fields: Any) -> Json:
177
+ return self._h("PATCH", f"/projects/{project_id}", json=fields)
178
+
179
+ def delete(self, project_id: str) -> None:
180
+ self._h("DELETE", f"/projects/{project_id}")
181
+
182
+
183
+ class _Runs:
184
+ def __init__(self, http: _Http) -> None:
185
+ self._h = http
186
+
187
+ def list(self, project_id: str, limit: int = 25) -> List[Json]:
188
+ return self._h("GET", f"/projects/{project_id}/runs", params={"limit": limit})
189
+
190
+ def get(self, project_id: str, run_id: str) -> Json:
191
+ return self._h("GET", f"/projects/{project_id}/runs/{run_id}")
192
+
193
+ def start(self, project_id: str, wait: bool = False, poll: float = 3.0, timeout: float = 3600) -> Json:
194
+ run = self._h("POST", f"/projects/{project_id}/runs", json={"trigger": "api"})
195
+ return self.wait(project_id, run["id"], poll, timeout) if wait else run
196
+
197
+ def wait(self, project_id: str, run_id: str, poll: float = 3.0, timeout: float = 3600) -> Json:
198
+ deadline = time.time() + timeout
199
+ while True:
200
+ run = self.get(project_id, run_id)
201
+ if run["status"] != "running" and not run.get("queued"):
202
+ return run
203
+ if time.time() >= deadline:
204
+ raise MeshArcTimeoutError(f"run {run_id} is still {run['status']} after {timeout}s", run_id)
205
+ time.sleep(poll)
206
+
207
+ def cancel(self, project_id: str, run_id: str) -> Json:
208
+ return self._h("POST", f"/projects/{project_id}/runs/{run_id}/cancel")
209
+
210
+
211
+ class Crawl:
212
+ """A crawl started by ``arc.crawl(url)``: a handle on a running job.
213
+
214
+ ``wait()`` blocks until it finishes; ``pages()`` yields pages as they
215
+ land, following the cursor, and ends when the job does; ``keep()``
216
+ turns the one-shot crawl into a project; ``cancel()`` stops it.
217
+ """
218
+
219
+ def __init__(self, http: _Http, envelope: Json) -> None:
220
+ self._h = http
221
+ self.id: str = envelope["id"]
222
+ self.url: str = envelope.get("url", "")
223
+ self.project_id: str = envelope.get("projectId", "")
224
+ #: Returned once, at creation: the secret the crawl's webhook messages are signed with.
225
+ self.webhook_secret: str = envelope.get("webhookSecret", "")
226
+ self.envelope: Json = envelope
227
+
228
+ def __repr__(self) -> str:
229
+ return f"<Crawl {self.id[:8]} {self.url} {self.envelope.get('status')}>"
230
+
231
+ @property
232
+ def status(self) -> str:
233
+ return self.envelope.get("status", "queued")
234
+
235
+ def refresh(self, formats: str = "markdown") -> Json:
236
+ """The envelope as it stands now, without its pages."""
237
+ self.envelope = self._h("GET", f"/crawl/{self.id}", params={"limit": 1, "formats": formats})
238
+ return self.envelope
239
+
240
+ def wait(self, poll: float = 3.0, timeout: float = 3600, formats: str = "markdown") -> Json:
241
+ """Block until the crawl finishes. Returns the envelope."""
242
+ deadline = time.time() + timeout
243
+ while True:
244
+ e = self.refresh(formats)
245
+ if not _running(e["status"]):
246
+ return e
247
+ if time.time() >= deadline:
248
+ raise MeshArcTimeoutError(f"crawl {self.id} is still {e['status']} after {timeout}s", self.id)
249
+ time.sleep(poll)
250
+
251
+ def pages(self, formats: str = "markdown", limit: int = 25, wait: bool = True, poll: float = 3.0,
252
+ timeout: float = 3600) -> Iterator[Json]:
253
+ """Every page of the crawl, oldest first.
254
+
255
+ While the crawl runs this waits for more pages rather than
256
+ stopping; ``wait=False`` yields what exists and returns.
257
+ """
258
+ deadline = time.time() + timeout
259
+ cursor: Optional[str] = None
260
+ while True:
261
+ params: Dict[str, Any] = {"formats": formats, "limit": limit}
262
+ if cursor:
263
+ params["cursor"] = cursor
264
+ page = self._h("GET", f"/crawl/{self.id}", params=params)
265
+ self.envelope = {k: v for k, v in page.items() if k != "data"}
266
+ for row in page["data"]:
267
+ yield row
268
+ # The cursor marks where this page ended, so a running crawl is never re-read from the top.
269
+ cursor = page.get("cursor") or cursor
270
+ if page.get("next"):
271
+ cursor = _cursor_of(page["next"]) or cursor
272
+ continue
273
+ if not wait or not _running(page["status"]):
274
+ return
275
+ if time.time() >= deadline:
276
+ raise MeshArcTimeoutError(f"crawl {self.id} is still {page['status']} after {timeout}s", self.id)
277
+ time.sleep(poll)
278
+
279
+ def keep(self, name: Optional[str] = None, schedule: Optional[str] = None,
280
+ retention: Optional[str] = None) -> Json:
281
+ """Make this one-shot crawl a project. Its run and pages are already in place."""
282
+ body = {k: v for k, v in (("name", name), ("schedule", schedule), ("retention", retention)) if v}
283
+ return self._h("POST", f"/crawl/{self.id}/keep", json=body)
284
+
285
+ def cancel(self) -> Json:
286
+ self._h("DELETE", f"/crawl/{self.id}")
287
+ self.envelope["status"] = "cancelled"
288
+ return self.envelope
289
+
290
+
291
+ class MeshArc:
292
+ """One client, one API key, one workspace.
293
+
294
+ ``api_key`` falls back to the ``MESHARC_API_KEY`` environment variable.
295
+ ``timeout`` is the HTTP timeout per request in seconds; ``max_retries``
296
+ how many times a request that is safe to repeat is retried on 429,
297
+ 502, 503, 504 or a network failure. ``base_url`` is for MeshArc's own
298
+ test environments; the hosted API needs nothing there.
299
+ """
300
+
301
+ def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None,
302
+ timeout: float = DEFAULT_TIMEOUT, max_retries: int = DEFAULT_MAX_RETRIES) -> None:
303
+ key = api_key or os.environ.get("MESHARC_API_KEY") or ""
304
+ if not key:
305
+ raise ValueError("An API key is required: MeshArc('mesharc_...') or set MESHARC_API_KEY.")
306
+ base = base_url or os.environ.get("MESHARC_API_URL") or DEFAULT_BASE
307
+ self._h = _Http(key, base, timeout, max_retries)
308
+ self.projects = _Projects(self._h)
309
+ self.runs = _Runs(self._h)
310
+
311
+ # ------------------------------------------------------------ one or many URLs
312
+
313
+ def extract(self, url: str, config: Optional[Json] = None, wait: bool = True, poll: float = 2.0,
314
+ timeout: float = 300) -> Json:
315
+ """One URL with every format, as the app's playground reads it."""
316
+ body: Json = {"url": url}
317
+ if config:
318
+ body["config"] = config
319
+ job = self._h("POST", "/playground/extract", json=body)
320
+ if not wait:
321
+ return job
322
+ deadline = time.time() + timeout
323
+ while True:
324
+ r = self._h("GET", f"/playground/{job['id']}")
325
+ if not _running(r["status"]):
326
+ return r
327
+ if time.time() >= deadline:
328
+ raise MeshArcTimeoutError(f"extraction {job['id']} is still {r['status']} after {timeout}s", job["id"])
329
+ time.sleep(poll)
330
+
331
+ def scrape(self, urls: Union[str, Iterable[str]], config: Optional[Json] = None,
332
+ webhook_url: Optional[str] = None, formats: str = "markdown", wait: bool = True,
333
+ poll: float = 3.0, timeout: float = 3600, idempotency_key: Optional[str] = None) -> Json:
334
+ """URLs in, their content out; no project.
335
+
336
+ One URL (a string) returns the page itself: the API holds the
337
+ request open until the page comes back. A list is a batch and
338
+ returns the finished batch, one row per URL, unless ``wait`` is
339
+ False.
340
+ """
341
+ if isinstance(urls, str):
342
+ return self.scrape_one(urls, config=config, formats=formats, wait=wait, poll=poll,
343
+ timeout=timeout, idempotency_key=idempotency_key)
344
+ body: Json = {"urls": list(urls)}
345
+ if config:
346
+ body["config"] = config
347
+ if webhook_url:
348
+ body["webhook_url"] = webhook_url
349
+ batch = self._h("POST", "/scrape", json=body, idempotency_key=idempotency_key)
350
+ if not wait:
351
+ return batch
352
+ return self.batch(batch["id"], formats=formats, wait=True, poll=poll, timeout=timeout)
353
+
354
+ def scrape_one(self, url: str, config: Optional[Json] = None, formats: str = "markdown", wait: bool = True,
355
+ timeout_s: int = 60, poll: float = 2.0, timeout: float = 600,
356
+ idempotency_key: Optional[str] = None) -> Json:
357
+ """One URL, waited for. Returns the page; ``wait=False`` returns the job envelope.
358
+
359
+ ``timeout_s`` is how long the API holds the request open for the
360
+ page (60 by default, 120 at most); a slower page comes back as a
361
+ job, which is then polled every ``poll`` seconds for up to
362
+ ``timeout`` seconds.
363
+ """
364
+ body: Json = {"url": url, "formats": formats, "timeout": timeout_s}
365
+ if config:
366
+ body["config"] = config
367
+ out = self._h("POST", "/scrape", json=body, idempotency_key=idempotency_key)
368
+ if not wait:
369
+ return out
370
+ deadline = time.time() + timeout
371
+ while True:
372
+ if out.get("status") == "done":
373
+ return (out.get("data") or [{}])[0]
374
+ if not _running(out.get("status")):
375
+ raise MeshArcError(502, out.get("error") or f"scrape {out.get('status')}", "job_failed")
376
+ if time.time() >= deadline:
377
+ raise MeshArcTimeoutError(f"scrape {out['id']} is still {out['status']} after {timeout}s", out["id"])
378
+ time.sleep(poll)
379
+ out = self._h("GET", f"/scrape/{out['id']}", params={"formats": formats})
380
+
381
+ def batch(self, batch_id: str, formats: str = "markdown", wait: bool = False, poll: float = 3.0,
382
+ timeout: float = 3600) -> Json:
383
+ """A batch started earlier. With ``wait``, returns once every row is in."""
384
+ deadline = time.time() + timeout
385
+ while True:
386
+ r = self._h("GET", f"/scrape/{batch_id}", params={"formats": formats})
387
+ if not wait or not _running(r["status"]):
388
+ return r
389
+ if time.time() >= deadline:
390
+ raise MeshArcTimeoutError(f"batch {batch_id} is still {r['status']} after {timeout}s", batch_id)
391
+ time.sleep(poll)
392
+
393
+ # ---------------------------------------------------------------- a whole site
394
+
395
+ def crawl(self, url: str, wait: bool = False, poll: float = 3.0, timeout: float = 3600,
396
+ idempotency_key: Optional[str] = None, **opts: Any) -> Crawl:
397
+ """Crawl a site once, with no project to set up first.
398
+
399
+ Returns a ``Crawl`` handle as soon as the job is queued; ``wait=True``
400
+ blocks until it finishes. ``opts`` are the request's own names --
401
+ ``limit``, ``maxDepth``, ``includePaths``, ``excludePaths``,
402
+ ``crawlMode``, ``maxAge``, ``maxTier``, ``scrapeOptions``,
403
+ ``webhook`` -- and ``config`` takes any project setting directly.
404
+ """
405
+ job = Crawl(self._h, self._h("POST", "/crawl", json={"url": url, **opts}, idempotency_key=idempotency_key))
406
+ if wait:
407
+ job.wait(poll=poll, timeout=timeout)
408
+ return job
409
+
410
+ def get_crawl(self, crawl_id: str) -> Crawl:
411
+ """A handle on a crawl started earlier or elsewhere."""
412
+ return Crawl(self._h, self._h("GET", f"/crawl/{crawl_id}", params={"limit": 1}))
413
+
414
+ def map(self, url: str, **opts: Any) -> List[Json]:
415
+ """Every URL a site declares in its sitemaps, as ``{url, lastmod, changefreq, source, file, section}``."""
416
+ return self.map_details(url, **opts)["data"]
417
+
418
+ def map_details(self, url: str, search: Optional[str] = None, limit: Optional[int] = None,
419
+ timeout_s: int = 10, poll: float = 2.0, timeout: float = 300, **opts: Any) -> Json:
420
+ """A map with everything the API said about it: how the sitemaps
421
+ were found, the totals, what robots.txt allowed, and the URLs
422
+ under ``data``. ``timeout_s`` is how long the API waits for the
423
+ sitemap tree before answering with a job (10 by default, 60 at
424
+ most), which is then polled.
425
+ """
426
+ body: Json = {"url": url, "timeout": timeout_s, **opts}
427
+ if search:
428
+ body["search"] = search
429
+ if limit:
430
+ body["limit"] = limit
431
+ out = self._h("POST", "/map", json=body)
432
+ params = {k: v for k, v in (("search", search), ("limit", limit)) if v}
433
+ deadline = time.time() + timeout
434
+ while out.get("status") == "running":
435
+ if time.time() >= deadline:
436
+ raise MeshArcTimeoutError(f"map {out['id']} is still reading {url} after {timeout}s", out["id"])
437
+ time.sleep(poll)
438
+ out = self._h("GET", f"/map/{out['id']}", params=params or None)
439
+ if out.get("status") != "done":
440
+ raise MeshArcError(502, out.get("error") or "no sitemap could be read", "job_failed")
441
+ return out
442
+
443
+ # ---------------------------------------------------------- what a project holds
444
+
445
+ def pages(self, project_id: str, run_id: Optional[str] = None) -> Json:
446
+ """The pages of a run (the latest finished run by default)."""
447
+ return self._h("GET", f"/projects/{project_id}/pages", params={"run_id": run_id} if run_id else None)
448
+
449
+ def page(self, project_id: str, url: str, run_id: Optional[str] = None) -> Json:
450
+ """One page in full: bodies, head fields, structured fields, versions."""
451
+ params: Dict[str, Any] = {"url": url}
452
+ if run_id:
453
+ params["run_id"] = run_id
454
+ return self._h("GET", f"/projects/{project_id}/pages/content", params=params)
455
+
456
+ def changes(self, project_id: str, run_id: Optional[str] = None) -> Json:
457
+ """The change record of a run against the run before it."""
458
+ return self._h("GET", f"/projects/{project_id}/changes", params={"run_id": run_id} if run_id else None)
459
+
460
+ def page_diff(self, project_id: str, url: str, run_id: Optional[str] = None) -> Json:
461
+ """The word-level diff of one page against the run before."""
462
+ params: Dict[str, Any] = {"url": url}
463
+ if run_id:
464
+ params["run_id"] = run_id
465
+ return self._h("GET", f"/projects/{project_id}/changes/page", params=params)
466
+
467
+ def search(self, project_id: str, q: str, mode: str = "content", run_id: Optional[str] = None) -> Json:
468
+ """Which pages say this (``content``: words, "phrases") or contain this (``selector``: CSS or XPath)."""
469
+ return self._h("POST", f"/projects/{project_id}/pages/search", json={"mode": mode, "q": q, "run_id": run_id})
470
+
471
+ def recrawl(self, project_id: str, urls: Iterable[str]) -> Json:
472
+ """Fetch these pages again now, as a scoped run."""
473
+ return self._h("POST", f"/projects/{project_id}/pages/recrawl", json={"urls": list(urls)})
474
+
475
+ def sources(self, project_id: str) -> Json:
476
+ """The seed, sitemap, URL list, feeds and patterns, with what the last run found through each."""
477
+ return self._h("GET", f"/projects/{project_id}/sources")
478
+
479
+ def export(self, project_id: str, path: str, dataset: str = "pages", fmt: str = "jsonl",
480
+ run_id: Optional[str] = None, urls: Optional[Iterable[str]] = None) -> str:
481
+ """Stream a dataset (pages, markdown, changes, fields, sitemap) to ``path`` as jsonl or csv.
482
+
483
+ ``urls`` restricts the export to those pages. Returns ``path``.
484
+ """
485
+ if urls:
486
+ ctx = self._h.stream("POST", f"/projects/{project_id}/export",
487
+ json={"dataset": dataset, "format": fmt, "run_id": run_id, "urls": list(urls)})
488
+ else:
489
+ params: Dict[str, Any] = {"dataset": dataset, "format": fmt}
490
+ if run_id:
491
+ params["run_id"] = run_id
492
+ ctx = self._h.stream("GET", f"/projects/{project_id}/export", params=params)
493
+ with ctx as r:
494
+ if r.status_code >= 400:
495
+ r.read()
496
+ raise self._h._error(r)
497
+ with open(path, "wb") as fh:
498
+ for chunk in r.iter_bytes():
499
+ fh.write(chunk)
500
+ return path
501
+
502
+ # --------------------------------------------------------------------- workspace
503
+
504
+ def me(self) -> Json:
505
+ """The workspace, its plan and limits, and what this key may do."""
506
+ return self._h("GET", "/me")
507
+
508
+ def keys(self) -> List[Json]:
509
+ return self._h("GET", "/me/keys")
510
+
511
+ def create_key(self, name: str = "default", scopes: Optional[List[str]] = None,
512
+ projects: Optional[List[str]] = None, expires_in_days: Optional[int] = None,
513
+ rpm: Optional[int] = None) -> Json:
514
+ """A new API key. The plaintext is in the response under ``key``, once."""
515
+ body: Json = {"name": name}
516
+ for key, value in (("scopes", scopes), ("projects", projects),
517
+ ("expires_in_days", expires_in_days), ("rpm", rpm)):
518
+ if value is not None:
519
+ body[key] = value
520
+ return self._h("POST", "/me/keys", json=body)
521
+
522
+ def revoke_key(self, key_id: str) -> None:
523
+ self._h("DELETE", f"/me/keys/{key_id}")
524
+
525
+ def usage(self) -> Json:
526
+ return self._h("GET", "/me/usage")
527
+
528
+ def monitor(self) -> Json:
529
+ return self._h("GET", "/me/monitor")
530
+
531
+ def meta(self) -> Json:
532
+ """Verdict meanings, engine costs, the configuration defaults and the ladder."""
533
+ return self._h("GET", "/meta")
534
+
535
+ def close(self) -> None:
536
+ self._h.close()
537
+
538
+ def __enter__(self) -> "MeshArc":
539
+ return self
540
+
541
+ def __exit__(self, *exc: Any) -> None:
542
+ self.close()
mesharc/mcp.py ADDED
@@ -0,0 +1,208 @@
1
+ """MeshArc as an MCP server: the API's verbs as tools an agent can call.
2
+
3
+ pip install "mesharc[mcp]" # Python 3.10+
4
+ MESHARC_API_KEY=mesharc_... mesharc-mcp # stdio, for Claude Desktop, Claude Code, Cursor
5
+
6
+ claude mcp add mesharc -e MESHARC_API_KEY=mesharc_... -- mesharc-mcp
7
+
8
+ Tools return the API's JSON, trimmed where a body would swamp a context
9
+ window (markdown is capped per page; ask for one page to get all of it).
10
+ Every tool is a call through the Python client, so what the agent gets
11
+ is what the API gives.
12
+ """
13
+
14
+ import os
15
+
16
+ try:
17
+ from mcp.server.mcpserver import MCPServer
18
+ except ImportError as exc: # pragma: no cover
19
+ raise SystemExit('The MCP server needs the "mcp" package: pip install "mesharc[mcp]"') from exc
20
+
21
+ from mesharc import MeshArc, MeshArcError
22
+
23
+ MARKDOWN_CAP = 12_000
24
+ PAGES_CAP = 50
25
+
26
+ server = MCPServer(
27
+ "mesharc",
28
+ instructions=(
29
+ "MeshArc turns URLs into clean content and keeps a record of what changed. "
30
+ "Use scrape_urls for a list of pages, extract_url for one page with every format, "
31
+ "map_site to see what URLs a site declares before fetching any of them, and "
32
+ "crawl_site to crawl a whole site once without setting a project up first "
33
+ "(keep_crawl_as_project turns one of those into a watched project afterwards). "
34
+ "The project tools are for a site watched over time: its pages, its change record, "
35
+ "and search inside a run. Blocked pages are reported as blocked, never as missing."
36
+ ),
37
+ )
38
+
39
+
40
+ def _client():
41
+ key = os.environ.get("MESHARC_API_KEY") or ""
42
+ if not key:
43
+ raise RuntimeError("MESHARC_API_KEY is not set")
44
+ return MeshArc(key, base_url=os.environ.get("MESHARC_API_URL") or None)
45
+
46
+
47
+ def _trim_page(p):
48
+ if isinstance(p, dict):
49
+ for k in ("markdown", "text", "cleanHtml", "html"):
50
+ if isinstance(p.get(k), str) and len(p[k]) > MARKDOWN_CAP:
51
+ p[k] = p[k][:MARKDOWN_CAP] + f"\n… [{len(p[k]) - MARKDOWN_CAP} more characters; fetch this page alone for all of it]"
52
+ p.pop("response", None)
53
+ return p
54
+
55
+
56
+ def _safe(fn):
57
+ try:
58
+ return fn()
59
+ except MeshArcError as exc:
60
+ return {"error": exc.detail, "status": exc.status}
61
+ except Exception as exc: # noqa: BLE001
62
+ return {"error": f"{type(exc).__name__}: {exc}"}
63
+
64
+
65
+ @server.tool(description="Scrape a list of URLs (up to 500) into markdown, no project needed. Waits for the batch. "
66
+ "One URL is answered in the same request where the page is quick. "
67
+ "`config` is any subset of a project config, e.g. {\"formats\": [\"markdown\", \"text\"], \"concurrency\": 4, \"render_js\": \"always\"}.")
68
+ def scrape_urls(urls: list[str], config: dict | None = None, formats: str = "markdown") -> dict:
69
+ def go():
70
+ with _client() as s:
71
+ if len(urls) == 1:
72
+ # One URL takes the synchronous path: the API holds the
73
+ # request open for the page rather than making us poll.
74
+ return {"status": "done", "urls": 1,
75
+ "pages": [_trim_page(s.scrape(urls[0], config=config, formats=formats))]}
76
+ b = s.scrape(urls, config=config, formats=formats)
77
+ b["pages"] = [_trim_page(p) for p in b.get("pages", [])[:PAGES_CAP]]
78
+ return b
79
+ return _safe(go)
80
+
81
+
82
+ @server.tool(description="Extract one URL with every format a project can produce (markdown, text, cleanHtml, "
83
+ "json fields, screenshot), through the fetch ladder: plain http first, a browser only "
84
+ "when needed or when render_js is 'always'. Browser `actions` (click, type, select, press, "
85
+ "wait, scroll; a click with repeat 'until_gone' for Load-more buttons; `each` to click "
86
+ "every match of a selector and run nested steps) run before the page is read.")
87
+ def extract_url(url: str, config: dict | None = None) -> dict:
88
+ def go():
89
+ with _client() as s:
90
+ r = s.extract(url, config=config)
91
+ if r.get("page"):
92
+ r["page"] = _trim_page(r["page"])
93
+ r["page"].pop("links", None)
94
+ return r
95
+ return _safe(go)
96
+
97
+
98
+ @server.tool(description="Every URL a site declares in its sitemaps -- robots.txt, the well-known paths, and every "
99
+ "index file walked to its children -- without fetching any of the pages. Cheap, and the right "
100
+ "first step before crawling: it says how big a site is and what sections it has. `search` "
101
+ "narrows to URLs containing a string.")
102
+ def map_site(url: str, search: str | None = None, limit: int = 1000) -> dict:
103
+ def go():
104
+ with _client() as s:
105
+ out = s.map_details(url, search=search, limit=min(limit, 5000))
106
+ return {"url": out["url"], "method": out["method"], "totals": out["totals"],
107
+ "urls": [u["url"] for u in out["data"]],
108
+ "sections": sorted({u.get("section", "") for u in out["data"] if u.get("section")})}
109
+ return _safe(go)
110
+
111
+
112
+ @server.tool(description="Crawl a whole site once and return its pages -- no project needed. Follows links from the "
113
+ "URL given, reads the sitemap, and stops at `limit` pages. Returns when the crawl finishes "
114
+ "(minutes for a large limit); the pages come back with markdown, capped per page. "
115
+ "`crawl_id` in the result can be handed to keep_crawl_as_project.")
116
+ def crawl_site(url: str, limit: int = 50, max_depth: int = 3, include_paths: list[str] | None = None,
117
+ exclude_paths: list[str] | None = None, config: dict | None = None) -> dict:
118
+ def go():
119
+ with _client() as s:
120
+ opts = {"limit": min(limit, 5000), "maxDepth": max_depth}
121
+ if include_paths:
122
+ opts["includePaths"] = include_paths
123
+ if exclude_paths:
124
+ opts["excludePaths"] = exclude_paths
125
+ if config:
126
+ opts["config"] = config
127
+ job = s.crawl(url, **opts)
128
+ pages = [_trim_page(p) for p in job.pages(limit=50)][:PAGES_CAP]
129
+ e = job.envelope
130
+ return {"crawl_id": job.id, "status": e.get("status"), "url": url,
131
+ "pages": pages, "counts": e.get("counts"), "stop": e.get("stop", ""),
132
+ "note": ("this crawl is kept for a day unless keep_crawl_as_project is called"
133
+ if e.get("ephemeral") else "")}
134
+ return _safe(go)
135
+
136
+
137
+ @server.tool(description="Keep a crawl from crawl_site as a project, so the site is watched over time and its changes "
138
+ "are recorded. Nothing is re-fetched: the crawl's pages become the project's first run. "
139
+ "schedule: manual | hourly | daily | weekly.")
140
+ def keep_crawl_as_project(crawl_id: str, name: str | None = None, schedule: str = "manual") -> dict:
141
+ def go():
142
+ with _client() as s:
143
+ return s.get_crawl(crawl_id).keep(name=name, schedule=schedule)
144
+ return _safe(go)
145
+
146
+
147
+ @server.tool(description="The workspace's projects: sites watched over time, with their last run's counts.")
148
+ def list_projects() -> list | dict:
149
+ return _safe(lambda: [{k: p.get(k) for k in ("id", "name", "seed", "host", "schedule", "pages", "coverage", "lastRun", "health")}
150
+ for p in _client().projects.list()])
151
+
152
+
153
+ @server.tool(description="Create a project for a site (seed URL) so it is crawled on a schedule and its changes recorded. "
154
+ "schedule: manual | hourly | daily | weekly.")
155
+ def create_project(seed: str, name: str | None = None, schedule: str = "manual", config: dict | None = None) -> dict:
156
+ return _safe(lambda: _client().projects.create(seed, name=name, schedule=schedule, config=config))
157
+
158
+
159
+ @server.tool(description="Start a crawl of a project now. With wait=true, returns the finished run (may take minutes).")
160
+ def start_run(project_id: str, wait: bool = False) -> dict:
161
+ return _safe(lambda: _client().runs.start(project_id, wait=wait))
162
+
163
+
164
+ @server.tool(description="The pages of a project's last finished run (or run_id): url, status, depth, words, when changed.")
165
+ def list_pages(project_id: str, run_id: str | None = None) -> dict:
166
+ def go():
167
+ r = _client().pages(project_id, run_id)
168
+ r["pages"] = r.get("pages", [])[:500]
169
+ return r
170
+ return _safe(go)
171
+
172
+
173
+ @server.tool(description="One stored page in full: markdown, head fields, fields, and its versions across runs.")
174
+ def get_page(project_id: str, url: str, run_id: str | None = None) -> dict:
175
+ return _safe(lambda: _trim_page(_client().page(project_id, url, run_id)))
176
+
177
+
178
+ @server.tool(description="What changed in a project's last run against the run before it: pages added, modified, "
179
+ "removed (withheld when the crawl reached under 90% of the site), and head-field changes.")
180
+ def get_changes(project_id: str, run_id: str | None = None) -> dict:
181
+ def go():
182
+ r = _client().changes(project_id, run_id)
183
+ ch = r.get("change") or {}
184
+ for k in ("feed", "fields", "withheld"):
185
+ if isinstance(ch.get(k), list):
186
+ ch[k] = ch[k][:200]
187
+ return r
188
+ return _safe(go)
189
+
190
+
191
+ @server.tool(description="Search inside a project's run. mode 'content': every word must appear, \"quoted phrases\" as written, "
192
+ "over the extracted markdown. mode 'selector': a CSS selector or XPath (starting with / or () over the stored html.")
193
+ def search_pages(project_id: str, q: str, mode: str = "content", run_id: str | None = None) -> dict:
194
+ return _safe(lambda: _client().search(project_id, q, mode=mode, run_id=run_id))
195
+
196
+
197
+ @server.tool(description="Fetch listed pages of a project again now, as a run of their own compared against the last full run.")
198
+ def recrawl_pages(project_id: str, urls: list[str]) -> dict:
199
+ return _safe(lambda: _client().recrawl(project_id, urls))
200
+
201
+
202
+ def main() -> None:
203
+ """The `mesharc-mcp` command: serve over stdio."""
204
+ server.run("stdio")
205
+
206
+
207
+ if __name__ == "__main__":
208
+ main()
mesharc/py.typed ADDED
File without changes
@@ -0,0 +1,274 @@
1
+ Metadata-Version: 2.4
2
+ Name: mesharc
3
+ Version: 0.1.2
4
+ Summary: Python client for the MeshArc API: a URL in, clean content out, and a record of what changed.
5
+ License: MIT
6
+ Project-URL: Homepage, https://mesharc.dev
7
+ Project-URL: Documentation, https://mesharc.dev/docs/sdks
8
+ Project-URL: Source, https://github.com/mesharc-org/mesharc-python
9
+ Project-URL: Issues, https://github.com/mesharc-org/mesharc-python/issues
10
+ Keywords: scraping,crawler,web-data,change-detection,sitemap,markdown,llm
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3 :: Only
13
+ Classifier: Typing :: Typed
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Topic :: Internet :: WWW/HTTP
17
+ Requires-Python: >=3.9
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: httpx>=0.25
21
+ Provides-Extra: mcp
22
+ Requires-Dist: mcp>=1.0; python_version >= "3.10" and extra == "mcp"
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=8; extra == "dev"
25
+ Requires-Dist: build; extra == "dev"
26
+ Requires-Dist: twine; extra == "dev"
27
+ Dynamic: license-file
28
+
29
+ # mesharc
30
+
31
+ The Python client for the [MeshArc](https://mesharc.dev) API: a URL in, clean content out, and a record of what changed.
32
+
33
+ - **Scrape** one page or a batch — markdown, text, HTML, links, structured fields, a screenshot.
34
+ - **Crawl** a whole site with no project to set up first, and keep it as one if it turns out to be worth watching.
35
+ - **Map** what a site declares in its sitemaps before fetching any of it.
36
+ - **Watch** a site over time: projects, scheduled runs, and a change record — pages added, removed, modified, field by field.
37
+
38
+ Python 3.9 or newer. One dependency (`httpx`). Fully typed.
39
+
40
+ ## Install
41
+
42
+ ```bash
43
+ pip install mesharc
44
+ ```
45
+
46
+ ## Authentication
47
+
48
+ Every call needs an API key. Create one in the app under **Settings → API keys** — it is shown once — and give it to the client, or put it in `MESHARC_API_KEY` and construct the client with nothing:
49
+
50
+ ```python
51
+ from mesharc import MeshArc
52
+
53
+ arc = MeshArc("mesharc_...")
54
+ # or, with MESHARC_API_KEY in the environment:
55
+ arc = MeshArc()
56
+ # options: MeshArc(api_key, timeout=150.0, max_retries=2)
57
+ ```
58
+
59
+ The key is only ever sent as a bearer header to `api.mesharc.dev`.
60
+
61
+ A key carries the scopes it was made with (`read`, `write`, `admin`), optionally a set of projects it may see, an expiry and a rate limit. A route the key may not use answers `403`; a project it may not see answers `404`.
62
+
63
+ ## Quick start
64
+
65
+ ```python
66
+ from mesharc import MeshArc
67
+
68
+ arc = MeshArc("mesharc_...")
69
+
70
+ page = arc.scrape("https://example.com/pricing")
71
+ print(page["markdown"])
72
+ print(page["verdict"], page["method"], page["credits"]) # ok crawler 1
73
+ ```
74
+
75
+ `scrape` holds the request open until the page comes back (60 s by default), so there is nothing to poll for an ordinary page.
76
+
77
+ ## Reading pages
78
+
79
+ ### One page
80
+
81
+ ```python
82
+ page = arc.scrape("https://quotes.toscrape.com/js/", config={"render_js": "always"})
83
+ ```
84
+
85
+ `config` is any setting a project takes, by its API name (`render_js`, `only_main_content`, `formats`, `max_tier`, `wait_for_selector`, `actions`, …). The full list, with defaults, is at [mesharc.dev/docs/configuration](https://mesharc.dev/docs/configuration).
86
+
87
+ ```python
88
+ page = arc.scrape_one(
89
+ "https://example.com/",
90
+ formats="markdown,text,cleanHtml", # which bodies to return
91
+ timeout_s=120, # how long the API holds the request (120 max)
92
+ idempotency_key="pricing-2026-09-18", # the same key returns the first answer for 24 h
93
+ )
94
+ ```
95
+
96
+ ### Many pages
97
+
98
+ A list of URLs is a batch: grouped by host, fetched in parallel where the config allows, and returned as one row per URL.
99
+
100
+ ```python
101
+ batch = arc.scrape(["https://a.com/", "https://b.com/x"], config={"concurrency": 4})
102
+ for row in batch["pages"]:
103
+ print(row["url"], row["httpStatus"], row["verdict"], row["credits"])
104
+ ```
105
+
106
+ `arc.scrape(urls, wait=False)` returns the batch id at once; `arc.batch(id, wait=True)` finishes it later. Pass `webhook_url=` to be told instead of polling (`batch.finished`, signed with a secret returned once).
107
+
108
+ ### What a page looks like
109
+
110
+ Every page row carries the same fields, whether it came from a scrape, a crawl or a project:
111
+
112
+ | Field | Meaning |
113
+ |---|---|
114
+ | `markdown`, `text`, `cleanHtml`, `html`, `links`, `fields`, `screenshot` | The bodies you asked for |
115
+ | `httpStatus` | The status the site answered with |
116
+ | `verdict` | `ok`, `thin` (short, but a page), `blocked` (refused, or a 404), `skipped` |
117
+ | `errorCode` | `OK`, or what went wrong: `BLOCKED`, `NOT_FOUND`, `TIER_LIMIT`, `CAPTCHA`, `LOGIN_REQUIRED`, `RATE_LIMITED`, `TIMEOUT` … |
118
+ | `shape`, `warnings`, `signals` | `listing` / `table` / `form` for a short page whose markup says what it is; `short`; why the judge decided as it did |
119
+ | `method`, `tier`, `climbedTo` | The engine that read it (`crawler`, `tls`, `minted`, `browser`, `browser-residential`, …), its tier, and how far a refused page climbed |
120
+ | `credits`, `billedAs` | What it cost; the rung it is priced at when not the one that fetched it |
121
+ | `words`, `language`, `head`, `reason`, `crawledAt` | Size, language, the head fields, the judge's sentence, when |
122
+
123
+ A page the site refused costs 0, and so does a 404.
124
+
125
+ ## Crawling a site
126
+
127
+ ```python
128
+ job = arc.crawl(
129
+ "https://docs.example.com",
130
+ limit=200, # page budget
131
+ maxDepth=3, # link hops from the seed
132
+ includePaths=["/docs/*"],
133
+ crawlMode="sitemap_first", # what the sitemap declares first, then links
134
+ maxTier="browser", # how far a refused page may climb
135
+ scrapeOptions={"formats": ["markdown", "links"]},
136
+ config={"crawl_delay_ms": 500}, # any project setting, directly
137
+ )
138
+
139
+ for page in job.pages(): # follows the cursor while the crawl runs
140
+ print(page["url"], page["words"])
141
+ print(job.status, job.envelope["counts"], job.envelope["creditsUsed"])
142
+ ```
143
+
144
+ `crawl` returns a handle immediately; `job.pages()` yields pages as they land and ends when the crawl does. `wait=True` blocks until it finishes; `job.wait()`, `job.refresh()`, `job.cancel()` do what they say; `arc.get_crawl(id)` reattaches to a crawl started elsewhere.
145
+
146
+ A one-shot crawl expires after 30 days. If the site is worth watching:
147
+
148
+ ```python
149
+ project = job.keep(name="Docs", schedule="weekly")
150
+ ```
151
+
152
+ A `webhook=` in the options (`{"url", "events", "metadata"}`) is told about `crawl.started`, `crawl.page` (fifty pages a message) and `crawl.completed`; its signing secret comes back once as `job.webhook_secret`.
153
+
154
+ ## Mapping a site
155
+
156
+ ```python
157
+ for u in arc.map("https://docs.example.com"):
158
+ print(u["url"], u["lastmod"])
159
+
160
+ details = arc.map_details("https://www.gov.uk/", search="visa", limit=500)
161
+ print(details["totals"], details["creditsUsed"]) # {'files': 29, 'urls': 508431, …} 29
162
+ ```
163
+
164
+ A map costs one credit per sitemap file read — most sites are one file.
165
+
166
+ ## Watching a site: projects and runs
167
+
168
+ ```python
169
+ project = arc.projects.create(
170
+ "https://docs.example.com",
171
+ name="Docs",
172
+ schedule="weekly", # manual | hourly | daily | weekly
173
+ config={"max_pages": 300, "include_paths": ["/docs/*"]},
174
+ )
175
+
176
+ run = arc.runs.start(project["id"], wait=True) # the first run
177
+ # ...a week later, or arc.runs.start again: the second run produces the change record
178
+
179
+ record = arc.changes(project["id"])
180
+ print(record["change"]["counts"]) # {'added': …, 'removed': …, 'modified': …, 'withheld': …}
181
+
182
+ diff = arc.page_diff(project["id"], "https://docs.example.com/pricing")
183
+ ```
184
+
185
+ | Method | What it does |
186
+ |---|---|
187
+ | `projects.list()` · `projects.get(id)` · `projects.update(id, name=, schedule=, retention=, config=)` · `projects.delete(id)` | The projects |
188
+ | `runs.list(project_id)` · `runs.start(project_id, wait=)` · `runs.wait(project_id, run_id)` · `runs.get(project_id, run_id)` · `runs.cancel(project_id, run_id)` | Runs |
189
+ | `pages(project_id, run_id=None)` · `page(project_id, url, run_id=None)` | The pages of a run; one page in full |
190
+ | `changes(project_id, run_id=None)` · `page_diff(project_id, url, run_id=None)` | The change record; one page's word-level diff |
191
+ | `search(project_id, q, mode="content" \| "selector", run_id=None)` | Which pages say this (words, `"phrases"`) or contain this (CSS / XPath) |
192
+ | `recrawl(project_id, urls)` | Fetch these pages again, now |
193
+ | `sources(project_id)` | The seed, sitemap, URL list, feeds and patterns with what the last run found through each |
194
+ | `export(project_id, path, dataset="pages", fmt="jsonl", run_id=None, urls=None)` | Stream a dataset (`pages`, `markdown`, `changes`, `fields`, `sitemap`) as `jsonl` or `csv` to a file |
195
+
196
+ ```python
197
+ arc.export(project["id"], "pages.csv", dataset="pages", fmt="csv")
198
+ ```
199
+
200
+ ## The workspace
201
+
202
+ ```python
203
+ me = arc.me() # the workspace, its plan and limits, credits used and remaining, what this key may do
204
+ usage = arc.usage() # pages per day, this month by engine
205
+ monitor = arc.monitor() # what is queued and running
206
+ meta = arc.meta() # verdict meanings, engine costs, the config defaults
207
+ keys = arc.keys()
208
+ key = arc.create_key("ci", scopes=["read", "write"], projects=[project["id"]], expires_in_days=90) # key["key"], once
209
+ arc.revoke_key(key["id"])
210
+ ```
211
+
212
+ ## Errors
213
+
214
+ Every failure raises `MeshArcError`:
215
+
216
+ ```python
217
+ from mesharc import MeshArc, MeshArcError
218
+
219
+ try:
220
+ arc.crawl("https://example.com", limit=1_000_000)
221
+ except MeshArcError as exc:
222
+ print(exc.status, exc.code, exc.detail, exc.request_id)
223
+ ```
224
+
225
+ | `code` | Status | Meaning |
226
+ |---|---|---|
227
+ | `validation` | 400 / 422 | Something in the request is wrong; `detail` says what |
228
+ | `unauthorized` | 401 | No key, or a revoked or expired one |
229
+ | `plan_limit` | 402 | The plan does not include this, or the credits are spent |
230
+ | `forbidden` | 403 | The key's scopes do not allow it |
231
+ | `not_found` | 404 | No such thing — or not one this key may see |
232
+ | `conflict` | 409 | The request contradicts current state |
233
+ | `rate_limited` | 429 | Over the key's rate limit; `X-RateLimit-Reset` says when |
234
+ | `internal` | 500 | Quote `request_id` to support |
235
+
236
+ `request_id` is the id the API put on the response and in its own logs, so a support conversation starts from one string.
237
+
238
+ Two more cases: a network failure or a request that hits `timeout` raises `MeshArcError` with `status == 0` and `code` `network` or `timeout`; a job the client stopped waiting for raises `MeshArcTimeoutError` — both a `MeshArcError` and a `TimeoutError` — which carries `job_id` so you can poll it later (`arc.get_crawl(id)`, `arc.batch(id)`).
239
+
240
+ ## Idempotency and timeouts
241
+
242
+ - `scrape`, `scrape_one` and `crawl` take `idempotency_key=`: send the same key again within 24 hours and you get the first answer back rather than a second job.
243
+ - Waiting calls take `wait=`, `poll=` (seconds between polls) and `timeout=` (seconds before `TimeoutError`). `wait=False` returns the envelope at once; the default polls every 3 s for up to an hour.
244
+ - `timeout_s` on a single scrape is how long the API itself holds the request open (60 s by default, 120 at most); a slower page comes back as an id and is polled.
245
+ - `MeshArc(..., timeout=150.0)` is the HTTP timeout per request. A request is retried on 429, 502, 503, 504 and network failures when it is safe to repeat — a GET, a DELETE, or a POST with an idempotency key — up to `max_retries` times (2), honouring `Retry-After`.
246
+
247
+ ## Credits
248
+
249
+ Every response says what it cost: `credits` on a page, `creditsUsed` on a job envelope, `X-MeshArc-Credits` on the HTTP response. A page costs the engine that read it — a plain fetch 1, a render 4 — and a refused page or a 404 costs nothing. The schedule and the plans are at [mesharc.dev/docs/billing](https://mesharc.dev/docs/billing).
250
+
251
+ ## The MCP server
252
+
253
+ The package also ships MeshArc as an MCP server, so Claude Desktop, Claude Code, Cursor and any MCP client can scrape, crawl, map and read change records as tools. Python 3.10+.
254
+
255
+ ```bash
256
+ pip install "mesharc[mcp]"
257
+ MESHARC_API_KEY=mesharc_... mesharc-mcp # serves over stdio
258
+
259
+ # Claude Code
260
+ claude mcp add mesharc -e MESHARC_API_KEY=mesharc_... -- mesharc-mcp
261
+ ```
262
+
263
+ Tools: `scrape_urls`, `extract_url`, `map_site`, `crawl_site`, `keep_crawl_as_project`, `list_projects`, `create_project`, `start_run`, `list_pages`, `get_page`, `get_changes`, `search_pages`, `recrawl_pages`. Every tool is a call through this client, trimmed where a body would swamp a context window (markdown is capped per page; ask for one page to get all of it).
264
+
265
+ ## Anything else
266
+
267
+ The client is a thin wrapper: every method is one API call and returns the API's JSON as a `dict`. The full reference is at [mesharc.dev/docs/api](https://mesharc.dev/docs/api). Call `arc.close()` when you are done, or use the client as a context manager.
268
+
269
+ - Documentation: [mesharc.dev/docs](https://mesharc.dev/docs)
270
+ - Node client: `npm install mesharc` — [mesharc-node](https://github.com/mesharc-org/mesharc-node)
271
+ - Issues and pull requests: [mesharc-python](https://github.com/mesharc-org/mesharc-python)
272
+ - Questions: hello@mesharc.dev
273
+
274
+ MIT.
@@ -0,0 +1,9 @@
1
+ mesharc/__init__.py,sha256=hmvqwrTHauN2Tqvg3MjrjMLnONhxcGxYLnkXbOJfJoA,24301
2
+ mesharc/mcp.py,sha256=GTL8mWD1rpO5uKBdg5Wr2fOGR0XMr4Dv9EXWbhcccCM,10464
3
+ mesharc/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
+ mesharc-0.1.2.dist-info/licenses/LICENSE,sha256=gUvBXupjjMPMIBGEtujNeVFxFKCDu8RAGWpOgfV0Nas,1085
5
+ mesharc-0.1.2.dist-info/METADATA,sha256=jKyPYYy0nE-45pDVuOJj7Vq4q8YvE55v1eJ4VehlSXI,13448
6
+ mesharc-0.1.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
7
+ mesharc-0.1.2.dist-info/entry_points.txt,sha256=zqwmWOlh9U__HARPXqxApyRYJQoeHhsRbQyyCh3WNHQ,49
8
+ mesharc-0.1.2.dist-info/top_level.txt,sha256=KXErx_IOhEs5npgSe59IMxFKY68k1jHDYX2bfU7-jKI,8
9
+ mesharc-0.1.2.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ mesharc-mcp = mesharc.mcp:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 MeshArc
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ mesharc