urlpipe 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- urlpipe-0.1.0/.gitignore +13 -0
- urlpipe-0.1.0/CHANGELOG.md +13 -0
- urlpipe-0.1.0/LICENSE +21 -0
- urlpipe-0.1.0/PKG-INFO +262 -0
- urlpipe-0.1.0/README.md +226 -0
- urlpipe-0.1.0/pyproject.toml +60 -0
- urlpipe-0.1.0/src/urlpipe/__init__.py +78 -0
- urlpipe-0.1.0/src/urlpipe/_client.py +645 -0
- urlpipe-0.1.0/src/urlpipe/_core.py +483 -0
- urlpipe-0.1.0/src/urlpipe/_models.py +147 -0
- urlpipe-0.1.0/src/urlpipe/_version.py +1 -0
- urlpipe-0.1.0/src/urlpipe/errors.py +187 -0
- urlpipe-0.1.0/src/urlpipe/py.typed +0 -0
- urlpipe-0.1.0/src/urlpipe/webhooks.py +107 -0
- urlpipe-0.1.0/tests/conftest.py +200 -0
- urlpipe-0.1.0/tests/test_client.py +107 -0
- urlpipe-0.1.0/tests/test_errors.py +323 -0
- urlpipe-0.1.0/tests/test_live.py +22 -0
- urlpipe-0.1.0/tests/test_operations.py +396 -0
- urlpipe-0.1.0/tests/test_polling.py +93 -0
- urlpipe-0.1.0/tests/test_retries.py +130 -0
- urlpipe-0.1.0/tests/test_webhooks.py +111 -0
urlpipe-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 — 2026-09-25
|
|
4
|
+
|
|
5
|
+
The first release.
|
|
6
|
+
|
|
7
|
+
- `Client` and `AsyncClient` with one method per operation: `markdown`, `html`, `summarize`, `screenshot`, `meta`, `keywords`, `console`, `lighthouse` and `scrape`, plus `result` and `wait` for tokens.
|
|
8
|
+
- Calls wait for their result by default; a call that outlives the API's 60-second sync window is polled to completion.
|
|
9
|
+
- Screenshots decoded to bytes, with the MIME type and the key-free image URL.
|
|
10
|
+
- Response metadata parsed from headers: cache status and age, processing time, credits spent and left, concurrency limit.
|
|
11
|
+
- Automatic retries for connection errors, rate and concurrency limits and 500/502/503, each call under one `Idempotency-Key`.
|
|
12
|
+
- A typed error for every failure the API reports.
|
|
13
|
+
- `verify_webhook` for signed webhook deliveries.
|
urlpipe-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aliat Partner S.L.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
urlpipe-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: urlpipe
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Official Python client for the URLpipe API: turn a URL into Markdown, HTML, a screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit.
|
|
5
|
+
Project-URL: Homepage, https://urlpipe.dev
|
|
6
|
+
Project-URL: Documentation, https://urlpipe.dev/docs
|
|
7
|
+
Project-URL: Source, https://github.com/URLpipe/urlpipe-python
|
|
8
|
+
Project-URL: Changelog, https://github.com/URLpipe/urlpipe-python/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: URLpipe <contact@urlpipe.dev>
|
|
10
|
+
Maintainer-email: URLpipe <contact@urlpipe.dev>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: api client,html,lighthouse,llm,markdown,screenshot,urlpipe,web scraping
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Framework :: AsyncIO
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
27
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
28
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
29
|
+
Classifier: Typing :: Typed
|
|
30
|
+
Requires-Python: >=3.9
|
|
31
|
+
Requires-Dist: httpx<1,>=0.23
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: mypy>=1.5; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# urlpipe
|
|
38
|
+
|
|
39
|
+
Turn any URL into clean Markdown, rendered HTML, a full-page screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit — from Python, in one call. Pages are rendered in real Chrome, so JavaScript-heavy sites come back complete.
|
|
40
|
+
|
|
41
|
+
This is the official Python client for the [URLpipe API](https://urlpipe.dev/docs). It works sync or async, is fully typed, and depends only on `httpx`.
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
pip install urlpipe
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Python 3.9 or later.
|
|
50
|
+
|
|
51
|
+
## Quickstart
|
|
52
|
+
|
|
53
|
+
Create a project in the [dashboard](https://urlpipe.dev) and copy its API key. The Free plan gives you 1,000 credits a month, no card needed.
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
export URLPIPE_API_KEY=your_project_key
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import urlpipe
|
|
61
|
+
|
|
62
|
+
client = urlpipe.Client() # reads URLPIPE_API_KEY
|
|
63
|
+
|
|
64
|
+
page = client.markdown("https://example.com")
|
|
65
|
+
print(page.data) # "# Example Domain\n\nThis domain is for use in …"
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Every method waits for the result and returns a `Response`:
|
|
69
|
+
|
|
70
|
+
| Field | What it holds |
|
|
71
|
+
|---|---|
|
|
72
|
+
| `status` | `"completed"`, `"accepted"` (an async request was taken) or `"processing"` (still running) |
|
|
73
|
+
| `data` | the result, typed per operation; `None` unless completed |
|
|
74
|
+
| `token` | the request's token — fetch the result again for free with `client.result(token)` |
|
|
75
|
+
| `labels` | the labels the request was made with |
|
|
76
|
+
| `meta` | `cache`, `cache_age`, `processing_time_ms`, `quota` (`cost`, `limit`, `remaining`, `overage`, `resets_at`), `concurrency_limit`, `result_url`, `idempotent_replayed` |
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
page.meta.cache # "hit" — served from cache, cost nothing
|
|
80
|
+
page.meta.quota.remaining # 943, or "unlimited"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Operations
|
|
84
|
+
|
|
85
|
+
Every method takes the URL first; everything else is an optional keyword.
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
client.markdown("https://example.com").data # str: the main content as Markdown
|
|
89
|
+
client.html("https://example.com").data # str: the HTML after JavaScript ran
|
|
90
|
+
client.summarize("https://example.com").data # str: an AI summary, in Markdown
|
|
91
|
+
client.meta("https://example.com").data # dict: title, description, language, author, …
|
|
92
|
+
client.keywords("https://example.com").data # list[str], most relevant first
|
|
93
|
+
client.console("https://example.com").data # [{"type": "error", "text": "…"}, …]
|
|
94
|
+
client.lighthouse("https://example.com", device="desktop", include_audits=True).data
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Screenshots
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
shot = client.screenshot(
|
|
101
|
+
"https://example.com/pricing",
|
|
102
|
+
screenshot_options={"viewport_width": 390, "format": "webp"},
|
|
103
|
+
).data
|
|
104
|
+
|
|
105
|
+
shot.save("pricing.webp")
|
|
106
|
+
shot.mime_type # "image/webp", read from the image bytes
|
|
107
|
+
shot.result_url # a link to the image that needs no API key, for an <img> tag
|
|
108
|
+
shot.data # the decoded bytes
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Several operations off one page visit
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
result = client.scrape("https://example.com", ["markdown", "meta", "screenshot"]).data
|
|
115
|
+
result["operations"]["meta"]["result"]["title"]
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Each operation has its own `success`, `result`, `error` and `cached`; one failing never affects the others. A screenshot inside a scrape is Base64 text, as the API returns it.
|
|
119
|
+
|
|
120
|
+
### Options every method takes
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
client.markdown(
|
|
124
|
+
"https://example.com/blog",
|
|
125
|
+
max_age="1 hour", # or seconds; 0 skips the cache
|
|
126
|
+
labels={"client": "acme"}, # your own ids, returned with the result
|
|
127
|
+
residential=True, # fetch from a home broadband address
|
|
128
|
+
page_options={"block_cookie_banners": True, "remove_selectors": [".promo"]},
|
|
129
|
+
idempotency_key="import-2026-09-25-17",
|
|
130
|
+
)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`page_options` is not accepted by `lighthouse`. Options the API adds after this release can be sent with `extra`, which is merged into the request body as given:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
client.markdown("https://example.com", extra={"some_option": True})
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The client passes option values through unchanged; the API validates them and answers `InvalidRequestError` naming what to change.
|
|
140
|
+
|
|
141
|
+
## Async requests and `wait`
|
|
142
|
+
|
|
143
|
+
Pass `sync=False` and the call returns straight away with a token, while the work carries on. Collect the result by polling, or have it POSTed to a webhook with `report_to`.
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
accepted = client.lighthouse("https://example.com", sync=False)
|
|
147
|
+
accepted.status # "accepted"
|
|
148
|
+
|
|
149
|
+
report = client.wait(accepted.token, operation="lighthouse") # polls every 2 s
|
|
150
|
+
report.data["categories"]["performance"]["score"]
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`client.result(token)` makes a single check instead: it returns a `"processing"` Response while the work is running.
|
|
154
|
+
|
|
155
|
+
`GET /result` does not say which operation made a token, so pass `operation=` to get typed `data`. Without it, JSON comes back parsed and text as a string — a screenshot stays Base64 until you ask for `wait(token, operation="screenshot")`.
|
|
156
|
+
|
|
157
|
+
A synchronous call that runs past the API's 60-second window is not an error: the client polls for the result itself and returns it, for up to `wait_timeout` seconds (5 minutes by default) before raising `WaitTimeoutError`. The work keeps running either way, so `client.wait(error.token)` picks it up later.
|
|
158
|
+
|
|
159
|
+
### asyncio
|
|
160
|
+
|
|
161
|
+
`AsyncClient` has the same methods, as coroutines:
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
import asyncio
|
|
165
|
+
import urlpipe
|
|
166
|
+
|
|
167
|
+
async def main() -> None:
|
|
168
|
+
async with urlpipe.AsyncClient() as client:
|
|
169
|
+
pages = await asyncio.gather(
|
|
170
|
+
client.markdown("https://example.com/a"),
|
|
171
|
+
client.markdown("https://example.com/b"),
|
|
172
|
+
)
|
|
173
|
+
print([p.data for p in pages])
|
|
174
|
+
|
|
175
|
+
asyncio.run(main())
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Webhooks
|
|
179
|
+
|
|
180
|
+
With webhook signing on (project **Settings → Webhook Signing**), check each delivery came from URLpipe before you trust it. `verify_webhook` needs no client:
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
import os
|
|
184
|
+
import urlpipe
|
|
185
|
+
from flask import Flask, request, abort
|
|
186
|
+
|
|
187
|
+
app = Flask(__name__)
|
|
188
|
+
|
|
189
|
+
@app.post("/webhooks/urlpipe")
|
|
190
|
+
def urlpipe_webhook():
|
|
191
|
+
try:
|
|
192
|
+
delivery = urlpipe.verify_webhook(
|
|
193
|
+
request.get_data(), # the raw body bytes
|
|
194
|
+
request.headers,
|
|
195
|
+
os.environ["URLPIPE_WEBHOOK_SECRET"], # whsec_…
|
|
196
|
+
)
|
|
197
|
+
except urlpipe.WebhookVerificationError:
|
|
198
|
+
abort(401)
|
|
199
|
+
enqueue(delivery["token"], delivery["result"])
|
|
200
|
+
return "", 200
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Give it the **raw request body** exactly as received (`request.get_data()` in Flask, `await request.body()` in FastAPI, `request.body` in Django). Parsed and re-serialised JSON has different bytes, and the signature will not match. Header names are matched case-insensitively; deliveries signed more than `tolerance` seconds ago (300 by default) are refused, and during a secret rotation either signature is accepted.
|
|
204
|
+
|
|
205
|
+
It returns the payload: `token`, `operation`, `labels`, `success`, `result`, `result_url`, `error` and `meta`.
|
|
206
|
+
|
|
207
|
+
## Errors
|
|
208
|
+
|
|
209
|
+
Every error is a `urlpipe.UrlpipeError`, with `status`, `code`, `message`, `body` and `token`.
|
|
210
|
+
|
|
211
|
+
| Error | When |
|
|
212
|
+
|---|---|
|
|
213
|
+
| `AuthenticationError` | 401: the API key is missing or wrong |
|
|
214
|
+
| `EmailUnverifiedError` | 403: confirm the email address on the account |
|
|
215
|
+
| `InvalidRequestError` | 422: a parameter was refused (`invalid_url`, `invalid_options`, …); nothing ran |
|
|
216
|
+
| `AnalysisFailedError` | 422: the page could not be analysed; `message` says why |
|
|
217
|
+
| `QuotaExceededError` | 429: the Free plan's credits are spent (`limit`, `used`, `needed`, `resets_at`) |
|
|
218
|
+
| `ConcurrencyLimitError` | 429: too many of your requests running (`limit`, `running`) |
|
|
219
|
+
| `RateLimitedError` | 429: sending too fast (`retry_after`) |
|
|
220
|
+
| `NotFoundError` | 404: no result for this token |
|
|
221
|
+
| `StaleResultError` | 410: the result is past the 30-day window |
|
|
222
|
+
| `ServerError` | 5xx |
|
|
223
|
+
| `APIConnectionError` | no HTTP answer at all |
|
|
224
|
+
| `WaitTimeoutError` | the result was not ready within the wait timeout (`token`) |
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
try:
|
|
228
|
+
page = client.markdown(url)
|
|
229
|
+
except urlpipe.AnalysisFailedError as error:
|
|
230
|
+
print("Could not read the page:", error.message)
|
|
231
|
+
except urlpipe.QuotaExceededError as error:
|
|
232
|
+
print("Out of credits until", error.resets_at)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
A failed analysis costs nothing, and neither does a cache hit.
|
|
236
|
+
|
|
237
|
+
## Retries and idempotency
|
|
238
|
+
|
|
239
|
+
The client retries connection errors, rate limits (after `Retry-After`, up to 60 s), concurrency limits and 500/502/503 responses — twice by default, with exponential backoff. It never retries a 4xx other than those, or a spent quota.
|
|
240
|
+
|
|
241
|
+
A retry is only safe if it cannot run the work twice, so every analysis the client may retry carries an `Idempotency-Key`: yours if you pass `idempotency_key=`, otherwise one generated for that call and reused by all of its retries. The API answers a repeated key with the first request's result — one run, one charge, one webhook. When you retry across processes yourself (a job that ran twice), pass the same key each time.
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
client = urlpipe.Client(
|
|
245
|
+
api_key="…", # default: URLPIPE_API_KEY
|
|
246
|
+
timeout=90, # seconds per HTTP request
|
|
247
|
+
max_retries=2, # 0 turns retries off
|
|
248
|
+
wait_timeout=300, # how long a slow sync call keeps polling
|
|
249
|
+
)
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
For a proxy, custom TLS or test transports, pass your own `httpx.Client` (or `httpx.AsyncClient`) as `http_client=`. You keep ownership of it; closing the URLpipe client leaves it open.
|
|
253
|
+
|
|
254
|
+
## Links
|
|
255
|
+
|
|
256
|
+
- API docs: https://urlpipe.dev/docs
|
|
257
|
+
- Pricing: https://urlpipe.dev/pricing
|
|
258
|
+
- MCP server, for using URLpipe from AI assistants: https://github.com/URLpipe/mcp
|
|
259
|
+
|
|
260
|
+
## License
|
|
261
|
+
|
|
262
|
+
MIT © Aliat Partner S.L.
|
urlpipe-0.1.0/README.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# urlpipe
|
|
2
|
+
|
|
3
|
+
Turn any URL into clean Markdown, rendered HTML, a full-page screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit — from Python, in one call. Pages are rendered in real Chrome, so JavaScript-heavy sites come back complete.
|
|
4
|
+
|
|
5
|
+
This is the official Python client for the [URLpipe API](https://urlpipe.dev/docs). It works sync or async, is fully typed, and depends only on `httpx`.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
pip install urlpipe
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Python 3.9 or later.
|
|
14
|
+
|
|
15
|
+
## Quickstart
|
|
16
|
+
|
|
17
|
+
Create a project in the [dashboard](https://urlpipe.dev) and copy its API key. The Free plan gives you 1,000 credits a month, no card needed.
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
export URLPIPE_API_KEY=your_project_key
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
import urlpipe
|
|
25
|
+
|
|
26
|
+
client = urlpipe.Client() # reads URLPIPE_API_KEY
|
|
27
|
+
|
|
28
|
+
page = client.markdown("https://example.com")
|
|
29
|
+
print(page.data) # "# Example Domain\n\nThis domain is for use in …"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Every method waits for the result and returns a `Response`:
|
|
33
|
+
|
|
34
|
+
| Field | What it holds |
|
|
35
|
+
|---|---|
|
|
36
|
+
| `status` | `"completed"`, `"accepted"` (an async request was taken) or `"processing"` (still running) |
|
|
37
|
+
| `data` | the result, typed per operation; `None` unless completed |
|
|
38
|
+
| `token` | the request's token — fetch the result again for free with `client.result(token)` |
|
|
39
|
+
| `labels` | the labels the request was made with |
|
|
40
|
+
| `meta` | `cache`, `cache_age`, `processing_time_ms`, `quota` (`cost`, `limit`, `remaining`, `overage`, `resets_at`), `concurrency_limit`, `result_url`, `idempotent_replayed` |
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
page.meta.cache # "hit" — served from cache, cost nothing
|
|
44
|
+
page.meta.quota.remaining # 943, or "unlimited"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Operations
|
|
48
|
+
|
|
49
|
+
Every method takes the URL first; everything else is an optional keyword.
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
client.markdown("https://example.com").data # str: the main content as Markdown
|
|
53
|
+
client.html("https://example.com").data # str: the HTML after JavaScript ran
|
|
54
|
+
client.summarize("https://example.com").data # str: an AI summary, in Markdown
|
|
55
|
+
client.meta("https://example.com").data # dict: title, description, language, author, …
|
|
56
|
+
client.keywords("https://example.com").data # list[str], most relevant first
|
|
57
|
+
client.console("https://example.com").data # [{"type": "error", "text": "…"}, …]
|
|
58
|
+
client.lighthouse("https://example.com", device="desktop", include_audits=True).data
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Screenshots
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
shot = client.screenshot(
|
|
65
|
+
"https://example.com/pricing",
|
|
66
|
+
screenshot_options={"viewport_width": 390, "format": "webp"},
|
|
67
|
+
).data
|
|
68
|
+
|
|
69
|
+
shot.save("pricing.webp")
|
|
70
|
+
shot.mime_type # "image/webp", read from the image bytes
|
|
71
|
+
shot.result_url # a link to the image that needs no API key, for an <img> tag
|
|
72
|
+
shot.data # the decoded bytes
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Several operations off one page visit
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
result = client.scrape("https://example.com", ["markdown", "meta", "screenshot"]).data
|
|
79
|
+
result["operations"]["meta"]["result"]["title"]
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Each operation has its own `success`, `result`, `error` and `cached`; one failing never affects the others. A screenshot inside a scrape is Base64 text, as the API returns it.
|
|
83
|
+
|
|
84
|
+
### Options every method takes
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
client.markdown(
|
|
88
|
+
"https://example.com/blog",
|
|
89
|
+
max_age="1 hour", # or seconds; 0 skips the cache
|
|
90
|
+
labels={"client": "acme"}, # your own ids, returned with the result
|
|
91
|
+
residential=True, # fetch from a home broadband address
|
|
92
|
+
page_options={"block_cookie_banners": True, "remove_selectors": [".promo"]},
|
|
93
|
+
idempotency_key="import-2026-09-25-17",
|
|
94
|
+
)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`page_options` is not accepted by `lighthouse`. Options the API adds after this release can be sent with `extra`, which is merged into the request body as given:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
client.markdown("https://example.com", extra={"some_option": True})
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The client passes option values through unchanged; the API validates them and answers `InvalidRequestError` naming what to change.
|
|
104
|
+
|
|
105
|
+
## Async requests and `wait`
|
|
106
|
+
|
|
107
|
+
Pass `sync=False` and the call returns straight away with a token, while the work carries on. Collect the result by polling, or have it POSTed to a webhook with `report_to`.
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
accepted = client.lighthouse("https://example.com", sync=False)
|
|
111
|
+
accepted.status # "accepted"
|
|
112
|
+
|
|
113
|
+
report = client.wait(accepted.token, operation="lighthouse") # polls every 2 s
|
|
114
|
+
report.data["categories"]["performance"]["score"]
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`client.result(token)` makes a single check instead: it returns a `"processing"` Response while the work is running.
|
|
118
|
+
|
|
119
|
+
`GET /result` does not say which operation made a token, so pass `operation=` to get typed `data`. Without it, JSON comes back parsed and text as a string — a screenshot stays Base64 until you ask for `wait(token, operation="screenshot")`.
|
|
120
|
+
|
|
121
|
+
A synchronous call that runs past the API's 60-second window is not an error: the client polls for the result itself and returns it, for up to `wait_timeout` seconds (5 minutes by default) before raising `WaitTimeoutError`. The work keeps running either way, so `client.wait(error.token)` picks it up later.
|
|
122
|
+
|
|
123
|
+
### asyncio
|
|
124
|
+
|
|
125
|
+
`AsyncClient` has the same methods, as coroutines:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
import asyncio
|
|
129
|
+
import urlpipe
|
|
130
|
+
|
|
131
|
+
async def main() -> None:
|
|
132
|
+
async with urlpipe.AsyncClient() as client:
|
|
133
|
+
pages = await asyncio.gather(
|
|
134
|
+
client.markdown("https://example.com/a"),
|
|
135
|
+
client.markdown("https://example.com/b"),
|
|
136
|
+
)
|
|
137
|
+
print([p.data for p in pages])
|
|
138
|
+
|
|
139
|
+
asyncio.run(main())
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Webhooks
|
|
143
|
+
|
|
144
|
+
With webhook signing on (project **Settings → Webhook Signing**), check each delivery came from URLpipe before you trust it. `verify_webhook` needs no client:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
import os
|
|
148
|
+
import urlpipe
|
|
149
|
+
from flask import Flask, request, abort
|
|
150
|
+
|
|
151
|
+
app = Flask(__name__)
|
|
152
|
+
|
|
153
|
+
@app.post("/webhooks/urlpipe")
|
|
154
|
+
def urlpipe_webhook():
|
|
155
|
+
try:
|
|
156
|
+
delivery = urlpipe.verify_webhook(
|
|
157
|
+
request.get_data(), # the raw body bytes
|
|
158
|
+
request.headers,
|
|
159
|
+
os.environ["URLPIPE_WEBHOOK_SECRET"], # whsec_…
|
|
160
|
+
)
|
|
161
|
+
except urlpipe.WebhookVerificationError:
|
|
162
|
+
abort(401)
|
|
163
|
+
enqueue(delivery["token"], delivery["result"])
|
|
164
|
+
return "", 200
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Give it the **raw request body** exactly as received (`request.get_data()` in Flask, `await request.body()` in FastAPI, `request.body` in Django). Parsed and re-serialised JSON has different bytes, and the signature will not match. Header names are matched case-insensitively; deliveries signed more than `tolerance` seconds ago (300 by default) are refused, and during a secret rotation either signature is accepted.
|
|
168
|
+
|
|
169
|
+
It returns the payload: `token`, `operation`, `labels`, `success`, `result`, `result_url`, `error` and `meta`.
|
|
170
|
+
|
|
171
|
+
## Errors
|
|
172
|
+
|
|
173
|
+
Every error is a `urlpipe.UrlpipeError`, with `status`, `code`, `message`, `body` and `token`.
|
|
174
|
+
|
|
175
|
+
| Error | When |
|
|
176
|
+
|---|---|
|
|
177
|
+
| `AuthenticationError` | 401: the API key is missing or wrong |
|
|
178
|
+
| `EmailUnverifiedError` | 403: confirm the email address on the account |
|
|
179
|
+
| `InvalidRequestError` | 422: a parameter was refused (`invalid_url`, `invalid_options`, …); nothing ran |
|
|
180
|
+
| `AnalysisFailedError` | 422: the page could not be analysed; `message` says why |
|
|
181
|
+
| `QuotaExceededError` | 429: the Free plan's credits are spent (`limit`, `used`, `needed`, `resets_at`) |
|
|
182
|
+
| `ConcurrencyLimitError` | 429: too many of your requests running (`limit`, `running`) |
|
|
183
|
+
| `RateLimitedError` | 429: sending too fast (`retry_after`) |
|
|
184
|
+
| `NotFoundError` | 404: no result for this token |
|
|
185
|
+
| `StaleResultError` | 410: the result is past the 30-day window |
|
|
186
|
+
| `ServerError` | 5xx |
|
|
187
|
+
| `APIConnectionError` | no HTTP answer at all |
|
|
188
|
+
| `WaitTimeoutError` | the result was not ready within the wait timeout (`token`) |
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
try:
|
|
192
|
+
page = client.markdown(url)
|
|
193
|
+
except urlpipe.AnalysisFailedError as error:
|
|
194
|
+
print("Could not read the page:", error.message)
|
|
195
|
+
except urlpipe.QuotaExceededError as error:
|
|
196
|
+
print("Out of credits until", error.resets_at)
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
A failed analysis costs nothing, and neither does a cache hit.
|
|
200
|
+
|
|
201
|
+
## Retries and idempotency
|
|
202
|
+
|
|
203
|
+
The client retries connection errors, rate limits (after `Retry-After`, up to 60 s), concurrency limits and 500/502/503 responses — twice by default, with exponential backoff. It never retries a 4xx other than those, or a spent quota.
|
|
204
|
+
|
|
205
|
+
A retry is only safe if it cannot run the work twice, so every analysis the client may retry carries an `Idempotency-Key`: yours if you pass `idempotency_key=`, otherwise one generated for that call and reused by all of its retries. The API answers a repeated key with the first request's result — one run, one charge, one webhook. When you retry across processes yourself (a job that ran twice), pass the same key each time.
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
client = urlpipe.Client(
|
|
209
|
+
api_key="…", # default: URLPIPE_API_KEY
|
|
210
|
+
timeout=90, # seconds per HTTP request
|
|
211
|
+
max_retries=2, # 0 turns retries off
|
|
212
|
+
wait_timeout=300, # how long a slow sync call keeps polling
|
|
213
|
+
)
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
For a proxy, custom TLS or test transports, pass your own `httpx.Client` (or `httpx.AsyncClient`) as `http_client=`. You keep ownership of it; closing the URLpipe client leaves it open.
|
|
217
|
+
|
|
218
|
+
## Links
|
|
219
|
+
|
|
220
|
+
- API docs: https://urlpipe.dev/docs
|
|
221
|
+
- Pricing: https://urlpipe.dev/pricing
|
|
222
|
+
- MCP server, for using URLpipe from AI assistants: https://github.com/URLpipe/mcp
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
MIT © Aliat Partner S.L.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "urlpipe"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Official Python client for the URLpipe API: turn a URL into Markdown, HTML, a screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
authors = [{ name = "URLpipe", email = "contact@urlpipe.dev" }]
|
|
14
|
+
maintainers = [{ name = "URLpipe", email = "contact@urlpipe.dev" }]
|
|
15
|
+
keywords = ["urlpipe", "markdown", "screenshot", "html", "lighthouse", "web scraping", "llm", "api client"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Programming Language :: Python :: 3.14",
|
|
29
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
30
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
31
|
+
"Typing :: Typed",
|
|
32
|
+
"Framework :: AsyncIO",
|
|
33
|
+
]
|
|
34
|
+
dependencies = ["httpx>=0.23,<1"]
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
dev = ["pytest>=7", "mypy>=1.5"]
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://urlpipe.dev"
|
|
41
|
+
Documentation = "https://urlpipe.dev/docs"
|
|
42
|
+
Source = "https://github.com/URLpipe/urlpipe-python"
|
|
43
|
+
Changelog = "https://github.com/URLpipe/urlpipe-python/blob/main/CHANGELOG.md"
|
|
44
|
+
|
|
45
|
+
[tool.hatch.version]
|
|
46
|
+
path = "src/urlpipe/_version.py"
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.wheel]
|
|
49
|
+
packages = ["src/urlpipe"]
|
|
50
|
+
|
|
51
|
+
[tool.hatch.build.targets.sdist]
|
|
52
|
+
include = ["src/urlpipe", "tests", "README.md", "CHANGELOG.md", "LICENSE", "pyproject.toml"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
addopts = "-ra"
|
|
57
|
+
|
|
58
|
+
[tool.mypy]
|
|
59
|
+
strict = true
|
|
60
|
+
files = ["src/urlpipe"]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""The official Python client for the URLpipe API: turn a URL into Markdown,
|
|
2
|
+
HTML, a screenshot, metadata, a summary, keywords, console errors or a
|
|
3
|
+
Lighthouse audit.
|
|
4
|
+
|
|
5
|
+
import urlpipe
|
|
6
|
+
|
|
7
|
+
client = urlpipe.Client() # reads URLPIPE_API_KEY
|
|
8
|
+
print(client.markdown("https://example.com").data)
|
|
9
|
+
|
|
10
|
+
Docs: https://urlpipe.dev/docs
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from . import errors
|
|
14
|
+
from ._client import AsyncClient, Client
|
|
15
|
+
from ._models import (
|
|
16
|
+
ConsoleEntry,
|
|
17
|
+
Operation,
|
|
18
|
+
PageMeta,
|
|
19
|
+
Quota,
|
|
20
|
+
Response,
|
|
21
|
+
ResponseMeta,
|
|
22
|
+
ResponseStatus,
|
|
23
|
+
ScrapeOperationResult,
|
|
24
|
+
ScrapeResult,
|
|
25
|
+
Screenshot,
|
|
26
|
+
)
|
|
27
|
+
from ._version import __version__
|
|
28
|
+
from .errors import (
|
|
29
|
+
AnalysisFailedError,
|
|
30
|
+
APIConnectionError,
|
|
31
|
+
AuthenticationError,
|
|
32
|
+
ConcurrencyLimitError,
|
|
33
|
+
ConfigurationError,
|
|
34
|
+
EmailUnverifiedError,
|
|
35
|
+
InvalidRequestError,
|
|
36
|
+
NotFoundError,
|
|
37
|
+
QuotaExceededError,
|
|
38
|
+
RateLimitedError,
|
|
39
|
+
ServerError,
|
|
40
|
+
StaleResultError,
|
|
41
|
+
UrlpipeError,
|
|
42
|
+
WaitTimeoutError,
|
|
43
|
+
WebhookVerificationError,
|
|
44
|
+
)
|
|
45
|
+
from .webhooks import verify_webhook
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"APIConnectionError",
|
|
49
|
+
"AnalysisFailedError",
|
|
50
|
+
"AsyncClient",
|
|
51
|
+
"AuthenticationError",
|
|
52
|
+
"Client",
|
|
53
|
+
"ConcurrencyLimitError",
|
|
54
|
+
"ConfigurationError",
|
|
55
|
+
"ConsoleEntry",
|
|
56
|
+
"EmailUnverifiedError",
|
|
57
|
+
"InvalidRequestError",
|
|
58
|
+
"NotFoundError",
|
|
59
|
+
"Operation",
|
|
60
|
+
"PageMeta",
|
|
61
|
+
"Quota",
|
|
62
|
+
"QuotaExceededError",
|
|
63
|
+
"RateLimitedError",
|
|
64
|
+
"Response",
|
|
65
|
+
"ResponseMeta",
|
|
66
|
+
"ResponseStatus",
|
|
67
|
+
"ScrapeOperationResult",
|
|
68
|
+
"ScrapeResult",
|
|
69
|
+
"Screenshot",
|
|
70
|
+
"ServerError",
|
|
71
|
+
"StaleResultError",
|
|
72
|
+
"UrlpipeError",
|
|
73
|
+
"WaitTimeoutError",
|
|
74
|
+
"WebhookVerificationError",
|
|
75
|
+
"__version__",
|
|
76
|
+
"errors",
|
|
77
|
+
"verify_webhook",
|
|
78
|
+
]
|