urlpipe 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ .venv*/
2
+ venv/
3
+ dist/
4
+ build/
5
+ *.egg-info/
6
+ __pycache__/
7
+ *.py[cod]
8
+ .pytest_cache/
9
+ .mypy_cache/
10
+ .ruff_cache/
11
+ .coverage
12
+ htmlcov/
13
+ .DS_Store
@@ -0,0 +1,13 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 — 2026-09-25
4
+
5
+ The first release.
6
+
7
+ - `Client` and `AsyncClient` with one method per operation: `markdown`, `html`, `summarize`, `screenshot`, `meta`, `keywords`, `console`, `lighthouse` and `scrape`, plus `result` and `wait` for tokens.
8
+ - Calls wait for their result by default; a call that outlives the API's 60-second sync window is polled to completion.
9
+ - Screenshots decoded to bytes, with the MIME type and the key-free image URL.
10
+ - Response metadata parsed from headers: cache status and age, processing time, credits spent and left, concurrency limit.
11
+ - Automatic retries for connection errors, rate and concurrency limits and 500/502/503, each call under one `Idempotency-Key`.
12
+ - A typed error for every failure the API reports.
13
+ - `verify_webhook` for signed webhook deliveries.
urlpipe-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Aliat Partner S.L.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
urlpipe-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,262 @@
1
+ Metadata-Version: 2.5
2
+ Name: urlpipe
3
+ Version: 0.1.0
4
+ Summary: Official Python client for the URLpipe API: turn a URL into Markdown, HTML, a screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit.
5
+ Project-URL: Homepage, https://urlpipe.dev
6
+ Project-URL: Documentation, https://urlpipe.dev/docs
7
+ Project-URL: Source, https://github.com/URLpipe/urlpipe-python
8
+ Project-URL: Changelog, https://github.com/URLpipe/urlpipe-python/blob/main/CHANGELOG.md
9
+ Author-email: URLpipe <contact@urlpipe.dev>
10
+ Maintainer-email: URLpipe <contact@urlpipe.dev>
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ Keywords: api client,html,lighthouse,llm,markdown,screenshot,urlpipe,web scraping
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Framework :: AsyncIO
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Programming Language :: Python :: 3.9
22
+ Classifier: Programming Language :: Python :: 3.10
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Classifier: Programming Language :: Python :: 3.13
26
+ Classifier: Programming Language :: Python :: 3.14
27
+ Classifier: Topic :: Internet :: WWW/HTTP
28
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
29
+ Classifier: Typing :: Typed
30
+ Requires-Python: >=3.9
31
+ Requires-Dist: httpx<1,>=0.23
32
+ Provides-Extra: dev
33
+ Requires-Dist: mypy>=1.5; extra == 'dev'
34
+ Requires-Dist: pytest>=7; extra == 'dev'
35
+ Description-Content-Type: text/markdown
36
+
37
+ # urlpipe
38
+
39
+ Turn any URL into clean Markdown, rendered HTML, a full-page screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit — from Python, in one call. Pages are rendered in real Chrome, so JavaScript-heavy sites come back complete.
40
+
41
+ This is the official Python client for the [URLpipe API](https://urlpipe.dev/docs). It works sync or async, is fully typed, and depends only on `httpx`.
42
+
43
+ ## Install
44
+
45
+ ```sh
46
+ pip install urlpipe
47
+ ```
48
+
49
+ Python 3.9 or later.
50
+
51
+ ## Quickstart
52
+
53
+ Create a project in the [dashboard](https://urlpipe.dev) and copy its API key. The Free plan gives you 1,000 credits a month, no card needed.
54
+
55
+ ```sh
56
+ export URLPIPE_API_KEY=your_project_key
57
+ ```
58
+
59
+ ```python
60
+ import urlpipe
61
+
62
+ client = urlpipe.Client() # reads URLPIPE_API_KEY
63
+
64
+ page = client.markdown("https://example.com")
65
+ print(page.data) # "# Example Domain\n\nThis domain is for use in …"
66
+ ```
67
+
68
+ Every method waits for the result and returns a `Response`:
69
+
70
+ | Field | What it holds |
71
+ |---|---|
72
+ | `status` | `"completed"`, `"accepted"` (an async request was taken) or `"processing"` (still running) |
73
+ | `data` | the result, typed per operation; `None` unless completed |
74
+ | `token` | the request's token — fetch the result again for free with `client.result(token)` |
75
+ | `labels` | the labels the request was made with |
76
+ | `meta` | `cache`, `cache_age`, `processing_time_ms`, `quota` (`cost`, `limit`, `remaining`, `overage`, `resets_at`), `concurrency_limit`, `result_url`, `idempotent_replayed` |
77
+
78
+ ```python
79
+ page.meta.cache # "hit" — served from cache, cost nothing
80
+ page.meta.quota.remaining # 943, or "unlimited"
81
+ ```
82
+
83
+ ## Operations
84
+
85
+ Every method takes the URL first; everything else is an optional keyword.
86
+
87
+ ```python
88
+ client.markdown("https://example.com").data # str: the main content as Markdown
89
+ client.html("https://example.com").data # str: the HTML after JavaScript ran
90
+ client.summarize("https://example.com").data # str: an AI summary, in Markdown
91
+ client.meta("https://example.com").data # dict: title, description, language, author, …
92
+ client.keywords("https://example.com").data # list[str], most relevant first
93
+ client.console("https://example.com").data # [{"type": "error", "text": "…"}, …]
94
+ client.lighthouse("https://example.com", device="desktop", include_audits=True).data
95
+ ```
96
+
97
+ ### Screenshots
98
+
99
+ ```python
100
+ shot = client.screenshot(
101
+ "https://example.com/pricing",
102
+ screenshot_options={"viewport_width": 390, "format": "webp"},
103
+ ).data
104
+
105
+ shot.save("pricing.webp")
106
+ shot.mime_type # "image/webp", read from the image bytes
107
+ shot.result_url # a link to the image that needs no API key, for an <img> tag
108
+ shot.data # the decoded bytes
109
+ ```
110
+
111
+ ### Several operations off one page visit
112
+
113
+ ```python
114
+ result = client.scrape("https://example.com", ["markdown", "meta", "screenshot"]).data
115
+ result["operations"]["meta"]["result"]["title"]
116
+ ```
117
+
118
+ Each operation has its own `success`, `result`, `error` and `cached`; one failing never affects the others. A screenshot inside a scrape is Base64 text, as the API returns it.
119
+
120
+ ### Options every method takes
121
+
122
+ ```python
123
+ client.markdown(
124
+ "https://example.com/blog",
125
+ max_age="1 hour", # or seconds; 0 skips the cache
126
+ labels={"client": "acme"}, # your own ids, returned with the result
127
+ residential=True, # fetch from a home broadband address
128
+ page_options={"block_cookie_banners": True, "remove_selectors": [".promo"]},
129
+ idempotency_key="import-2026-09-25-17",
130
+ )
131
+ ```
132
+
133
+ `page_options` is not accepted by `lighthouse`. Options the API adds after this release can be sent with `extra`, which is merged into the request body as given:
134
+
135
+ ```python
136
+ client.markdown("https://example.com", extra={"some_option": True})
137
+ ```
138
+
139
+ The client passes option values through unchanged; the API validates them and answers `InvalidRequestError` naming what to change.
140
+
141
+ ## Async requests and `wait`
142
+
143
+ Pass `sync=False` and the call returns straight away with a token, while the work carries on. Collect the result by polling, or have it POSTed to a webhook with `report_to`.
144
+
145
+ ```python
146
+ accepted = client.lighthouse("https://example.com", sync=False)
147
+ accepted.status # "accepted"
148
+
149
+ report = client.wait(accepted.token, operation="lighthouse") # polls every 2 s
150
+ report.data["categories"]["performance"]["score"]
151
+ ```
152
+
153
+ `client.result(token)` makes a single check instead: it returns a `"processing"` Response while the work is running.
154
+
155
+ `GET /result` does not say which operation made a token, so pass `operation=` to get typed `data`. Without it, JSON comes back parsed and text as a string — a screenshot stays Base64 until you ask for `wait(token, operation="screenshot")`.
156
+
157
+ A synchronous call that runs past the API's 60-second window is not an error: the client polls for the result itself and returns it, for up to `wait_timeout` seconds (5 minutes by default) before raising `WaitTimeoutError`. The work keeps running either way, so `client.wait(error.token)` picks it up later.
158
+
159
+ ### asyncio
160
+
161
+ `AsyncClient` has the same methods, as coroutines:
162
+
163
+ ```python
164
+ import asyncio
165
+ import urlpipe
166
+
167
+ async def main() -> None:
168
+ async with urlpipe.AsyncClient() as client:
169
+ pages = await asyncio.gather(
170
+ client.markdown("https://example.com/a"),
171
+ client.markdown("https://example.com/b"),
172
+ )
173
+ print([p.data for p in pages])
174
+
175
+ asyncio.run(main())
176
+ ```
177
+
178
+ ## Webhooks
179
+
180
+ With webhook signing on (project **Settings → Webhook Signing**), check each delivery came from URLpipe before you trust it. `verify_webhook` needs no client:
181
+
182
+ ```python
183
+ import os
184
+ import urlpipe
185
+ from flask import Flask, request, abort
186
+
187
+ app = Flask(__name__)
188
+
189
+ @app.post("/webhooks/urlpipe")
190
+ def urlpipe_webhook():
191
+ try:
192
+ delivery = urlpipe.verify_webhook(
193
+ request.get_data(), # the raw body bytes
194
+ request.headers,
195
+ os.environ["URLPIPE_WEBHOOK_SECRET"], # whsec_…
196
+ )
197
+ except urlpipe.WebhookVerificationError:
198
+ abort(401)
199
+ enqueue(delivery["token"], delivery["result"])
200
+ return "", 200
201
+ ```
202
+
203
+ Give it the **raw request body** exactly as received (`request.get_data()` in Flask, `await request.body()` in FastAPI, `request.body` in Django). Parsed and re-serialised JSON has different bytes, and the signature will not match. Header names are matched case-insensitively; deliveries signed more than `tolerance` seconds ago (300 by default) are refused, and during a secret rotation either signature is accepted.
204
+
205
+ It returns the payload: `token`, `operation`, `labels`, `success`, `result`, `result_url`, `error` and `meta`.
206
+
207
+ ## Errors
208
+
209
+ Every error is a `urlpipe.UrlpipeError`, with `status`, `code`, `message`, `body` and `token`.
210
+
211
+ | Error | When |
212
+ |---|---|
213
+ | `AuthenticationError` | 401: the API key is missing or wrong |
214
+ | `EmailUnverifiedError` | 403: confirm the email address on the account |
215
+ | `InvalidRequestError` | 422: a parameter was refused (`invalid_url`, `invalid_options`, …); nothing ran |
216
+ | `AnalysisFailedError` | 422: the page could not be analysed; `message` says why |
217
+ | `QuotaExceededError` | 429: the Free plan's credits are spent (`limit`, `used`, `needed`, `resets_at`) |
218
+ | `ConcurrencyLimitError` | 429: too many of your requests running (`limit`, `running`) |
219
+ | `RateLimitedError` | 429: sending too fast (`retry_after`) |
220
+ | `NotFoundError` | 404: no result for this token |
221
+ | `StaleResultError` | 410: the result is past the 30-day window |
222
+ | `ServerError` | 5xx |
223
+ | `APIConnectionError` | no HTTP answer at all |
224
+ | `WaitTimeoutError` | the result was not ready within the wait timeout (`token`) |
225
+
226
+ ```python
227
+ try:
228
+ page = client.markdown(url)
229
+ except urlpipe.AnalysisFailedError as error:
230
+ print("Could not read the page:", error.message)
231
+ except urlpipe.QuotaExceededError as error:
232
+ print("Out of credits until", error.resets_at)
233
+ ```
234
+
235
+ A failed analysis costs nothing, and neither does a cache hit.
236
+
237
+ ## Retries and idempotency
238
+
239
+ The client retries connection errors, rate limits (after `Retry-After`, up to 60 s), concurrency limits and 500/502/503 responses — twice by default, with exponential backoff. It never retries a 4xx other than those, or a spent quota.
240
+
241
+ A retry is only safe if it cannot run the work twice, so every analysis the client may retry carries an `Idempotency-Key`: yours if you pass `idempotency_key=`, otherwise one generated for that call and reused by all of its retries. The API answers a repeated key with the first request's result — one run, one charge, one webhook. When you retry across processes yourself (a job that ran twice), pass the same key each time.
242
+
243
+ ```python
244
+ client = urlpipe.Client(
245
+ api_key="…", # default: URLPIPE_API_KEY
246
+ timeout=90, # seconds per HTTP request
247
+ max_retries=2, # 0 turns retries off
248
+ wait_timeout=300, # how long a slow sync call keeps polling
249
+ )
250
+ ```
251
+
252
+ For a proxy, custom TLS or test transports, pass your own `httpx.Client` (or `httpx.AsyncClient`) as `http_client=`. You keep ownership of it; closing the URLpipe client leaves it open.
253
+
254
+ ## Links
255
+
256
+ - API docs: https://urlpipe.dev/docs
257
+ - Pricing: https://urlpipe.dev/pricing
258
+ - MCP server, for using URLpipe from AI assistants: https://github.com/URLpipe/mcp
259
+
260
+ ## License
261
+
262
+ MIT © Aliat Partner S.L.
@@ -0,0 +1,226 @@
1
+ # urlpipe
2
+
3
+ Turn any URL into clean Markdown, rendered HTML, a full-page screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit — from Python, in one call. Pages are rendered in real Chrome, so JavaScript-heavy sites come back complete.
4
+
5
+ This is the official Python client for the [URLpipe API](https://urlpipe.dev/docs). It works sync or async, is fully typed, and depends only on `httpx`.
6
+
7
+ ## Install
8
+
9
+ ```sh
10
+ pip install urlpipe
11
+ ```
12
+
13
+ Python 3.9 or later.
14
+
15
+ ## Quickstart
16
+
17
+ Create a project in the [dashboard](https://urlpipe.dev) and copy its API key. The Free plan gives you 1,000 credits a month, no card needed.
18
+
19
+ ```sh
20
+ export URLPIPE_API_KEY=your_project_key
21
+ ```
22
+
23
+ ```python
24
+ import urlpipe
25
+
26
+ client = urlpipe.Client() # reads URLPIPE_API_KEY
27
+
28
+ page = client.markdown("https://example.com")
29
+ print(page.data) # "# Example Domain\n\nThis domain is for use in …"
30
+ ```
31
+
32
+ Every method waits for the result and returns a `Response`:
33
+
34
+ | Field | What it holds |
35
+ |---|---|
36
+ | `status` | `"completed"`, `"accepted"` (an async request was taken) or `"processing"` (still running) |
37
+ | `data` | the result, typed per operation; `None` unless completed |
38
+ | `token` | the request's token — fetch the result again for free with `client.result(token)` |
39
+ | `labels` | the labels the request was made with |
40
+ | `meta` | `cache`, `cache_age`, `processing_time_ms`, `quota` (`cost`, `limit`, `remaining`, `overage`, `resets_at`), `concurrency_limit`, `result_url`, `idempotent_replayed` |
41
+
42
+ ```python
43
+ page.meta.cache # "hit" — served from cache, cost nothing
44
+ page.meta.quota.remaining # 943, or "unlimited"
45
+ ```
46
+
47
+ ## Operations
48
+
49
+ Every method takes the URL first; everything else is an optional keyword.
50
+
51
+ ```python
52
+ client.markdown("https://example.com").data # str: the main content as Markdown
53
+ client.html("https://example.com").data # str: the HTML after JavaScript ran
54
+ client.summarize("https://example.com").data # str: an AI summary, in Markdown
55
+ client.meta("https://example.com").data # dict: title, description, language, author, …
56
+ client.keywords("https://example.com").data # list[str], most relevant first
57
+ client.console("https://example.com").data # [{"type": "error", "text": "…"}, …]
58
+ client.lighthouse("https://example.com", device="desktop", include_audits=True).data
59
+ ```
60
+
61
+ ### Screenshots
62
+
63
+ ```python
64
+ shot = client.screenshot(
65
+ "https://example.com/pricing",
66
+ screenshot_options={"viewport_width": 390, "format": "webp"},
67
+ ).data
68
+
69
+ shot.save("pricing.webp")
70
+ shot.mime_type # "image/webp", read from the image bytes
71
+ shot.result_url # a link to the image that needs no API key, for an <img> tag
72
+ shot.data # the decoded bytes
73
+ ```
74
+
75
+ ### Several operations off one page visit
76
+
77
+ ```python
78
+ result = client.scrape("https://example.com", ["markdown", "meta", "screenshot"]).data
79
+ result["operations"]["meta"]["result"]["title"]
80
+ ```
81
+
82
+ Each operation has its own `success`, `result`, `error` and `cached`; one failing never affects the others. A screenshot inside a scrape is Base64 text, as the API returns it.
83
+
84
+ ### Options every method takes
85
+
86
+ ```python
87
+ client.markdown(
88
+ "https://example.com/blog",
89
+ max_age="1 hour", # or seconds; 0 skips the cache
90
+ labels={"client": "acme"}, # your own ids, returned with the result
91
+ residential=True, # fetch from a home broadband address
92
+ page_options={"block_cookie_banners": True, "remove_selectors": [".promo"]},
93
+ idempotency_key="import-2026-09-25-17",
94
+ )
95
+ ```
96
+
97
+ `page_options` is not accepted by `lighthouse`. Options the API adds after this release can be sent with `extra`, which is merged into the request body as given:
98
+
99
+ ```python
100
+ client.markdown("https://example.com", extra={"some_option": True})
101
+ ```
102
+
103
+ The client passes option values through unchanged; the API validates them and answers `InvalidRequestError` naming what to change.
104
+
105
+ ## Async requests and `wait`
106
+
107
+ Pass `sync=False` and the call returns straight away with a token, while the work carries on. Collect the result by polling, or have it POSTed to a webhook with `report_to`.
108
+
109
+ ```python
110
+ accepted = client.lighthouse("https://example.com", sync=False)
111
+ accepted.status # "accepted"
112
+
113
+ report = client.wait(accepted.token, operation="lighthouse") # polls every 2 s
114
+ report.data["categories"]["performance"]["score"]
115
+ ```
116
+
117
+ `client.result(token)` makes a single check instead: it returns a `"processing"` Response while the work is running.
118
+
119
+ `GET /result` does not say which operation made a token, so pass `operation=` to get typed `data`. Without it, JSON comes back parsed and text as a string — a screenshot stays Base64 until you ask for `wait(token, operation="screenshot")`.
120
+
121
+ A synchronous call that runs past the API's 60-second window is not an error: the client polls for the result itself and returns it, for up to `wait_timeout` seconds (5 minutes by default) before raising `WaitTimeoutError`. The work keeps running either way, so `client.wait(error.token)` picks it up later.
122
+
123
+ ### asyncio
124
+
125
+ `AsyncClient` has the same methods, as coroutines:
126
+
127
+ ```python
128
+ import asyncio
129
+ import urlpipe
130
+
131
+ async def main() -> None:
132
+ async with urlpipe.AsyncClient() as client:
133
+ pages = await asyncio.gather(
134
+ client.markdown("https://example.com/a"),
135
+ client.markdown("https://example.com/b"),
136
+ )
137
+ print([p.data for p in pages])
138
+
139
+ asyncio.run(main())
140
+ ```
141
+
142
+ ## Webhooks
143
+
144
+ With webhook signing on (project **Settings → Webhook Signing**), check each delivery came from URLpipe before you trust it. `verify_webhook` needs no client:
145
+
146
+ ```python
147
+ import os
148
+ import urlpipe
149
+ from flask import Flask, request, abort
150
+
151
+ app = Flask(__name__)
152
+
153
+ @app.post("/webhooks/urlpipe")
154
+ def urlpipe_webhook():
155
+ try:
156
+ delivery = urlpipe.verify_webhook(
157
+ request.get_data(), # the raw body bytes
158
+ request.headers,
159
+ os.environ["URLPIPE_WEBHOOK_SECRET"], # whsec_…
160
+ )
161
+ except urlpipe.WebhookVerificationError:
162
+ abort(401)
163
+ enqueue(delivery["token"], delivery["result"])
164
+ return "", 200
165
+ ```
166
+
167
+ Give it the **raw request body** exactly as received (`request.get_data()` in Flask, `await request.body()` in FastAPI, `request.body` in Django). Parsed and re-serialised JSON has different bytes, and the signature will not match. Header names are matched case-insensitively; deliveries signed more than `tolerance` seconds ago (300 by default) are refused, and during a secret rotation either signature is accepted.
168
+
169
+ It returns the payload: `token`, `operation`, `labels`, `success`, `result`, `result_url`, `error` and `meta`.
170
+
171
+ ## Errors
172
+
173
+ Every error is a `urlpipe.UrlpipeError`, with `status`, `code`, `message`, `body` and `token`.
174
+
175
+ | Error | When |
176
+ |---|---|
177
+ | `AuthenticationError` | 401: the API key is missing or wrong |
178
+ | `EmailUnverifiedError` | 403: confirm the email address on the account |
179
+ | `InvalidRequestError` | 422: a parameter was refused (`invalid_url`, `invalid_options`, …); nothing ran |
180
+ | `AnalysisFailedError` | 422: the page could not be analysed; `message` says why |
181
+ | `QuotaExceededError` | 429: the Free plan's credits are spent (`limit`, `used`, `needed`, `resets_at`) |
182
+ | `ConcurrencyLimitError` | 429: too many of your requests running (`limit`, `running`) |
183
+ | `RateLimitedError` | 429: sending too fast (`retry_after`) |
184
+ | `NotFoundError` | 404: no result for this token |
185
+ | `StaleResultError` | 410: the result is past the 30-day window |
186
+ | `ServerError` | 5xx |
187
+ | `APIConnectionError` | no HTTP answer at all |
188
+ | `WaitTimeoutError` | the result was not ready within the wait timeout (`token`) |
189
+
190
+ ```python
191
+ try:
192
+ page = client.markdown(url)
193
+ except urlpipe.AnalysisFailedError as error:
194
+ print("Could not read the page:", error.message)
195
+ except urlpipe.QuotaExceededError as error:
196
+ print("Out of credits until", error.resets_at)
197
+ ```
198
+
199
+ A failed analysis costs nothing, and neither does a cache hit.
200
+
201
+ ## Retries and idempotency
202
+
203
+ The client retries connection errors, rate limits (after `Retry-After`, up to 60 s), concurrency limits and 500/502/503 responses — twice by default, with exponential backoff. It never retries a 4xx other than those, or a spent quota.
204
+
205
+ A retry is only safe if it cannot run the work twice, so every analysis the client may retry carries an `Idempotency-Key`: yours if you pass `idempotency_key=`, otherwise one generated for that call and reused by all of its retries. The API answers a repeated key with the first request's result — one run, one charge, one webhook. When you retry across processes yourself (a job that ran twice), pass the same key each time.
206
+
207
+ ```python
208
+ client = urlpipe.Client(
209
+ api_key="…", # default: URLPIPE_API_KEY
210
+ timeout=90, # seconds per HTTP request
211
+ max_retries=2, # 0 turns retries off
212
+ wait_timeout=300, # how long a slow sync call keeps polling
213
+ )
214
+ ```
215
+
216
+ For a proxy, custom TLS or test transports, pass your own `httpx.Client` (or `httpx.AsyncClient`) as `http_client=`. You keep ownership of it; closing the URLpipe client leaves it open.
217
+
218
+ ## Links
219
+
220
+ - API docs: https://urlpipe.dev/docs
221
+ - Pricing: https://urlpipe.dev/pricing
222
+ - MCP server, for using URLpipe from AI assistants: https://github.com/URLpipe/mcp
223
+
224
+ ## License
225
+
226
+ MIT © Aliat Partner S.L.
@@ -0,0 +1,60 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.27"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "urlpipe"
7
+ dynamic = ["version"]
8
+ description = "Official Python client for the URLpipe API: turn a URL into Markdown, HTML, a screenshot, metadata, a summary, keywords, console errors or a Lighthouse audit."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.9"
13
+ authors = [{ name = "URLpipe", email = "contact@urlpipe.dev" }]
14
+ maintainers = [{ name = "URLpipe", email = "contact@urlpipe.dev" }]
15
+ keywords = ["urlpipe", "markdown", "screenshot", "html", "lighthouse", "web scraping", "llm", "api client"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "Operating System :: OS Independent",
20
+ "Programming Language :: Python",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3 :: Only",
23
+ "Programming Language :: Python :: 3.9",
24
+ "Programming Language :: Python :: 3.10",
25
+ "Programming Language :: Python :: 3.11",
26
+ "Programming Language :: Python :: 3.12",
27
+ "Programming Language :: Python :: 3.13",
28
+ "Programming Language :: Python :: 3.14",
29
+ "Topic :: Internet :: WWW/HTTP",
30
+ "Topic :: Software Development :: Libraries :: Python Modules",
31
+ "Typing :: Typed",
32
+ "Framework :: AsyncIO",
33
+ ]
34
+ dependencies = ["httpx>=0.23,<1"]
35
+
36
+ [project.optional-dependencies]
37
+ dev = ["pytest>=7", "mypy>=1.5"]
38
+
39
+ [project.urls]
40
+ Homepage = "https://urlpipe.dev"
41
+ Documentation = "https://urlpipe.dev/docs"
42
+ Source = "https://github.com/URLpipe/urlpipe-python"
43
+ Changelog = "https://github.com/URLpipe/urlpipe-python/blob/main/CHANGELOG.md"
44
+
45
+ [tool.hatch.version]
46
+ path = "src/urlpipe/_version.py"
47
+
48
+ [tool.hatch.build.targets.wheel]
49
+ packages = ["src/urlpipe"]
50
+
51
+ [tool.hatch.build.targets.sdist]
52
+ include = ["src/urlpipe", "tests", "README.md", "CHANGELOG.md", "LICENSE", "pyproject.toml"]
53
+
54
+ [tool.pytest.ini_options]
55
+ testpaths = ["tests"]
56
+ addopts = "-ra"
57
+
58
+ [tool.mypy]
59
+ strict = true
60
+ files = ["src/urlpipe"]
@@ -0,0 +1,78 @@
1
+ """The official Python client for the URLpipe API: turn a URL into Markdown,
2
+ HTML, a screenshot, metadata, a summary, keywords, console errors or a
3
+ Lighthouse audit.
4
+
5
+ import urlpipe
6
+
7
+ client = urlpipe.Client() # reads URLPIPE_API_KEY
8
+ print(client.markdown("https://example.com").data)
9
+
10
+ Docs: https://urlpipe.dev/docs
11
+ """
12
+
13
+ from . import errors
14
+ from ._client import AsyncClient, Client
15
+ from ._models import (
16
+ ConsoleEntry,
17
+ Operation,
18
+ PageMeta,
19
+ Quota,
20
+ Response,
21
+ ResponseMeta,
22
+ ResponseStatus,
23
+ ScrapeOperationResult,
24
+ ScrapeResult,
25
+ Screenshot,
26
+ )
27
+ from ._version import __version__
28
+ from .errors import (
29
+ AnalysisFailedError,
30
+ APIConnectionError,
31
+ AuthenticationError,
32
+ ConcurrencyLimitError,
33
+ ConfigurationError,
34
+ EmailUnverifiedError,
35
+ InvalidRequestError,
36
+ NotFoundError,
37
+ QuotaExceededError,
38
+ RateLimitedError,
39
+ ServerError,
40
+ StaleResultError,
41
+ UrlpipeError,
42
+ WaitTimeoutError,
43
+ WebhookVerificationError,
44
+ )
45
+ from .webhooks import verify_webhook
46
+
47
+ __all__ = [
48
+ "APIConnectionError",
49
+ "AnalysisFailedError",
50
+ "AsyncClient",
51
+ "AuthenticationError",
52
+ "Client",
53
+ "ConcurrencyLimitError",
54
+ "ConfigurationError",
55
+ "ConsoleEntry",
56
+ "EmailUnverifiedError",
57
+ "InvalidRequestError",
58
+ "NotFoundError",
59
+ "Operation",
60
+ "PageMeta",
61
+ "Quota",
62
+ "QuotaExceededError",
63
+ "RateLimitedError",
64
+ "Response",
65
+ "ResponseMeta",
66
+ "ResponseStatus",
67
+ "ScrapeOperationResult",
68
+ "ScrapeResult",
69
+ "Screenshot",
70
+ "ServerError",
71
+ "StaleResultError",
72
+ "UrlpipeError",
73
+ "WaitTimeoutError",
74
+ "WebhookVerificationError",
75
+ "__version__",
76
+ "errors",
77
+ "verify_webhook",
78
+ ]