tinyfish 0.2.6__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. tinyfish-0.4.0/CHANGELOG.md +17 -0
  2. {tinyfish-0.2.6 → tinyfish-0.4.0}/PKG-INFO +97 -2
  3. {tinyfish-0.2.6 → tinyfish-0.4.0}/README.md +96 -1
  4. {tinyfish-0.2.6 → tinyfish-0.4.0}/RELEASE.md +16 -16
  5. {tinyfish-0.2.6 → tinyfish-0.4.0}/pyproject.toml +2 -1
  6. tinyfish-0.4.0/src/tinyfish/fetch/__init__.py +273 -0
  7. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/fetch/types.py +14 -0
  8. tinyfish-0.4.0/src/tinyfish/search/__init__.py +260 -0
  9. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/search/types.py +7 -0
  10. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_fetch.py +155 -0
  11. tinyfish-0.4.0/tests/test_search.py +530 -0
  12. {tinyfish-0.2.6 → tinyfish-0.4.0}/uv.lock +4 -0
  13. tinyfish-0.2.6/src/tinyfish/fetch/__init__.py +0 -154
  14. tinyfish-0.2.6/src/tinyfish/search/__init__.py +0 -115
  15. tinyfish-0.2.6/tests/test_search.py +0 -252
  16. {tinyfish-0.2.6 → tinyfish-0.4.0}/.gitignore +0 -0
  17. {tinyfish-0.2.6 → tinyfish-0.4.0}/.pre-commit-config.yaml +0 -0
  18. {tinyfish-0.2.6 → tinyfish-0.4.0}/AGENTS.md +0 -0
  19. {tinyfish-0.2.6 → tinyfish-0.4.0}/CLAUDE.md +0 -0
  20. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/exceptions-and-errors-guide.md +0 -0
  21. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/internal/api-integration-header.md +0 -0
  22. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/internal/exceptions-and-errors-guide.md +0 -0
  23. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/internal/publishing-private-guide.md +0 -0
  24. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/pagination-guide.md +0 -0
  25. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/proxy-and-browser-profiles.md +0 -0
  26. {tinyfish-0.2.6 → tinyfish-0.4.0}/docs/streaming-guide.md +0 -0
  27. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/__init__.py +0 -0
  28. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/__init__.py +0 -0
  29. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/client/__init__.py +0 -0
  30. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/client/_base.py +0 -0
  31. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/client/async_.py +0 -0
  32. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/client/sync.py +0 -0
  33. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/exceptions.py +0 -0
  34. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/resource.py +0 -0
  35. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/_utils/sse_parser.py +0 -0
  36. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/agent/__init__.py +0 -0
  37. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/agent/types.py +0 -0
  38. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/browser/__init__.py +0 -0
  39. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/browser/types.py +0 -0
  40. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/client.py +0 -0
  41. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/py.typed +0 -0
  42. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/runs/__init__.py +0 -0
  43. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/runs/types.py +0 -0
  44. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/vault/__init__.py +0 -0
  45. {tinyfish-0.2.6 → tinyfish-0.4.0}/src/tinyfish/vault/types.py +0 -0
  46. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/__init__.py +0 -0
  47. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/conftest.py +0 -0
  48. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_agent.py +0 -0
  49. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_browser.py +0 -0
  50. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_client.py +0 -0
  51. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_errors.py +0 -0
  52. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_runs.py +0 -0
  53. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/test_vault.py +0 -0
  54. {tinyfish-0.2.6 → tinyfish-0.4.0}/tests/testing_guide.md +0 -0
@@ -0,0 +1,17 @@
1
+ # Changelog
2
+
3
+ ## [0.4.0](https://github.com/tinyfish-io/ux-labs/compare/sdk-py/v0.3.0...sdk-py/v0.4.0) (2026-07-23)
4
+
5
+
6
+ ### Features
7
+
8
+ * **search:** add pub_year_min/pub_year_max publication-year filter ([#3691](https://github.com/tinyfish-io/ux-labs/issues/3691)) ([8728754](https://github.com/tinyfish-io/ux-labs/commit/8728754206fe4b7cf3a4ee8226c74bb364532869))
9
+ * **search:** expose include_domains/exclude_domains in SDKs, CLI, MCP prose (ML-2084) ([#3692](https://github.com/tinyfish-io/ux-labs/issues/3692)) ([6c2079f](https://github.com/tinyfish-io/ux-labs/commit/6c2079f52872f9c9472cde8ec95d70280f25defa))
10
+
11
+ ## [0.3.0](https://github.com/tinyfish-io/ux-labs/compare/sdk-py/v0.2.6...sdk-py/v0.3.0) (2026-07-22)
12
+
13
+
14
+ ### Features
15
+
16
+ * **ci:** automate SDK/CLI releases via release-please ([#3632](https://github.com/tinyfish-io/ux-labs/issues/3632)) ([1b311ea](https://github.com/tinyfish-io/ux-labs/commit/1b311ea6d6277fad21b57baf0b7cfdc6619a3b93))
17
+ * **sdk:** add domain_type to search.query() in TypeScript and Python SDKs ([#3432](https://github.com/tinyfish-io/ux-labs/issues/3432)) ([1c2954c](https://github.com/tinyfish-io/ux-labs/commit/1c2954cb192324cdca1ce206515de72df2cd231d))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tinyfish
3
- Version: 0.2.6
3
+ Version: 0.4.0
4
4
  Summary: Official Python SDK for the TinyFish API
5
5
  Requires-Python: >=3.11
6
6
  Requires-Dist: httpx>=0.27.0
@@ -53,8 +53,10 @@ Every method below is available on both `TinyFish` (sync) and `AsyncTinyFish` (a
53
53
  | [`agent.run()`](#agentrun--block-until-done) | Run an automation, wait for the result | `AgentRunResponse` | Yes |
54
54
  | [`agent.queue()`](#agentqueue--fire-and-forget) | Start an automation, return immediately | `AgentRunAsyncResponse` | No |
55
55
  | [`agent.stream()`](#agentstream--real-time-events) | Stream live SSE events as the agent works | `AgentStream` | No |
56
+ | [`fetch.get_contents()`](#fetchget_contents--clean-content) | Fetch extracted page content without running a browser agent | `FetchResponse` | Yes |
56
57
  | [`runs.get()`](#runsget--retrieve-a-single-run) | Retrieve a single run by ID | `Run` | — |
57
58
  | [`runs.list()`](#runslist--list-and-filter-runs) | List runs with filtering, sorting, pagination | `RunListResponse` | — |
59
+ | [`search.query()`](#searchquery--search-the-web) | Search the web for ranked results | `SearchQueryResponse` | Yes |
58
60
 
59
61
  ---
60
62
 
@@ -183,6 +185,41 @@ See the [Streaming Guide](docs/streaming-guide.md) for the full event lifecycle,
183
185
 
184
186
  ---
185
187
 
188
+ ### `fetch.get_contents()` — clean content
189
+
190
+ Fetch extracted content from one or more URLs without a browser-agent run.
191
+
192
+ ```python
193
+ from tinyfish import TinyFish
194
+
195
+ client = TinyFish()
196
+
197
+ response = client.fetch.get_contents(
198
+ [
199
+ "https://example.com",
200
+ "https://example.org",
201
+ ],
202
+ format="markdown",
203
+ links=True,
204
+ image_links=False,
205
+ per_url_timeout_ms=45_000,
206
+ )
207
+
208
+ print(response.results)
209
+ print(response.errors)
210
+ ```
211
+
212
+ `fetch.get_contents()` accepts 1 to 10 URLs. Set `per_url_timeout_ms` to apply an independent timeout budget to each URL in the batch; slow URLs return in `errors` with `timeout` while siblings can still complete.
213
+
214
+ **Returns `FetchResponse`:**
215
+
216
+ | Field | Type | Description |
217
+ |-------|------|-------------|
218
+ | `results` | `list[FetchResult]` | Successfully fetched URLs |
219
+ | `errors` | `list[FetchError]` | URLs that failed to fetch or extract |
220
+
221
+ ---
222
+
186
223
  ### `runs.get()` — retrieve a single run
187
224
 
188
225
  Fetch the full details of a run by its ID.
@@ -258,6 +295,64 @@ See the [Pagination Guide](docs/pagination-guide.md) for full pagination loop ex
258
295
 
259
296
  ---
260
297
 
298
+ ### `search.query()` — search the web
299
+
300
+ Returns ranked web search results with titles, snippets, and URLs.
301
+
302
+ ```python
303
+ from tinyfish import TinyFish
304
+
305
+ client = TinyFish()
306
+
307
+ response = client.search.query("FIFA")
308
+
309
+ print(response.query)
310
+ print(response.total_results)
311
+ print(response.results[0].title if response.results else "No results")
312
+ ```
313
+
314
+ Optional parameters:
315
+
316
+ - `location` — country code for geo-targeted results (e.g. `"US"`, `"GB"`)
317
+ - `language` — language code (e.g. `"en"`, `"fr"`)
318
+ - `page` — page number, 0-indexed, max `10`
319
+ - `recency_minutes` — freshness window in minutes (`1` to `5256000`)
320
+ - `after_date` / `before_date` — calendar date range in `YYYY-MM-DD`
321
+ - `domain_type` — result category: `"web"` (default), `"news"`, or `"research_paper"`
322
+ - `pub_year_min` / `pub_year_max` — publication-year range, inclusive (`0` to `9999`). Only supported for `domain_type="research_paper"`
323
+
324
+ ```python
325
+ # geo-targeted
326
+ response = client.search.query("FIFA", location="US", language="en")
327
+
328
+ # freshness window
329
+ response = client.search.query("FIFA", recency_minutes=60)
330
+
331
+ # calendar date range
332
+ response = client.search.query("FIFA", after_date="2026-06-01", before_date="2026-06-18")
333
+
334
+ # domain type
335
+ response = client.search.query("FIFA", domain_type="news")
336
+ response = client.search.query("machine learning", domain_type="research_paper")
337
+
338
+ # publication-year range (research_paper only)
339
+ response = client.search.query(
340
+ "transformer architecture", domain_type="research_paper", pub_year_min=2019, pub_year_max=2022
341
+ )
342
+ ```
343
+
344
+ Filter validation rules:
345
+
346
+ - `recency_minutes` must be an integer from `1` to `5256000`
347
+ - `after_date` and `before_date` must use `YYYY-MM-DD`
348
+ - `recency_minutes` cannot be combined with `after_date` or `before_date`
349
+ - if both dates are present, `after_date` must be less than or equal to `before_date`
350
+ - `domain_type` must be one of `"web"`, `"news"`, or `"research_paper"`
351
+ - `pub_year_min` and `pub_year_max` must be integers from `0` to `9999`
352
+ - if both are present, `pub_year_min` must be less than or equal to `pub_year_max`
353
+
354
+ ---
355
+
261
356
  ## Sync vs Async
262
357
 
263
358
  Use `AsyncTinyFish` when you're in an async context (FastAPI, aiohttp, etc.):
@@ -280,7 +375,7 @@ client = AsyncTinyFish()
280
375
  response = await client.agent.run(goal="...", url="...")
281
376
  ```
282
377
 
283
- All five methods (`agent.run()`, `agent.queue()`, `agent.stream()`, `runs.get()`, `runs.list()`) work the same way — same parameters, just `await`-ed.
378
+ All seven methods (`agent.run()`, `agent.queue()`, `agent.stream()`, `fetch.get_contents()`, `runs.get()`, `runs.list()`, `search.query()`) work the same way — same parameters, just `await`-ed.
284
379
 
285
380
  ## Configuration
286
381
 
@@ -43,8 +43,10 @@ Every method below is available on both `TinyFish` (sync) and `AsyncTinyFish` (a
43
43
  | [`agent.run()`](#agentrun--block-until-done) | Run an automation, wait for the result | `AgentRunResponse` | Yes |
44
44
  | [`agent.queue()`](#agentqueue--fire-and-forget) | Start an automation, return immediately | `AgentRunAsyncResponse` | No |
45
45
  | [`agent.stream()`](#agentstream--real-time-events) | Stream live SSE events as the agent works | `AgentStream` | No |
46
+ | [`fetch.get_contents()`](#fetchget_contents--clean-content) | Fetch extracted page content without running a browser agent | `FetchResponse` | Yes |
46
47
  | [`runs.get()`](#runsget--retrieve-a-single-run) | Retrieve a single run by ID | `Run` | — |
47
48
  | [`runs.list()`](#runslist--list-and-filter-runs) | List runs with filtering, sorting, pagination | `RunListResponse` | — |
49
+ | [`search.query()`](#searchquery--search-the-web) | Search the web for ranked results | `SearchQueryResponse` | Yes |
48
50
 
49
51
  ---
50
52
 
@@ -173,6 +175,41 @@ See the [Streaming Guide](docs/streaming-guide.md) for the full event lifecycle,
173
175
 
174
176
  ---
175
177
 
178
+ ### `fetch.get_contents()` — clean content
179
+
180
+ Fetch extracted content from one or more URLs without a browser-agent run.
181
+
182
+ ```python
183
+ from tinyfish import TinyFish
184
+
185
+ client = TinyFish()
186
+
187
+ response = client.fetch.get_contents(
188
+ [
189
+ "https://example.com",
190
+ "https://example.org",
191
+ ],
192
+ format="markdown",
193
+ links=True,
194
+ image_links=False,
195
+ per_url_timeout_ms=45_000,
196
+ )
197
+
198
+ print(response.results)
199
+ print(response.errors)
200
+ ```
201
+
202
+ `fetch.get_contents()` accepts 1 to 10 URLs. Set `per_url_timeout_ms` to apply an independent timeout budget to each URL in the batch; slow URLs return in `errors` with `timeout` while siblings can still complete.
203
+
204
+ **Returns `FetchResponse`:**
205
+
206
+ | Field | Type | Description |
207
+ |-------|------|-------------|
208
+ | `results` | `list[FetchResult]` | Successfully fetched URLs |
209
+ | `errors` | `list[FetchError]` | URLs that failed to fetch or extract |
210
+
211
+ ---
212
+
176
213
  ### `runs.get()` — retrieve a single run
177
214
 
178
215
  Fetch the full details of a run by its ID.
@@ -248,6 +285,64 @@ See the [Pagination Guide](docs/pagination-guide.md) for full pagination loop ex
248
285
 
249
286
  ---
250
287
 
288
+ ### `search.query()` — search the web
289
+
290
+ Returns ranked web search results with titles, snippets, and URLs.
291
+
292
+ ```python
293
+ from tinyfish import TinyFish
294
+
295
+ client = TinyFish()
296
+
297
+ response = client.search.query("FIFA")
298
+
299
+ print(response.query)
300
+ print(response.total_results)
301
+ print(response.results[0].title if response.results else "No results")
302
+ ```
303
+
304
+ Optional parameters:
305
+
306
+ - `location` — country code for geo-targeted results (e.g. `"US"`, `"GB"`)
307
+ - `language` — language code (e.g. `"en"`, `"fr"`)
308
+ - `page` — page number, 0-indexed, max `10`
309
+ - `recency_minutes` — freshness window in minutes (`1` to `5256000`)
310
+ - `after_date` / `before_date` — calendar date range in `YYYY-MM-DD`
311
+ - `domain_type` — result category: `"web"` (default), `"news"`, or `"research_paper"`
312
+ - `pub_year_min` / `pub_year_max` — publication-year range, inclusive (`0` to `9999`). Only supported for `domain_type="research_paper"`
313
+
314
+ ```python
315
+ # geo-targeted
316
+ response = client.search.query("FIFA", location="US", language="en")
317
+
318
+ # freshness window
319
+ response = client.search.query("FIFA", recency_minutes=60)
320
+
321
+ # calendar date range
322
+ response = client.search.query("FIFA", after_date="2026-06-01", before_date="2026-06-18")
323
+
324
+ # domain type
325
+ response = client.search.query("FIFA", domain_type="news")
326
+ response = client.search.query("machine learning", domain_type="research_paper")
327
+
328
+ # publication-year range (research_paper only)
329
+ response = client.search.query(
330
+ "transformer architecture", domain_type="research_paper", pub_year_min=2019, pub_year_max=2022
331
+ )
332
+ ```
333
+
334
+ Filter validation rules:
335
+
336
+ - `recency_minutes` must be an integer from `1` to `5256000`
337
+ - `after_date` and `before_date` must use `YYYY-MM-DD`
338
+ - `recency_minutes` cannot be combined with `after_date` or `before_date`
339
+ - if both dates are present, `after_date` must be less than or equal to `before_date`
340
+ - `domain_type` must be one of `"web"`, `"news"`, or `"research_paper"`
341
+ - `pub_year_min` and `pub_year_max` must be integers from `0` to `9999`
342
+ - if both are present, `pub_year_min` must be less than or equal to `pub_year_max`
343
+
344
+ ---
345
+
251
346
  ## Sync vs Async
252
347
 
253
348
  Use `AsyncTinyFish` when you're in an async context (FastAPI, aiohttp, etc.):
@@ -270,7 +365,7 @@ client = AsyncTinyFish()
270
365
  response = await client.agent.run(goal="...", url="...")
271
366
  ```
272
367
 
273
- All five methods (`agent.run()`, `agent.queue()`, `agent.stream()`, `runs.get()`, `runs.list()`) work the same way — same parameters, just `await`-ed.
368
+ All seven methods (`agent.run()`, `agent.queue()`, `agent.stream()`, `fetch.get_contents()`, `runs.get()`, `runs.list()`, `search.query()`) work the same way — same parameters, just `await`-ed.
274
369
 
275
370
  ## Configuration
276
371
 
@@ -19,25 +19,22 @@ This is a one-time change. Once merged and applied, the workflow will have acces
19
19
 
20
20
  ## Release process
21
21
 
22
- ### Step 1: Bump the version
22
+ ### Step 1: Merge SDK changes to main
23
23
 
24
- Edit `sdk/sdk-python/pyproject.toml` and update the `version` field:
24
+ Use conventional commits (`feat:`, `fix:`, `feat!:`) for changes under `sdk/sdk-python/`.
25
+ After merge, `release-please.yml` opens or updates a Python SDK release PR with:
25
26
 
26
- ```toml
27
- [project]
28
- version = "0.2.0" # bump this
29
- ```
27
+ - `sdk/sdk-python/pyproject.toml` version bump
28
+ - `sdk/sdk-python/CHANGELOG.md` release notes
29
+ - `.github/release-please/manifest.json` version tracking
30
+
31
+ Review and merge that release PR when you're ready to publish.
30
32
 
31
- Commit and merge to main.
33
+ ### Step 2: Release Please creates the GitHub Release
32
34
 
33
- ### Step 2: Create a GitHub Release
35
+ Merging the release PR creates a GitHub Release tagged with the Python SDK component:
34
36
 
35
- 1. Go to the `ux-labs` repo on GitHub **Releases** → **Draft a new release**
36
- 2. Set the tag to match the version exactly, prefixed with `sdk-py/v`:
37
- - Version `0.2.0` → tag `sdk-py/v0.2.0`
38
- 3. Set the target to `main`
39
- 4. Write release notes summarizing what changed
40
- 5. Click **Publish release**
37
+ - Version `0.2.7` → tag `sdk-py/v0.2.7`
41
38
 
42
39
  The CD workflow (`CD_sdk_python.yml`) only triggers on tags prefixed with `sdk-py/v`. It validates that the version portion of the tag matches `pyproject.toml` — if they don't match, the build fails before anything is published.
43
40
 
@@ -52,15 +49,18 @@ Monitor progress at: `github.com/tinyfish-io/ux-labs/actions`
52
49
 
53
50
  ### Step 4: Verify
54
51
 
55
- After the workflow completes, verify the release:
52
+ After the workflow completes, verify the release (replace `0.2.7` with the released version):
56
53
 
57
54
  ```bash
58
- pip install tinyfish==0.2.0
55
+ pip install tinyfish==0.2.7
59
56
  python -c "from tinyfish import TinyFish; print('ok')"
60
57
  ```
61
58
 
62
59
  ## Troubleshooting
63
60
 
61
+ **Manual release escape hatch**
62
+ Use `workflow_dispatch` on `CD_sdk_python.yml` at a `sdk-py/v*` tag if Release Please created the tag but the publish workflow did not run.
63
+
64
64
  **Tag does not match pyproject.toml version**
65
65
  The build job validates that the git tag (e.g., `sdk-py/v0.2.0`) matches the version in `pyproject.toml` (e.g., `0.2.0`). Fix by updating `pyproject.toml` to match the tag before publishing, then delete and recreate the release.
66
66
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tinyfish"
3
- version = "0.2.6"
3
+ version = "0.4.0"
4
4
  description = "Official Python SDK for the TinyFish API"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -11,6 +11,7 @@ dependencies = [
11
11
  ]
12
12
 
13
13
  [tool.uv]
14
+ exclude-newer = "7 days"
14
15
  constraint-dependencies = [
15
16
  "pygments>=2.20.0",
16
17
  ]
@@ -0,0 +1,273 @@
1
+ """Fetch resource for extracting clean content from URLs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from tinyfish._utils.exceptions import SDKError
6
+ from tinyfish._utils.resource import BaseAsyncAPIResource, BaseSyncAPIResource
7
+
8
+ from .types import FetchFormat, FetchResponse
9
+
10
+ MIN_URLS = 1
11
+ MAX_URLS = 10
12
+ MIN_PER_URL_TIMEOUT_MS = 1
13
+ MAX_PER_URL_TIMEOUT_MS = 110_000
14
+ FETCH_PURPOSE_MAX_LENGTH = 2000
15
+
16
+
17
+ class FetchResource(BaseSyncAPIResource):
18
+ """Fetch and extract clean content from URLs."""
19
+
20
+ def get_contents(
21
+ self,
22
+ urls: list[str],
23
+ *,
24
+ purpose: str | None = None,
25
+ format: FetchFormat | None = None,
26
+ include_html_head: bool | None = None,
27
+ links: bool | None = None,
28
+ image_links: bool | None = None,
29
+ ttl: int | None = None,
30
+ per_url_timeout_ms: int | None = None,
31
+ if_none_match: str | None = None,
32
+ if_modified_since: str | None = None,
33
+ include_etag_and_last_modified: bool | None = None,
34
+ ) -> FetchResponse:
35
+ """Fetch and extract content from one or more URLs.
36
+
37
+ Args:
38
+ urls: List of URLs to fetch (1-10 items).
39
+ purpose: Optional short statement of why these URLs are being
40
+ fetched — the goal or task the content will be used for.
41
+ Freeform; gives the fetch additional intent signal for
42
+ better-quality results. When supplied it must be non-empty;
43
+ max 2000 characters.
44
+ format: Output format — "markdown", "html", or "json".
45
+ include_html_head: Whether to wrap html output in a complete document with a head tag.
46
+ links: Whether to extract links from the page.
47
+ image_links: Whether to extract image links from the page.
48
+ ttl: Caller freshness tolerance in seconds for the cached entry.
49
+ Omit (default) for unlimited tolerance — any cached entry is
50
+ acceptable. Pass 0 to prefer a live fetch; the cached entry
51
+ is still served if the origin's Cache-Control: max-age
52
+ covers its age, or the host is on the small allowlist of
53
+ operator-pinned never-expire domains. Pass N > 0 to accept
54
+ an entry whose age is below N seconds; the upstream max-age
55
+ and the never-expire list may extend (never shorten) this.
56
+ per_url_timeout_ms: Wall-clock timeout budget in milliseconds
57
+ applied independently to each URL.
58
+ if_none_match: ETag validator from a prior fetch, forwarded
59
+ verbatim as the If-None-Match header on the origin request.
60
+ Single-URL only — combining it with a batch of URLs is
61
+ rejected. Validators are not persisted; the caller replays
62
+ them on the next request.
63
+ if_modified_since: Last-Modified validator from a prior fetch,
64
+ forwarded verbatim as the If-Modified-Since header on the
65
+ origin request. Single-URL only — combining it with a batch
66
+ of URLs is rejected. Validators are not persisted; the caller
67
+ replays them on the next request.
68
+ include_etag_and_last_modified: Opt-in to receiving the ``etag`` /
69
+ ``last_modified`` validators (and ``not_modified`` detection)
70
+ on each result. Defaults to false — those fields come back
71
+ only when this is set. Works with a single URL or a batch.
72
+
73
+ Returns:
74
+ FetchResponse with results and errors lists.
75
+
76
+ Raises:
77
+ SDKError: urls is empty or has more than 10 items, purpose is
78
+ empty or over 2000 characters, ttl is negative or not an
79
+ integer, or per_url_timeout_ms is outside 1..110000.
80
+ AuthenticationError: Invalid API key.
81
+ """
82
+ _validate_urls(urls)
83
+ _validate_purpose(purpose)
84
+ _validate_ttl(ttl)
85
+ _validate_per_url_timeout_ms(per_url_timeout_ms)
86
+ _validate_conditional_headers(urls, if_none_match, if_modified_since)
87
+ body = _build_body(
88
+ urls,
89
+ purpose=purpose,
90
+ format=format,
91
+ include_html_head=include_html_head,
92
+ links=links,
93
+ image_links=image_links,
94
+ ttl=ttl,
95
+ per_url_timeout_ms=per_url_timeout_ms,
96
+ if_none_match=if_none_match,
97
+ if_modified_since=if_modified_since,
98
+ include_etag_and_last_modified=include_etag_and_last_modified,
99
+ )
100
+ return self._post("/v1/fetch", json=body, cast_to=FetchResponse)
101
+
102
+
103
+ class AsyncFetchResource(BaseAsyncAPIResource):
104
+ """Async fetch and extract clean content from URLs."""
105
+
106
+ async def get_contents(
107
+ self,
108
+ urls: list[str],
109
+ *,
110
+ purpose: str | None = None,
111
+ format: FetchFormat | None = None,
112
+ include_html_head: bool | None = None,
113
+ links: bool | None = None,
114
+ image_links: bool | None = None,
115
+ ttl: int | None = None,
116
+ per_url_timeout_ms: int | None = None,
117
+ if_none_match: str | None = None,
118
+ if_modified_since: str | None = None,
119
+ include_etag_and_last_modified: bool | None = None,
120
+ ) -> FetchResponse:
121
+ """Fetch and extract content from one or more URLs.
122
+
123
+ Async version of `FetchResource.get_contents()`.
124
+
125
+ Args:
126
+ urls: List of URLs to fetch (1-10 items).
127
+ purpose: Optional short statement of why these URLs are being
128
+ fetched — the goal or task the content will be used for.
129
+ Freeform; gives the fetch additional intent signal for
130
+ better-quality results. When supplied it must be non-empty;
131
+ max 2000 characters.
132
+ format: Output format — "markdown", "html", or "json".
133
+ include_html_head: Whether to wrap html output in a complete document with a head tag.
134
+ links: Whether to extract links from the page.
135
+ image_links: Whether to extract image links from the page.
136
+ ttl: Caller freshness tolerance in seconds for the cached entry.
137
+ See FetchResource.get_contents for full semantics.
138
+ per_url_timeout_ms: Wall-clock timeout budget in milliseconds
139
+ applied independently to each URL.
140
+ if_none_match: ETag validator from a prior fetch, forwarded
141
+ verbatim as the If-None-Match header on the origin request.
142
+ Single-URL only. See FetchResource.get_contents for full
143
+ semantics.
144
+ if_modified_since: Last-Modified validator from a prior fetch,
145
+ forwarded verbatim as the If-Modified-Since header on the
146
+ origin request. Single-URL only. See FetchResource.get_contents
147
+ for full semantics.
148
+ include_etag_and_last_modified: Opt-in to receiving the ``etag`` /
149
+ ``last_modified`` validators (and ``not_modified`` detection)
150
+ on each result. Defaults to false. Works with a single URL or
151
+ a batch.
152
+
153
+ Returns:
154
+ FetchResponse with results and errors lists.
155
+
156
+ Raises:
157
+ SDKError: urls is empty or has more than 10 items, purpose is
158
+ empty or over 2000 characters, ttl is negative or not an
159
+ integer, or per_url_timeout_ms is outside 1..110000.
160
+ AuthenticationError: Invalid API key.
161
+ """
162
+ _validate_urls(urls)
163
+ _validate_purpose(purpose)
164
+ _validate_ttl(ttl)
165
+ _validate_per_url_timeout_ms(per_url_timeout_ms)
166
+ _validate_conditional_headers(urls, if_none_match, if_modified_since)
167
+ body = _build_body(
168
+ urls,
169
+ purpose=purpose,
170
+ format=format,
171
+ include_html_head=include_html_head,
172
+ links=links,
173
+ image_links=image_links,
174
+ ttl=ttl,
175
+ per_url_timeout_ms=per_url_timeout_ms,
176
+ if_none_match=if_none_match,
177
+ if_modified_since=if_modified_since,
178
+ include_etag_and_last_modified=include_etag_and_last_modified,
179
+ )
180
+ return await self._post("/v1/fetch", json=body, cast_to=FetchResponse)
181
+
182
+
183
+ def _validate_purpose(purpose: str | None) -> None:
184
+ if purpose is None:
185
+ return
186
+ if not purpose.strip():
187
+ raise SDKError("purpose must be a non-empty string")
188
+ if len(purpose) > FETCH_PURPOSE_MAX_LENGTH:
189
+ raise SDKError(f"purpose must be at most {FETCH_PURPOSE_MAX_LENGTH} characters")
190
+
191
+
192
+ def _validate_urls(urls: list[str]) -> None:
193
+ if len(urls) < MIN_URLS:
194
+ raise SDKError("urls must be a non-empty array")
195
+ if len(urls) > MAX_URLS:
196
+ raise SDKError("urls must contain at most 10 items")
197
+
198
+
199
+ def _validate_ttl(ttl: int | None) -> None:
200
+ """Reject negative or non-integer ttl values client-side.
201
+
202
+ Matches the TypeScript SDK's `z.int().min(0)`. `bool` is excluded
203
+ explicitly (Python treats `bool` as a subclass of `int`, so the
204
+ `isinstance(ttl, int)` check alone would accept `True`/`False`).
205
+ """
206
+ if ttl is None:
207
+ return
208
+ if isinstance(ttl, bool) or not isinstance(ttl, int):
209
+ raise SDKError("ttl must be a non-negative integer")
210
+ if ttl < 0:
211
+ raise SDKError("ttl must be a non-negative integer")
212
+
213
+
214
+ def _validate_per_url_timeout_ms(per_url_timeout_ms: int | None) -> None:
215
+ if per_url_timeout_ms is None:
216
+ return
217
+ if isinstance(per_url_timeout_ms, bool) or not isinstance(per_url_timeout_ms, int):
218
+ raise SDKError("per_url_timeout_ms must be an integer between 1 and 110000")
219
+ if not MIN_PER_URL_TIMEOUT_MS <= per_url_timeout_ms <= MAX_PER_URL_TIMEOUT_MS:
220
+ raise SDKError("per_url_timeout_ms must be an integer between 1 and 110000")
221
+
222
+
223
+ def _validate_conditional_headers(urls: list[str], if_none_match: str | None, if_modified_since: str | None) -> None:
224
+ """Reject `if_none_match` / `if_modified_since` when combined with a batch of URLs.
225
+
226
+ Conditional GETs only make sense against a single resource. Mirrors the
227
+ API's own validation (`api/models.py` `_check_flags`) so the error
228
+ surfaces client-side before the request is sent.
229
+ """
230
+ if len(urls) <= 1:
231
+ return
232
+ if if_none_match is not None:
233
+ raise SDKError("if_none_match can only be used with a single URL, not a batch")
234
+ if if_modified_since is not None:
235
+ raise SDKError("if_modified_since can only be used with a single URL, not a batch")
236
+
237
+
238
+ def _build_body(
239
+ urls: list[str],
240
+ *,
241
+ purpose: str | None,
242
+ format: FetchFormat | None,
243
+ include_html_head: bool | None,
244
+ links: bool | None,
245
+ image_links: bool | None,
246
+ ttl: int | None,
247
+ per_url_timeout_ms: int | None,
248
+ if_none_match: str | None,
249
+ if_modified_since: str | None,
250
+ include_etag_and_last_modified: bool | None,
251
+ ) -> dict[str, object]:
252
+ body: dict[str, object] = {"urls": urls}
253
+ if purpose is not None:
254
+ body["purpose"] = purpose
255
+ if format is not None:
256
+ body["format"] = format
257
+ if include_html_head is not None:
258
+ body["include_html_head"] = include_html_head
259
+ if links is not None:
260
+ body["links"] = links
261
+ if image_links is not None:
262
+ body["image_links"] = image_links
263
+ if ttl is not None:
264
+ body["ttl"] = ttl
265
+ if per_url_timeout_ms is not None:
266
+ body["per_url_timeout_ms"] = per_url_timeout_ms
267
+ if if_none_match is not None:
268
+ body["if_none_match"] = if_none_match
269
+ if if_modified_since is not None:
270
+ body["if_modified_since"] = if_modified_since
271
+ if include_etag_and_last_modified is not None:
272
+ body["include_etag_and_last_modified"] = include_etag_and_last_modified
273
+ return body
@@ -24,6 +24,20 @@ class FetchResult(BaseModel):
24
24
  links: list[str] = Field(default_factory=list, description="Extracted links")
25
25
  image_links: list[str] = Field(default_factory=list, description="Extracted image links")
26
26
  latency_ms: float | None = Field(None, description="Fetch latency in milliseconds")
27
+ not_modified: bool | None = Field(
28
+ None,
29
+ description=(
30
+ "True only for a successful 304 returned against the caller-supplied "
31
+ "if_none_match / if_modified_since validators. Omitted (None) when "
32
+ "conditional headers were not used or not requested via "
33
+ "include_etag_and_last_modified."
34
+ ),
35
+ )
36
+ etag: str | None = Field(None, description="Origin ETag validator; omitted when not applicable")
37
+ last_modified: str | None = Field(
38
+ None,
39
+ description="Origin Last-Modified validator; omitted when not applicable",
40
+ )
27
41
 
28
42
 
29
43
  class FetchError(BaseModel):