hydrafetch 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/PKG-INFO +40 -1
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/README.md +39 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/pyproject.toml +1 -1
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/src/hydrafetch/__init__.py +1 -1
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/src/hydrafetch/_client.py +40 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/tests/test_client.py +42 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/uv.lock +4 -4
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/.github/workflows/ci.yml +0 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/.github/workflows/publish.yml +0 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/.gitignore +0 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/LICENSE +0 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/src/hydrafetch/_schema.py +0 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/src/hydrafetch/errors.py +0 -0
- {hydrafetch-0.4.0 → hydrafetch-0.6.0}/src/hydrafetch/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: hydrafetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.
|
|
5
5
|
Project-URL: Homepage, https://hydrafetch.com
|
|
6
6
|
Project-URL: Documentation, https://docs.hydrafetch.com
|
|
@@ -165,6 +165,42 @@ for r in res["results"]:
|
|
|
165
165
|
print(((r.get("data") or {}).get("markdown") or "")[:500])
|
|
166
166
|
```
|
|
167
167
|
|
|
168
|
+
## News
|
|
169
|
+
|
|
170
|
+
Find recent news articles about anything, newest first, each with its outlet and publish time:
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
found = hf.news("acme raises funding", timeRange="week", limit=10)
|
|
174
|
+
|
|
175
|
+
for article in found["results"]:
|
|
176
|
+
print(article["publishedAt"], article["source"], article["title"], article["url"])
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
A news search costs 1 credit, and one that finds nothing is free. Pass `scrapeResults=True` to also read each article as clean data, at 1 extra credit per article.
|
|
180
|
+
|
|
181
|
+
## Jobs
|
|
182
|
+
|
|
183
|
+
Search open jobs collected from company career sites and hiring platforms, by the words in the job title:
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
jobs = hf.search_jobs("data engineer", location="Berlin", postedWithinDays=14, limit=20)
|
|
187
|
+
|
|
188
|
+
for job in jobs:
|
|
189
|
+
print(job["title"], job["company"], job["location"], job["url"])
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
See what changed in a company's hiring since a date: what is open now, what opened and what closed, by department, seniority and location, with the jobs themselves:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
[acme] = hf.hiring_changes("acme.com", "2026-10-01")
|
|
196
|
+
|
|
197
|
+
print(acme["totals"]) # {"open": 42, "opened": 6, "closed": 3}
|
|
198
|
+
for job in acme["opened"]:
|
|
199
|
+
print(job["title"], job["seniority"], job["firstSeenAt"])
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
A search costs 1 credit per 10 jobs returned, and an empty one is free. Hiring changes cost 1 credit per company we have jobs for; companies we do not track yet come back empty and are free.
|
|
203
|
+
|
|
168
204
|
## Brand data
|
|
169
205
|
|
|
170
206
|
```python
|
|
@@ -270,6 +306,9 @@ hf = Hydrafetch(
|
|
|
270
306
|
| `markdown(url, **options)` | `str` | 1 |
|
|
271
307
|
| `map(url, **options)` | links dict | 1 |
|
|
272
308
|
| `search(query, **options)` | results dict | 1 + 1 per scraped result |
|
|
309
|
+
| `news(query, **options)` | articles, newest first | 1 + 1 per article read; free when empty |
|
|
310
|
+
| `search_jobs(query, **options)` | list of jobs | 1 per 10 jobs |
|
|
311
|
+
| `hiring_changes(domains, since, **options)` | list of companies | 1 per company with jobs |
|
|
273
312
|
| `extract(urls, **options)` | dict with `"results"` | 5 per URL |
|
|
274
313
|
| `brand(domain)` | brand dict | 5 |
|
|
275
314
|
| `logo(domain, **options)` | logo dict | 1 |
|
|
@@ -142,6 +142,42 @@ for r in res["results"]:
|
|
|
142
142
|
print(((r.get("data") or {}).get("markdown") or "")[:500])
|
|
143
143
|
```
|
|
144
144
|
|
|
145
|
+
## News
|
|
146
|
+
|
|
147
|
+
Find recent news articles about anything, newest first, each with its outlet and publish time:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
found = hf.news("acme raises funding", timeRange="week", limit=10)
|
|
151
|
+
|
|
152
|
+
for article in found["results"]:
|
|
153
|
+
print(article["publishedAt"], article["source"], article["title"], article["url"])
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
A news search costs 1 credit, and one that finds nothing is free. Pass `scrapeResults=True` to also read each article as clean data, at 1 extra credit per article.
|
|
157
|
+
|
|
158
|
+
## Jobs
|
|
159
|
+
|
|
160
|
+
Search open jobs collected from company career sites and hiring platforms, by the words in the job title:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
jobs = hf.search_jobs("data engineer", location="Berlin", postedWithinDays=14, limit=20)
|
|
164
|
+
|
|
165
|
+
for job in jobs:
|
|
166
|
+
print(job["title"], job["company"], job["location"], job["url"])
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
See what changed in a company's hiring since a date: what is open now, what opened and what closed, by department, seniority and location, with the jobs themselves:
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
[acme] = hf.hiring_changes("acme.com", "2026-10-01")
|
|
173
|
+
|
|
174
|
+
print(acme["totals"]) # {"open": 42, "opened": 6, "closed": 3}
|
|
175
|
+
for job in acme["opened"]:
|
|
176
|
+
print(job["title"], job["seniority"], job["firstSeenAt"])
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
A search costs 1 credit per 10 jobs returned, and an empty one is free. Hiring changes cost 1 credit per company we have jobs for; companies we do not track yet come back empty and are free.
|
|
180
|
+
|
|
145
181
|
## Brand data
|
|
146
182
|
|
|
147
183
|
```python
|
|
@@ -247,6 +283,9 @@ hf = Hydrafetch(
|
|
|
247
283
|
| `markdown(url, **options)` | `str` | 1 |
|
|
248
284
|
| `map(url, **options)` | links dict | 1 |
|
|
249
285
|
| `search(query, **options)` | results dict | 1 + 1 per scraped result |
|
|
286
|
+
| `news(query, **options)` | articles, newest first | 1 + 1 per article read; free when empty |
|
|
287
|
+
| `search_jobs(query, **options)` | list of jobs | 1 per 10 jobs |
|
|
288
|
+
| `hiring_changes(domains, since, **options)` | list of companies | 1 per company with jobs |
|
|
250
289
|
| `extract(urls, **options)` | dict with `"results"` | 5 per URL |
|
|
251
290
|
| `brand(domain)` | brand dict | 5 |
|
|
252
291
|
| `logo(domain, **options)` | logo dict | 1 |
|
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
import random
|
|
5
5
|
import time
|
|
6
6
|
from collections.abc import Iterable, Mapping
|
|
7
|
+
from datetime import datetime
|
|
7
8
|
from typing import Any, Callable, Literal, TypeVar
|
|
8
9
|
|
|
9
10
|
import httpx
|
|
@@ -130,6 +131,13 @@ class _Base:
|
|
|
130
131
|
body["schema"] = to_json_schema(body["schema"])
|
|
131
132
|
return {"urls": url_list, **body}
|
|
132
133
|
|
|
134
|
+
def _changes_body(
|
|
135
|
+
self, domains: str | Iterable[str], since: str | datetime, options: Mapping[str, Any]
|
|
136
|
+
) -> dict[str, Any]:
|
|
137
|
+
domain_list = [domains] if isinstance(domains, str) else list(domains)
|
|
138
|
+
when = since.isoformat() if isinstance(since, datetime) else since
|
|
139
|
+
return {"domains": domain_list, "since": when, **_drop_none(options)}
|
|
140
|
+
|
|
133
141
|
def _job_id(self, payload: Any, *keys: str) -> str:
|
|
134
142
|
if isinstance(payload, dict):
|
|
135
143
|
for key in (*keys, "id"):
|
|
@@ -206,6 +214,21 @@ class Hydrafetch(_Base):
|
|
|
206
214
|
def search(self, query: str, **options: Any) -> dict[str, Any]:
|
|
207
215
|
return self._request("POST", "/v1/web/search", json={"query": query, **_drop_none(options)})
|
|
208
216
|
|
|
217
|
+
def news(self, query: str, **options: Any) -> dict[str, Any]:
|
|
218
|
+
return self._request("POST", "/v1/web/news", json={"query": query, **_drop_none(options)})
|
|
219
|
+
|
|
220
|
+
def search_jobs(self, query: str, **options: Any) -> list[dict[str, Any]]:
|
|
221
|
+
return self._request(
|
|
222
|
+
"POST", "/v1/web/index/jobs/search", json={"query": query, **_drop_none(options)}
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
def hiring_changes(
|
|
226
|
+
self, domains: str | Iterable[str], since: str | datetime, **options: Any
|
|
227
|
+
) -> list[dict[str, Any]]:
|
|
228
|
+
return self._request(
|
|
229
|
+
"POST", "/v1/web/index/jobs/changes", json=self._changes_body(domains, since, options)
|
|
230
|
+
)
|
|
231
|
+
|
|
209
232
|
def extract(self, urls: str | Iterable[str], **options: Any) -> Any:
|
|
210
233
|
return self._request("POST", "/v1/web/extract", json=self._extract_body(urls, options))
|
|
211
234
|
|
|
@@ -366,6 +389,23 @@ class AsyncHydrafetch(_Base):
|
|
|
366
389
|
"POST", "/v1/web/search", json={"query": query, **_drop_none(options)}
|
|
367
390
|
)
|
|
368
391
|
|
|
392
|
+
async def news(self, query: str, **options: Any) -> dict[str, Any]:
|
|
393
|
+
return await self._request(
|
|
394
|
+
"POST", "/v1/web/news", json={"query": query, **_drop_none(options)}
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
async def search_jobs(self, query: str, **options: Any) -> list[dict[str, Any]]:
|
|
398
|
+
return await self._request(
|
|
399
|
+
"POST", "/v1/web/index/jobs/search", json={"query": query, **_drop_none(options)}
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
async def hiring_changes(
|
|
403
|
+
self, domains: str | Iterable[str], since: str | datetime, **options: Any
|
|
404
|
+
) -> list[dict[str, Any]]:
|
|
405
|
+
return await self._request(
|
|
406
|
+
"POST", "/v1/web/index/jobs/changes", json=self._changes_body(domains, since, options)
|
|
407
|
+
)
|
|
408
|
+
|
|
369
409
|
async def extract(self, urls: str | Iterable[str], **options: Any) -> Any:
|
|
370
410
|
return await self._request(
|
|
371
411
|
"POST", "/v1/web/extract", json=self._extract_body(urls, options)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
from datetime import datetime, timezone
|
|
4
5
|
from typing import Any, Optional
|
|
5
6
|
|
|
6
7
|
import httpx
|
|
@@ -293,3 +294,44 @@ def test_rate_result_can_rate_by_url_and_needs_a_target() -> None:
|
|
|
293
294
|
assert json.loads(rec.last.content) == {"url": "https://a.com", "rating": "good"}
|
|
294
295
|
with pytest.raises(HydrafetchError):
|
|
295
296
|
client.rate_result("good")
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def test_search_jobs_posts_the_query_and_returns_the_list() -> None:
|
|
300
|
+
job = {"title": "Data Engineer", "domain": "acme.com", "board": "greenhouse"}
|
|
301
|
+
rec = Recorder({"success": True, "data": [job]})
|
|
302
|
+
client = make_client(rec)
|
|
303
|
+
jobs = client.search_jobs("data engineer", location="Berlin", limit=20, remoteOnly=None)
|
|
304
|
+
assert jobs == [job]
|
|
305
|
+
assert rec.last.url.path == "/v1/web/index/jobs/search"
|
|
306
|
+
assert json.loads(rec.last.content) == {
|
|
307
|
+
"query": "data engineer",
|
|
308
|
+
"location": "Berlin",
|
|
309
|
+
"limit": 20,
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def test_hiring_changes_takes_one_domain_or_many_and_a_datetime() -> None:
|
|
314
|
+
rec = Recorder({"success": True, "data": []})
|
|
315
|
+
client = make_client(rec)
|
|
316
|
+
client.hiring_changes("stripe.com", datetime(2026, 10, 1, tzinfo=timezone.utc))
|
|
317
|
+
assert rec.last.url.path == "/v1/web/index/jobs/changes"
|
|
318
|
+
assert json.loads(rec.last.content) == {
|
|
319
|
+
"domains": ["stripe.com"],
|
|
320
|
+
"since": "2026-10-01T00:00:00+00:00",
|
|
321
|
+
}
|
|
322
|
+
client.hiring_changes(["a.com", "b.com"], "2026-10-01", includeDescriptions=True)
|
|
323
|
+
assert json.loads(rec.last.content) == {
|
|
324
|
+
"domains": ["a.com", "b.com"],
|
|
325
|
+
"since": "2026-10-01",
|
|
326
|
+
"includeDescriptions": True,
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def test_news_posts_the_query_and_window() -> None:
|
|
331
|
+
found = {"query": "acme raises", "status": "ok", "results": [{"title": "Acme raises $10M"}]}
|
|
332
|
+
rec = Recorder({"success": True, "data": found})
|
|
333
|
+
client = make_client(rec)
|
|
334
|
+
out = client.news("acme raises", timeRange="day", limit=5, country=None)
|
|
335
|
+
assert out == found
|
|
336
|
+
assert rec.last.url.path == "/v1/web/news"
|
|
337
|
+
assert json.loads(rec.last.content) == {"query": "acme raises", "timeRange": "day", "limit": 5}
|
|
@@ -75,11 +75,11 @@ wheels = [
|
|
|
75
75
|
|
|
76
76
|
[[package]]
|
|
77
77
|
name = "certifi"
|
|
78
|
-
version = "2026.
|
|
78
|
+
version = "2026.10.19"
|
|
79
79
|
source = { registry = "https://pypi.org/simple" }
|
|
80
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
80
|
+
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
|
81
81
|
wheels = [
|
|
82
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
82
|
+
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
|
83
83
|
]
|
|
84
84
|
|
|
85
85
|
[[package]]
|
|
@@ -143,7 +143,7 @@ wheels = [
|
|
|
143
143
|
|
|
144
144
|
[[package]]
|
|
145
145
|
name = "hydrafetch"
|
|
146
|
-
version = "0.
|
|
146
|
+
version = "0.6.0"
|
|
147
147
|
source = { editable = "." }
|
|
148
148
|
dependencies = [
|
|
149
149
|
{ name = "httpx" },
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|