hydrafetch 0.2.0__tar.gz → 0.4.0__tar.gz

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hydrafetch
-Version: 0.2.0
+Version: 0.4.0
 Summary: Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.
 Project-URL: Homepage, https://hydrafetch.com
 Project-URL: Documentation, https://docs.hydrafetch.com
@@ -48,7 +48,7 @@ page = hf.scrape("https://example.com/article")
 print(page["markdown"])
 ```
 
-Create a key at [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
+Copy your key from [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
 
 Both clients are context managers, which closes the connection pool deterministically:
 
@@ -171,10 +171,29 @@ for r in res["results"]:
 hf.brand("stripe.com")                              # logos, colours, fonts, socials
 hf.logo("stripe.com", theme="dark", type="icon")    # one asset
 hf.styleguide("stripe.com")                         # computed design system
+hf.styleguide("https://stripe.com/pricing", maxAge=0)  # one page, read fresh
 ```
 
 For logos in a browser use [`@hydrafetch/client-sdk`](https://github.com/Hydrafetch/client-sdk) with a publishable key. Those bill against logo pulls rather than credits.
 
+## Rating results
+
+Tell us when a result was wrong, so it gets looked at. It is free, and rating the same request again replaces the earlier rating.
+
+```python
+page = hf.scrape("https://stripe.com/pricing")
+hf.rate_result(
+    "bad",
+    request_id=hf.last_request_id,
+    reasons=["missing_content"],
+    note="The enterprise tier is missing",
+)
+
+hf.rate_result("good", url="https://stripe.com/pricing")  # by URL, last 24 hours
+```
+
+Reasons are `missing_content`, `wrong_content`, `blocked`, `slow`, `formatting` and `other`.
+
 ## Async
 
 `AsyncHydrafetch` mirrors the same surface. Use it when several calls can run concurrently.
@@ -254,13 +273,14 @@ hf = Hydrafetch(
 | `extract(urls, **options)` | dict with `"results"` | 5 per URL |
 | `brand(domain)` | brand dict | 5 |
 | `logo(domain, **options)` | logo dict | 1 |
-| `styleguide(domain)` | design system dict | 10 |
+| `styleguide(domain_or_url, **options)` | design system dict | 10 |
 | `screenshot(url, **options)` | screenshot dict | 5 |
 | `images(url)`, `links(url)` | page assets | 1 |
 | `crawl(url, **options)` | job dict, polled to completion | 1 per page |
 | `batch(urls, **options)` | job dict, polled to completion | 1 per page |
 | `start_crawl`, `start_batch` | job id `str` | 1 per page |
 | `crawl_status(id)`, `batch_status(id)` | job dict | free |
+| `rate_result(rating, request_id=, url=, reasons=, note=)` | rating dict | free |
 
 Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
 

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/README.md

@@ -25,7 +25,7 @@ page = hf.scrape("https://example.com/article")
 print(page["markdown"])
 ```
 
-Create a key at [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
+Copy your key from [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
 
 Both clients are context managers, which closes the connection pool deterministically:
 
@@ -148,10 +148,29 @@ for r in res["results"]:
 hf.brand("stripe.com")                              # logos, colours, fonts, socials
 hf.logo("stripe.com", theme="dark", type="icon")    # one asset
 hf.styleguide("stripe.com")                         # computed design system
+hf.styleguide("https://stripe.com/pricing", maxAge=0)  # one page, read fresh
 ```
 
 For logos in a browser use [`@hydrafetch/client-sdk`](https://github.com/Hydrafetch/client-sdk) with a publishable key. Those bill against logo pulls rather than credits.
 
+## Rating results
+
+Tell us when a result was wrong, so it gets looked at. It is free, and rating the same request again replaces the earlier rating.
+
+```python
+page = hf.scrape("https://stripe.com/pricing")
+hf.rate_result(
+    "bad",
+    request_id=hf.last_request_id,
+    reasons=["missing_content"],
+    note="The enterprise tier is missing",
+)
+
+hf.rate_result("good", url="https://stripe.com/pricing")  # by URL, last 24 hours
+```
+
+Reasons are `missing_content`, `wrong_content`, `blocked`, `slow`, `formatting` and `other`.
+
 ## Async
 
 `AsyncHydrafetch` mirrors the same surface. Use it when several calls can run concurrently.
@@ -231,13 +250,14 @@ hf = Hydrafetch(
 | `extract(urls, **options)` | dict with `"results"` | 5 per URL |
 | `brand(domain)` | brand dict | 5 |
 | `logo(domain, **options)` | logo dict | 1 |
-| `styleguide(domain)` | design system dict | 10 |
+| `styleguide(domain_or_url, **options)` | design system dict | 10 |
 | `screenshot(url, **options)` | screenshot dict | 5 |
 | `images(url)`, `links(url)` | page assets | 1 |
 | `crawl(url, **options)` | job dict, polled to completion | 1 per page |
 | `batch(urls, **options)` | job dict, polled to completion | 1 per page |
 | `start_crawl`, `start_batch` | job id `str` | 1 per page |
 | `crawl_status(id)`, `batch_status(id)` | job dict | free |
+| `rate_result(rating, request_id=, url=, reasons=, note=)` | rating dict | free |
 
 Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
 

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hydrafetch"
-version = "0.2.0"
+version = "0.4.0"
 description = "Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data."
 readme = "README.md"
 license = "MIT"
@@ -46,3 +46,9 @@ target-version = "py39"
 # PYI019 wants typing.Self on the context managers. Self landed in 3.11 and this
 # package supports 3.9, so the bound TypeVars stay until the floor moves.
 ignore = ["PYI019"]
+
+[tool.ruff.lint.pyupgrade]
+# `from __future__ import annotations` normally makes `X | None` safe on 3.9, which is why ruff
+# offers to rewrite Optional. Pydantic evaluates annotations at runtime, so on 3.9 the rewrite
+# turns a working model into a TypeError at import. Only the 3.9 CI leg catches it.
+keep-runtime-typing = true

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/src/hydrafetch/__init__.py

@@ -4,4 +4,4 @@ from ._client import AsyncHydrafetch, Hydrafetch
 from .errors import HydrafetchError, HydrafetchTimeout
 
 __all__ = ["AsyncHydrafetch", "Hydrafetch", "HydrafetchError", "HydrafetchTimeout"]
-__version__ = "0.1.0"
+__version__ = "0.4.0"

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/src/hydrafetch/_client.py

@@ -88,10 +88,39 @@ class _Base:
         self._base_url = base_url.rstrip("/")
         self._timeout = timeout
         self._max_retries = max_retries
+        self.last_request_id: str | None = None
+
+    def _rate_body(
+        self,
+        rating: str,
+        request_id: str | None,
+        url: str | None,
+        reasons: Iterable[str] | None,
+        note: str | None,
+    ) -> dict[str, Any]:
+        if request_id is None and url is None:
+            raise HydrafetchError(
+                "Pass request_id or url to say which result you are rating.",
+                code="VALIDATION_ERROR",
+                status=400,
+            )
+        return _drop_none(
+            {
+                "requestId": request_id,
+                "url": url,
+                "rating": rating,
+                "reasons": list(reasons) if reasons is not None else None,
+                "note": note,
+            }
+        )
 
     def _scrape_body(self, url: str, options: Mapping[str, Any]) -> dict[str, Any]:
         return {"url": url, **_drop_none(options)}
 
+    def _styleguide_params(self, target: str, options: Mapping[str, Any]) -> dict[str, Any]:
+        key = "url" if target.lower().startswith(("http://", "https://")) else "domain"
+        return {key: target, **_drop_none(options)}
+
     def _extract_body(
         self, urls: str | Iterable[str], options: Mapping[str, Any]
     ) -> dict[str, Any]:
@@ -146,6 +175,7 @@ class Hydrafetch(_Base):
                     "or use the crawl and batch job endpoints for long work."
                 ) from exc
 
+            self.last_request_id = response.headers.get("x-request-id") or self.last_request_id
             try:
                 return _unwrap(response)
             except HydrafetchError as err:
@@ -158,6 +188,18 @@ class Hydrafetch(_Base):
     def scrape(self, url: str, **options: Any) -> dict[str, Any]:
         return self._request("POST", "/v1/web/scrape", json=self._scrape_body(url, options))
 
+    def rate_result(
+        self,
+        rating: str,
+        *,
+        request_id: str | None = None,
+        url: str | None = None,
+        reasons: Iterable[str] | None = None,
+        note: str | None = None,
+    ) -> dict[str, Any]:
+        body = self._rate_body(rating, request_id, url, reasons, note)
+        return self._request("POST", "/v1/web/feedback", json=body)
+
     def map(self, url: str, **options: Any) -> dict[str, Any]:
         return self._request("POST", "/v1/web/map", json=self._scrape_body(url, options))
 
@@ -173,8 +215,10 @@ class Hydrafetch(_Base):
     def logo(self, domain: str, **options: Any) -> dict[str, Any]:
         return self._request("GET", "/v1/web/brand/logo", params={"domain": domain, **options})
 
-    def styleguide(self, domain: str) -> dict[str, Any]:
-        return self._request("GET", "/v1/web/styleguide", params={"domain": domain})
+    def styleguide(self, domain_or_url: str, **options: Any) -> dict[str, Any]:
+        return self._request(
+            "GET", "/v1/web/styleguide", params=self._styleguide_params(domain_or_url, options)
+        )
 
     def screenshot(self, url: str, **options: Any) -> dict[str, Any]:
         return self._request("POST", "/v1/web/screenshot", json=self._scrape_body(url, options))
@@ -289,6 +333,7 @@ class AsyncHydrafetch(_Base):
                     "or use the crawl and batch job endpoints for long work."
                 ) from exc
 
+            self.last_request_id = response.headers.get("x-request-id") or self.last_request_id
             try:
                 return _unwrap(response)
             except HydrafetchError as err:
@@ -301,6 +346,18 @@ class AsyncHydrafetch(_Base):
     async def scrape(self, url: str, **options: Any) -> dict[str, Any]:
         return await self._request("POST", "/v1/web/scrape", json=self._scrape_body(url, options))
 
+    async def rate_result(
+        self,
+        rating: str,
+        *,
+        request_id: str | None = None,
+        url: str | None = None,
+        reasons: Iterable[str] | None = None,
+        note: str | None = None,
+    ) -> dict[str, Any]:
+        body = self._rate_body(rating, request_id, url, reasons, note)
+        return await self._request("POST", "/v1/web/feedback", json=body)
+
     async def map(self, url: str, **options: Any) -> dict[str, Any]:
         return await self._request("POST", "/v1/web/map", json=self._scrape_body(url, options))
 
@@ -322,8 +379,10 @@ class AsyncHydrafetch(_Base):
             "GET", "/v1/web/brand/logo", params={"domain": domain, **options}
         )
 
-    async def styleguide(self, domain: str) -> dict[str, Any]:
-        return await self._request("GET", "/v1/web/styleguide", params={"domain": domain})
+    async def styleguide(self, domain_or_url: str, **options: Any) -> dict[str, Any]:
+        return await self._request(
+            "GET", "/v1/web/styleguide", params=self._styleguide_params(domain_or_url, options)
+        )
 
     async def screenshot(self, url: str, **options: Any) -> dict[str, Any]:
         return await self._request(

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/tests/test_client.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import json
-from typing import Any
+from typing import Any, Optional
 
 import httpx
 import pytest
@@ -102,6 +102,16 @@ class TestRequestShaping:
         assert rec.last.url.params["theme"] == "dark"
         assert not rec.last.content
 
+    def test_styleguide_reads_a_domain_or_one_page(self) -> None:
+        rec = Recorder({"success": True, "data": {}})
+        hf = make_client(rec)
+        hf.styleguide("stripe.com")
+        assert rec.last.url.params["domain"] == "stripe.com"
+        hf.styleguide("https://stripe.com/pricing", maxAge=0)
+        assert rec.last.url.params["url"] == "https://stripe.com/pricing"
+        assert rec.last.url.params["maxAge"] == "0"
+        assert "domain" not in rec.last.url.params
+
 
 class TestErrors:
     def test_surfaces_code_status_and_request_id(self) -> None:
@@ -238,7 +248,7 @@ def test_extract_still_takes_plain_json_schema():
 
 
 class Node(BaseModel):
-    child: Node | None = None
+    child: Optional[Node] = None
 
 
 def test_extract_rejects_a_recursive_model_rather_than_sending_a_ref():
@@ -246,3 +256,40 @@ def test_extract_rejects_a_recursive_model_rather_than_sending_a_ref():
     with pytest.raises(HydrafetchError) as err:
         client.extract("https://example.com", schema=Node)
     assert err.value.code == "SCHEMA_CONVERSION_FAILED"
+
+
+def test_rate_result_posts_the_rating_for_the_last_request() -> None:
+    seen: list[httpx.Request] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(request)
+        if request.url.path == "/v1/web/feedback":
+            return ok({"success": True, "data": {"id": "f1", "requestId": "r1", "rating": "bad"}})
+        return httpx.Response(
+            200, json={"success": True, "data": {}}, headers={"x-request-id": "r1"}
+        )
+
+    client = make_client(handler)
+    client.scrape("https://a.com")
+    assert client.last_request_id == "r1"
+
+    out = client.rate_result(
+        "bad", request_id=client.last_request_id, reasons=["missing_content"], note="no prices"
+    )
+
+    assert out == {"id": "f1", "requestId": "r1", "rating": "bad"}
+    assert json.loads(seen[-1].content) == {
+        "requestId": "r1",
+        "rating": "bad",
+        "reasons": ["missing_content"],
+        "note": "no prices",
+    }
+
+
+def test_rate_result_can_rate_by_url_and_needs_a_target() -> None:
+    rec = Recorder({"success": True, "data": {"id": "f2", "requestId": "r2", "rating": "good"}})
+    client = make_client(rec)
+    client.rate_result("good", url="https://a.com")
+    assert json.loads(rec.last.content) == {"url": "https://a.com", "rating": "good"}
+    with pytest.raises(HydrafetchError):
+        client.rate_result("good")

{hydrafetch-0.2.0 → hydrafetch-0.4.0}/uv.lock

@@ -143,7 +143,7 @@ wheels = [
 
 [[package]]
 name = "hydrafetch"
-version = "0.2.0"
+version = "0.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },