hydrafetch 0.1.1__tar.gz → 0.3.0__tar.gz

{hydrafetch-0.1.1 → hydrafetch-0.3.0}/.github/workflows/publish.yml

@@ -11,7 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     environment: pypi
     permissions:
-      contents: read
+      # contents: write is what lets the release be created; the publish itself needs only read.
+      contents: write
       id-token: write
     steps:
       - uses: actions/checkout@v4
@@ -25,3 +26,10 @@ jobs:
           [ "$TAG" = "$PKG" ] || { echo "tag $TAG != pyproject $PKG"; exit 1; }
       - run: uv build
       - uses: pypa/gh-action-pypi-publish@release/v1
+
+      # After the publish, never before: a release pointing at a version that is not on PyPI is
+      # worse than no release at all.
+      - name: Create the GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true

{hydrafetch-0.1.1 → hydrafetch-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hydrafetch
-Version: 0.1.1
+Version: 0.3.0
 Summary: Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.
 Project-URL: Homepage, https://hydrafetch.com
 Project-URL: Documentation, https://docs.hydrafetch.com
@@ -105,6 +105,22 @@ for item in out["results"]:
 
 Pass `prompt=` instead of, or alongside, `schema=` to describe the fields in plain language.
 
+### From a pydantic model
+
+`schema=` also takes a pydantic model, so the shape you already parse into is the shape you ask for. A `Field` description is carried through and tells the model what the field means, which is worth more on an ambiguous page than the field name alone.
+
+```python
+from pydantic import BaseModel, Field
+
+class Product(BaseModel):
+    name: str = Field(description="The product being sold on this page, not a recommended one")
+    price_usd: float = Field(description="Current price in USD, after any discount")
+
+out = hf.extract("https://example.com/product/1", schema=Product)
+```
+
+Nested models work and are flattened before sending. pydantic is not a dependency and is imported only if you pass a model. A model that refers to itself cannot be flattened and is refused rather than sent.
+
 ## Discovery and bulk work
 
 `map` lists a site's URLs for one credit without fetching any page.
@@ -155,6 +171,7 @@ for r in res["results"]:
 hf.brand("stripe.com")                              # logos, colours, fonts, socials
 hf.logo("stripe.com", theme="dark", type="icon")    # one asset
 hf.styleguide("stripe.com")                         # computed design system
+hf.styleguide("https://stripe.com/pricing", maxAge=0)  # one page, read fresh
 ```
 
 For logos in a browser use [`@hydrafetch/client-sdk`](https://github.com/Hydrafetch/client-sdk) with a publishable key. Those bill against logo pulls rather than credits.
@@ -238,7 +255,7 @@ hf = Hydrafetch(
 | `extract(urls, **options)` | dict with `"results"` | 5 per URL |
 | `brand(domain)` | brand dict | 5 |
 | `logo(domain, **options)` | logo dict | 1 |
-| `styleguide(domain)` | design system dict | 10 |
+| `styleguide(domain_or_url, **options)` | design system dict | 10 |
 | `screenshot(url, **options)` | screenshot dict | 5 |
 | `images(url)`, `links(url)` | page assets | 1 |
 | `crawl(url, **options)` | job dict, polled to completion | 1 per page |

{hydrafetch-0.1.1 → hydrafetch-0.3.0}/README.md

@@ -82,6 +82,22 @@ for item in out["results"]:
 
 Pass `prompt=` instead of, or alongside, `schema=` to describe the fields in plain language.
 
+### From a pydantic model
+
+`schema=` also takes a pydantic model, so the shape you already parse into is the shape you ask for. A `Field` description is carried through and tells the model what the field means, which is worth more on an ambiguous page than the field name alone.
+
+```python
+from pydantic import BaseModel, Field
+
+class Product(BaseModel):
+    name: str = Field(description="The product being sold on this page, not a recommended one")
+    price_usd: float = Field(description="Current price in USD, after any discount")
+
+out = hf.extract("https://example.com/product/1", schema=Product)
+```
+
+Nested models work and are flattened before sending. pydantic is not a dependency and is imported only if you pass a model. A model that refers to itself cannot be flattened and is refused rather than sent.
+
 ## Discovery and bulk work
 
 `map` lists a site's URLs for one credit without fetching any page.
@@ -132,6 +148,7 @@ for r in res["results"]:
 hf.brand("stripe.com")                              # logos, colours, fonts, socials
 hf.logo("stripe.com", theme="dark", type="icon")    # one asset
 hf.styleguide("stripe.com")                         # computed design system
+hf.styleguide("https://stripe.com/pricing", maxAge=0)  # one page, read fresh
 ```
 
 For logos in a browser use [`@hydrafetch/client-sdk`](https://github.com/Hydrafetch/client-sdk) with a publishable key. Those bill against logo pulls rather than credits.
@@ -215,7 +232,7 @@ hf = Hydrafetch(
 | `extract(urls, **options)` | dict with `"results"` | 5 per URL |
 | `brand(domain)` | brand dict | 5 |
 | `logo(domain, **options)` | logo dict | 1 |
-| `styleguide(domain)` | design system dict | 10 |
+| `styleguide(domain_or_url, **options)` | design system dict | 10 |
 | `screenshot(url, **options)` | screenshot dict | 5 |
 | `images(url)`, `links(url)` | page assets | 1 |
 | `crawl(url, **options)` | job dict, polled to completion | 1 per page |

{hydrafetch-0.1.1 → hydrafetch-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hydrafetch"
-version = "0.1.1"
+version = "0.3.0"
 description = "Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data."
 readme = "README.md"
 license = "MIT"
@@ -32,7 +32,7 @@ build-backend = "hatchling.build"
 packages = ["src/hydrafetch"]
 
 [dependency-groups]
-dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.6"]
+dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.6", "pydantic>=2"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
@@ -46,3 +46,9 @@ target-version = "py39"
 # PYI019 wants typing.Self on the context managers. Self landed in 3.11 and this
 # package supports 3.9, so the bound TypeVars stay until the floor moves.
 ignore = ["PYI019"]
+
+[tool.ruff.lint.pyupgrade]
+# `from __future__ import annotations` normally makes `X | None` safe on 3.9, which is why ruff
+# offers to rewrite Optional. Pydantic evaluates annotations at runtime, so on 3.9 the rewrite
+# turns a working model into a TypeError at import. Only the 3.9 CI leg catches it.
+keep-runtime-typing = true

{hydrafetch-0.1.1 → hydrafetch-0.3.0}/src/hydrafetch/_client.py

@@ -8,6 +8,7 @@ from typing import Any, Callable, Literal, TypeVar
 
 import httpx
 
+from ._schema import to_json_schema
 from .errors import HydrafetchError, HydrafetchTimeout
 
 DEFAULT_BASE_URL = "https://api.hydrafetch.com"
@@ -91,11 +92,18 @@ class _Base:
     def _scrape_body(self, url: str, options: Mapping[str, Any]) -> dict[str, Any]:
         return {"url": url, **_drop_none(options)}
 
+    def _styleguide_params(self, target: str, options: Mapping[str, Any]) -> dict[str, Any]:
+        key = "url" if target.lower().startswith(("http://", "https://")) else "domain"
+        return {key: target, **_drop_none(options)}
+
     def _extract_body(
         self, urls: str | Iterable[str], options: Mapping[str, Any]
     ) -> dict[str, Any]:
         url_list = [urls] if isinstance(urls, str) else list(urls)
-        return {"urls": url_list, **_drop_none(options)}
+        body = _drop_none(options)
+        if "schema" in body:
+            body["schema"] = to_json_schema(body["schema"])
+        return {"urls": url_list, **body}
 
     def _job_id(self, payload: Any, *keys: str) -> str:
         if isinstance(payload, dict):
@@ -169,8 +177,10 @@ class Hydrafetch(_Base):
     def logo(self, domain: str, **options: Any) -> dict[str, Any]:
         return self._request("GET", "/v1/web/brand/logo", params={"domain": domain, **options})
 
-    def styleguide(self, domain: str) -> dict[str, Any]:
-        return self._request("GET", "/v1/web/styleguide", params={"domain": domain})
+    def styleguide(self, domain_or_url: str, **options: Any) -> dict[str, Any]:
+        return self._request(
+            "GET", "/v1/web/styleguide", params=self._styleguide_params(domain_or_url, options)
+        )
 
     def screenshot(self, url: str, **options: Any) -> dict[str, Any]:
         return self._request("POST", "/v1/web/screenshot", json=self._scrape_body(url, options))
@@ -318,8 +328,10 @@ class AsyncHydrafetch(_Base):
             "GET", "/v1/web/brand/logo", params={"domain": domain, **options}
         )
 
-    async def styleguide(self, domain: str) -> dict[str, Any]:
-        return await self._request("GET", "/v1/web/styleguide", params={"domain": domain})
+    async def styleguide(self, domain_or_url: str, **options: Any) -> dict[str, Any]:
+        return await self._request(
+            "GET", "/v1/web/styleguide", params=self._styleguide_params(domain_or_url, options)
+        )
 
     async def screenshot(self, url: str, **options: Any) -> dict[str, Any]:
         return await self._request(

hydrafetch-0.3.0/src/hydrafetch/_schema.py

@@ -0,0 +1,69 @@
+"""Turn whatever the caller has into the JSON Schema the API expects.
+
+Pydantic is not a dependency and is never imported. A model is recognised by the method it carries,
+so any object that can describe itself as JSON Schema works, and someone who has no use for
+pydantic still installs this package with one dependency.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .errors import HydrafetchError
+
+MAX_REF_DEPTH = 12
+
+
+def to_json_schema(schema: Any) -> dict[str, Any]:
+    if isinstance(schema, Mapping):
+        return inline_refs(dict(schema))
+
+    for method in ("model_json_schema", "schema"):
+        describe = getattr(schema, method, None)
+        if callable(describe):
+            return inline_refs(describe())
+
+    raise HydrafetchError(
+        "schema must be a dict of JSON Schema or a pydantic model.",
+        code="SCHEMA_CONVERSION_FAILED",
+        status=0,
+    )
+
+
+def inline_refs(schema: dict[str, Any]) -> dict[str, Any]:
+    """Replace every local $ref with the thing it points at, and drop the definitions block.
+
+    A nested model is the ordinary case, not an exotic one, and pydantic always describes it as a
+    $ref into $defs. That indirection does not survive the trip to a model's tool schema, so
+    forwarding it turns a normal two-model extraction into a confusing failure from somewhere the
+    caller cannot see. Resolving it here costs nothing and keeps the surface honest.
+    """
+    defs = {**schema.get("$defs", {}), **schema.get("definitions", {})}
+    if not defs:
+        return schema
+
+    def resolve(node: Any, depth: int) -> Any:
+        if isinstance(node, list):
+            return [resolve(item, depth) for item in node]
+        if not isinstance(node, dict):
+            return node
+
+        target = node.get("$ref")
+        if isinstance(target, str) and target.startswith("#/"):
+            name = target.rsplit("/", 1)[-1]
+            # A self-referential model has no finite inlining. Leaving the $ref in place is worse
+            # than saying so, because the failure would surface as an opaque error from the API.
+            if depth >= MAX_REF_DEPTH or name not in defs:
+                raise HydrafetchError(
+                    f"schema reference {target} is recursive or missing, so it cannot be inlined. "
+                    "Flatten the model or pass JSON Schema directly.",
+                    code="SCHEMA_CONVERSION_FAILED",
+                    status=0,
+                )
+            merged = {**defs[name], **{k: v for k, v in node.items() if k != "$ref"}}
+            return resolve(merged, depth + 1)
+
+        return {k: resolve(v, depth) for k, v in node.items() if k not in ("$defs", "definitions")}
+
+    return resolve(schema, 0)

{hydrafetch-0.1.1 → hydrafetch-0.3.0}/tests/test_client.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import json
-from typing import Any
+from typing import Any, Optional
 
 import httpx
 import pytest
+from pydantic import BaseModel
 
 from hydrafetch import AsyncHydrafetch, Hydrafetch, HydrafetchError, HydrafetchTimeout
 
@@ -20,6 +21,16 @@ def make_client(handler, **kwargs: Any) -> Hydrafetch:
     return client
 
 
+def _recording_client() -> tuple[Hydrafetch, list[dict[str, Any]]]:
+    seen: list[dict[str, Any]] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(json.loads(request.content))
+        return httpx.Response(200, json={"success": True, "data": {"results": []}})
+
+    return make_client(handler), seen
+
+
 def ok(body: Any, status: int = 200) -> httpx.Response:
     return httpx.Response(status, json=body)
 
@@ -91,6 +102,16 @@ class TestRequestShaping:
         assert rec.last.url.params["theme"] == "dark"
         assert not rec.last.content
 
+    def test_styleguide_reads_a_domain_or_one_page(self) -> None:
+        rec = Recorder({"success": True, "data": {}})
+        hf = make_client(rec)
+        hf.styleguide("stripe.com")
+        assert rec.last.url.params["domain"] == "stripe.com"
+        hf.styleguide("https://stripe.com/pricing", maxAge=0)
+        assert rec.last.url.params["url"] == "https://stripe.com/pricing"
+        assert rec.last.url.params["maxAge"] == "0"
+        assert "domain" not in rec.last.url.params
+
 
 class TestErrors:
     def test_surfaces_code_status_and_request_id(self) -> None:
@@ -200,3 +221,38 @@ class TestAsyncClient:
             out = await client.scrape("https://example.com")
         assert out["markdown"] == "# async"
         assert rec.last.headers["x-api-key"] == "hf_test"
+
+
+def test_extract_accepts_a_pydantic_model_and_inlines_nested_ones():
+    class Address(BaseModel):
+        city: str
+
+    class Person(BaseModel):
+        name: str
+        home: Address
+
+    client, seen = _recording_client()
+    client.extract("https://example.com", schema=Person)
+
+    schema = seen[0]["schema"]
+    assert schema["properties"]["name"]["type"] == "string"
+    assert schema["properties"]["home"]["properties"]["city"]["type"] == "string"
+    assert "$defs" not in schema and "$ref" not in str(schema)
+
+
+def test_extract_still_takes_plain_json_schema():
+    client, seen = _recording_client()
+    plain = {"type": "object", "properties": {"name": {"type": "string"}}}
+    client.extract("https://example.com", schema=plain)
+    assert seen[0]["schema"] == plain
+
+
+class Node(BaseModel):
+    child: Optional[Node] = None
+
+
+def test_extract_rejects_a_recursive_model_rather_than_sending_a_ref():
+    client, _ = _recording_client()
+    with pytest.raises(HydrafetchError) as err:
+        client.extract("https://example.com", schema=Node)
+    assert err.value.code == "SCHEMA_CONVERSION_FAILED"