hydrafetch 0.1.1__tar.gz → 0.2.0__tar.gz

{hydrafetch-0.1.1 → hydrafetch-0.2.0}/.github/workflows/publish.yml

@@ -11,7 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     environment: pypi
     permissions:
-      contents: read
+      # contents: write is what lets the release be created; the publish itself needs only read.
+      contents: write
       id-token: write
     steps:
       - uses: actions/checkout@v4
@@ -25,3 +26,10 @@ jobs:
           [ "$TAG" = "$PKG" ] || { echo "tag $TAG != pyproject $PKG"; exit 1; }
       - run: uv build
       - uses: pypa/gh-action-pypi-publish@release/v1
+
+      # After the publish, never before: a release pointing at a version that is not on PyPI is
+      # worse than no release at all.
+      - name: Create the GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true

{hydrafetch-0.1.1 → hydrafetch-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hydrafetch
-Version: 0.1.1
+Version: 0.2.0
 Summary: Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.
 Project-URL: Homepage, https://hydrafetch.com
 Project-URL: Documentation, https://docs.hydrafetch.com
@@ -105,6 +105,22 @@ for item in out["results"]:
 
 Pass `prompt=` instead of, or alongside, `schema=` to describe the fields in plain language.
 
+### From a pydantic model
+
+`schema=` also takes a pydantic model, so the shape you already parse into is the shape you ask for. A `Field` description is carried through and tells the model what the field means, which is worth more on an ambiguous page than the field name alone.
+
+```python
+from pydantic import BaseModel, Field
+
+class Product(BaseModel):
+    name: str = Field(description="The product being sold on this page, not a recommended one")
+    price_usd: float = Field(description="Current price in USD, after any discount")
+
+out = hf.extract("https://example.com/product/1", schema=Product)
+```
+
+Nested models work and are flattened before sending. pydantic is not a dependency and is imported only if you pass a model. A model that refers to itself cannot be flattened and is refused rather than sent.
+
 ## Discovery and bulk work
 
 `map` lists a site's URLs for one credit without fetching any page.

{hydrafetch-0.1.1 → hydrafetch-0.2.0}/README.md

@@ -82,6 +82,22 @@ for item in out["results"]:
 
 Pass `prompt=` instead of, or alongside, `schema=` to describe the fields in plain language.
 
+### From a pydantic model
+
+`schema=` also takes a pydantic model, so the shape you already parse into is the shape you ask for. A `Field` description is carried through and tells the model what the field means, which is worth more on an ambiguous page than the field name alone.
+
+```python
+from pydantic import BaseModel, Field
+
+class Product(BaseModel):
+    name: str = Field(description="The product being sold on this page, not a recommended one")
+    price_usd: float = Field(description="Current price in USD, after any discount")
+
+out = hf.extract("https://example.com/product/1", schema=Product)
+```
+
+Nested models work and are flattened before sending. pydantic is not a dependency and is imported only if you pass a model. A model that refers to itself cannot be flattened and is refused rather than sent.
+
 ## Discovery and bulk work
 
 `map` lists a site's URLs for one credit without fetching any page.

{hydrafetch-0.1.1 → hydrafetch-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hydrafetch"
-version = "0.1.1"
+version = "0.2.0"
 description = "Official Python client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data."
 readme = "README.md"
 license = "MIT"
@@ -32,7 +32,7 @@ build-backend = "hatchling.build"
 packages = ["src/hydrafetch"]
 
 [dependency-groups]
-dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.6"]
+dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.6", "pydantic>=2"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"

{hydrafetch-0.1.1 → hydrafetch-0.2.0}/src/hydrafetch/_client.py

@@ -8,6 +8,7 @@ from typing import Any, Callable, Literal, TypeVar
 
 import httpx
 
+from ._schema import to_json_schema
 from .errors import HydrafetchError, HydrafetchTimeout
 
 DEFAULT_BASE_URL = "https://api.hydrafetch.com"
@@ -95,7 +96,10 @@ class _Base:
         self, urls: str | Iterable[str], options: Mapping[str, Any]
     ) -> dict[str, Any]:
         url_list = [urls] if isinstance(urls, str) else list(urls)
-        return {"urls": url_list, **_drop_none(options)}
+        body = _drop_none(options)
+        if "schema" in body:
+            body["schema"] = to_json_schema(body["schema"])
+        return {"urls": url_list, **body}
 
     def _job_id(self, payload: Any, *keys: str) -> str:
         if isinstance(payload, dict):

hydrafetch-0.2.0/src/hydrafetch/_schema.py

@@ -0,0 +1,69 @@
+"""Turn whatever the caller has into the JSON Schema the API expects.
+
+Pydantic is not a dependency and is never imported. A model is recognised by the method it carries,
+so any object that can describe itself as JSON Schema works, and someone who has no use for
+pydantic still installs this package with one dependency.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .errors import HydrafetchError
+
+MAX_REF_DEPTH = 12
+
+
+def to_json_schema(schema: Any) -> dict[str, Any]:
+    if isinstance(schema, Mapping):
+        return inline_refs(dict(schema))
+
+    for method in ("model_json_schema", "schema"):
+        describe = getattr(schema, method, None)
+        if callable(describe):
+            return inline_refs(describe())
+
+    raise HydrafetchError(
+        "schema must be a dict of JSON Schema or a pydantic model.",
+        code="SCHEMA_CONVERSION_FAILED",
+        status=0,
+    )
+
+
+def inline_refs(schema: dict[str, Any]) -> dict[str, Any]:
+    """Replace every local $ref with the thing it points at, and drop the definitions block.
+
+    A nested model is the ordinary case, not an exotic one, and pydantic always describes it as a
+    $ref into $defs. That indirection does not survive the trip to a model's tool schema, so
+    forwarding it turns a normal two-model extraction into a confusing failure from somewhere the
+    caller cannot see. Resolving it here costs nothing and keeps the surface honest.
+    """
+    defs = {**schema.get("$defs", {}), **schema.get("definitions", {})}
+    if not defs:
+        return schema
+
+    def resolve(node: Any, depth: int) -> Any:
+        if isinstance(node, list):
+            return [resolve(item, depth) for item in node]
+        if not isinstance(node, dict):
+            return node
+
+        target = node.get("$ref")
+        if isinstance(target, str) and target.startswith("#/"):
+            name = target.rsplit("/", 1)[-1]
+            # A self-referential model has no finite inlining. Leaving the $ref in place is worse
+            # than saying so, because the failure would surface as an opaque error from the API.
+            if depth >= MAX_REF_DEPTH or name not in defs:
+                raise HydrafetchError(
+                    f"schema reference {target} is recursive or missing, so it cannot be inlined. "
+                    "Flatten the model or pass JSON Schema directly.",
+                    code="SCHEMA_CONVERSION_FAILED",
+                    status=0,
+                )
+            merged = {**defs[name], **{k: v for k, v in node.items() if k != "$ref"}}
+            return resolve(merged, depth + 1)
+
+        return {k: resolve(v, depth) for k, v in node.items() if k not in ("$defs", "definitions")}
+
+    return resolve(schema, 0)

{hydrafetch-0.1.1 → hydrafetch-0.2.0}/tests/test_client.py

@@ -5,6 +5,7 @@ from typing import Any
 
 import httpx
 import pytest
+from pydantic import BaseModel
 
 from hydrafetch import AsyncHydrafetch, Hydrafetch, HydrafetchError, HydrafetchTimeout
 
@@ -20,6 +21,16 @@ def make_client(handler, **kwargs: Any) -> Hydrafetch:
     return client
 
 
+def _recording_client() -> tuple[Hydrafetch, list[dict[str, Any]]]:
+    seen: list[dict[str, Any]] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(json.loads(request.content))
+        return httpx.Response(200, json={"success": True, "data": {"results": []}})
+
+    return make_client(handler), seen
+
+
 def ok(body: Any, status: int = 200) -> httpx.Response:
     return httpx.Response(status, json=body)
 
@@ -200,3 +211,38 @@ class TestAsyncClient:
             out = await client.scrape("https://example.com")
         assert out["markdown"] == "# async"
         assert rec.last.headers["x-api-key"] == "hf_test"
+
+
+def test_extract_accepts_a_pydantic_model_and_inlines_nested_ones():
+    class Address(BaseModel):
+        city: str
+
+    class Person(BaseModel):
+        name: str
+        home: Address
+
+    client, seen = _recording_client()
+    client.extract("https://example.com", schema=Person)
+
+    schema = seen[0]["schema"]
+    assert schema["properties"]["name"]["type"] == "string"
+    assert schema["properties"]["home"]["properties"]["city"]["type"] == "string"
+    assert "$defs" not in schema and "$ref" not in str(schema)
+
+
+def test_extract_still_takes_plain_json_schema():
+    client, seen = _recording_client()
+    plain = {"type": "object", "properties": {"name": {"type": "string"}}}
+    client.extract("https://example.com", schema=plain)
+    assert seen[0]["schema"] == plain
+
+
+class Node(BaseModel):
+    child: Node | None = None
+
+
+def test_extract_rejects_a_recursive_model_rather_than_sending_a_ref():
+    client, _ = _recording_client()
+    with pytest.raises(HydrafetchError) as err:
+        client.extract("https://example.com", schema=Node)
+    assert err.value.code == "SCHEMA_CONVERSION_FAILED"