scrapeunblocker-haystack 0.4.0__tar.gz → 0.4.2__tar.gz

{scrapeunblocker_haystack-0.4.0 → scrapeunblocker_haystack-0.4.2}/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.4.2
+
+- `ScrapeUnblockerWebSearch` adds `sitelinks` (the extra links under a result:
+  `title`, `url`, `description`), `rating` (`value`, `max`, `reviews`, `url`)
+  and `rich_snippet` (the facts line under the snippet) to each Document's
+  metadata, from the new `/serpApi` fields.
+
+## 0.4.1
+
+- `ScrapeUnblockerWebSearch` adds `displayed_url`, `domain`, `source` (the
+  site's name) and `date` to each Document's metadata. The API now returns
+  every organic result with its real URL, also those Google shows without a
+  snippet; such a Document's content is the result's title.
+
 ## 0.4.0
 
 - `ScrapeUnblockerFetcher` can return pages as clean Markdown for LLMs and RAG.

{scrapeunblocker_haystack-0.4.0 → scrapeunblocker_haystack-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scrapeunblocker-haystack
-Version: 0.4.0
+Version: 0.4.2
 Summary: Haystack integration for ScrapeUnblocker - scrape pages behind anti-bot protections
 Project-URL: Homepage, https://www.scrapeunblocker.com
 Project-URL: Documentation, https://docs.scrapeunblocker.com
@@ -69,7 +69,7 @@ print(result["documents"][0].content[:200])
 | --- | --- | --- |
 | `api_key` | `SCRAPEUNBLOCKER_API_KEY` env var | ScrapeUnblocker API key |
 | `parsed_data` | `False` | Return AI-parsed structured JSON instead of raw HTML |
-| `proxy_country` | `None` | Two-letter country code for the exit IP |
+| `proxy_country` | `None` | Two-letter country code of the country the request is made from |
 | `time_sleep` | `None` | Seconds to wait after load before capturing |
 | `steps` | `None` | Browser actions to run before the HTML is captured (see below) |
 | `list_elements` | `False` | Return the parsed elements JSON instead of HTML (see below) |
@@ -185,7 +185,9 @@ print(doc.content)                # {"url": ..., "count": ..., "elements": [...]
 ### ScrapeUnblockerWebSearch
 
 Searches Google and returns the organic results as `Document` objects, with the
-snippet as content and `title` / `link` / `position` in the metadata.
+snippet as content and `title` / `link` / `position` / `displayed_url` / `domain` /
+`source` (the site's name) / `date` / `sitelinks` / `rating` / `rich_snippet` (the
+facts line under the snippet) in the metadata.
 
 ```python
 from scrapeunblocker_haystack import ScrapeUnblockerWebSearch

{scrapeunblocker_haystack-0.4.0 → scrapeunblocker_haystack-0.4.2}/README.md

@@ -44,7 +44,7 @@ print(result["documents"][0].content[:200])
 | --- | --- | --- |
 | `api_key` | `SCRAPEUNBLOCKER_API_KEY` env var | ScrapeUnblocker API key |
 | `parsed_data` | `False` | Return AI-parsed structured JSON instead of raw HTML |
-| `proxy_country` | `None` | Two-letter country code for the exit IP |
+| `proxy_country` | `None` | Two-letter country code of the country the request is made from |
 | `time_sleep` | `None` | Seconds to wait after load before capturing |
 | `steps` | `None` | Browser actions to run before the HTML is captured (see below) |
 | `list_elements` | `False` | Return the parsed elements JSON instead of HTML (see below) |
@@ -160,7 +160,9 @@ print(doc.content)                # {"url": ..., "count": ..., "elements": [...]
 ### ScrapeUnblockerWebSearch
 
 Searches Google and returns the organic results as `Document` objects, with the
-snippet as content and `title` / `link` / `position` in the metadata.
+snippet as content and `title` / `link` / `position` / `displayed_url` / `domain` /
+`source` (the site's name) / `date` / `sitelinks` / `rating` / `rich_snippet` (the
+facts line under the snippet) in the metadata.
 
 ```python
 from scrapeunblocker_haystack import ScrapeUnblockerWebSearch

{scrapeunblocker_haystack-0.4.0 → scrapeunblocker_haystack-0.4.2}/src/scrapeunblocker_haystack/search.py

@@ -17,8 +17,9 @@ class ScrapeUnblockerWebSearch:
     """
     Searches Google through the ScrapeUnblocker API and returns the organic results.
 
-    Each result becomes a Document whose content is the snippet, with the title,
-    link and position in the metadata.
+    Each result becomes a Document whose content is the snippet (the title when
+    Google shows none), with the title, link, position, displayed URL, domain,
+    site name, date, sitelinks, star rating and facts line in the metadata.
 
     ### Usage example
 
@@ -107,6 +108,13 @@ class ScrapeUnblockerWebSearch:
                     "title": result.get("title"),
                     "link": result.get("url"),
                     "position": result.get("position"),
+                    "displayed_url": result.get("displayedUrl"),
+                    "domain": result.get("domain"),
+                    "source": result.get("source"),
+                    "date": result.get("date"),
+                    "sitelinks": result.get("sitelinks") or [],
+                    "rating": result.get("rating"),
+                    "rich_snippet": result.get("richSnippet"),
                     "query": query,
                 },
             )

scrapeunblocker_haystack-0.4.2/src/scrapeunblocker_haystack/version.py

@@ -0,0 +1 @@
+__version__ = "0.4.2"

{scrapeunblocker_haystack-0.4.0 → scrapeunblocker_haystack-0.4.2}/tests/test_components.py

@@ -211,7 +211,12 @@ class TestWebSearch:
         mock_post.return_value = _response(
             json_data={
                 "organic": [
-                    {"title": "First", "url": "https://a.com", "description": "snippet a", "position": "1"},
+                    {"title": "First", "url": "https://a.com", "description": "snippet a", "position": "1",
+                     "displayedUrl": "https://a.com › docs", "domain": "a.com", "source": "A",
+                     "date": "Oct 2, 2026",
+                     "sitelinks": [{"title": "Docs", "url": "https://a.com/docs", "description": None}],
+                     "rating": {"value": 4.5, "max": 5.0, "reviews": 120, "url": None},
+                     "richSnippet": "4.5 (120) · Free"},
                     {"title": "Second", "url": "https://b.com", "description": "snippet b", "position": "2"},
                 ]
             }
@@ -225,6 +230,15 @@ class TestWebSearch:
         assert result["documents"][0].meta["title"] == "First"
         assert result["documents"][0].meta["link"] == "https://a.com"
         assert result["documents"][0].meta["query"] == "test query"
+        assert result["documents"][0].meta["domain"] == "a.com"
+        assert result["documents"][0].meta["source"] == "A"
+        assert result["documents"][0].meta["date"] == "Oct 2, 2026"
+        assert result["documents"][1].meta["date"] is None
+        assert result["documents"][0].meta["sitelinks"][0]["url"] == "https://a.com/docs"
+        assert result["documents"][0].meta["rating"]["value"] == 4.5
+        assert result["documents"][0].meta["rich_snippet"] == "4.5 (120) · Free"
+        assert result["documents"][1].meta["sitelinks"] == []
+        assert result["documents"][1].meta["rating"] is None
 
         _, kwargs = mock_post.call_args
         assert kwargs["params"]["keyword"] == "test query"

scrapeunblocker_haystack-0.4.0/src/scrapeunblocker_haystack/version.py

@@ -1 +0,0 @@
-__version__ = "0.4.0"