scrapeunblocker-haystack 0.4.1__tar.gz → 0.5.0__tar.gz

{scrapeunblocker_haystack-0.4.1 → scrapeunblocker_haystack-0.5.0}/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.5.0
+
+- `ScrapeUnblockerWebSearch` takes `full_urls` (default `False`): wait until
+  Google gives out every result's real URL. Google releases a results page's
+  links a few at a time, the last ones up to about 40 seconds after the search.
+  Without the flag the call returns sooner, and a result whose link has not
+  come yet has the site's home page as its `link` (or a Google redirect link
+  when Google shows no site). With it, the call takes up to about 40 seconds
+  longer. Billed the same. Sent to `/serpApi` only when `True`, and saved with
+  the component in `to_dict()`.
+
+## 0.4.2
+
+- `ScrapeUnblockerWebSearch` adds `sitelinks` (the extra links under a result:
+  `title`, `url`, `description`), `rating` (`value`, `max`, `reviews`, `url`)
+  and `rich_snippet` (the facts line under the snippet) to each Document's
+  metadata, from the new `/serpApi` fields.
+
 ## 0.4.1
 
 - `ScrapeUnblockerWebSearch` adds `displayed_url`, `domain`, `source` (the

{scrapeunblocker_haystack-0.4.1 → scrapeunblocker_haystack-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scrapeunblocker-haystack
-Version: 0.4.1
+Version: 0.5.0
 Summary: Haystack integration for ScrapeUnblocker - scrape pages behind anti-bot protections
 Project-URL: Homepage, https://www.scrapeunblocker.com
 Project-URL: Documentation, https://docs.scrapeunblocker.com
@@ -186,7 +186,8 @@ print(doc.content)                # {"url": ..., "count": ..., "elements": [...]
 
 Searches Google and returns the organic results as `Document` objects, with the
 snippet as content and `title` / `link` / `position` / `displayed_url` / `domain` /
-`source` (the site's name) / `date` in the metadata.
+`source` (the site's name) / `date` / `sitelinks` / `rating` / `rich_snippet` (the
+facts line under the snippet) in the metadata.
 
 ```python
 from scrapeunblocker_haystack import ScrapeUnblockerWebSearch
@@ -204,6 +205,24 @@ for doc in result["documents"]:
 | `pages_to_check` | `1` | How many result pages to scrape |
 | `proxy_country` | `None` | Two-letter country code for localised results |
 | `top_k` | `None` | Keep at most this many results |
+| `full_urls` | `False` | Wait until Google gives out every result's real URL (see below) |
+| `base_url` | `https://api.scrapeunblocker.com` | API base URL |
+| `timeout` | `180` | HTTP timeout in seconds |
+
+#### Every result's real URL
+
+Google releases a results page's links a few at a time, the last ones up to
+about 40 seconds after the search. By default the search returns sooner, so
+most results carry their real URL in `meta["link"]`, and a result whose link
+has not come yet has the site's home page there (or a Google redirect link when
+Google shows no site). Set `full_urls=True` when you need the exact page of
+every result, for example to fetch or cite them:
+
+```python
+search = ScrapeUnblockerWebSearch(full_urls=True)
+```
+
+The call then takes up to about 40 seconds longer. It is billed the same.
 
 ## In a pipeline
 

{scrapeunblocker_haystack-0.4.1 → scrapeunblocker_haystack-0.5.0}/README.md

@@ -161,7 +161,8 @@ print(doc.content)                # {"url": ..., "count": ..., "elements": [...]
 
 Searches Google and returns the organic results as `Document` objects, with the
 snippet as content and `title` / `link` / `position` / `displayed_url` / `domain` /
-`source` (the site's name) / `date` in the metadata.
+`source` (the site's name) / `date` / `sitelinks` / `rating` / `rich_snippet` (the
+facts line under the snippet) in the metadata.
 
 ```python
 from scrapeunblocker_haystack import ScrapeUnblockerWebSearch
@@ -179,6 +180,24 @@ for doc in result["documents"]:
 | `pages_to_check` | `1` | How many result pages to scrape |
 | `proxy_country` | `None` | Two-letter country code for localised results |
 | `top_k` | `None` | Keep at most this many results |
+| `full_urls` | `False` | Wait until Google gives out every result's real URL (see below) |
+| `base_url` | `https://api.scrapeunblocker.com` | API base URL |
+| `timeout` | `180` | HTTP timeout in seconds |
+
+#### Every result's real URL
+
+Google releases a results page's links a few at a time, the last ones up to
+about 40 seconds after the search. By default the search returns sooner, so
+most results carry their real URL in `meta["link"]`, and a result whose link
+has not come yet has the site's home page there (or a Google redirect link when
+Google shows no site). Set `full_urls=True` when you need the exact page of
+every result, for example to fetch or cite them:
+
+```python
+search = ScrapeUnblockerWebSearch(full_urls=True)
+```
+
+The call then takes up to about 40 seconds longer. It is billed the same.
 
 ## In a pipeline
 

{scrapeunblocker_haystack-0.4.1 → scrapeunblocker_haystack-0.5.0}/src/scrapeunblocker_haystack/search.py

@@ -19,7 +19,7 @@ class ScrapeUnblockerWebSearch:
 
     Each result becomes a Document whose content is the snippet (the title when
     Google shows none), with the title, link, position, displayed URL, domain,
-    site name and date in the metadata.
+    site name, date, sitelinks, star rating and facts line in the metadata.
 
     ### Usage example
 
@@ -41,6 +41,7 @@ class ScrapeUnblockerWebSearch:
         top_k: Optional[int] = None,
         base_url: str = DEFAULT_BASE_URL,
         timeout: int = DEFAULT_TIMEOUT,
+        full_urls: bool = False,
     ) -> None:
         """
         :param api_key: ScrapeUnblocker API key. Read from `SCRAPEUNBLOCKER_API_KEY` by default.
@@ -49,6 +50,11 @@ class ScrapeUnblockerWebSearch:
         :param top_k: Keep at most this many results. `None` keeps all of them.
         :param base_url: API base URL. Override to target a different environment.
         :param timeout: HTTP timeout in seconds.
+        :param full_urls: Wait until Google gives out every result's real URL. Google releases a
+            results page's links a few at a time, the last ones up to about 40 seconds after the
+            search. Without this flag the call returns sooner, and a result whose link has not come
+            yet has the site's home page as its `link` (or a Google redirect link when Google shows
+            no site). With it, the call takes up to about 40 seconds longer. Billed the same.
         """
         self.api_key = api_key
         self.pages_to_check = pages_to_check
@@ -56,6 +62,7 @@ class ScrapeUnblockerWebSearch:
         self.top_k = top_k
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
+        self.full_urls = full_urls
 
     def to_dict(self) -> Dict[str, Any]:
         """Serialize this component to a dictionary."""
@@ -67,6 +74,7 @@ class ScrapeUnblockerWebSearch:
             top_k=self.top_k,
             base_url=self.base_url,
             timeout=self.timeout,
+            full_urls=self.full_urls,
         )
 
     @classmethod
@@ -86,6 +94,8 @@ class ScrapeUnblockerWebSearch:
         params: Dict[str, Any] = {"keyword": query, "pages_to_check": self.pages_to_check}
         if self.proxy_country:
             params["proxy_country"] = self.proxy_country
+        if self.full_urls:
+            params["full_urls"] = "true"
 
         response = requests.post(
             f"{self.base_url}/serpApi",
@@ -112,6 +122,9 @@ class ScrapeUnblockerWebSearch:
                     "domain": result.get("domain"),
                     "source": result.get("source"),
                     "date": result.get("date"),
+                    "sitelinks": result.get("sitelinks") or [],
+                    "rating": result.get("rating"),
+                    "rich_snippet": result.get("richSnippet"),
                     "query": query,
                 },
             )

scrapeunblocker_haystack-0.5.0/src/scrapeunblocker_haystack/version.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

{scrapeunblocker_haystack-0.4.1 → scrapeunblocker_haystack-0.5.0}/tests/test_components.py

@@ -213,7 +213,10 @@ class TestWebSearch:
                 "organic": [
                     {"title": "First", "url": "https://a.com", "description": "snippet a", "position": "1",
                      "displayedUrl": "https://a.com › docs", "domain": "a.com", "source": "A",
-                     "date": "Oct 2, 2026"},
+                     "date": "Oct 2, 2026",
+                     "sitelinks": [{"title": "Docs", "url": "https://a.com/docs", "description": None}],
+                     "rating": {"value": 4.5, "max": 5.0, "reviews": 120, "url": None},
+                     "richSnippet": "4.5 (120) · Free"},
                     {"title": "Second", "url": "https://b.com", "description": "snippet b", "position": "2"},
                 ]
             }
@@ -231,6 +234,11 @@ class TestWebSearch:
         assert result["documents"][0].meta["source"] == "A"
         assert result["documents"][0].meta["date"] == "Oct 2, 2026"
         assert result["documents"][1].meta["date"] is None
+        assert result["documents"][0].meta["sitelinks"][0]["url"] == "https://a.com/docs"
+        assert result["documents"][0].meta["rating"]["value"] == 4.5
+        assert result["documents"][0].meta["rich_snippet"] == "4.5 (120) · Free"
+        assert result["documents"][1].meta["sitelinks"] == []
+        assert result["documents"][1].meta["rating"] is None
 
         _, kwargs = mock_post.call_args
         assert kwargs["params"]["keyword"] == "test query"
@@ -251,12 +259,42 @@ class TestWebSearch:
 
         assert search.run(query="q")["documents"] == []
 
+    @patch("scrapeunblocker_haystack.search.requests.post")
+    def test_full_urls_off_by_default(self, mock_post):
+        mock_post.return_value = _response(json_data={"organic": []})
+        search = ScrapeUnblockerWebSearch(api_key=API_KEY)
+        assert search.full_urls is False
+
+        search.run(query="q")
+
+        _, kwargs = mock_post.call_args
+        assert "full_urls" not in kwargs["params"]
+
+    @patch("scrapeunblocker_haystack.search.requests.post")
+    def test_full_urls_sent_when_on(self, mock_post):
+        mock_post.return_value = _response(json_data={"organic": []})
+        search = ScrapeUnblockerWebSearch(api_key=API_KEY, full_urls=True)
+
+        search.run(query="q")
+
+        _, kwargs = mock_post.call_args
+        assert kwargs["params"]["full_urls"] == "true"
+
     def test_to_dict_and_back(self):
-        search = ScrapeUnblockerWebSearch(api_key=ENV_KEY, top_k=5, proxy_country="us")
+        search = ScrapeUnblockerWebSearch(api_key=ENV_KEY, top_k=5, proxy_country="us", full_urls=True)
         data = search.to_dict()
+        assert data["init_parameters"]["full_urls"] is True
         restored = ScrapeUnblockerWebSearch.from_dict(data)
         assert restored.top_k == 5
         assert restored.proxy_country == "us"
+        assert restored.full_urls is True
+
+    def test_from_dict_of_older_version(self):
+        data = ScrapeUnblockerWebSearch(api_key=ENV_KEY).to_dict()
+        del data["init_parameters"]["full_urls"]
+
+        restored = ScrapeUnblockerWebSearch.from_dict(data)
+        assert restored.full_urls is False
 
 
 class TestTargetNotFound:

scrapeunblocker_haystack-0.4.1/src/scrapeunblocker_haystack/version.py

@@ -1 +0,0 @@
-__version__ = "0.4.1"