scrapeunblocker-haystack 0.4.2__tar.gz → 0.5.0__tar.gz

{scrapeunblocker_haystack-0.4.2 → scrapeunblocker_haystack-0.5.0}/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.5.0
+
+- `ScrapeUnblockerWebSearch` takes `full_urls` (default `False`): wait until
+  Google gives out every result's real URL. Google releases a results page's
+  links a few at a time, the last ones up to about 40 seconds after the search.
+  Without the flag the call returns sooner, and a result whose link has not
+  come yet has the site's home page as its `link` (or a Google redirect link
+  when Google shows no site). With it, the call takes up to about 40 seconds
+  longer. Billed the same. Sent to `/serpApi` only when `True`, and saved with
+  the component in `to_dict()`.
+
 ## 0.4.2
 
 - `ScrapeUnblockerWebSearch` adds `sitelinks` (the extra links under a result:

{scrapeunblocker_haystack-0.4.2 → scrapeunblocker_haystack-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scrapeunblocker-haystack
-Version: 0.4.2
+Version: 0.5.0
 Summary: Haystack integration for ScrapeUnblocker - scrape pages behind anti-bot protections
 Project-URL: Homepage, https://www.scrapeunblocker.com
 Project-URL: Documentation, https://docs.scrapeunblocker.com
@@ -205,6 +205,24 @@ for doc in result["documents"]:
 | `pages_to_check` | `1` | How many result pages to scrape |
 | `proxy_country` | `None` | Two-letter country code for localised results |
 | `top_k` | `None` | Keep at most this many results |
+| `full_urls` | `False` | Wait until Google gives out every result's real URL (see below) |
+| `base_url` | `https://api.scrapeunblocker.com` | API base URL |
+| `timeout` | `180` | HTTP timeout in seconds |
+
+#### Every result's real URL
+
+Google releases a results page's links a few at a time, the last ones up to
+about 40 seconds after the search. By default the search returns sooner, so
+most results carry their real URL in `meta["link"]`, and a result whose link
+has not come yet has the site's home page there (or a Google redirect link when
+Google shows no site). Set `full_urls=True` when you need the exact page of
+every result, for example to fetch or cite them:
+
+```python
+search = ScrapeUnblockerWebSearch(full_urls=True)
+```
+
+The call then takes up to about 40 seconds longer. It is billed the same.
 
 ## In a pipeline
 

{scrapeunblocker_haystack-0.4.2 → scrapeunblocker_haystack-0.5.0}/README.md

@@ -180,6 +180,24 @@ for doc in result["documents"]:
 | `pages_to_check` | `1` | How many result pages to scrape |
 | `proxy_country` | `None` | Two-letter country code for localised results |
 | `top_k` | `None` | Keep at most this many results |
+| `full_urls` | `False` | Wait until Google gives out every result's real URL (see below) |
+| `base_url` | `https://api.scrapeunblocker.com` | API base URL |
+| `timeout` | `180` | HTTP timeout in seconds |
+
+#### Every result's real URL
+
+Google releases a results page's links a few at a time, the last ones up to
+about 40 seconds after the search. By default the search returns sooner, so
+most results carry their real URL in `meta["link"]`, and a result whose link
+has not come yet has the site's home page there (or a Google redirect link when
+Google shows no site). Set `full_urls=True` when you need the exact page of
+every result, for example to fetch or cite them:
+
+```python
+search = ScrapeUnblockerWebSearch(full_urls=True)
+```
+
+The call then takes up to about 40 seconds longer. It is billed the same.
 
 ## In a pipeline
 

{scrapeunblocker_haystack-0.4.2 → scrapeunblocker_haystack-0.5.0}/src/scrapeunblocker_haystack/search.py

@@ -41,6 +41,7 @@ class ScrapeUnblockerWebSearch:
         top_k: Optional[int] = None,
         base_url: str = DEFAULT_BASE_URL,
         timeout: int = DEFAULT_TIMEOUT,
+        full_urls: bool = False,
     ) -> None:
         """
         :param api_key: ScrapeUnblocker API key. Read from `SCRAPEUNBLOCKER_API_KEY` by default.
@@ -49,6 +50,11 @@ class ScrapeUnblockerWebSearch:
         :param top_k: Keep at most this many results. `None` keeps all of them.
         :param base_url: API base URL. Override to target a different environment.
         :param timeout: HTTP timeout in seconds.
+        :param full_urls: Wait until Google gives out every result's real URL. Google releases a
+            results page's links a few at a time, the last ones up to about 40 seconds after the
+            search. Without this flag the call returns sooner, and a result whose link has not come
+            yet has the site's home page as its `link` (or a Google redirect link when Google shows
+            no site). With it, the call takes up to about 40 seconds longer. Billed the same.
         """
         self.api_key = api_key
         self.pages_to_check = pages_to_check
@@ -56,6 +62,7 @@ class ScrapeUnblockerWebSearch:
         self.top_k = top_k
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
+        self.full_urls = full_urls
 
     def to_dict(self) -> Dict[str, Any]:
         """Serialize this component to a dictionary."""
@@ -67,6 +74,7 @@ class ScrapeUnblockerWebSearch:
             top_k=self.top_k,
             base_url=self.base_url,
             timeout=self.timeout,
+            full_urls=self.full_urls,
         )
 
     @classmethod
@@ -86,6 +94,8 @@ class ScrapeUnblockerWebSearch:
         params: Dict[str, Any] = {"keyword": query, "pages_to_check": self.pages_to_check}
         if self.proxy_country:
             params["proxy_country"] = self.proxy_country
+        if self.full_urls:
+            params["full_urls"] = "true"
 
         response = requests.post(
             f"{self.base_url}/serpApi",

scrapeunblocker_haystack-0.5.0/src/scrapeunblocker_haystack/version.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

{scrapeunblocker_haystack-0.4.2 → scrapeunblocker_haystack-0.5.0}/tests/test_components.py

@@ -259,12 +259,42 @@ class TestWebSearch:
 
         assert search.run(query="q")["documents"] == []
 
+    @patch("scrapeunblocker_haystack.search.requests.post")
+    def test_full_urls_off_by_default(self, mock_post):
+        mock_post.return_value = _response(json_data={"organic": []})
+        search = ScrapeUnblockerWebSearch(api_key=API_KEY)
+        assert search.full_urls is False
+
+        search.run(query="q")
+
+        _, kwargs = mock_post.call_args
+        assert "full_urls" not in kwargs["params"]
+
+    @patch("scrapeunblocker_haystack.search.requests.post")
+    def test_full_urls_sent_when_on(self, mock_post):
+        mock_post.return_value = _response(json_data={"organic": []})
+        search = ScrapeUnblockerWebSearch(api_key=API_KEY, full_urls=True)
+
+        search.run(query="q")
+
+        _, kwargs = mock_post.call_args
+        assert kwargs["params"]["full_urls"] == "true"
+
     def test_to_dict_and_back(self):
-        search = ScrapeUnblockerWebSearch(api_key=ENV_KEY, top_k=5, proxy_country="us")
+        search = ScrapeUnblockerWebSearch(api_key=ENV_KEY, top_k=5, proxy_country="us", full_urls=True)
         data = search.to_dict()
+        assert data["init_parameters"]["full_urls"] is True
         restored = ScrapeUnblockerWebSearch.from_dict(data)
         assert restored.top_k == 5
         assert restored.proxy_country == "us"
+        assert restored.full_urls is True
+
+    def test_from_dict_of_older_version(self):
+        data = ScrapeUnblockerWebSearch(api_key=ENV_KEY).to_dict()
+        del data["init_parameters"]["full_urls"]
+
+        restored = ScrapeUnblockerWebSearch.from_dict(data)
+        assert restored.full_urls is False
 
 
 class TestTargetNotFound:

scrapeunblocker_haystack-0.4.2/src/scrapeunblocker_haystack/version.py

@@ -1 +0,0 @@
-__version__ = "0.4.2"