scrapeunblocker-haystack 0.2.0__tar.gz → 0.2.1__tar.gz

{scrapeunblocker_haystack-0.2.0 → scrapeunblocker_haystack-0.2.1}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.2.1
+
+- A URL whose page does not exist (the site answers HTTP 404 or 410) no longer
+  shows up as a generic request failure, and an older API response that carried
+  the not-found page as a 200 no longer becomes a Document. The fetcher skips it
+  with a clear warning, or raises the new `TargetNotFoundError` (`url`,
+  `origin_status`, `body`) when `raise_on_failure=True`. The call is billed by
+  ScrapeUnblocker and retrying returns the same answer.
+
 ## 0.2.0
 
 - `ScrapeUnblockerFetcher` now supports **browser steps** via the `steps`

{scrapeunblocker_haystack-0.2.0 → scrapeunblocker_haystack-0.2.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.5
 Name: scrapeunblocker-haystack
-Version: 0.2.0
+Version: 0.2.1
 Summary: Haystack integration for ScrapeUnblocker - scrape pages behind anti-bot protections
 Project-URL: Homepage, https://www.scrapeunblocker.com
-Project-URL: Documentation, https://developers.scrapeunblocker.com
+Project-URL: Documentation, https://docs.scrapeunblocker.com
 Project-URL: Source, https://github.com/ScrapeUnblocker/scrapeunblocker-haystack
 Project-URL: Issues, https://github.com/ScrapeUnblocker/scrapeunblocker-haystack/issues
 Author-email: ScrapeUnblocker <info@scrapeunblocker.com>
@@ -120,6 +120,14 @@ Steps are **non-idempotent**. If a step fails, the fetch raises
 which is logged and skipped by default, or re-raised when
 `raise_on_failure=True`.
 
+#### Pages that do not exist
+
+If the site answers HTTP 404 or 410, there is no page to index: the URL is
+skipped with a warning, or `TargetNotFoundError` (with `url`, `origin_status`
+and the site's own `body`) is raised when `raise_on_failure=True`. That is the
+site's answer, not a block - the call is billed and a retry returns the same
+result.
+
 #### List elements
 
 Set `list_elements=True` to return the parsed elements JSON
@@ -206,7 +214,7 @@ reference, not as its value.
 ## Links
 
 - ScrapeUnblocker: [www.scrapeunblocker.com](https://www.scrapeunblocker.com?utm_source=haystack&utm_medium=integration&utm_campaign=haystack-integration)
-- API documentation: [developers.scrapeunblocker.com](https://developers.scrapeunblocker.com?utm_source=haystack&utm_medium=integration&utm_campaign=haystack-integration)
+- API documentation: [docs.scrapeunblocker.com](https://docs.scrapeunblocker.com?utm_source=haystack&utm_medium=integration&utm_campaign=haystack-integration)
 - Haystack: https://haystack.deepset.ai
 
 ## License

{scrapeunblocker_haystack-0.2.0 → scrapeunblocker_haystack-0.2.1}/README.md

@@ -95,6 +95,14 @@ Steps are **non-idempotent**. If a step fails, the fetch raises
 which is logged and skipped by default, or re-raised when
 `raise_on_failure=True`.
 
+#### Pages that do not exist
+
+If the site answers HTTP 404 or 410, there is no page to index: the URL is
+skipped with a warning, or `TargetNotFoundError` (with `url`, `origin_status`
+and the site's own `body`) is raised when `raise_on_failure=True`. That is the
+site's answer, not a block - the call is billed and a retry returns the same
+result.
+
 #### List elements
 
 Set `list_elements=True` to return the parsed elements JSON
@@ -181,7 +189,7 @@ reference, not as its value.
 ## Links
 
 - ScrapeUnblocker: [www.scrapeunblocker.com](https://www.scrapeunblocker.com?utm_source=haystack&utm_medium=integration&utm_campaign=haystack-integration)
-- API documentation: [developers.scrapeunblocker.com](https://developers.scrapeunblocker.com?utm_source=haystack&utm_medium=integration&utm_campaign=haystack-integration)
+- API documentation: [docs.scrapeunblocker.com](https://docs.scrapeunblocker.com?utm_source=haystack&utm_medium=integration&utm_campaign=haystack-integration)
 - Haystack: https://haystack.deepset.ai
 
 ## License

{scrapeunblocker_haystack-0.2.0 → scrapeunblocker_haystack-0.2.1}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = ["haystack-ai>=2.0.0", "requests>=2.31.0"]
 
 [project.urls]
 Homepage = "https://www.scrapeunblocker.com"
-Documentation = "https://developers.scrapeunblocker.com"
+Documentation = "https://docs.scrapeunblocker.com"
 Source = "https://github.com/ScrapeUnblocker/scrapeunblocker-haystack"
 Issues = "https://github.com/ScrapeUnblocker/scrapeunblocker-haystack/issues"
 

scrapeunblocker_haystack-0.2.1/src/scrapeunblocker_haystack/__init__.py

@@ -0,0 +1,13 @@
+from scrapeunblocker_haystack.fetcher import (
+    ScrapeUnblockerFetcher,
+    StepExecutionError,
+    TargetNotFoundError,
+)
+from scrapeunblocker_haystack.search import ScrapeUnblockerWebSearch
+
+__all__ = [
+    "ScrapeUnblockerFetcher",
+    "ScrapeUnblockerWebSearch",
+    "StepExecutionError",
+    "TargetNotFoundError",
+]

{scrapeunblocker_haystack-0.2.0 → scrapeunblocker_haystack-0.2.1}/src/scrapeunblocker_haystack/fetcher.py

@@ -13,6 +13,42 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_BASE_URL = "https://api.scrapeunblocker.com"
 DEFAULT_TIMEOUT = 180
+_TARGET_GONE = frozenset({404, 410})
+
+
+class TargetNotFoundError(Exception):
+    """Raised when the target page does not exist (the site answered 404/410).
+
+    This is the website's own answer, not a block or an API failure: the call
+    is billed, and retrying returns the same result. ``body`` is the target's
+    own not-found page (can be empty).
+    """
+
+    def __init__(self, url: str, origin_status: int, body: str = "") -> None:
+        self.url = url
+        self.origin_status = origin_status
+        self.body = body
+        super().__init__(
+            f"Target page does not exist: {url} answered HTTP {origin_status}. "
+            "This is the site's own answer, not a block or an API failure; the "
+            "call was billed and retrying returns the same result."
+        )
+
+
+def _target_gone_status(response: requests.Response) -> Optional[int]:
+    """The target's own 404/410 answer, or None.
+
+    ``/getPageSource`` always exists, so a 404/410 from it is the target site's
+    answer, sent with ``X-Origin-Status``. Older API versions returned that
+    same answer as a 200 carrying the header, so the header is checked first.
+    """
+    try:
+        origin = int(response.headers.get("X-Origin-Status", ""))
+    except (TypeError, ValueError):
+        origin = None
+    if origin in _TARGET_GONE:
+        return origin
+    return response.status_code if response.status_code in _TARGET_GONE else None
 
 
 class StepExecutionError(Exception):
@@ -102,7 +138,7 @@ class ScrapeUnblockerFetcher:
         :param base_url: API base URL. Override to target a different environment.
         :param timeout: HTTP timeout in seconds.
         :param raise_on_failure: Raise instead of skipping when a URL cannot be fetched
-            (including when a browser step fails).
+            (including when a browser step fails or the page does not exist).
         """
         self.api_key = api_key
         self.parsed_data = parsed_data
@@ -155,6 +191,10 @@ class ScrapeUnblockerFetcher:
             timeout=self.timeout,
         )
 
+        gone = _target_gone_status(response)
+        if gone is not None:
+            raise TargetNotFoundError(url, gone, response.text)
+
         # A failed browser step returns 422 with a structured JSON error.
         if response.status_code == 422:
             error = self._parse_step_error(response)
@@ -184,6 +224,9 @@ class ScrapeUnblockerFetcher:
         """
         Fetch each URL and return one Document per successfully fetched page.
 
+        A URL whose page does not exist (the site answers 404/410) yields no
+        Document; with `raise_on_failure` it raises `TargetNotFoundError`.
+
         :param urls: URLs to fetch.
         :returns: A dictionary with a `documents` key holding the fetched pages.
         """
@@ -194,6 +237,12 @@ class ScrapeUnblockerFetcher:
         for url in urls:
             try:
                 response = self._fetch(url)
+            except TargetNotFoundError as exc:
+                if self.raise_on_failure:
+                    raise
+                # A page that does not exist has no content to index.
+                logger.warning("ScrapeUnblocker skipped {url}: {error}", url=url, error=str(exc))
+                continue
             except Exception as exc:
                 if self.raise_on_failure:
                     raise

scrapeunblocker_haystack-0.2.1/src/scrapeunblocker_haystack/version.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

{scrapeunblocker_haystack-0.2.0 → scrapeunblocker_haystack-0.2.1}/tests/test_components.py

@@ -11,6 +11,7 @@ from scrapeunblocker_haystack import (
     ScrapeUnblockerFetcher,
     ScrapeUnblockerWebSearch,
     StepExecutionError,
+    TargetNotFoundError,
 )
 
 API_KEY = Secret.from_token("test_key")
@@ -249,3 +250,50 @@ class TestWebSearch:
         restored = ScrapeUnblockerWebSearch.from_dict(data)
         assert restored.top_k == 5
         assert restored.proxy_country == "us"
+
+
+class TestTargetNotFound:
+    NOT_FOUND = "<html><title>404 Not Found</title></html>"
+
+    def _gone(self, status_code: int, origin: str) -> MagicMock:
+        response = _response(text=self.NOT_FOUND, status_code=status_code)
+        response.headers = {"Content-Type": "text/html", "X-Origin-Status": origin}
+        return response
+
+    @pytest.mark.parametrize("status", [404, 410])
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_missing_page_is_skipped(self, mock_post, status):
+        mock_post.side_effect = [self._gone(status, str(status)), _response()]
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY)
+
+        result = fetcher.run(urls=["https://example.com/gone", "https://example.com"])
+
+        assert [d.meta["url"] for d in result["documents"]] == ["https://example.com"]
+
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_legacy_200_with_origin_404_is_skipped(self, mock_post):
+        mock_post.return_value = self._gone(200, "404")
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY)
+
+        assert fetcher.run(urls=["https://example.com/gone"])["documents"] == []
+
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_missing_page_raises_when_asked(self, mock_post):
+        mock_post.return_value = self._gone(410, "410")
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY, raise_on_failure=True)
+
+        with pytest.raises(TargetNotFoundError) as exc:
+            fetcher.run(urls=["https://example.com/gone"])
+        assert exc.value.origin_status == 410
+        assert exc.value.url == "https://example.com/gone"
+        assert "404 Not Found" in exc.value.body
+        assert "billed" in str(exc.value)
+
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_dead_origin_5xx_page_is_still_a_document(self, mock_post):
+        response = _response(text="<html>502 Bad Gateway</html>")
+        response.headers = {"Content-Type": "text/html", "X-Origin-Status": "502"}
+        mock_post.return_value = response
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY)
+
+        assert len(fetcher.run(urls=["https://example.com"])["documents"]) == 1

scrapeunblocker_haystack-0.2.0/src/scrapeunblocker_haystack/__init__.py

@@ -1,4 +0,0 @@
-from scrapeunblocker_haystack.fetcher import ScrapeUnblockerFetcher, StepExecutionError
-from scrapeunblocker_haystack.search import ScrapeUnblockerWebSearch
-
-__all__ = ["ScrapeUnblockerFetcher", "ScrapeUnblockerWebSearch", "StepExecutionError"]

scrapeunblocker_haystack-0.2.0/src/scrapeunblocker_haystack/version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0"