scrapeunblocker-haystack 0.2.1__tar.gz → 0.3.0__tar.gz

{scrapeunblocker_haystack-0.2.1 → scrapeunblocker_haystack-0.3.0}/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.3.0
+
+- With `parsed_data=True`, a page that loads but holds no structured data is
+  now a Document instead of being skipped. The API answers it as HTTP 200 with
+  `data_extracted: false` and the rendered HTML (billed like a plain fetch),
+  replacing the 422 `no_data_extracted` handled in 0.2.2. The Document's JSON
+  carries that HTML, and `meta["data_extracted"]` is `False` (`True` for a
+  successful parse). `NoDataExtractedError` is no longer raised; it stays
+  importable so existing `except` clauses keep working.
+
+## 0.2.2
+
+- With `parsed_data=True`, a page that loads but holds no structured data
+  (the API answers HTTP 422 `no_data_extracted`, not billed, no HTML) no longer
+  shows up as a generic request failure. The fetcher skips it with a clear
+  warning, or raises the new `NoDataExtractedError` when
+  `raise_on_failure=True`. Fetch without `parsed_data` to get the HTML.
+
 ## 0.2.1
 
 - A URL whose page does not exist (the site answers HTTP 404 or 410) no longer

{scrapeunblocker_haystack-0.2.1 → scrapeunblocker_haystack-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scrapeunblocker-haystack
-Version: 0.2.1
+Version: 0.3.0
 Summary: Haystack integration for ScrapeUnblocker - scrape pages behind anti-bot protections
 Project-URL: Homepage, https://www.scrapeunblocker.com
 Project-URL: Documentation, https://docs.scrapeunblocker.com
@@ -120,6 +120,13 @@ Steps are **non-idempotent**. If a step fails, the fetch raises
 which is logged and skipped by default, or re-raised when
 `raise_on_failure=True`.
 
+#### Pages with nothing to parse
+
+With `parsed_data=True`, a page that loads but holds no structured data still
+yields a Document. Its JSON carries `"data_extracted": false` and the rendered
+page in `html`, and `meta["data_extracted"]` is `False`, so you can route it to
+an HTML converter. The call is billed like a plain HTML fetch.
+
 #### Pages that do not exist
 
 If the site answers HTTP 404 or 410, there is no page to index: the URL is

{scrapeunblocker_haystack-0.2.1 → scrapeunblocker_haystack-0.3.0}/README.md

@@ -95,6 +95,13 @@ Steps are **non-idempotent**. If a step fails, the fetch raises
 which is logged and skipped by default, or re-raised when
 `raise_on_failure=True`.
 
+#### Pages with nothing to parse
+
+With `parsed_data=True`, a page that loads but holds no structured data still
+yields a Document. Its JSON carries `"data_extracted": false` and the rendered
+page in `html`, and `meta["data_extracted"]` is `False`, so you can route it to
+an HTML converter. The call is billed like a plain HTML fetch.
+
 #### Pages that do not exist
 
 If the site answers HTTP 404 or 410, there is no page to index: the URL is

{scrapeunblocker_haystack-0.2.1 → scrapeunblocker_haystack-0.3.0}/src/scrapeunblocker_haystack/__init__.py

@@ -1,4 +1,5 @@
 from scrapeunblocker_haystack.fetcher import (
+    NoDataExtractedError,
     ScrapeUnblockerFetcher,
     StepExecutionError,
     TargetNotFoundError,
@@ -6,6 +7,7 @@ from scrapeunblocker_haystack.fetcher import (
 from scrapeunblocker_haystack.search import ScrapeUnblockerWebSearch
 
 __all__ = [
+    "NoDataExtractedError",
     "ScrapeUnblockerFetcher",
     "ScrapeUnblockerWebSearch",
     "StepExecutionError",

{scrapeunblocker_haystack-0.2.1 → scrapeunblocker_haystack-0.3.0}/src/scrapeunblocker_haystack/fetcher.py

@@ -35,6 +35,19 @@ class TargetNotFoundError(Exception):
         )
 
 
+class NoDataExtractedError(Exception):
+    """No longer raised; kept so existing ``except`` clauses still import.
+
+    A ``parsed_data`` page with no structured data comes back as a billed
+    HTTP 200 carrying ``data_extracted: false`` and the rendered HTML, so the
+    fetcher returns it as a Document (``meta["data_extracted"]`` is False).
+    """
+
+    def __init__(self, url: str) -> None:
+        self.url = url
+        super().__init__(f"No structured data could be extracted from {url}.")
+
+
 def _target_gone_status(response: requests.Response) -> Optional[int]:
     """The target's own 404/410 answer, or None.
 
@@ -225,7 +238,10 @@ class ScrapeUnblockerFetcher:
         Fetch each URL and return one Document per successfully fetched page.
 
         A URL whose page does not exist (the site answers 404/410) yields no
-        Document; with `raise_on_failure` it raises `TargetNotFoundError`.
+        Document; with `raise_on_failure` it raises `TargetNotFoundError`. With
+        `parsed_data`, a page holding no structured data still yields a
+        Document: its JSON carries `data_extracted: false` and the rendered
+        `html`, and `meta["data_extracted"]` is False.
 
         :param urls: URLs to fetch.
         :returns: A dictionary with a `documents` key holding the fetched pages.
@@ -265,6 +281,8 @@ class ScrapeUnblockerFetcher:
                     content = json.dumps(payload, ensure_ascii=False)
                     if self.list_elements and isinstance(payload, dict):
                         meta["count"] = payload.get("count")
+                    elif self.parsed_data and isinstance(payload, dict):
+                        meta["data_extracted"] = payload.get("data_extracted") is not False
                 except ValueError:
                     content = response.text
             else:

scrapeunblocker_haystack-0.3.0/src/scrapeunblocker_haystack/version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{scrapeunblocker_haystack-0.2.1 → scrapeunblocker_haystack-0.3.0}/tests/test_components.py

@@ -10,6 +10,7 @@ from haystack.utils import Secret
 from scrapeunblocker_haystack import (
     ScrapeUnblockerFetcher,
     ScrapeUnblockerWebSearch,
+    NoDataExtractedError,
     StepExecutionError,
     TargetNotFoundError,
 )
@@ -297,3 +298,50 @@ class TestTargetNotFound:
         fetcher = ScrapeUnblockerFetcher(api_key=API_KEY)
 
         assert len(fetcher.run(urls=["https://example.com"])["documents"]) == 1
+
+
+class TestNoDataExtracted:
+    BODY = {
+        "data": {"page_type": "unknown", "data": {}},
+        "data_extracted": False,
+        "detail": "The page was rendered, but no structured data could be extracted from it.",
+        "html": "<html><title>Example Domain</title></html>",
+    }
+
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_empty_parse_is_a_document_with_the_html(self, mock_post):
+        mock_post.side_effect = [
+            _response(json_data=self.BODY),
+            _response(json_data={"data": {"page_type": "product", "data": {"title": "x"}}}),
+        ]
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY, parsed_data=True)
+
+        empty, parsed = fetcher.run(urls=["https://example.com/empty", "https://example.com"])[
+            "documents"
+        ]
+
+        assert empty.meta["data_extracted"] is False
+        assert json.loads(empty.content)["html"] == self.BODY["html"]
+        assert parsed.meta["data_extracted"] is True
+
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_empty_parse_does_not_raise(self, mock_post):
+        mock_post.return_value = _response(json_data=self.BODY)
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY, parsed_data=True, raise_on_failure=True)
+
+        assert len(fetcher.run(urls=["https://example.com/empty"])["documents"]) == 1
+
+    def test_error_class_still_importable(self):
+        assert issubclass(NoDataExtractedError, Exception)
+
+    @patch("scrapeunblocker_haystack.fetcher.requests.post")
+    def test_dead_url_with_parsed_body_is_skipped(self, mock_post):
+        response = _response(
+            status_code=404, json_data={"data": {"page_type": "not_found", "data": {}}}
+        )
+        response.text = '{"data":{"page_type":"not_found","data":{}}}'
+        response.headers = {"Content-Type": "application/json", "X-Origin-Status": "404"}
+        mock_post.return_value = response
+        fetcher = ScrapeUnblockerFetcher(api_key=API_KEY, parsed_data=True)
+
+        assert fetcher.run(urls=["https://example.com/gone"])["documents"] == []

scrapeunblocker_haystack-0.2.1/src/scrapeunblocker_haystack/version.py

@@ -1 +0,0 @@
-__version__ = "0.2.1"