scrapeunblocker-haystack 0.2.2__tar.gz → 0.3.0__tar.gz

{scrapeunblocker_haystack-0.2.2 → scrapeunblocker_haystack-0.3.0}/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.3.0
+
+- With `parsed_data=True`, a page that loads but holds no structured data is
+  now a Document instead of being skipped. The API answers it as HTTP 200 with
+  `data_extracted: false` and the rendered HTML (billed like a plain fetch),
+  replacing the 422 `no_data_extracted` handled in 0.2.2. The Document's JSON
+  carries that HTML, and `meta["data_extracted"]` is `False` (`True` for a
+  successful parse). `NoDataExtractedError` is no longer raised; it stays
+  importable so existing `except` clauses keep working.
+
 ## 0.2.2
 
 - With `parsed_data=True`, a page that loads but holds no structured data

{scrapeunblocker_haystack-0.2.2 → scrapeunblocker_haystack-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scrapeunblocker-haystack
-Version: 0.2.2
+Version: 0.3.0
 Summary: Haystack integration for ScrapeUnblocker - scrape pages behind anti-bot protections
 Project-URL: Homepage, https://www.scrapeunblocker.com
 Project-URL: Documentation, https://docs.scrapeunblocker.com
@@ -122,10 +122,10 @@ which is logged and skipped by default, or re-raised when
 
 #### Pages with nothing to parse
 
-With `parsed_data=True`, a page that loads but holds no structured data yields
-no Document: it is skipped with a warning, or `NoDataExtractedError` is raised
-when `raise_on_failure=True`. The call is not billed; fetch the URL without
-`parsed_data` to get the HTML.
+With `parsed_data=True`, a page that loads but holds no structured data still
+yields a Document. Its JSON carries `"data_extracted": false` and the rendered
+page in `html`, and `meta["data_extracted"]` is `False`, so you can route it to
+an HTML converter. The call is billed like a plain HTML fetch.
 
 #### Pages that do not exist
 

{scrapeunblocker_haystack-0.2.2 → scrapeunblocker_haystack-0.3.0}/README.md

@@ -97,10 +97,10 @@ which is logged and skipped by default, or re-raised when
 
 #### Pages with nothing to parse
 
-With `parsed_data=True`, a page that loads but holds no structured data yields
-no Document: it is skipped with a warning, or `NoDataExtractedError` is raised
-when `raise_on_failure=True`. The call is not billed; fetch the URL without
-`parsed_data` to get the HTML.
+With `parsed_data=True`, a page that loads but holds no structured data still
+yields a Document. Its JSON carries `"data_extracted": false` and the rendered
+page in `html`, and `meta["data_extracted"]` is `False`, so you can route it to
+an HTML converter. The call is billed like a plain HTML fetch.
 
 #### Pages that do not exist
 

{scrapeunblocker_haystack-0.2.2 → scrapeunblocker_haystack-0.3.0}/src/scrapeunblocker_haystack/fetcher.py

@@ -36,20 +36,16 @@ class TargetNotFoundError(Exception):
 
 
 class NoDataExtractedError(Exception):
-    """Raised when ``parsed_data`` found no structured data on the page.
+    """No longer raised; kept so existing ``except`` clauses still import.
 
-    The API answers this as HTTP 422 ``no_data_extracted``: the page loaded but
-    held nothing to extract. It is not billed and carries no HTML; fetch the
-    URL without ``parsed_data`` to get the page itself.
+    A ``parsed_data`` page with no structured data comes back as a billed
+    HTTP 200 carrying ``data_extracted: false`` and the rendered HTML, so the
+    fetcher returns it as a Document (``meta["data_extracted"]`` is False).
     """
 
     def __init__(self, url: str) -> None:
         self.url = url
-        super().__init__(
-            f"No structured data could be extracted from {url}: the page loaded, but "
-            "nothing on it matched a structured shape. The call was not billed; fetch "
-            "it without parsed_data to get the HTML."
-        )
+        super().__init__(f"No structured data could be extracted from {url}.")
 
 
 def _target_gone_status(response: requests.Response) -> Optional[int]:
@@ -155,8 +151,7 @@ class ScrapeUnblockerFetcher:
         :param base_url: API base URL. Override to target a different environment.
         :param timeout: HTTP timeout in seconds.
         :param raise_on_failure: Raise instead of skipping when a URL cannot be fetched
-            (including when a browser step fails, the page does not exist, or
-            `parsed_data` finds nothing to extract).
+            (including when a browser step fails or the page does not exist).
         """
         self.api_key = api_key
         self.parsed_data = parsed_data
@@ -213,11 +208,8 @@ class ScrapeUnblockerFetcher:
         if gone is not None:
             raise TargetNotFoundError(url, gone, response.text)
 
-        # A failed browser step returns 422 with a structured JSON error; so does
-        # a parsed_data call that found nothing to extract.
+        # A failed browser step returns 422 with a structured JSON error.
         if response.status_code == 422:
-            if self._is_no_data_extracted(response):
-                raise NoDataExtractedError(url)
             error = self._parse_step_error(response)
             if error is not None:
                 raise error
@@ -225,14 +217,6 @@ class ScrapeUnblockerFetcher:
         response.raise_for_status()
         return response
 
-    @staticmethod
-    def _is_no_data_extracted(response: requests.Response) -> bool:
-        try:
-            payload = response.json()
-        except ValueError:
-            return False
-        return isinstance(payload, dict) and payload.get("error") == "no_data_extracted"
-
     @staticmethod
     def _parse_step_error(response: requests.Response) -> Optional[StepExecutionError]:
         try:
@@ -255,8 +239,9 @@ class ScrapeUnblockerFetcher:
 
         A URL whose page does not exist (the site answers 404/410) yields no
         Document; with `raise_on_failure` it raises `TargetNotFoundError`. With
-        `parsed_data`, a page holding no structured data yields no Document
-        either, or raises `NoDataExtractedError`.
+        `parsed_data`, a page holding no structured data still yields a
+        Document: its JSON carries `data_extracted: false` and the rendered
+        `html`, and `meta["data_extracted"]` is False.
 
         :param urls: URLs to fetch.
         :returns: A dictionary with a `documents` key holding the fetched pages.
@@ -268,10 +253,10 @@ class ScrapeUnblockerFetcher:
         for url in urls:
             try:
                 response = self._fetch(url)
-            except (TargetNotFoundError, NoDataExtractedError) as exc:
+            except TargetNotFoundError as exc:
                 if self.raise_on_failure:
                     raise
-                # A page that does not exist, or has nothing to parse, has no content to index.
+                # A page that does not exist has no content to index.
                 logger.warning("ScrapeUnblocker skipped {url}: {error}", url=url, error=str(exc))
                 continue
             except Exception as exc:
@@ -296,6 +281,8 @@ class ScrapeUnblockerFetcher:
                     content = json.dumps(payload, ensure_ascii=False)
                     if self.list_elements and isinstance(payload, dict):
                         meta["count"] = payload.get("count")
+                    elif self.parsed_data and isinstance(payload, dict):
+                        meta["data_extracted"] = payload.get("data_extracted") is not False
                 except ValueError:
                     content = response.text
             else:

scrapeunblocker_haystack-0.3.0/src/scrapeunblocker_haystack/version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{scrapeunblocker_haystack-0.2.2 → scrapeunblocker_haystack-0.3.0}/tests/test_components.py

@@ -302,31 +302,37 @@ class TestTargetNotFound:
 
 class TestNoDataExtracted:
     BODY = {
-        "error": "no_data_extracted",
+        "data": {"page_type": "unknown", "data": {}},
+        "data_extracted": False,
         "detail": "The page was rendered, but no structured data could be extracted from it.",
+        "html": "<html><title>Example Domain</title></html>",
     }
 
     @patch("scrapeunblocker_haystack.fetcher.requests.post")
-    def test_empty_parse_is_skipped(self, mock_post):
+    def test_empty_parse_is_a_document_with_the_html(self, mock_post):
         mock_post.side_effect = [
-            _response(status_code=422, json_data=self.BODY),
-            _response(json_data={"data": {"page_type": "product"}}),
+            _response(json_data=self.BODY),
+            _response(json_data={"data": {"page_type": "product", "data": {"title": "x"}}}),
         ]
         fetcher = ScrapeUnblockerFetcher(api_key=API_KEY, parsed_data=True)
 
-        result = fetcher.run(urls=["https://example.com/empty", "https://example.com"])
+        empty, parsed = fetcher.run(urls=["https://example.com/empty", "https://example.com"])[
+            "documents"
+        ]
 
-        assert [d.meta["url"] for d in result["documents"]] == ["https://example.com"]
+        assert empty.meta["data_extracted"] is False
+        assert json.loads(empty.content)["html"] == self.BODY["html"]
+        assert parsed.meta["data_extracted"] is True
 
     @patch("scrapeunblocker_haystack.fetcher.requests.post")
-    def test_empty_parse_raises_when_asked(self, mock_post):
-        mock_post.return_value = _response(status_code=422, json_data=self.BODY)
+    def test_empty_parse_does_not_raise(self, mock_post):
+        mock_post.return_value = _response(json_data=self.BODY)
         fetcher = ScrapeUnblockerFetcher(api_key=API_KEY, parsed_data=True, raise_on_failure=True)
 
-        with pytest.raises(NoDataExtractedError) as exc:
-            fetcher.run(urls=["https://example.com/empty"])
-        assert exc.value.url == "https://example.com/empty"
-        assert "not billed" in str(exc.value)
+        assert len(fetcher.run(urls=["https://example.com/empty"])["documents"]) == 1
+
+    def test_error_class_still_importable(self):
+        assert issubclass(NoDataExtractedError, Exception)
 
     @patch("scrapeunblocker_haystack.fetcher.requests.post")
     def test_dead_url_with_parsed_body_is_skipped(self, mock_post):

scrapeunblocker_haystack-0.2.2/src/scrapeunblocker_haystack/version.py

@@ -1 +0,0 @@
-__version__ = "0.2.2"