scrapegraphai 2.2.1__tar.gz → 2.2.2__tar.gz

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/.github/workflows/test-suite.yml

@@ -42,6 +42,7 @@ jobs:
           tests/test_batch_api.py
           tests/test_csv_scraper_multi_graph.py
           tests/test_depth_search_graph.py
+          tests/test_error_page_detection.py
           tests/test_json_scraper_graph.py
           tests/test_minimax_models.py
           tests/test_scrape_do.py

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [2.2.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.1...v2.2.2) (2026-08-23)
+
+
+### Bug Fixes
+
+* **fetch:** surface HTTP errors and missing content instead of answering NA ([adc92f7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/adc92f7eff9aa39d70e3848c6867328c3ba2ba2d)), closes [#1102](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1102) [#1102](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1102)
+
 ## [2.2.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0...v2.2.1) (2026-08-21)
 
 

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scrapegraphai
-Version: 2.2.1
+Version: 2.2.2
 Summary: A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines.
 Author-email: Marco Vinciguerra <mvincig11@gmail.com>, Lorenzo Padoan <lorenzo.padoan977@gmail.com>
 Keywords: ai,artificial intelligence,gpt,graph,langchain,llm,machine learning,natural language processing,nlp,openai,rag,scrapegraph,scrapegraphai,scraping,web scraping,web scraping library,web scraping tool,webscraping

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "scrapegraphai"
 
-version = "2.2.1"
+version = "2.2.2"
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/docloaders/chromium.py

@@ -10,6 +10,32 @@ from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy
 logger = get_logger("web-loader")
 
 
+def _warn_on_error_status(response: Any, url: str) -> None:
+    """Log a warning when a navigation returned an HTTP error status.
+
+    Playwright's ``page.goto()`` returns the main-frame ``Response``, but the
+    scrapers only keep ``page.content()``. Without this check an error page
+    (404, 403, 500, a captcha wall, a login redirect) is indistinguishable
+    from the intended document once it reaches the LLM, which then produces a
+    confidently wrong answer with no signal that anything went wrong.
+
+    This mirrors the behaviour of the ``use_soup=True`` path in ``FetchNode``:
+    it warns rather than raising, so scraping error pages on purpose keeps
+    working.
+
+    Args:
+        response: The ``Response`` returned by ``page.goto()``; may be ``None``
+            (for example on a same-document navigation) or lack a usable status.
+        url: The URL that was requested, used in the warning message.
+    """
+    status = getattr(response, "status", None)
+    if isinstance(status, int) and status >= 400:
+        logger.warning(
+            f"Received HTTP {status} for {url}; the scraped content is likely "
+            "an error page, not the intended document."
+        )
+
+
 class ChromiumLoader:
     """Scrapes HTML pages from URLs using a (headless) instance of the
     Chromium web driver with proxy protection.
@@ -251,7 +277,8 @@ class ChromiumLoader:
                     context = await browser.new_context()
                     await Malenia.apply_stealth(context)
                     page = await context.new_page()
-                    await page.goto(url, wait_until="domcontentloaded")
+                    response = await page.goto(url, wait_until="domcontentloaded")
+                    _warn_on_error_status(response, url)
                     await page.wait_for_load_state(self.load_state)
 
                     previous_height = None
@@ -364,7 +391,8 @@ class ChromiumLoader:
                     )
                     await Malenia.apply_stealth(context)
                     page = await context.new_page()
-                    await page.goto(url, wait_until="domcontentloaded")
+                    response = await page.goto(url, wait_until="domcontentloaded")
+                    _warn_on_error_status(response, url)
                     await page.wait_for_load_state(self.load_state)
                     results = await page.content()
                     logger.info("Content scraped")
@@ -421,7 +449,8 @@ class ChromiumLoader:
                         storage_state=self.storage_state
                     )
                     page = await context.new_page()
-                    await page.goto(url, wait_until="networkidle")
+                    response = await page.goto(url, wait_until="networkidle")
+                    _warn_on_error_status(response, url)
                     results = await page.content()
                     logger.info("Content scraped after JavaScript rendering")
                     return results

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/code_generator_graph.py

@@ -93,7 +93,11 @@ class CodeGeneratorGraph(AbstractGraph):
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"llm_model": self.llm_model, "chunk_size": self.model_token},
+            node_config={
+                "llm_model": self.llm_model,
+                "chunk_size": self.model_token,
+                "schema": self.schema,
+            },
         )
 
         generate_validation_answer_node = GenerateAnswerNode(

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/document_scraper_graph.py

@@ -76,6 +76,7 @@ class DocumentScraperGraph(AbstractGraph):
                 "parse_html": False,
                 "chunk_size": self.model_token,
                 "llm_model": self.llm_model,
+                "schema": self.schema,
             },
         )
         generate_answer_node = GenerateAnswerNode(

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/omni_scraper_graph.py

@@ -83,6 +83,7 @@ class OmniScraperGraph(AbstractGraph):
                 "chunk_size": self.model_token,
                 "parse_urls": True,
                 "llm_model": self.llm_model,
+                "schema": self.schema,
             },
         )
 

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/script_creator_graph.py

@@ -82,6 +82,7 @@ class ScriptCreatorGraph(AbstractGraph):
                 "chunk_size": self.model_token,
                 "parse_html": False,
                 "llm_model": self.llm_model,
+                "schema": self.schema,
             },
         )
 

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/smart_scraper_graph.py

@@ -110,7 +110,11 @@ class SmartScraperGraph(AbstractGraph):
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"llm_model": self.llm_model, "chunk_size": self.model_token},
+            node_config={
+                "llm_model": self.llm_model,
+                "chunk_size": self.model_token,
+                "schema": self.schema,
+            },
         )
 
         generate_answer_node = GenerateAnswerNode(
@@ -152,6 +156,7 @@ class SmartScraperGraph(AbstractGraph):
                 node_config={
                     "llm_model": self.llm_model,
                     "chunk_size": self.model_token,
+                    "schema": self.schema,
                 },
             )
 

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/smart_scraper_lite_graph.py

@@ -74,7 +74,11 @@ class SmartScraperLiteGraph(AbstractGraph):
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"llm_model": self.llm_model, "chunk_size": self.model_token},
+            node_config={
+                "llm_model": self.llm_model,
+                "chunk_size": self.model_token,
+                "schema": self.schema,
+            },
         )
 
         return BaseGraph(

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/smart_scraper_multi_batch_graph.py

@@ -5,9 +5,8 @@ A scraping pipeline that uses the OpenAI Batch API for LLM calls,
 providing 50% cost savings compared to real-time API calls.
 """
 
-import asyncio
 from copy import deepcopy
-from typing import Dict, List, Optional, Type
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -17,7 +16,6 @@ from ..nodes.merge_answers_node import MergeAnswersNode
 from ..utils.copy import safe_deepcopy
 from .abstract_graph import AbstractGraph
 from .base_graph import BaseGraph
-from .smart_scraper_graph import SmartScraperGraph
 
 
 class _FetchParseOnlyGraph(AbstractGraph):
@@ -57,6 +55,7 @@ class _FetchParseOnlyGraph(AbstractGraph):
             node_config={
                 "llm_model": self.llm_model,
                 "chunk_size": self.model_token,
+                "schema": self.schema,
             },
         )
 

{scrapegraphai-2.2.1 → scrapegraphai-2.2.2}/scrapegraphai/graphs/speech_graph.py

@@ -70,7 +70,11 @@ class SpeechGraph(AbstractGraph):
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token, "llm_model": self.llm_model},
+            node_config={
+                "chunk_size": self.model_token,
+                "llm_model": self.llm_model,
+                "schema": self.schema,
+            },
         )
 
         generate_answer_node = GenerateAnswerNode(