scrapegraphai 2.2.0b3__tar.gz → 2.2.0b5__tar.gz

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [2.2.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.4...v2.2.0-beta.5) (2026-06-23)
+
+
+### Bug Fixes
+
+* lazy imports to prevent torchcodec FFmpeg DLL crash on Windows ([#1089](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1089)) ([2f55377](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2f55377010fb16c81655aa1c8d08b88daa880797))
+
+## [2.2.0-beta.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.3...v2.2.0-beta.4) (2026-06-11)
+
+
+### Bug Fixes
+
+* **nodes:** tolerate doubled-brace JSON output from models like DeepSeek ([#1085](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1085)) ([aaa5d2c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/aaa5d2cf6d2657f8e8a3b9020ab8b40cdb311c46))
+
 ## [2.2.0-beta.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.2...v2.2.0-beta.3) (2026-06-01)
 
 

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scrapegraphai
-Version: 2.2.0b3
+Version: 2.2.0b5
 Summary: A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines.
 Author-email: Marco Vinciguerra <mvincig11@gmail.com>, Lorenzo Padoan <lorenzo.padoan977@gmail.com>
 Keywords: ai,artificial intelligence,gpt,graph,langchain,llm,machine learning,natural language processing,nlp,openai,rag,scrapegraph,scrapegraphai,scraping,web scraping,web scraping library,web scraping tool,webscraping

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "scrapegraphai"
 
-version = "2.2.0b3"
+version = "2.2.0b5"
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [

scrapegraphai-2.2.0b5/scrapegraphai/docloaders/__init__.py

@@ -0,0 +1,30 @@
+"""
+This module handles document loading functionalities for the ScrapeGraphAI application.
+
+Note: ChromiumLoader and PlasmateLoader are lazy-imported to avoid triggering
+torchcodec/FFmpeg DLL loading at import time (sentence_transformers -> torchcodec chain).
+"""
+
+from .browser_base import browser_base_fetch
+from .scrape_do import scrape_do_fetch
+
+_LAZY_MODULES = {
+    "ChromiumLoader": ".chromium",
+    "PlasmateLoader": ".plasmate",
+}
+
+
+def __getattr__(name):
+    if name in _LAZY_MODULES:
+        import importlib
+        module = importlib.import_module(_LAZY_MODULES[name], __package__)
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "browser_base_fetch",
+    "ChromiumLoader",
+    "PlasmateLoader",
+    "scrape_do_fetch",
+]

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/scrapegraphai/docloaders/chromium.py

@@ -3,7 +3,6 @@ from typing import Any, AsyncIterator, Iterator, List, Optional, Union
 
 import aiohttp
 import async_timeout
-from langchain_community.document_loaders.base import BaseLoader
 from langchain_core.documents import Document
 
 from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy
@@ -11,7 +10,7 @@ from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy
 logger = get_logger("web-loader")
 
 
-class ChromiumLoader(BaseLoader):
+class ChromiumLoader:
     """Scrapes HTML pages from URLs using a (headless) instance of the
     Chromium web driver with proxy protection.
 
@@ -436,6 +435,14 @@ class ChromiumLoader(BaseLoader):
             finally:
                 await browser.close()
 
+    def load(self) -> List[Document]:
+        """Load all documents synchronously."""
+        return list(self.lazy_load())
+
+    async def aload(self) -> List[Document]:
+        """Load all documents asynchronously."""
+        return [doc async for doc in self.alazy_load()]
+
     def lazy_load(self) -> Iterator[Document]:
         """
         Lazily load text content from the provided URLs.

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/scrapegraphai/nodes/fetch_node.py

@@ -7,7 +7,6 @@ from typing import List, Optional
 import concurrent.futures
 
 import requests
-from langchain_community.document_loaders import PyPDFLoader
 from langchain_core.documents import Document
 from langchain_openai import AzureChatOpenAI, ChatOpenAI
 
@@ -182,6 +181,7 @@ class FetchNode(BaseNode):
         """
 
         if input_type == "pdf":
+            from langchain_community.document_loaders import PyPDFLoader
             loader = PyPDFLoader(source)
             # PyPDFLoader.load() can be blocking for large PDFs. Run it in a thread and
             # enforce the configured timeout if provided.

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/scrapegraphai/nodes/generate_answer_node.py

@@ -9,7 +9,6 @@ from typing import List, Optional
 from langchain_core.prompts import PromptTemplate
 from langchain_aws import ChatBedrock
 from langchain_ollama import ChatOllama
-from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.runnables import RunnableParallel
 from langchain_openai import ChatOpenAI
 from requests.exceptions import Timeout
@@ -23,7 +22,10 @@ from ..prompts import (
     TEMPLATE_NO_CHUNKS,
     TEMPLATE_NO_CHUNKS_MD,
 )
-from ..utils.output_parser import get_pydantic_output_parser
+from ..utils.output_parser import (
+    TolerantJsonOutputParser,
+    get_pydantic_output_parser,
+)
 from .base_node import BaseNode
 
 
@@ -148,7 +150,7 @@ class GenerateAnswerNode(BaseNode):
                     format_instructions = ""
         else:
             if not isinstance(self.llm_model, ChatBedrock):
-                output_parser = JsonOutputParser()
+                output_parser = TolerantJsonOutputParser()
                 format_instructions = (
                     "You must respond with a JSON object. Your response should be formatted as a valid JSON "
                     "with a 'content' field containing your analysis. For example:\n"

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/scrapegraphai/nodes/robots_node.py

@@ -7,7 +7,6 @@ from urllib.parse import urlparse
 
 from langchain_core.output_parsers import CommaSeparatedListOutputParser
 from langchain_core.prompts import PromptTemplate
-from langchain_community.document_loaders import AsyncChromiumLoader
 
 from ..helpers import robots_dictionary
 from ..prompts import TEMPLATE_ROBOT
@@ -90,6 +89,7 @@ class RobotsNode(BaseNode):
         else:
             parsed_url = urlparse(source)
             base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+            from langchain_community.document_loaders import AsyncChromiumLoader
             loader = AsyncChromiumLoader(f"{base_url}/robots.txt")
             document = loader.load()
             if "ollama" in self.llm_model.model:

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/scrapegraphai/utils/output_parser.py

@@ -2,13 +2,53 @@
 Functions to retrieve the correct output parser and format instructions for the LLM model.
 """
 
-from typing import Any, Callable, Dict, Type, Union
+from typing import Any, Callable, Dict, List, Type, Union
 
+from langchain_core.exceptions import OutputParserException
+from langchain_core.outputs import Generation
 from langchain_core.output_parsers import JsonOutputParser
 from pydantic import BaseModel as BaseModelV2
 from pydantic.v1 import BaseModel as BaseModelV1
 
 
+def _strip_doubled_braces(text: str) -> str:
+    """Strip one layer of the doubled braces some models echo from the prompt.
+
+    The default ``format_instructions`` show the expected shape using LangChain's
+    escaped braces, e.g. ``{{"content": "..."}}``. Strongly instruction-following
+    models (GPT-4o, etc.) emit single braces, but some models (notably DeepSeek)
+    copy the doubled braces verbatim, producing ``{{"content": "..."}}`` which is
+    not valid JSON. This normalizes that single case and is a no-op otherwise.
+    """
+    stripped = text.strip()
+    if stripped.startswith("{{") and stripped.endswith("}}"):
+        return stripped[1:-1]
+    return text
+
+
+class TolerantJsonOutputParser(JsonOutputParser):
+    """A :class:`JsonOutputParser` tolerant of doubled-brace output.
+
+    Behaviour is unchanged on the happy path: valid JSON is parsed by the parent
+    parser exactly as before. Only when parsing fails AND the output is wrapped in
+    doubled braces (``{{ ... }}``) does it retry once with a single layer of braces
+    removed. This keeps providers like DeepSeek working without altering output for
+    any model that already returns clean JSON.
+    """
+
+    def parse_result(self, result: List[Generation], *, partial: bool = False) -> Any:
+        try:
+            return super().parse_result(result, partial=partial)
+        except OutputParserException:
+            text = result[0].text
+            normalized = _strip_doubled_braces(text)
+            if normalized != text:
+                return super().parse_result(
+                    [Generation(text=normalized)], partial=partial
+                )
+            raise
+
+
 def get_structured_output_parser(
     schema: Union[Dict[str, Any], Type[BaseModelV1 | BaseModelV2], Type],
 ) -> Callable:

{scrapegraphai-2.2.0b3 → scrapegraphai-2.2.0b5}/tests/conftest.py

@@ -14,9 +14,24 @@ from pathlib import Path
 from typing import Any, Dict
 from unittest.mock import Mock
 
+import sys
+import types
+
 import pytest
 from dotenv import load_dotenv
 
+# Mock torchcodec to prevent FFmpeg DLL crashes at import time.
+# sentence_transformers -> torchcodec -> FFmpeg native DLLs can't load on some systems.
+_tc = types.ModuleType("torchcodec")
+_tc.__version__ = "0.0.0"
+_tc.__file__ = "<mock>"
+_tc.__spec__ = types.ModuleType("spec")
+_tc.__spec__.name = "torchcodec"
+_tc.__spec__.loader = None
+_tc.__spec__.submodule_search_locations = []
+if "torchcodec" not in sys.modules:
+    sys.modules["torchcodec"] = _tc
+
 # Load environment variables
 load_dotenv()
 

scrapegraphai-2.2.0b5/tests/utils/output_parser_test.py

@@ -0,0 +1,44 @@
+"""Tests for scrapegraphai.utils.output_parser.TolerantJsonOutputParser."""
+
+import pytest
+
+from scrapegraphai.utils.output_parser import (
+    TolerantJsonOutputParser,
+    _strip_doubled_braces,
+)
+
+
+def test_strip_doubled_braces_unwraps_single_layer():
+    assert _strip_doubled_braces('{{"content": "hi"}}') == '{"content": "hi"}'
+
+
+def test_strip_doubled_braces_is_noop_for_clean_json():
+    text = '{"content": "hi"}'
+    assert _strip_doubled_braces(text) == text
+
+
+def test_strip_doubled_braces_ignores_unbalanced():
+    text = '{{"content": "hi"}'
+    assert _strip_doubled_braces(text) == text
+
+
+def test_tolerant_parser_parses_clean_json_unchanged():
+    parser = TolerantJsonOutputParser()
+    assert parser.parse('{"content": "hi"}') == {"content": "hi"}
+
+
+def test_tolerant_parser_recovers_doubled_braces():
+    """Models such as DeepSeek echo the prompt's escaped braces verbatim."""
+    parser = TolerantJsonOutputParser()
+    assert parser.parse('{{"content": "hi"}}') == {"content": "hi"}
+
+
+def test_tolerant_parser_recovers_doubled_braces_with_whitespace():
+    parser = TolerantJsonOutputParser()
+    assert parser.parse('  {{"content": "hi"}}  ') == {"content": "hi"}
+
+
+def test_tolerant_parser_still_raises_on_irrecoverable_output():
+    parser = TolerantJsonOutputParser()
+    with pytest.raises(Exception):
+        parser.parse("this is not json at all")

scrapegraphai-2.2.0b3/scrapegraphai/docloaders/__init__.py

@@ -1,15 +0,0 @@
-"""
-This module handles document loading functionalities for the ScrapeGraphAI application.
-"""
-
-from .browser_base import browser_base_fetch
-from .chromium import ChromiumLoader
-from .plasmate import PlasmateLoader
-from .scrape_do import scrape_do_fetch
-
-__all__ = [
-    "browser_base_fetch",
-    "ChromiumLoader",
-    "PlasmateLoader",
-    "scrape_do_fetch",
-]