scrapegraphai 1.2.1__tar.gz → 1.2.2__tar.gz

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [1.2.2](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.2.1...v1.2.2) (2024-05-15)
+
+
+### Bug Fixes
+
+* come back to the old version ([cc5adef](https://github.com/VinciGit00/Scrapegraph-ai/commit/cc5adefd29eb2d0d7127515c4a4a72eabbc7eaa8))
+
 ## [1.2.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.2.0...v1.2.1) (2024-05-15)
 
 

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scrapegraphai
-Version: 1.2.1
+Version: 1.2.2
 Summary: A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines.
 Author-email: Marco Vinciguerra <mvincig11@gmail.com>, Marco Perini <perinim.98@gmail.com>, Lorenzo Padoan <lorenzo.padoan977@gmail.com>
 License-Expression: MIT
@@ -10,7 +10,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: <3.12,>=3.9
+Requires-Python: >=3.9
 Requires-Dist: beautifulsoup4==4.12.3
 Requires-Dist: faiss-cpu==1.8.0
 Requires-Dist: free-proxy==1.1.1

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/docs/source/getting_started/installation.rst

@@ -25,13 +25,11 @@ The library is available on PyPI, so it can be installed using the following com
    
    It is higly recommended to install the library in a virtual environment (conda, venv, etc.)
 
-If you clone the repository, you can install the library using `rye <https://rye-up.com/>`_. Follow the installation instruction from the website and then run:
+If your clone the repository, you can install the library using `poetry <https://python-poetry.org/docs/>`_:
 
 .. code-block:: bash
 
-   rye pin 3.10
-   rye sync
-   rye build
+   poetry install
 
 Additionally on Windows when using WSL
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/manual deployment/deploy_on_pip.sh

@@ -2,9 +2,6 @@
 cd ..
 
 rye self update
-
-rye pin 3.10
-
 # Install dependencies using Poetry
 rye sync
 

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "scrapegraphai"
 
-version = "1.2.1"
+version = "1.2.2"
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [
@@ -10,6 +10,7 @@ authors = [
     { name = "Lorenzo Padoan", email = "lorenzo.padoan977@gmail.com" }
 ]
 dependencies = [
+    # python = ">=3.9, <3.12"
     "langchain==0.1.15",
     "langchain-openai==0.1.6",
     "langchain-google-genai==1.0.3",
@@ -61,14 +62,12 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Operating System :: OS Independent",
 ]
-requires-python = ">= 3.9, < 3.12"
-
+requires-python = ">= 3.9"
 
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-
 [tool.rye]
 managed = true
 dev-dependencies = [

scrapegraphai-1.2.2/scrapegraphai/builders/__init__.py

@@ -0,0 +1,5 @@
+"""
+    __init__.py file for builders folder
+"""
+
+from .graph_builder import GraphBuilder

scrapegraphai-1.2.2/scrapegraphai/builders/graph_builder.py

@@ -0,0 +1,168 @@
+""" 
+GraphBuilder Module
+"""
+
+from langchain_core.prompts import ChatPromptTemplate
+from langchain.chains import create_extraction_chain
+from ..models import OpenAI, Gemini
+from ..helpers import nodes_metadata, graph_schema
+
+
+class GraphBuilder:
+    """
+    GraphBuilder is a dynamic tool for constructing web scraping graphs based on user prompts. 
+    It utilizes a natural language understanding model to interpret user prompts and 
+    automatically generates a graph configuration for scraping web content.
+
+    Attributes:
+        prompt (str): The user's natural language prompt for the scraping task.
+        llm (ChatOpenAI): An instance of the ChatOpenAI class configured 
+        with the specified llm_config.
+        nodes_description (str): A string description of all available nodes and their arguments.
+        chain (LLMChain): The extraction chain responsible for 
+        processing the prompt and creating the graph.
+
+    Methods:
+        build_graph(): Executes the graph creation process based on the user prompt 
+        and returns the graph configuration.
+        convert_json_to_graphviz(json_data): Converts a JSON graph configuration 
+        to a Graphviz object for visualization.
+
+    Args:
+        prompt (str): The user's natural language prompt describing the desired scraping operation.
+        url (str): The target URL from which data is to be scraped.
+        llm_config (dict): Configuration parameters for the 
+            language model, where 'api_key' is mandatory, 
+            and 'model_name', 'temperature', and 'streaming' can be optionally included.
+
+    Raises:
+        ValueError: If 'api_key' is not included in llm_config.
+    """
+
+    def __init__(self, user_prompt: str, config: dict):
+        """
+        Initializes the GraphBuilder with a user prompt and language model configuration.
+        """
+        self.user_prompt = user_prompt
+        self.config = config
+        self.llm = self._create_llm(config["llm"])
+        self.nodes_description = self._generate_nodes_description()
+        self.chain = self._create_extraction_chain()
+
+    def _create_llm(self, llm_config: dict):
+        """
+        Creates an instance of the OpenAI class with the provided language model configuration.
+
+        Returns:
+            OpenAI: An instance of the OpenAI class.
+
+        Raises:
+            ValueError: If 'api_key' is not provided in llm_config.
+        """
+        llm_defaults = {
+            "temperature": 0,
+            "streaming": True
+        }
+        # Update defaults with any LLM parameters that were provided
+        llm_params = {**llm_defaults, **llm_config}
+        if "api_key" not in llm_params:
+            raise ValueError("LLM configuration must include an 'api_key'.")
+
+        # select the model based on the model name
+        if "gpt-" in llm_params["model"]:
+            return OpenAI(llm_params)
+        elif "gemini" in llm_params["model"]:
+            return Gemini(llm_params)
+        raise ValueError("Model not supported")
+
+    def _generate_nodes_description(self):
+        """
+        Generates a string description of all available nodes and their arguments.
+
+        Returns:
+            str: A string description of all available nodes and their arguments.
+        """
+
+        return "\n".join([
+            f"""- {node}: {data["description"]} (Type: {data["type"]}, 
+            Args: {", ".join(data["args"].keys())})"""
+            for node, data in nodes_metadata.items()
+        ])
+
+    def _create_extraction_chain(self):
+        """
+        Creates an extraction chain for processing the user prompt and 
+        generating the graph configuration.
+
+        Returns:
+            LLMChain: An instance of the LLMChain class.
+        """
+
+        create_graph_prompt_template = """
+        You are an AI that designs direct graphs for web scraping tasks. 
+        Your goal is to create a web scraping pipeline that is efficient and tailored to the user's requirements. 
+        You have access to a set of default nodes, each with specific capabilities:
+
+        {nodes_description}
+
+        Based on the user's input: "{input}", identify the essential nodes required for the task and suggest a graph configuration that outlines the flow between the chosen nodes.
+        """.format(nodes_description=self.nodes_description, input="{input}")
+        extraction_prompt = ChatPromptTemplate.from_template(
+            create_graph_prompt_template)
+        return create_extraction_chain(prompt=extraction_prompt, schema=graph_schema, llm=self.llm)
+
+    def build_graph(self):
+        """
+        Executes the graph creation process based on the user prompt and
+         returns the graph configuration.
+
+        Returns:
+            dict: A JSON representation of the graph configuration.
+        """
+        return self.chain.invoke(self.user_prompt)
+
+    @staticmethod
+    def convert_json_to_graphviz(json_data, format: str = 'pdf'):
+        """
+        Converts a JSON graph configuration to a Graphviz object for visualization.
+
+        Args:
+            json_data (dict): A JSON representation of the graph configuration.
+
+        Returns:
+            graphviz.Digraph: A Graphviz object representing the graph configuration.
+        """
+        try:
+            import graphviz
+        except ImportError:
+            raise ImportError("The 'graphviz' library is required for this functionality. "
+                              "Please install it from 'https://graphviz.org/download/'.")
+
+        graph = graphviz.Digraph(comment='ScrapeGraphAI Generated Graph', format=format,
+                                 node_attr={'color': 'lightblue2', 'style': 'filled'})
+
+        graph_config = json_data["text"][0]
+
+        # Retrieve nodes, edges, and the entry point from the JSON data
+        nodes = graph_config.get('nodes', [])
+        edges = graph_config.get('edges', [])
+        entry_point = graph_config.get('entry_point')
+
+        # Add nodes to the graph
+        for node in nodes:
+            # If this node is the entry point, use a double circle to denote it
+            if node['node_name'] == entry_point:
+                graph.node(node['node_name'], shape='doublecircle')
+            else:
+                graph.node(node['node_name'])
+
+        # Add edges to the graph
+        for edge in edges:
+            # An edge could potentially have multiple 'to' nodes if it's from a conditional node
+            if isinstance(edge['to'], list):
+                for to_node in edge['to']:
+                    graph.edge(edge['from'], to_node)
+            else:
+                graph.edge(edge['from'], edge['to'])
+
+        return graph

scrapegraphai-1.2.2/scrapegraphai/docloaders/__init__.py

@@ -0,0 +1,3 @@
+"""__init__.py file for docloaders folder"""
+
+from .chromium import ChromiumLoader

scrapegraphai-1.2.2/scrapegraphai/docloaders/chromium.py

@@ -0,0 +1,126 @@
+import asyncio
+import logging
+from typing import Any, AsyncIterator, Iterator, List, Optional
+
+from langchain_community.document_loaders.base import BaseLoader
+from langchain_core.documents import Document
+
+from ..utils import Proxy, dynamic_import, parse_or_search_proxy
+
+
+logger = logging.getLogger(__name__)
+
+
+class ChromiumLoader(BaseLoader):
+    """scrapes HTML pages from URLs using a (headless) instance of the
+    Chromium web driver with proxy protection
+
+    Attributes:
+        backend: The web driver backend library; defaults to 'playwright'.
+        browser_config: A dictionary containing additional browser kwargs.
+        headless: whether to run browser in headless mode.
+        proxy: A dictionary containing proxy settings; None disables protection.
+        urls: A list of URLs to scrape content from.
+    """
+
+    def __init__(
+        self,
+        urls: List[str],
+        *,
+        backend: str = "playwright",
+        headless: bool = True,
+        proxy: Optional[Proxy] = None,
+        **kwargs: Any,
+    ):
+        """Initialize the loader with a list of URL paths.
+
+        Args:
+            backend: The web driver backend library; defaults to 'playwright'.
+            headless: whether to run browser in headless mode.
+            proxy: A dictionary containing proxy information; None disables protection.
+            urls: A list of URLs to scrape content from.
+            kwargs: A dictionary containing additional browser kwargs.
+
+        Raises:
+            ImportError: If the required backend package is not installed.
+        """
+        message = (
+            f"{backend} is required for ChromiumLoader. "
+            f"Please install it with `pip install {backend}`."
+        )
+
+        dynamic_import(backend, message)
+
+        self.backend = backend
+        self.browser_config = kwargs
+        self.headless = headless
+        self.proxy = parse_or_search_proxy(proxy) if proxy else None
+        self.urls = urls
+
+    async def ascrape_playwright(self, url: str) -> str:
+        """
+        Asynchronously scrape the content of a given URL using Playwright's async API.
+
+        Args:
+            url (str): The URL to scrape.
+
+        Returns:
+            str: The scraped HTML content or an error message if an exception occurs.
+
+        """
+        from playwright.async_api import async_playwright
+
+        logger.info("Starting scraping...")
+        results = ""
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(
+                headless=self.headless, proxy=self.proxy, **self.browser_config
+            )
+            try:
+                page = await browser.new_page()
+                await page.goto(url)
+                results = await page.content()  # Simply get the HTML content
+                logger.info("Content scraped")
+            except Exception as e:
+                results = f"Error: {e}"
+            await browser.close()
+        return results
+
+    def lazy_load(self) -> Iterator[Document]:
+        """
+        Lazily load text content from the provided URLs.
+
+        This method yields Documents one at a time as they're scraped,
+        instead of waiting to scrape all URLs before returning.
+
+        Yields:
+            Document: The scraped content encapsulated within a Document object.
+
+        """
+        scraping_fn = getattr(self, f"ascrape_{self.backend}")
+
+        for url in self.urls:
+            html_content = asyncio.run(scraping_fn(url))
+            metadata = {"source": url}
+            yield Document(page_content=html_content, metadata=metadata)
+
+    async def alazy_load(self) -> AsyncIterator[Document]:
+        """
+        Asynchronously load text content from the provided URLs.
+
+        This method leverages asyncio to initiate the scraping of all provided URLs
+        simultaneously. It improves performance by utilizing concurrent asynchronous
+        requests. Each Document is yielded as soon as its content is available,
+        encapsulating the scraped content.
+
+        Yields:
+            Document: A Document object containing the scraped content, along with its
+            source URL as metadata.
+        """
+        scraping_fn = getattr(self, f"ascrape_{self.backend}")
+
+        tasks = [scraping_fn(url) for url in self.urls]
+        results = await asyncio.gather(*tasks)
+        for url, content in zip(self.urls, results):
+            metadata = {"source": url}
+            yield Document(page_content=content, metadata=metadata)

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/graphs/__init__.py

@@ -5,6 +5,7 @@ __init__.py file for graphs folder
 from .abstract_graph import AbstractGraph
 from .base_graph import BaseGraph
 from .smart_scraper_graph import SmartScraperGraph
+from .deep_scraper_graph import DeepScraperGraph
 from .speech_graph import SpeechGraph
 from .search_graph import SearchGraph
 from .script_creator_graph import ScriptCreatorGraph

scrapegraphai-1.2.2/scrapegraphai/graphs/deep_scraper_graph.py

@@ -0,0 +1,116 @@
+"""
+DeepScraperGraph Module
+"""
+
+from .base_graph import BaseGraph
+from ..nodes import (
+    FetchNode,
+    SearchLinkNode,
+    ParseNode,
+    RAGNode,
+    GenerateAnswerNode
+)
+from .abstract_graph import AbstractGraph
+
+
+class DeepScraperGraph(AbstractGraph):
+    """
+    [WIP]
+
+    DeepScraper is a scraping pipeline that automates the process of 
+    extracting information from web pages
+    using a natural language model to interpret and answer prompts.
+
+    Unlike SmartScraper, DeepScraper can navigate to the links within the input webpage,
+    to fuflfil the task within the prompt.
+
+    
+    Attributes:
+        prompt (str): The prompt for the graph.
+        source (str): The source of the graph.
+        config (dict): Configuration parameters for the graph.
+        llm_model: An instance of a language model client, configured for generating answers.
+        embedder_model: An instance of an embedding model client, 
+        configured for generating embeddings.
+        verbose (bool): A flag indicating whether to show print statements during execution.
+        headless (bool): A flag indicating whether to run the graph in headless mode.
+    Args:
+        prompt (str): The prompt for the graph.
+        source (str): The source of the graph.
+        config (dict): Configuration parameters for the graph.
+    Example:
+        >>> deep_scraper = DeepScraperGraph(
+        ...     "List me all the job titles and detailed job description.",
+        ...     "https://www.google.com/about/careers/applications/jobs/results/?location=Bangalore%20India",
+        ...     {"llm": {"model": "gpt-3.5-turbo"}}
+        ... )
+        >>> result = deep_scraper.run()
+        )
+    """
+
+    def __init__(self, prompt: str, source: str, config: dict):
+        super().__init__(prompt, config, source)
+
+        self.input_key = "url" if source.startswith("http") else "local_dir"
+
+    def _create_graph(self) -> BaseGraph:
+        """
+        Creates the graph of nodes representing the workflow for web scraping.
+        Returns:
+            BaseGraph: A graph instance representing the web scraping workflow.
+        """
+        fetch_node = FetchNode(
+            input="url | local_dir",
+            output=["doc", "link_urls", "img_urls"]
+        )
+        parse_node = ParseNode(
+            input="doc",
+            output=["parsed_doc"],
+            node_config={
+                "chunk_size": self.model_token
+            }
+        )
+        rag_node = RAGNode(
+            input="user_prompt & (parsed_doc | doc)",
+            output=["relevant_chunks"],
+            node_config={
+                "llm_model": self.llm_model,
+                "embedder_model": self.embedder_model
+            }
+        )
+        search_node = SearchLinkNode(
+            input="user_prompt & relevant_chunks",
+            output=["relevant_links"],
+            node_config={
+                "llm_model": self.llm_model,
+                "embedder_model": self.embedder_model
+            }
+        )
+
+        return BaseGraph(
+            nodes=[
+                fetch_node,
+                parse_node,
+                rag_node,
+                search_node
+            ],
+            edges=[
+                (fetch_node, parse_node),
+                (parse_node, rag_node),
+                (rag_node, search_node)
+
+            ],
+            entry_point=fetch_node
+        )
+
+    def run(self) -> str:
+        """
+        Executes the scraping process and returns the answer to the prompt.
+        Returns:
+            str: The answer to the prompt.
+        """
+
+        inputs = {"user_prompt": self.prompt, self.input_key: self.source}
+        self.final_state, self.execution_info = self.graph.execute(inputs)
+
+        return self.final_state.get("answer", "No answer found.")

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/graphs/omni_search_graph.py

@@ -2,7 +2,7 @@
 OmniSearchGraph Module
 """
 
-from copy import copy
+from copy import deepcopy
 
 from .base_graph import BaseGraph
 from ..nodes import (
@@ -43,7 +43,7 @@ class OmniSearchGraph(AbstractGraph):
     def __init__(self, prompt: str, config: dict):
 
         self.max_results = config.get("max_results", 3)
-        self.copy_config = copy(config)
+        self.copy_config = deepcopy(config)
 
         super().__init__(prompt, config)
 

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/graphs/search_graph.py

@@ -2,7 +2,7 @@
 SearchGraph Module
 """
 
-from copy import copy
+from copy import deepcopy
 
 from .base_graph import BaseGraph
 from ..nodes import (
@@ -42,7 +42,7 @@ class SearchGraph(AbstractGraph):
     def __init__(self, prompt: str, config: dict):
 
         self.max_results = config.get("max_results", 3)
-        self.copy_config = copy(config)
+        self.copy_config = deepcopy(config)
 
         super().__init__(prompt, config)
 

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/graphs/smart_scraper_graph.py

@@ -111,4 +111,4 @@ class SmartScraperGraph(AbstractGraph):
         inputs = {"user_prompt": self.prompt, self.input_key: self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 
-        return self.final_state.get("answer", "No answer found.")
+        return self.final_state.get("answer", "No answer found.")

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/nodes/__init__.py

@@ -19,5 +19,4 @@ from .generate_answer_csv_node import GenerateAnswerCSVNode
 from .generate_answer_pdf_node import GenerateAnswerPDFNode
 from .graph_iterator_node import GraphIteratorNode
 from .merge_answers_node import MergeAnswersNode
-from .generate_answer_omni_node import GenerateAnswerOmniNode
-from .search_node_with_context import SearchLinksWithContext
+from .generate_answer_omni_node import GenerateAnswerOmniNode

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/nodes/fetch_node.py

@@ -162,5 +162,4 @@ class FetchNode(BaseNode):
             ]
 
         state.update({self.output[0]: compressed_document, self.output[1]: link_urls, self.output[2]: image_urls})
-
         return state

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/nodes/generate_answer_node.py

@@ -38,7 +38,7 @@ class GenerateAnswerNode(BaseNode):
         super().__init__(node_name, "node", input, output, 2, node_config)
 
         self.llm_model = node_config["llm_model"]
-        self.verbose = True if node_config is None else node_config.get(
+        self.verbose = False if node_config is None else node_config.get(
             "verbose", False)
 
     def execute(self, state: dict) -> dict:

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/nodes/merge_answers_node.py

@@ -4,6 +4,7 @@ MergeAnswersNode Module
 
 # Imports from standard library
 from typing import List, Optional
+from tqdm import tqdm
 
 # Imports from Langchain
 from langchain.prompts import PromptTemplate
@@ -38,8 +39,7 @@ class MergeAnswersNode(BaseNode):
 
     def execute(self, state: dict) -> dict:
         """
-        Executes the node's logic to merge the answers from multiple graph instances into a 
-        single answer.
+        Executes the node's logic to merge the answers from multiple graph instances into a single answer.
 
         Args:
             state (dict): The current state of the graph. The input keys will be used

{scrapegraphai-1.2.1 → scrapegraphai-1.2.2}/scrapegraphai/nodes/robots_node.py

@@ -35,15 +35,12 @@ class RobotsNode(BaseNode):
     """
 
     def __init__(self, input: str, output: List[str],  node_config: Optional[dict]=None,
-
                  node_name: str = "Robots"):
         super().__init__(node_name, "node", input, output, 1)
 
         self.llm_model = node_config["llm_model"]
-
-        self.force_scraping = force_scraping
-        self.verbose = True if node_config is None else node_config.get(
-            "verbose", False)
+        self.force_scraping = False if node_config is None else node_config.get("force_scraping", False)
+        self.verbose = False if node_config is None else node_config.get("verbose", False)
 
     def execute(self, state: dict) -> dict:
         """
@@ -100,8 +97,7 @@ class RobotsNode(BaseNode):
             loader = AsyncChromiumLoader(f"{base_url}/robots.txt")
             document = loader.load()
             if "ollama" in self.llm_model.model_name:
-                self.llm_model.model_name = self.llm_model.model_name.split(
-                    "/")[-1]
+                self.llm_model.model_name = self.llm_model.model_name.split("/")[-1]
                 model = self.llm_model.model_name.split("/")[-1]
 
             else:
@@ -126,7 +122,7 @@ class RobotsNode(BaseNode):
             if "no" in is_scrapable:
                 if self.verbose:
                     print("\033[31m(Scraping this website is not allowed)\033[0m")
-
+                    
                 if not self.force_scraping:
                     raise ValueError(
                         'The website you selected is not scrapable')