scrapegraphai 0.2.6__tar.gz → 0.2.7__tar.gz

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scrapegraphai
-Version: 0.2.6
+Version: 0.2.7
 Summary: A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines.
 Home-page: https://scrapegraph-ai.readthedocs.io/
 License: MIT
@@ -38,6 +38,7 @@ Description-Content-Type: text/markdown
 [![Downloads](https://static.pepy.tech/badge/scrapegraphai)](https://pepy.tech/project/scrapegraphai)
 [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint)
 [![Pylint](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml)
+[![CodeQL](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![](https://dcbadge.vercel.app/api/server/gkxQDAjfeX)](https://discord.gg/gkxQDAjfeX)
 
@@ -88,12 +89,11 @@ graph_config = {
         "model": "ollama/mistral",
         "temperature": 0,
         "format": "json",  # Ollama needs the format to be specified explicitly
-        "base_url": "http://localhost:11434",  # set Ollama URL arbitrarily
+        "base_url": "http://localhost:11434",  # set Ollama URL
     },
     "embeddings": {
         "model": "ollama/nomic-embed-text",
-        "temperature": 0,
-        "base_url": "http://localhost:11434",  # set Ollama URL arbitrarily
+        "base_url": "http://localhost:11434",  # set Ollama URL
     }
 }
 
@@ -114,7 +114,7 @@ print(result)
 Note: before using the local model remember to create the docker container!
 ```text
     docker-compose up -d
-    docker exec -it ollama ollama run stablelm-zephyr
+    docker exec -it ollama ollama pull stablelm-zephyr
 ```
 You can use which models avaiable on Ollama or your own model instead of stablelm-zephyr
 ```python

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/README.md

@@ -3,6 +3,7 @@
 [![Downloads](https://static.pepy.tech/badge/scrapegraphai)](https://pepy.tech/project/scrapegraphai)
 [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint)
 [![Pylint](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml)
+[![CodeQL](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![](https://dcbadge.vercel.app/api/server/gkxQDAjfeX)](https://discord.gg/gkxQDAjfeX)
 
@@ -53,12 +54,11 @@ graph_config = {
         "model": "ollama/mistral",
         "temperature": 0,
         "format": "json",  # Ollama needs the format to be specified explicitly
-        "base_url": "http://localhost:11434",  # set Ollama URL arbitrarily
+        "base_url": "http://localhost:11434",  # set Ollama URL
     },
     "embeddings": {
         "model": "ollama/nomic-embed-text",
-        "temperature": 0,
-        "base_url": "http://localhost:11434",  # set Ollama URL arbitrarily
+        "base_url": "http://localhost:11434",  # set Ollama URL
     }
 }
 
@@ -79,7 +79,7 @@ print(result)
 Note: before using the local model remember to create the docker container!
 ```text
     docker-compose up -d
-    docker exec -it ollama ollama run stablelm-zephyr
+    docker exec -it ollama ollama pull stablelm-zephyr
 ```
 You can use which models avaiable on Ollama or your own model instead of stablelm-zephyr
 ```python

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "scrapegraphai"
-version = "0.2.6"
+version = "0.2.7"
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [
     "Marco Vinciguerra <mvincig11@gmail.com>",

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/scrapegraphai/helpers/models_tokens.py

@@ -9,6 +9,8 @@ models_tokens = {
         "gpt-3.5-turbo-instruct": 4096,
         "gpt-4-0125-preview": 128000,
         "gpt-4-turbo-preview": 128000,
+        "gpt-4-turbo": 128000,
+        "gpt-4-turbo-2024-04-09": 128000,
         "gpt-4-1106-preview": 128000,
         "gpt-4-vision-preview": 128000,
         "gpt-4": 8192,

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/scrapegraphai/nodes/generate_scraper_node.py

@@ -22,7 +22,7 @@ class GenerateScraperNode(BaseNode):
     an answer.
 
     Attributes:
-        llm (ChatOpenAI): An instance of a language model client, configured for generating answers.
+        llm: An instance of a language model client, configured for generating answers.
         node_name (str): The unique identifier name for the node, defaulting 
         to "GenerateScraperNode".
         node_type (str): The type of the node, set to "node" indicating a 

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/scrapegraphai/nodes/search_internet_node.py

@@ -29,7 +29,6 @@ class SearchInternetNode(BaseNode):
                              generated answer will be stored.
         model_config (dict): Configuration parameters for the language model client.
         node_name (str, optional): The unique identifier name for the node. 
-        Defaults to "GenerateAnswer".
 
     Methods:
         execute(state): Processes the input and document from the state to generate an answer,

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/scrapegraphai/utils/convert_to_csv.py

@@ -12,12 +12,10 @@ def convert_to_csv(data: dict, filename: str, position: str = None):
 
     Args:
     data (dict): Data to be converted to CSV.
-    filename (str): Name of the CSV file (without the .csv extension).
     position (str): Optional path where the file should be saved. If not provided,
     the directory of the caller script will be used.
 
     Raises:
-    ValueError: If the filename contains '.csv'.
     FileNotFoundError: If the specified directory does not exist.
     PermissionError: If the program lacks write permission for the directory.
     TypeError: If the input data is not a dictionary.
@@ -25,7 +23,7 @@ def convert_to_csv(data: dict, filename: str, position: str = None):
     """
 
     if ".csv" in filename:
-        raise ValueError("The filename should not contain '.csv'")
+        filename = filename.replace(".csv", "")  # Remove .csv extension
 
     # Get the directory of the caller script if position is not provided
     if position is None:

{scrapegraphai-0.2.6 → scrapegraphai-0.2.7}/scrapegraphai/utils/convert_to_json.py

@@ -22,7 +22,7 @@ def convert_to_json(data: dict, filename: str, position: str = None):
     PermissionError: If the program does not have permission to write to the directory.
     """
     if ".json" in filename:
-        raise ValueError("The filename should not contain '.json'")
+        filename = filename.replace(".json", "")  # Remove .csv extension
 
   # Get the directory of the caller script
     if position is None: