scrapegraphai 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/PKG-INFO +15 -15
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/README.md +13 -13
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/pyproject.toml +2 -2
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/LICENSE +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/builders/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/builders/graph_builder.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/abstract_graph.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/base_graph.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/script_creator_graph.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/search_graph.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/smart_scraper_graph.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/graphs/speech_graph.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/helpers/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/helpers/models_tokens.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/helpers/nodes_metadata.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/helpers/schemas.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/azure_openai.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/gemini.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/hugging_face.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/ollama.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/openai.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/openai_itt.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/models/openai_tts.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/base_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/conditional_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/fetch_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/generate_answer_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/generate_scraper_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/get_probable_tags_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/image_to_text_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/parse_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/rag_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/search_internet_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/nodes/text_to_speech_node.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/__init__.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/convert_to_csv.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/convert_to_json.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/parse_state_keys.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/prettify_exec_info.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/remover.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/research_web.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/save_audio_from_bytes.py +0 -0
- {scrapegraphai-0.2.0 → scrapegraphai-0.2.1}/scrapegraphai/utils/token_calculator.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: scrapegraphai
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines.
|
|
5
5
|
Home-page: https://scrapegraph-ai.readthedocs.io/
|
|
6
6
|
License: MIT
|
|
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
18
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
19
|
Requires-Dist: beautifulsoup4 (==4.12.3)
|
|
20
|
-
Requires-Dist: faiss-cpu (==1.
|
|
20
|
+
Requires-Dist: faiss-cpu (==1.8.0)
|
|
21
21
|
Requires-Dist: google (==3.0.0)
|
|
22
22
|
Requires-Dist: graphviz (==0.20.1)
|
|
23
23
|
Requires-Dist: html2text (==2020.1.16)
|
|
@@ -42,7 +42,7 @@ Description-Content-Type: text/markdown
|
|
|
42
42
|
[](https://discord.gg/gkxQDAjfeX)
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
ScrapeGraphAI is a *web scraping* python library
|
|
45
|
+
ScrapeGraphAI is a *web scraping* python library that uses LLM and direct graph logic to create scraping pipelines for websites, documents and XML files.
|
|
46
46
|
Just say which information you want to extract and the library will do it for you!
|
|
47
47
|
|
|
48
48
|
<p align="center">
|
|
@@ -52,7 +52,7 @@ Just say which information you want to extract and the library will do it for yo
|
|
|
52
52
|
|
|
53
53
|
## 🚀 Quick install
|
|
54
54
|
|
|
55
|
-
The reference page for Scrapegraph-ai is
|
|
55
|
+
The reference page for Scrapegraph-ai is available on the official page of pypy: [pypi](https://pypi.org/project/scrapegraphai/).
|
|
56
56
|
|
|
57
57
|
```bash
|
|
58
58
|
pip install scrapegraphai
|
|
@@ -78,7 +78,7 @@ Check out also the docusaurus [documentation](https://scrapegraph-doc.onrender.c
|
|
|
78
78
|
You can use the `SmartScraper` class to extract information from a website using a prompt.
|
|
79
79
|
|
|
80
80
|
The `SmartScraper` class is a direct graph implementation that uses the most common nodes present in a web scraping pipeline. For more information, please see the [documentation](https://scrapegraph-ai.readthedocs.io/en/latest/).
|
|
81
|
-
### Case 1: Extracting
|
|
81
|
+
### Case 1: Extracting information using Ollama
|
|
82
82
|
Remember to download the model on Ollama separately!
|
|
83
83
|
```python
|
|
84
84
|
from scrapegraphai.graphs import SmartScraperGraph
|
|
@@ -88,12 +88,12 @@ graph_config = {
|
|
|
88
88
|
"model": "ollama/mistral",
|
|
89
89
|
"temperature": 0,
|
|
90
90
|
"format": "json", # Ollama needs the format to be specified explicitly
|
|
91
|
-
"base_url": "http://localhost:11434", # set
|
|
91
|
+
"base_url": "http://localhost:11434", # set Ollama URL arbitrarily
|
|
92
92
|
},
|
|
93
93
|
"embeddings": {
|
|
94
94
|
"model": "ollama/nomic-embed-text",
|
|
95
95
|
"temperature": 0,
|
|
96
|
-
"base_url": "http://localhost:11434", # set
|
|
96
|
+
"base_url": "http://localhost:11434", # set Ollama URL arbitrarily
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
99
|
|
|
@@ -109,9 +109,9 @@ print(result)
|
|
|
109
109
|
|
|
110
110
|
```
|
|
111
111
|
|
|
112
|
-
### Case 2: Extracting
|
|
112
|
+
### Case 2: Extracting information using Docker
|
|
113
113
|
|
|
114
|
-
Note: before using the local model
|
|
114
|
+
Note: before using the local model remember to create the docker container!
|
|
115
115
|
```text
|
|
116
116
|
docker-compose up -d
|
|
117
117
|
docker exec -it ollama ollama run stablelm-zephyr
|
|
@@ -141,7 +141,7 @@ print(result)
|
|
|
141
141
|
```
|
|
142
142
|
|
|
143
143
|
|
|
144
|
-
### Case 3: Extracting
|
|
144
|
+
### Case 3: Extracting information using Openai model
|
|
145
145
|
```python
|
|
146
146
|
from scrapegraphai.graphs import SmartScraperGraph
|
|
147
147
|
OPENAI_API_KEY = "YOUR_API_KEY"
|
|
@@ -164,7 +164,7 @@ result = smart_scraper_graph.run()
|
|
|
164
164
|
print(result)
|
|
165
165
|
```
|
|
166
166
|
|
|
167
|
-
### Case 4: Extracting
|
|
167
|
+
### Case 4: Extracting information using Gemini
|
|
168
168
|
```python
|
|
169
169
|
from scrapegraphai.graphs import SmartScraperGraph
|
|
170
170
|
GOOGLE_APIKEY = "YOUR_API_KEY"
|
|
@@ -188,7 +188,7 @@ result = smart_scraper_graph.run()
|
|
|
188
188
|
print(result)
|
|
189
189
|
```
|
|
190
190
|
|
|
191
|
-
The output for
|
|
191
|
+
The output for all 3 the cases will be a dictionary with the extracted information, for example:
|
|
192
192
|
|
|
193
193
|
```bash
|
|
194
194
|
{
|
|
@@ -203,9 +203,9 @@ The output for alle 3 the cases will be a dictionary with the extracted informat
|
|
|
203
203
|
|
|
204
204
|
## 🤝 Contributing
|
|
205
205
|
|
|
206
|
-
|
|
206
|
+
Feel free to contribute and join our Discord server to discuss with us improvements and give us suggestions!
|
|
207
207
|
|
|
208
|
-
|
|
208
|
+
Please see the [contributing guidelines](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
|
209
209
|
|
|
210
210
|
[](https://discord.gg/gkxQDAjfeX)
|
|
211
211
|
[](https://www.linkedin.com/company/scrapegraphai/)
|
|
@@ -222,7 +222,7 @@ If you have used our library for research purposes please quote us with the foll
|
|
|
222
222
|
title = {Scrapegraph-ai},
|
|
223
223
|
year = {2024},
|
|
224
224
|
url = {https://github.com/VinciGit00/Scrapegraph-ai},
|
|
225
|
-
note = {A Python library for scraping
|
|
225
|
+
note = {A Python library for scraping leveraging large language models}
|
|
226
226
|
}
|
|
227
227
|
```
|
|
228
228
|
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
[](https://discord.gg/gkxQDAjfeX)
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
ScrapeGraphAI is a *web scraping* python library
|
|
10
|
+
ScrapeGraphAI is a *web scraping* python library that uses LLM and direct graph logic to create scraping pipelines for websites, documents and XML files.
|
|
11
11
|
Just say which information you want to extract and the library will do it for you!
|
|
12
12
|
|
|
13
13
|
<p align="center">
|
|
@@ -17,7 +17,7 @@ Just say which information you want to extract and the library will do it for yo
|
|
|
17
17
|
|
|
18
18
|
## 🚀 Quick install
|
|
19
19
|
|
|
20
|
-
The reference page for Scrapegraph-ai is
|
|
20
|
+
The reference page for Scrapegraph-ai is available on the official page of pypy: [pypi](https://pypi.org/project/scrapegraphai/).
|
|
21
21
|
|
|
22
22
|
```bash
|
|
23
23
|
pip install scrapegraphai
|
|
@@ -43,7 +43,7 @@ Check out also the docusaurus [documentation](https://scrapegraph-doc.onrender.c
|
|
|
43
43
|
You can use the `SmartScraper` class to extract information from a website using a prompt.
|
|
44
44
|
|
|
45
45
|
The `SmartScraper` class is a direct graph implementation that uses the most common nodes present in a web scraping pipeline. For more information, please see the [documentation](https://scrapegraph-ai.readthedocs.io/en/latest/).
|
|
46
|
-
### Case 1: Extracting
|
|
46
|
+
### Case 1: Extracting information using Ollama
|
|
47
47
|
Remember to download the model on Ollama separately!
|
|
48
48
|
```python
|
|
49
49
|
from scrapegraphai.graphs import SmartScraperGraph
|
|
@@ -53,12 +53,12 @@ graph_config = {
|
|
|
53
53
|
"model": "ollama/mistral",
|
|
54
54
|
"temperature": 0,
|
|
55
55
|
"format": "json", # Ollama needs the format to be specified explicitly
|
|
56
|
-
"base_url": "http://localhost:11434", # set
|
|
56
|
+
"base_url": "http://localhost:11434", # set Ollama URL arbitrarily
|
|
57
57
|
},
|
|
58
58
|
"embeddings": {
|
|
59
59
|
"model": "ollama/nomic-embed-text",
|
|
60
60
|
"temperature": 0,
|
|
61
|
-
"base_url": "http://localhost:11434", # set
|
|
61
|
+
"base_url": "http://localhost:11434", # set Ollama URL arbitrarily
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
|
|
@@ -74,9 +74,9 @@ print(result)
|
|
|
74
74
|
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
### Case 2: Extracting
|
|
77
|
+
### Case 2: Extracting information using Docker
|
|
78
78
|
|
|
79
|
-
Note: before using the local model
|
|
79
|
+
Note: before using the local model remember to create the docker container!
|
|
80
80
|
```text
|
|
81
81
|
docker-compose up -d
|
|
82
82
|
docker exec -it ollama ollama run stablelm-zephyr
|
|
@@ -106,7 +106,7 @@ print(result)
|
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
|
|
109
|
-
### Case 3: Extracting
|
|
109
|
+
### Case 3: Extracting information using Openai model
|
|
110
110
|
```python
|
|
111
111
|
from scrapegraphai.graphs import SmartScraperGraph
|
|
112
112
|
OPENAI_API_KEY = "YOUR_API_KEY"
|
|
@@ -129,7 +129,7 @@ result = smart_scraper_graph.run()
|
|
|
129
129
|
print(result)
|
|
130
130
|
```
|
|
131
131
|
|
|
132
|
-
### Case 4: Extracting
|
|
132
|
+
### Case 4: Extracting information using Gemini
|
|
133
133
|
```python
|
|
134
134
|
from scrapegraphai.graphs import SmartScraperGraph
|
|
135
135
|
GOOGLE_APIKEY = "YOUR_API_KEY"
|
|
@@ -153,7 +153,7 @@ result = smart_scraper_graph.run()
|
|
|
153
153
|
print(result)
|
|
154
154
|
```
|
|
155
155
|
|
|
156
|
-
The output for
|
|
156
|
+
The output for all 3 the cases will be a dictionary with the extracted information, for example:
|
|
157
157
|
|
|
158
158
|
```bash
|
|
159
159
|
{
|
|
@@ -168,9 +168,9 @@ The output for alle 3 the cases will be a dictionary with the extracted informat
|
|
|
168
168
|
|
|
169
169
|
## 🤝 Contributing
|
|
170
170
|
|
|
171
|
-
|
|
171
|
+
Feel free to contribute and join our Discord server to discuss with us improvements and give us suggestions!
|
|
172
172
|
|
|
173
|
-
|
|
173
|
+
Please see the [contributing guidelines](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
|
174
174
|
|
|
175
175
|
[](https://discord.gg/gkxQDAjfeX)
|
|
176
176
|
[](https://www.linkedin.com/company/scrapegraphai/)
|
|
@@ -187,7 +187,7 @@ If you have used our library for research purposes please quote us with the foll
|
|
|
187
187
|
title = {Scrapegraph-ai},
|
|
188
188
|
year = {2024},
|
|
189
189
|
url = {https://github.com/VinciGit00/Scrapegraph-ai},
|
|
190
|
-
note = {A Python library for scraping
|
|
190
|
+
note = {A Python library for scraping leveraging large language models}
|
|
191
191
|
}
|
|
192
192
|
```
|
|
193
193
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "scrapegraphai"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
|
|
5
5
|
authors = [
|
|
6
6
|
"Marco Vinciguerra <mvincig11@gmail.com>",
|
|
@@ -26,7 +26,7 @@ langchain = "0.1.14"
|
|
|
26
26
|
langchain-openai = "0.1.1"
|
|
27
27
|
langchain-google-genai = "1.0.1"
|
|
28
28
|
html2text = "2020.1.16"
|
|
29
|
-
faiss-cpu = "1.
|
|
29
|
+
faiss-cpu = "1.8.0"
|
|
30
30
|
beautifulsoup4 = "4.12.3"
|
|
31
31
|
pandas = "2.0.3"
|
|
32
32
|
python-dotenv = "1.0.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|