llama-index-tools-brightdata 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llama_index_tools_brightdata-0.1.0/.gitignore +0 -0
- llama_index_tools_brightdata-0.1.0/LICENSE +21 -0
- llama_index_tools_brightdata-0.1.0/PKG-INFO +161 -0
- llama_index_tools_brightdata-0.1.0/README.md +148 -0
- llama_index_tools_brightdata-0.1.0/llama_index/tools/brightdata/__init__.py +3 -0
- llama_index_tools_brightdata-0.1.0/llama_index/tools/brightdata/base.py +355 -0
- llama_index_tools_brightdata-0.1.0/pyproject.toml +65 -0
|
File without changes
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) Bright Data
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llama-index-tools-brightdata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: llama-index tools Bright Data integration
|
|
5
|
+
Author-email: Bright Data <you@example.com>
|
|
6
|
+
Maintainer: brightdata
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Python: <4.0,>=3.9
|
|
10
|
+
Requires-Dist: llama-index-core<0.13,>=0.12.0
|
|
11
|
+
Requires-Dist: llama-index>=0.12.34
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# LlamaIndex Tools Integration: Bright Data
|
|
15
|
+
|
|
16
|
+
This tool connects to [Bright Data](https://brightdata.com/) to enable your agent to crawl websites, search the web, and access structured data from platforms like LinkedIn, Amazon, and social media.
|
|
17
|
+
|
|
18
|
+
Bright Data's tools provide robust web scraping capabilities with built-in CAPTCHA solving and bot detection avoidance, allowing you to reliably extract data from the web.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install llama-index llama-index-core llama-index-tools-brightdata
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Authentication
|
|
27
|
+
|
|
28
|
+
To use this tool, you'll need a Bright Data API key. You can obtain one by signing up for a Bright Data account on their [website](https://brightdata.com/).
|
|
29
|
+
|
|
30
|
+
## Usage
|
|
31
|
+
|
|
32
|
+
Here's an example of how to use the BrightDataToolSpec with LlamaIndex:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
llm = OpenAI(model="gpt-4o", api_key="your-api-key")
|
|
36
|
+
|
|
37
|
+
brightdata_tool = BrightDataToolSpec(api_key="your-api-key", zone="unlocker")
|
|
38
|
+
|
|
39
|
+
tool_list = brightdata_tool.to_tool_list()
|
|
40
|
+
|
|
41
|
+
for tool in tool_list:
|
|
42
|
+
tool.original_description = tool.metadata.description
|
|
43
|
+
tool.metadata.description = "Bright Data web scraping tool"
|
|
44
|
+
|
|
45
|
+
agent = OpenAIAgent.from_tools(tools=tool_list, llm=llm)
|
|
46
|
+
|
|
47
|
+
query = (
|
|
48
|
+
"Find and summarize the latest news about AI from major tech news sites"
|
|
49
|
+
)
|
|
50
|
+
tool_descriptions = "\n\n".join(
|
|
51
|
+
[
|
|
52
|
+
f"Tool Name: {tool.metadata.name}\nTool Description: {tool.original_description}"
|
|
53
|
+
for tool in tool_list
|
|
54
|
+
]
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
query_with_descriptions = f"{tool_descriptions}\n\nQuery: {query}"
|
|
58
|
+
|
|
59
|
+
response = agent.chat(query_with_descriptions)
|
|
60
|
+
print(response)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Features
|
|
64
|
+
|
|
65
|
+
The Bright Data tool provides the following capabilities:
|
|
66
|
+
|
|
67
|
+
### Web Scraping
|
|
68
|
+
|
|
69
|
+
- `scrape_as_markdown`: Scrape a webpage and convert the content to Markdown format. This tool can bypass CAPTCHA and bot detection.
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
result = brightdata_tool.scrape_as_markdown("https://example.com")
|
|
73
|
+
print(result.text)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Visual Capture
|
|
77
|
+
|
|
78
|
+
- `get_screenshot`: Take a screenshot of a webpage and save it to a file.
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
screenshot_path = brightdata_tool.get_screenshot(
|
|
82
|
+
"https://example.com", output_path="example_screenshot.png"
|
|
83
|
+
)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Search Engine Access
|
|
87
|
+
|
|
88
|
+
- `search_engine`: Search Google, Bing, or Yandex and get structured search results as JSON or Markdown. Supports advanced parameters for more specific searches.
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
search_results = brightdata_tool.search_engine(
|
|
92
|
+
query="climate change solutions",
|
|
93
|
+
engine="google",
|
|
94
|
+
language="en",
|
|
95
|
+
country_code="us",
|
|
96
|
+
num_results=20,
|
|
97
|
+
)
|
|
98
|
+
print(search_results.text)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### Structured Web Data Extraction
|
|
102
|
+
|
|
103
|
+
- `web_data_feed`: Retrieve structured data from various platforms including LinkedIn, Amazon, Instagram, Facebook, X (Twitter), Zillow, and more.
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
linkedin_profile = brightdata_tool.web_data_feed(
|
|
107
|
+
source_type="linkedin_person_profile",
|
|
108
|
+
url="https://www.linkedin.com/in/username/",
|
|
109
|
+
)
|
|
110
|
+
print(linkedin_profile)
|
|
111
|
+
|
|
112
|
+
amazon_product = brightdata_tool.web_data_feed(
|
|
113
|
+
source_type="amazon_product", url="https://www.amazon.com/dp/B08N5KWB9H"
|
|
114
|
+
)
|
|
115
|
+
print(amazon_product)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Advanced Configuration
|
|
119
|
+
|
|
120
|
+
The Bright Data tool offers various configuration options for specialized use cases:
|
|
121
|
+
|
|
122
|
+
### Search Engine Parameters
|
|
123
|
+
|
|
124
|
+
The `search_engine` function supports advanced parameters like:
|
|
125
|
+
|
|
126
|
+
- Language targeting (`language` parameter)
|
|
127
|
+
- Country-specific search (`country_code` parameter)
|
|
128
|
+
- Different search types (images, shopping, news, etc.)
|
|
129
|
+
- Pagination controls
|
|
130
|
+
- Mobile device emulation
|
|
131
|
+
- Geolocation targeting
|
|
132
|
+
- Hotel search parameters
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
results = brightdata_tool.search_engine(
|
|
136
|
+
query="best hotels in paris",
|
|
137
|
+
engine="google",
|
|
138
|
+
language="fr",
|
|
139
|
+
country_code="fr",
|
|
140
|
+
search_type="shopping",
|
|
141
|
+
device="mobile",
|
|
142
|
+
hotel_dates="2025-06-01,2025-06-05",
|
|
143
|
+
hotel_occupancy=2,
|
|
144
|
+
)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### Supported Web Data Sources
|
|
148
|
+
|
|
149
|
+
The `web_data_feed` function supports retrieving structured data from:
|
|
150
|
+
|
|
151
|
+
- LinkedIn (profiles and companies)
|
|
152
|
+
- Amazon (products and reviews)
|
|
153
|
+
- Instagram (profiles, posts, reels, comments)
|
|
154
|
+
- Facebook (posts, marketplace listings, company reviews)
|
|
155
|
+
- X/Twitter (posts)
|
|
156
|
+
- Zillow (property listings)
|
|
157
|
+
- Booking.com (hotel listings)
|
|
158
|
+
- YouTube (videos)
|
|
159
|
+
- ZoomInfo (company profiles)
|
|
160
|
+
|
|
161
|
+
For more information, visit the [Bright Data documentation](https://docs.brightdata.com/).
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# LlamaIndex Tools Integration: Bright Data
|
|
2
|
+
|
|
3
|
+
This tool connects to [Bright Data](https://brightdata.com/) to enable your agent to crawl websites, search the web, and access structured data from platforms like LinkedIn, Amazon, and social media.
|
|
4
|
+
|
|
5
|
+
Bright Data's tools provide robust web scraping capabilities with built-in CAPTCHA solving and bot detection avoidance, allowing you to reliably extract data from the web.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install llama-index llama-index-core llama-index-tools-brightdata
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Authentication
|
|
14
|
+
|
|
15
|
+
To use this tool, you'll need a Bright Data API key. You can obtain one by signing up for a Bright Data account on their [website](https://brightdata.com/).
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
Here's an example of how to use the BrightDataToolSpec with LlamaIndex:
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
llm = OpenAI(model="gpt-4o", api_key="your-api-key")
|
|
23
|
+
|
|
24
|
+
brightdata_tool = BrightDataToolSpec(api_key="your-api-key", zone="unlocker")
|
|
25
|
+
|
|
26
|
+
tool_list = brightdata_tool.to_tool_list()
|
|
27
|
+
|
|
28
|
+
for tool in tool_list:
|
|
29
|
+
tool.original_description = tool.metadata.description
|
|
30
|
+
tool.metadata.description = "Bright Data web scraping tool"
|
|
31
|
+
|
|
32
|
+
agent = OpenAIAgent.from_tools(tools=tool_list, llm=llm)
|
|
33
|
+
|
|
34
|
+
query = (
|
|
35
|
+
"Find and summarize the latest news about AI from major tech news sites"
|
|
36
|
+
)
|
|
37
|
+
tool_descriptions = "\n\n".join(
|
|
38
|
+
[
|
|
39
|
+
f"Tool Name: {tool.metadata.name}\nTool Description: {tool.original_description}"
|
|
40
|
+
for tool in tool_list
|
|
41
|
+
]
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
query_with_descriptions = f"{tool_descriptions}\n\nQuery: {query}"
|
|
45
|
+
|
|
46
|
+
response = agent.chat(query_with_descriptions)
|
|
47
|
+
print(response)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Features
|
|
51
|
+
|
|
52
|
+
The Bright Data tool provides the following capabilities:
|
|
53
|
+
|
|
54
|
+
### Web Scraping
|
|
55
|
+
|
|
56
|
+
- `scrape_as_markdown`: Scrape a webpage and convert the content to Markdown format. This tool can bypass CAPTCHA and bot detection.
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
result = brightdata_tool.scrape_as_markdown("https://example.com")
|
|
60
|
+
print(result.text)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Visual Capture
|
|
64
|
+
|
|
65
|
+
- `get_screenshot`: Take a screenshot of a webpage and save it to a file.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
screenshot_path = brightdata_tool.get_screenshot(
|
|
69
|
+
"https://example.com", output_path="example_screenshot.png"
|
|
70
|
+
)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### Search Engine Access
|
|
74
|
+
|
|
75
|
+
- `search_engine`: Search Google, Bing, or Yandex and get structured search results as JSON or Markdown. Supports advanced parameters for more specific searches.
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
search_results = brightdata_tool.search_engine(
|
|
79
|
+
query="climate change solutions",
|
|
80
|
+
engine="google",
|
|
81
|
+
language="en",
|
|
82
|
+
country_code="us",
|
|
83
|
+
num_results=20,
|
|
84
|
+
)
|
|
85
|
+
print(search_results.text)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### Structured Web Data Extraction
|
|
89
|
+
|
|
90
|
+
- `web_data_feed`: Retrieve structured data from various platforms including LinkedIn, Amazon, Instagram, Facebook, X (Twitter), Zillow, and more.
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
linkedin_profile = brightdata_tool.web_data_feed(
|
|
94
|
+
source_type="linkedin_person_profile",
|
|
95
|
+
url="https://www.linkedin.com/in/username/",
|
|
96
|
+
)
|
|
97
|
+
print(linkedin_profile)
|
|
98
|
+
|
|
99
|
+
amazon_product = brightdata_tool.web_data_feed(
|
|
100
|
+
source_type="amazon_product", url="https://www.amazon.com/dp/B08N5KWB9H"
|
|
101
|
+
)
|
|
102
|
+
print(amazon_product)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Advanced Configuration
|
|
106
|
+
|
|
107
|
+
The Bright Data tool offers various configuration options for specialized use cases:
|
|
108
|
+
|
|
109
|
+
### Search Engine Parameters
|
|
110
|
+
|
|
111
|
+
The `search_engine` function supports advanced parameters like:
|
|
112
|
+
|
|
113
|
+
- Language targeting (`language` parameter)
|
|
114
|
+
- Country-specific search (`country_code` parameter)
|
|
115
|
+
- Different search types (images, shopping, news, etc.)
|
|
116
|
+
- Pagination controls
|
|
117
|
+
- Mobile device emulation
|
|
118
|
+
- Geolocation targeting
|
|
119
|
+
- Hotel search parameters
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
results = brightdata_tool.search_engine(
|
|
123
|
+
query="best hotels in paris",
|
|
124
|
+
engine="google",
|
|
125
|
+
language="fr",
|
|
126
|
+
country_code="fr",
|
|
127
|
+
search_type="shopping",
|
|
128
|
+
device="mobile",
|
|
129
|
+
hotel_dates="2025-06-01,2025-06-05",
|
|
130
|
+
hotel_occupancy=2,
|
|
131
|
+
)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Supported Web Data Sources
|
|
135
|
+
|
|
136
|
+
The `web_data_feed` function supports retrieving structured data from:
|
|
137
|
+
|
|
138
|
+
- LinkedIn (profiles and companies)
|
|
139
|
+
- Amazon (products and reviews)
|
|
140
|
+
- Instagram (profiles, posts, reels, comments)
|
|
141
|
+
- Facebook (posts, marketplace listings, company reviews)
|
|
142
|
+
- X/Twitter (posts)
|
|
143
|
+
- Zillow (property listings)
|
|
144
|
+
- Booking.com (hotel listings)
|
|
145
|
+
- YouTube (videos)
|
|
146
|
+
- ZoomInfo (company profiles)
|
|
147
|
+
|
|
148
|
+
For more information, visit the [Bright Data documentation](https://docs.brightdata.com/).
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""Bright Data tool spec for LlamaIndex."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, Optional
|
|
4
|
+
from llama_index.core.schema import Document
|
|
5
|
+
from llama_index.core.tools.tool_spec.base import BaseToolSpec
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BrightDataToolSpec(BaseToolSpec):
|
|
9
|
+
"""Bright Data tool spec for web scraping and search capabilities."""
|
|
10
|
+
|
|
11
|
+
spec_functions = ["scrape_as_markdown", "get_screenshot", "search_engine","web_data_feed"]
|
|
12
|
+
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
api_key: str,
|
|
16
|
+
zone: str = "unblocker",
|
|
17
|
+
verbose: bool = False,
|
|
18
|
+
) -> None:
|
|
19
|
+
"""
|
|
20
|
+
Initialize with API token and default zone.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
api_key (str): Your Bright Data API token
|
|
24
|
+
zone (str): Bright Data zone name
|
|
25
|
+
verbose (bool): Print additional information about requests
|
|
26
|
+
|
|
27
|
+
"""
|
|
28
|
+
self._headers = {
|
|
29
|
+
"Content-Type": "application/json",
|
|
30
|
+
"Authorization": f"Bearer {api_key}"
|
|
31
|
+
}
|
|
32
|
+
self._api_key = api_key
|
|
33
|
+
self._zone = zone
|
|
34
|
+
self._verbose = verbose
|
|
35
|
+
self._endpoint = "https://api.brightdata.com/request"
|
|
36
|
+
|
|
37
|
+
def _make_request(self, payload: Dict) -> str:
|
|
38
|
+
"""
|
|
39
|
+
Make a request to Bright Data API.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
payload (Dict): Request payload
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
str: Response text
|
|
46
|
+
|
|
47
|
+
"""
|
|
48
|
+
import requests
|
|
49
|
+
import json
|
|
50
|
+
|
|
51
|
+
if self._verbose:
|
|
52
|
+
print(f"[Bright Data] Request: {payload['url']}")
|
|
53
|
+
|
|
54
|
+
response = requests.post(
|
|
55
|
+
self._endpoint,
|
|
56
|
+
headers=self._headers,
|
|
57
|
+
data=json.dumps(payload)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if response.status_code != 200:
|
|
61
|
+
raise Exception(f"Failed to scrape: {response.status_code} - {response.text}")
|
|
62
|
+
|
|
63
|
+
return response.text
|
|
64
|
+
|
|
65
|
+
def scrape_as_markdown(self, url: str, zone: Optional[str] = None) -> Document:
|
|
66
|
+
"""
|
|
67
|
+
Scrape a webpage and return content in Markdown format.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
url (str): URL to scrape
|
|
71
|
+
zone (Optional[str]): Override default zone
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
Document: Scraped content as Markdown
|
|
75
|
+
|
|
76
|
+
"""
|
|
77
|
+
payload = {
|
|
78
|
+
"url": url,
|
|
79
|
+
"zone": zone or self._zone,
|
|
80
|
+
"format": "raw",
|
|
81
|
+
"data_format": "markdown"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
content = self._make_request(payload)
|
|
85
|
+
return Document(text=content, metadata={"url": url})
|
|
86
|
+
|
|
87
|
+
def get_screenshot(self, url: str, output_path: str, zone: Optional[str] = None) -> str:
|
|
88
|
+
"""
|
|
89
|
+
Take a screenshot of a webpage.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
url (str): URL to screenshot
|
|
93
|
+
output_path (str): Path to save the screenshot
|
|
94
|
+
zone (Optional[str]): Override default zone
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
str: Path to saved screenshot
|
|
98
|
+
|
|
99
|
+
"""
|
|
100
|
+
import requests
|
|
101
|
+
import json
|
|
102
|
+
|
|
103
|
+
payload = {
|
|
104
|
+
"url": url,
|
|
105
|
+
"zone": zone or self._zone,
|
|
106
|
+
"format": "raw",
|
|
107
|
+
"data_format": "screenshot"
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
response = requests.post(
|
|
111
|
+
self._endpoint,
|
|
112
|
+
headers=self._headers,
|
|
113
|
+
data=json.dumps(payload)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if response.status_code != 200:
|
|
117
|
+
raise Exception(f"Error {response.status_code}: {response.text}")
|
|
118
|
+
|
|
119
|
+
with open(output_path, "wb") as f:
|
|
120
|
+
f.write(response.content)
|
|
121
|
+
|
|
122
|
+
return output_path
|
|
123
|
+
|
|
124
|
+
def search_engine(
|
|
125
|
+
self,
|
|
126
|
+
query: str,
|
|
127
|
+
engine: str = "google",
|
|
128
|
+
zone: Optional[str] = None,
|
|
129
|
+
language: Optional[str] = None, # hl parameter, e.g., "en"
|
|
130
|
+
country_code: Optional[str] = None, # gl parameter, e.g., "us"
|
|
131
|
+
search_type: Optional[str] = None, # tbm parameter (images, shopping, news, etc.)
|
|
132
|
+
start: Optional[int] = None, # pagination start index
|
|
133
|
+
num_results: Optional[int] = 10, # number of results to return
|
|
134
|
+
location: Optional[str] = None, # uule parameter for geo-location
|
|
135
|
+
device: Optional[str] = None, # device type for user-agent
|
|
136
|
+
return_json: bool = False, # parse results as JSON
|
|
137
|
+
hotel_dates: Optional[str] = None, # check-in and check-out dates
|
|
138
|
+
hotel_occupancy: Optional[int] = None # number of guests
|
|
139
|
+
) -> Document:
|
|
140
|
+
"""
|
|
141
|
+
Search using Google, Bing, or Yandex with advanced parameters and return results in Markdown.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
query (str): Search query
|
|
145
|
+
engine (str): Search engine - 'google', 'bing', or 'yandex'
|
|
146
|
+
zone (Optional[str]): Override default zone
|
|
147
|
+
|
|
148
|
+
# Google SERP specific parameters
|
|
149
|
+
language (Optional[str]): Two-letter language code (hl parameter)
|
|
150
|
+
country_code (Optional[str]): Two-letter country code (gl parameter)
|
|
151
|
+
search_type (Optional[str]): Type of search (images, shopping, news, etc.)
|
|
152
|
+
start (Optional[int]): Results pagination offset (0=first page, 10=second page)
|
|
153
|
+
num_results (Optional[int]): Number of results to return (default 10)
|
|
154
|
+
location (Optional[str]): Location for search results (uule parameter)
|
|
155
|
+
device (Optional[str]): Device type (mobile, ios, android, ipad, android_tablet)
|
|
156
|
+
return_json (bool): Return parsed JSON instead of HTML/Markdown
|
|
157
|
+
|
|
158
|
+
# Hotel search parameters
|
|
159
|
+
hotel_dates (Optional[str]): Check-in and check-out dates (format: YYYY-MM-DD,YYYY-MM-DD)
|
|
160
|
+
hotel_occupancy (Optional[int]): Number of guests (1-4)
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
Document: Search results as Markdown or JSON
|
|
164
|
+
|
|
165
|
+
"""
|
|
166
|
+
encoded_query = self._encode_query(query)
|
|
167
|
+
|
|
168
|
+
base_urls = {
|
|
169
|
+
"google": f"https://www.google.com/search?q={encoded_query}",
|
|
170
|
+
"bing": f"https://www.bing.com/search?q={encoded_query}",
|
|
171
|
+
"yandex": f"https://yandex.com/search/?text={encoded_query}"
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if engine not in base_urls:
|
|
175
|
+
raise ValueError(f"Unsupported search engine: {engine}. Use 'google', 'bing', or 'yandex'")
|
|
176
|
+
|
|
177
|
+
search_url = base_urls[engine]
|
|
178
|
+
|
|
179
|
+
if engine == "google":
|
|
180
|
+
params = []
|
|
181
|
+
|
|
182
|
+
if language:
|
|
183
|
+
params.append(f"hl={language}")
|
|
184
|
+
|
|
185
|
+
if country_code:
|
|
186
|
+
params.append(f"gl={country_code}")
|
|
187
|
+
|
|
188
|
+
if search_type:
|
|
189
|
+
if search_type == "jobs":
|
|
190
|
+
params.append("ibp=htl;jobs")
|
|
191
|
+
else:
|
|
192
|
+
search_types = {
|
|
193
|
+
"images": "isch",
|
|
194
|
+
"shopping": "shop",
|
|
195
|
+
"news": "nws"
|
|
196
|
+
}
|
|
197
|
+
tbm_value = search_types.get(search_type, search_type)
|
|
198
|
+
params.append(f"tbm={tbm_value}")
|
|
199
|
+
|
|
200
|
+
if start is not None:
|
|
201
|
+
params.append(f"start={start}")
|
|
202
|
+
|
|
203
|
+
if num_results:
|
|
204
|
+
params.append(f"num={num_results}")
|
|
205
|
+
|
|
206
|
+
if location:
|
|
207
|
+
params.append(f"uule={self._encode_query(location)}")
|
|
208
|
+
|
|
209
|
+
if device:
|
|
210
|
+
device_value = "1"
|
|
211
|
+
|
|
212
|
+
if device in ["ios", "iphone"]:
|
|
213
|
+
device_value = "ios"
|
|
214
|
+
elif device == "ipad":
|
|
215
|
+
device_value = "ios_tablet"
|
|
216
|
+
elif device == "android":
|
|
217
|
+
device_value = "android"
|
|
218
|
+
elif device == "android_tablet":
|
|
219
|
+
device_value = "android_tablet"
|
|
220
|
+
|
|
221
|
+
params.append(f"brd_mobile={device_value}")
|
|
222
|
+
|
|
223
|
+
if return_json:
|
|
224
|
+
params.append("brd_json=1")
|
|
225
|
+
|
|
226
|
+
if hotel_dates:
|
|
227
|
+
params.append(f"hotel_dates={self._encode_query(hotel_dates)}")
|
|
228
|
+
|
|
229
|
+
if hotel_occupancy:
|
|
230
|
+
params.append(f"hotel_occupancy={hotel_occupancy}")
|
|
231
|
+
|
|
232
|
+
if params:
|
|
233
|
+
search_url += "&" + "&".join(params)
|
|
234
|
+
|
|
235
|
+
payload = {
|
|
236
|
+
"url": search_url,
|
|
237
|
+
"zone": zone or self._zone,
|
|
238
|
+
"format": "raw",
|
|
239
|
+
"data_format": "markdown" if not return_json else "raw"
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
content = self._make_request(payload)
|
|
243
|
+
return Document(text=content, metadata={
|
|
244
|
+
"query": query,
|
|
245
|
+
"engine": engine,
|
|
246
|
+
"url": search_url
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
def web_data_feed(
|
|
250
|
+
self,
|
|
251
|
+
source_type: str,
|
|
252
|
+
url: str,
|
|
253
|
+
num_of_reviews: Optional[int] = None,
|
|
254
|
+
timeout: int = 600,
|
|
255
|
+
polling_interval: int = 1
|
|
256
|
+
) -> Dict:
|
|
257
|
+
"""
|
|
258
|
+
Retrieve structured web data from various sources like LinkedIn, Amazon, Instagram, etc.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
source_type (str): Type of data source (e.g., 'linkedin_person_profile', 'amazon_product')
|
|
262
|
+
url (str): URL of the web resource to retrieve data from
|
|
263
|
+
num_of_reviews (Optional[int]): Number of reviews to retrieve (only for facebook_company_reviews)
|
|
264
|
+
timeout (int): Maximum time in seconds to wait for data retrieval
|
|
265
|
+
polling_interval (int): Time in seconds between polling attempts
|
|
266
|
+
|
|
267
|
+
Returns:
|
|
268
|
+
Dict: Structured data from the requested source
|
|
269
|
+
|
|
270
|
+
"""
|
|
271
|
+
import requests
|
|
272
|
+
import time
|
|
273
|
+
|
|
274
|
+
datasets = {
|
|
275
|
+
"amazon_product": "gd_l7q7dkf244hwjntr0",
|
|
276
|
+
"amazon_product_reviews": "gd_le8e811kzy4ggddlq",
|
|
277
|
+
"linkedin_person_profile": "gd_l1viktl72bvl7bjuj0",
|
|
278
|
+
"linkedin_company_profile": "gd_l1vikfnt1wgvvqz95w",
|
|
279
|
+
"zoominfo_company_profile": "gd_m0ci4a4ivx3j5l6nx",
|
|
280
|
+
"instagram_profiles": "gd_l1vikfch901nx3by4",
|
|
281
|
+
"instagram_posts": "gd_lk5ns7kz21pck8jpis",
|
|
282
|
+
"instagram_reels": "gd_lyclm20il4r5helnj",
|
|
283
|
+
"instagram_comments": "gd_ltppn085pokosxh13",
|
|
284
|
+
"facebook_posts": "gd_lyclm1571iy3mv57zw",
|
|
285
|
+
"facebook_marketplace_listings": "gd_lvt9iwuh6fbcwmx1a",
|
|
286
|
+
"facebook_company_reviews": "gd_m0dtqpiu1mbcyc2g86",
|
|
287
|
+
"x_posts": "gd_lwxkxvnf1cynvib9co",
|
|
288
|
+
"zillow_properties_listing": "gd_lfqkr8wm13ixtbd8f5",
|
|
289
|
+
"booking_hotel_listings": "gd_m5mbdl081229ln6t4a",
|
|
290
|
+
"youtube_videos": "gd_m5mbdl081229ln6t4a"
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
if source_type not in datasets:
|
|
294
|
+
valid_sources = ", ".join(datasets.keys())
|
|
295
|
+
raise ValueError(f"Invalid source_type: {source_type}. Valid options are: {valid_sources}")
|
|
296
|
+
|
|
297
|
+
dataset_id = datasets[source_type]
|
|
298
|
+
|
|
299
|
+
request_data = {"url": url}
|
|
300
|
+
if source_type == "facebook_company_reviews" and num_of_reviews is not None:
|
|
301
|
+
request_data["num_of_reviews"] = str(num_of_reviews)
|
|
302
|
+
|
|
303
|
+
trigger_response = requests.post(
|
|
304
|
+
"https://api.brightdata.com/datasets/v3/trigger",
|
|
305
|
+
params={"dataset_id": dataset_id, "include_errors": True},
|
|
306
|
+
headers=self._headers,
|
|
307
|
+
json=[request_data]
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
trigger_data = trigger_response.json()
|
|
311
|
+
if not trigger_data.get("snapshot_id"):
|
|
312
|
+
raise Exception("No snapshot ID returned from trigger request")
|
|
313
|
+
|
|
314
|
+
snapshot_id = trigger_data["snapshot_id"]
|
|
315
|
+
if self._verbose:
|
|
316
|
+
print(f"[Bright Data] {source_type} triggered with snapshot ID: {snapshot_id}")
|
|
317
|
+
|
|
318
|
+
attempts = 0
|
|
319
|
+
max_attempts = timeout
|
|
320
|
+
|
|
321
|
+
while attempts < max_attempts:
|
|
322
|
+
try:
|
|
323
|
+
snapshot_response = requests.get(
|
|
324
|
+
f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}",
|
|
325
|
+
params={"format": "json"},
|
|
326
|
+
headers=self._headers
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
snapshot_data = snapshot_response.json()
|
|
330
|
+
|
|
331
|
+
if isinstance(snapshot_data, dict) and snapshot_data.get("status") == "running":
|
|
332
|
+
if self._verbose:
|
|
333
|
+
print(f"[Bright Data] Snapshot not ready, polling again (attempt {attempts + 1}/{max_attempts})")
|
|
334
|
+
attempts += 1
|
|
335
|
+
time.sleep(polling_interval)
|
|
336
|
+
continue
|
|
337
|
+
|
|
338
|
+
if self._verbose:
|
|
339
|
+
print(f"[Bright Data] Data received after {attempts + 1} attempts")
|
|
340
|
+
|
|
341
|
+
return snapshot_data
|
|
342
|
+
|
|
343
|
+
except Exception as e:
|
|
344
|
+
if self._verbose:
|
|
345
|
+
print(f"[Bright Data] Polling error: {e!s}")
|
|
346
|
+
attempts += 1
|
|
347
|
+
time.sleep(polling_interval)
|
|
348
|
+
|
|
349
|
+
raise TimeoutError(f"Timeout after {max_attempts} seconds waiting for {source_type} data")
|
|
350
|
+
|
|
351
|
+
@staticmethod
|
|
352
|
+
def _encode_query(query: str) -> str:
|
|
353
|
+
"""URL encode a search query."""
|
|
354
|
+
from urllib.parse import quote
|
|
355
|
+
return quote(query)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[dependency-groups]
|
|
6
|
+
dev = [
|
|
7
|
+
"ipython==8.10.0",
|
|
8
|
+
"jupyter>=1.0.0,<2",
|
|
9
|
+
"mypy==0.991",
|
|
10
|
+
"pre-commit==3.2.0",
|
|
11
|
+
"pylint==2.15.10",
|
|
12
|
+
"pytest==7.2.1",
|
|
13
|
+
"pytest-mock==3.11.1",
|
|
14
|
+
"ruff==0.0.292",
|
|
15
|
+
"types-Deprecated>=0.1.0",
|
|
16
|
+
"types-PyYAML>=6.0.12.12,<7",
|
|
17
|
+
"types-protobuf>=4.24.0.4,<5",
|
|
18
|
+
"types-redis==4.5.5.0",
|
|
19
|
+
"types-requests==2.28.11.8",
|
|
20
|
+
"types-setuptools==67.1.0.0",
|
|
21
|
+
"black[jupyter]<=23.9.1,>=23.7.0",
|
|
22
|
+
"codespell[toml]>=v2.2.6",
|
|
23
|
+
"diff-cover>=9.2.0",
|
|
24
|
+
"pytest-cov>=6.1.1",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project]
|
|
28
|
+
name = "llama-index-tools-brightdata"
|
|
29
|
+
version = "0.1.0"
|
|
30
|
+
description = "llama-index tools Bright Data integration"
|
|
31
|
+
authors = [{name = "Bright Data", email = "you@example.com"}]
|
|
32
|
+
requires-python = ">=3.9,<4.0"
|
|
33
|
+
readme = "README.md"
|
|
34
|
+
license = "MIT"
|
|
35
|
+
maintainers = [{name = "brightdata"}]
|
|
36
|
+
dependencies = [
|
|
37
|
+
"llama-index>=0.12.34",
|
|
38
|
+
"llama-index-core>=0.12.0,<0.13",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[tool.codespell]
|
|
42
|
+
check-filenames = true
|
|
43
|
+
check-hidden = true
|
|
44
|
+
skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
|
|
45
|
+
|
|
46
|
+
[tool.hatch.build.targets.sdist]
|
|
47
|
+
include = ["llama_index/"]
|
|
48
|
+
exclude = ["**/BUILD"]
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
include = ["llama_index/"]
|
|
52
|
+
exclude = ["**/BUILD"]
|
|
53
|
+
|
|
54
|
+
[tool.llamahub]
|
|
55
|
+
contains_example = false
|
|
56
|
+
import_path = "llama_index.tools.brightdata"
|
|
57
|
+
|
|
58
|
+
[tool.llamahub.class_authors]
|
|
59
|
+
BrightDataToolSpec = "brightdata"
|
|
60
|
+
|
|
61
|
+
[tool.mypy]
|
|
62
|
+
disallow_untyped_defs = true
|
|
63
|
+
exclude = ["_static", "build", "examples", "notebooks", "venv"]
|
|
64
|
+
ignore_missing_imports = true
|
|
65
|
+
python_version = "3.8"
|