llama-index-tools-brightdata 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,21 @@
1
+ The MIT License
2
+
3
+ Copyright (c) Bright Data
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,161 @@
1
+ Metadata-Version: 2.4
2
+ Name: llama-index-tools-brightdata
3
+ Version: 0.1.0
4
+ Summary: llama-index tools Bright Data integration
5
+ Author-email: Bright Data <you@example.com>
6
+ Maintainer: brightdata
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Requires-Python: <4.0,>=3.9
10
+ Requires-Dist: llama-index-core<0.13,>=0.12.0
11
+ Requires-Dist: llama-index>=0.12.34
12
+ Description-Content-Type: text/markdown
13
+
14
+ # LlamaIndex Tools Integration: Bright Data
15
+
16
+ This tool connects to [Bright Data](https://brightdata.com/) to enable your agent to crawl websites, search the web, and access structured data from platforms like LinkedIn, Amazon, and social media.
17
+
18
+ Bright Data's tools provide robust web scraping capabilities with built-in CAPTCHA solving and bot detection avoidance, allowing you to reliably extract data from the web.
19
+
20
+ ## Installation
21
+
22
+ ```bash
23
+ pip install llama-index llama-index-core llama-index-tools-brightdata
24
+ ```
25
+
26
+ ## Authentication
27
+
28
+ To use this tool, you'll need a Bright Data API key. You can obtain one by signing up for a Bright Data account on their [website](https://brightdata.com/).
29
+
30
+ ## Usage
31
+
32
+ Here's an example of how to use the BrightDataToolSpec with LlamaIndex:
33
+
34
+ ```python
35
+ llm = OpenAI(model="gpt-4o", api_key="your-api-key")
36
+
37
+ brightdata_tool = BrightDataToolSpec(api_key="your-api-key", zone="unlocker")
38
+
39
+ tool_list = brightdata_tool.to_tool_list()
40
+
41
+ for tool in tool_list:
42
+ tool.original_description = tool.metadata.description
43
+ tool.metadata.description = "Bright Data web scraping tool"
44
+
45
+ agent = OpenAIAgent.from_tools(tools=tool_list, llm=llm)
46
+
47
+ query = (
48
+ "Find and summarize the latest news about AI from major tech news sites"
49
+ )
50
+ tool_descriptions = "\n\n".join(
51
+ [
52
+ f"Tool Name: {tool.metadata.name}\nTool Description: {tool.original_description}"
53
+ for tool in tool_list
54
+ ]
55
+ )
56
+
57
+ query_with_descriptions = f"{tool_descriptions}\n\nQuery: {query}"
58
+
59
+ response = agent.chat(query_with_descriptions)
60
+ print(response)
61
+ ```
62
+
63
+ ## Features
64
+
65
+ The Bright Data tool provides the following capabilities:
66
+
67
+ ### Web Scraping
68
+
69
+ - `scrape_as_markdown`: Scrape a webpage and convert the content to Markdown format. This tool can bypass CAPTCHA and bot detection.
70
+
71
+ ```python
72
+ result = brightdata_tool.scrape_as_markdown("https://example.com")
73
+ print(result.text)
74
+ ```
75
+
76
+ ### Visual Capture
77
+
78
+ - `get_screenshot`: Take a screenshot of a webpage and save it to a file.
79
+
80
+ ```python
81
+ screenshot_path = brightdata_tool.get_screenshot(
82
+ "https://example.com", output_path="example_screenshot.png"
83
+ )
84
+ ```
85
+
86
+ ### Search Engine Access
87
+
88
+ - `search_engine`: Search Google, Bing, or Yandex and get structured search results as JSON or Markdown. Supports advanced parameters for more specific searches.
89
+
90
+ ```python
91
+ search_results = brightdata_tool.search_engine(
92
+ query="climate change solutions",
93
+ engine="google",
94
+ language="en",
95
+ country_code="us",
96
+ num_results=20,
97
+ )
98
+ print(search_results.text)
99
+ ```
100
+
101
+ ### Structured Web Data Extraction
102
+
103
+ - `web_data_feed`: Retrieve structured data from various platforms including LinkedIn, Amazon, Instagram, Facebook, X (Twitter), Zillow, and more.
104
+
105
+ ```python
106
+ linkedin_profile = brightdata_tool.web_data_feed(
107
+ source_type="linkedin_person_profile",
108
+ url="https://www.linkedin.com/in/username/",
109
+ )
110
+ print(linkedin_profile)
111
+
112
+ amazon_product = brightdata_tool.web_data_feed(
113
+ source_type="amazon_product", url="https://www.amazon.com/dp/B08N5KWB9H"
114
+ )
115
+ print(amazon_product)
116
+ ```
117
+
118
+ ## Advanced Configuration
119
+
120
+ The Bright Data tool offers various configuration options for specialized use cases:
121
+
122
+ ### Search Engine Parameters
123
+
124
+ The `search_engine` function supports advanced parameters like:
125
+
126
+ - Language targeting (`language` parameter)
127
+ - Country-specific search (`country_code` parameter)
128
+ - Different search types (images, shopping, news, etc.)
129
+ - Pagination controls
130
+ - Mobile device emulation
131
+ - Geolocation targeting
132
+ - Hotel search parameters
133
+
134
+ ```python
135
+ results = brightdata_tool.search_engine(
136
+ query="best hotels in paris",
137
+ engine="google",
138
+ language="fr",
139
+ country_code="fr",
140
+ search_type="shopping",
141
+ device="mobile",
142
+ hotel_dates="2025-06-01,2025-06-05",
143
+ hotel_occupancy=2,
144
+ )
145
+ ```
146
+
147
+ ### Supported Web Data Sources
148
+
149
+ The `web_data_feed` function supports retrieving structured data from:
150
+
151
+ - LinkedIn (profiles and companies)
152
+ - Amazon (products and reviews)
153
+ - Instagram (profiles, posts, reels, comments)
154
+ - Facebook (posts, marketplace listings, company reviews)
155
+ - X/Twitter (posts)
156
+ - Zillow (property listings)
157
+ - Booking.com (hotel listings)
158
+ - YouTube (videos)
159
+ - ZoomInfo (company profiles)
160
+
161
+ For more information, visit the [Bright Data documentation](https://docs.brightdata.com/).
@@ -0,0 +1,148 @@
1
+ # LlamaIndex Tools Integration: Bright Data
2
+
3
+ This tool connects to [Bright Data](https://brightdata.com/) to enable your agent to crawl websites, search the web, and access structured data from platforms like LinkedIn, Amazon, and social media.
4
+
5
+ Bright Data's tools provide robust web scraping capabilities with built-in CAPTCHA solving and bot detection avoidance, allowing you to reliably extract data from the web.
6
+
7
+ ## Installation
8
+
9
+ ```bash
10
+ pip install llama-index llama-index-core llama-index-tools-brightdata
11
+ ```
12
+
13
+ ## Authentication
14
+
15
+ To use this tool, you'll need a Bright Data API key. You can obtain one by signing up for a Bright Data account on their [website](https://brightdata.com/).
16
+
17
+ ## Usage
18
+
19
+ Here's an example of how to use the BrightDataToolSpec with LlamaIndex:
20
+
21
+ ```python
22
+ llm = OpenAI(model="gpt-4o", api_key="your-api-key")
23
+
24
+ brightdata_tool = BrightDataToolSpec(api_key="your-api-key", zone="unlocker")
25
+
26
+ tool_list = brightdata_tool.to_tool_list()
27
+
28
+ for tool in tool_list:
29
+ tool.original_description = tool.metadata.description
30
+ tool.metadata.description = "Bright Data web scraping tool"
31
+
32
+ agent = OpenAIAgent.from_tools(tools=tool_list, llm=llm)
33
+
34
+ query = (
35
+ "Find and summarize the latest news about AI from major tech news sites"
36
+ )
37
+ tool_descriptions = "\n\n".join(
38
+ [
39
+ f"Tool Name: {tool.metadata.name}\nTool Description: {tool.original_description}"
40
+ for tool in tool_list
41
+ ]
42
+ )
43
+
44
+ query_with_descriptions = f"{tool_descriptions}\n\nQuery: {query}"
45
+
46
+ response = agent.chat(query_with_descriptions)
47
+ print(response)
48
+ ```
49
+
50
+ ## Features
51
+
52
+ The Bright Data tool provides the following capabilities:
53
+
54
+ ### Web Scraping
55
+
56
+ - `scrape_as_markdown`: Scrape a webpage and convert the content to Markdown format. This tool can bypass CAPTCHA and bot detection.
57
+
58
+ ```python
59
+ result = brightdata_tool.scrape_as_markdown("https://example.com")
60
+ print(result.text)
61
+ ```
62
+
63
+ ### Visual Capture
64
+
65
+ - `get_screenshot`: Take a screenshot of a webpage and save it to a file.
66
+
67
+ ```python
68
+ screenshot_path = brightdata_tool.get_screenshot(
69
+ "https://example.com", output_path="example_screenshot.png"
70
+ )
71
+ ```
72
+
73
+ ### Search Engine Access
74
+
75
+ - `search_engine`: Search Google, Bing, or Yandex and get structured search results as JSON or Markdown. Supports advanced parameters for more specific searches.
76
+
77
+ ```python
78
+ search_results = brightdata_tool.search_engine(
79
+ query="climate change solutions",
80
+ engine="google",
81
+ language="en",
82
+ country_code="us",
83
+ num_results=20,
84
+ )
85
+ print(search_results.text)
86
+ ```
87
+
88
+ ### Structured Web Data Extraction
89
+
90
+ - `web_data_feed`: Retrieve structured data from various platforms including LinkedIn, Amazon, Instagram, Facebook, X (Twitter), Zillow, and more.
91
+
92
+ ```python
93
+ linkedin_profile = brightdata_tool.web_data_feed(
94
+ source_type="linkedin_person_profile",
95
+ url="https://www.linkedin.com/in/username/",
96
+ )
97
+ print(linkedin_profile)
98
+
99
+ amazon_product = brightdata_tool.web_data_feed(
100
+ source_type="amazon_product", url="https://www.amazon.com/dp/B08N5KWB9H"
101
+ )
102
+ print(amazon_product)
103
+ ```
104
+
105
+ ## Advanced Configuration
106
+
107
+ The Bright Data tool offers various configuration options for specialized use cases:
108
+
109
+ ### Search Engine Parameters
110
+
111
+ The `search_engine` function supports advanced parameters like:
112
+
113
+ - Language targeting (`language` parameter)
114
+ - Country-specific search (`country_code` parameter)
115
+ - Different search types (images, shopping, news, etc.)
116
+ - Pagination controls
117
+ - Mobile device emulation
118
+ - Geolocation targeting
119
+ - Hotel search parameters
120
+
121
+ ```python
122
+ results = brightdata_tool.search_engine(
123
+ query="best hotels in paris",
124
+ engine="google",
125
+ language="fr",
126
+ country_code="fr",
127
+ search_type="shopping",
128
+ device="mobile",
129
+ hotel_dates="2025-06-01,2025-06-05",
130
+ hotel_occupancy=2,
131
+ )
132
+ ```
133
+
134
+ ### Supported Web Data Sources
135
+
136
+ The `web_data_feed` function supports retrieving structured data from:
137
+
138
+ - LinkedIn (profiles and companies)
139
+ - Amazon (products and reviews)
140
+ - Instagram (profiles, posts, reels, comments)
141
+ - Facebook (posts, marketplace listings, company reviews)
142
+ - X/Twitter (posts)
143
+ - Zillow (property listings)
144
+ - Booking.com (hotel listings)
145
+ - YouTube (videos)
146
+ - ZoomInfo (company profiles)
147
+
148
+ For more information, visit the [Bright Data documentation](https://docs.brightdata.com/).
@@ -0,0 +1,3 @@
1
+ from llama_index.tools.brightdata.base import BrightDataToolSpec
2
+
3
+ __all__ = ["BrightDataToolSpec"]
@@ -0,0 +1,355 @@
1
+ """Bright Data tool spec for LlamaIndex."""
2
+
3
+ from typing import Dict, Optional
4
+ from llama_index.core.schema import Document
5
+ from llama_index.core.tools.tool_spec.base import BaseToolSpec
6
+
7
+
8
+ class BrightDataToolSpec(BaseToolSpec):
9
+ """Bright Data tool spec for web scraping and search capabilities."""
10
+
11
+ spec_functions = ["scrape_as_markdown", "get_screenshot", "search_engine","web_data_feed"]
12
+
13
+ def __init__(
14
+ self,
15
+ api_key: str,
16
+ zone: str = "unblocker",
17
+ verbose: bool = False,
18
+ ) -> None:
19
+ """
20
+ Initialize with API token and default zone.
21
+
22
+ Args:
23
+ api_key (str): Your Bright Data API token
24
+ zone (str): Bright Data zone name
25
+ verbose (bool): Print additional information about requests
26
+
27
+ """
28
+ self._headers = {
29
+ "Content-Type": "application/json",
30
+ "Authorization": f"Bearer {api_key}"
31
+ }
32
+ self._api_key = api_key
33
+ self._zone = zone
34
+ self._verbose = verbose
35
+ self._endpoint = "https://api.brightdata.com/request"
36
+
37
+ def _make_request(self, payload: Dict) -> str:
38
+ """
39
+ Make a request to Bright Data API.
40
+
41
+ Args:
42
+ payload (Dict): Request payload
43
+
44
+ Returns:
45
+ str: Response text
46
+
47
+ """
48
+ import requests
49
+ import json
50
+
51
+ if self._verbose:
52
+ print(f"[Bright Data] Request: {payload['url']}")
53
+
54
+ response = requests.post(
55
+ self._endpoint,
56
+ headers=self._headers,
57
+ data=json.dumps(payload)
58
+ )
59
+
60
+ if response.status_code != 200:
61
+ raise Exception(f"Failed to scrape: {response.status_code} - {response.text}")
62
+
63
+ return response.text
64
+
65
+ def scrape_as_markdown(self, url: str, zone: Optional[str] = None) -> Document:
66
+ """
67
+ Scrape a webpage and return content in Markdown format.
68
+
69
+ Args:
70
+ url (str): URL to scrape
71
+ zone (Optional[str]): Override default zone
72
+
73
+ Returns:
74
+ Document: Scraped content as Markdown
75
+
76
+ """
77
+ payload = {
78
+ "url": url,
79
+ "zone": zone or self._zone,
80
+ "format": "raw",
81
+ "data_format": "markdown"
82
+ }
83
+
84
+ content = self._make_request(payload)
85
+ return Document(text=content, metadata={"url": url})
86
+
87
+ def get_screenshot(self, url: str, output_path: str, zone: Optional[str] = None) -> str:
88
+ """
89
+ Take a screenshot of a webpage.
90
+
91
+ Args:
92
+ url (str): URL to screenshot
93
+ output_path (str): Path to save the screenshot
94
+ zone (Optional[str]): Override default zone
95
+
96
+ Returns:
97
+ str: Path to saved screenshot
98
+
99
+ """
100
+ import requests
101
+ import json
102
+
103
+ payload = {
104
+ "url": url,
105
+ "zone": zone or self._zone,
106
+ "format": "raw",
107
+ "data_format": "screenshot"
108
+ }
109
+
110
+ response = requests.post(
111
+ self._endpoint,
112
+ headers=self._headers,
113
+ data=json.dumps(payload)
114
+ )
115
+
116
+ if response.status_code != 200:
117
+ raise Exception(f"Error {response.status_code}: {response.text}")
118
+
119
+ with open(output_path, "wb") as f:
120
+ f.write(response.content)
121
+
122
+ return output_path
123
+
124
+ def search_engine(
125
+ self,
126
+ query: str,
127
+ engine: str = "google",
128
+ zone: Optional[str] = None,
129
+ language: Optional[str] = None, # hl parameter, e.g., "en"
130
+ country_code: Optional[str] = None, # gl parameter, e.g., "us"
131
+ search_type: Optional[str] = None, # tbm parameter (images, shopping, news, etc.)
132
+ start: Optional[int] = None, # pagination start index
133
+ num_results: Optional[int] = 10, # number of results to return
134
+ location: Optional[str] = None, # uule parameter for geo-location
135
+ device: Optional[str] = None, # device type for user-agent
136
+ return_json: bool = False, # parse results as JSON
137
+ hotel_dates: Optional[str] = None, # check-in and check-out dates
138
+ hotel_occupancy: Optional[int] = None # number of guests
139
+ ) -> Document:
140
+ """
141
+ Search using Google, Bing, or Yandex with advanced parameters and return results in Markdown.
142
+
143
+ Args:
144
+ query (str): Search query
145
+ engine (str): Search engine - 'google', 'bing', or 'yandex'
146
+ zone (Optional[str]): Override default zone
147
+
148
+ # Google SERP specific parameters
149
+ language (Optional[str]): Two-letter language code (hl parameter)
150
+ country_code (Optional[str]): Two-letter country code (gl parameter)
151
+ search_type (Optional[str]): Type of search (images, shopping, news, etc.)
152
+ start (Optional[int]): Results pagination offset (0=first page, 10=second page)
153
+ num_results (Optional[int]): Number of results to return (default 10)
154
+ location (Optional[str]): Location for search results (uule parameter)
155
+ device (Optional[str]): Device type (mobile, ios, android, ipad, android_tablet)
156
+ return_json (bool): Return parsed JSON instead of HTML/Markdown
157
+
158
+ # Hotel search parameters
159
+ hotel_dates (Optional[str]): Check-in and check-out dates (format: YYYY-MM-DD,YYYY-MM-DD)
160
+ hotel_occupancy (Optional[int]): Number of guests (1-4)
161
+
162
+ Returns:
163
+ Document: Search results as Markdown or JSON
164
+
165
+ """
166
+ encoded_query = self._encode_query(query)
167
+
168
+ base_urls = {
169
+ "google": f"https://www.google.com/search?q={encoded_query}",
170
+ "bing": f"https://www.bing.com/search?q={encoded_query}",
171
+ "yandex": f"https://yandex.com/search/?text={encoded_query}"
172
+ }
173
+
174
+ if engine not in base_urls:
175
+ raise ValueError(f"Unsupported search engine: {engine}. Use 'google', 'bing', or 'yandex'")
176
+
177
+ search_url = base_urls[engine]
178
+
179
+ if engine == "google":
180
+ params = []
181
+
182
+ if language:
183
+ params.append(f"hl={language}")
184
+
185
+ if country_code:
186
+ params.append(f"gl={country_code}")
187
+
188
+ if search_type:
189
+ if search_type == "jobs":
190
+ params.append("ibp=htl;jobs")
191
+ else:
192
+ search_types = {
193
+ "images": "isch",
194
+ "shopping": "shop",
195
+ "news": "nws"
196
+ }
197
+ tbm_value = search_types.get(search_type, search_type)
198
+ params.append(f"tbm={tbm_value}")
199
+
200
+ if start is not None:
201
+ params.append(f"start={start}")
202
+
203
+ if num_results:
204
+ params.append(f"num={num_results}")
205
+
206
+ if location:
207
+ params.append(f"uule={self._encode_query(location)}")
208
+
209
+ if device:
210
+ device_value = "1"
211
+
212
+ if device in ["ios", "iphone"]:
213
+ device_value = "ios"
214
+ elif device == "ipad":
215
+ device_value = "ios_tablet"
216
+ elif device == "android":
217
+ device_value = "android"
218
+ elif device == "android_tablet":
219
+ device_value = "android_tablet"
220
+
221
+ params.append(f"brd_mobile={device_value}")
222
+
223
+ if return_json:
224
+ params.append("brd_json=1")
225
+
226
+ if hotel_dates:
227
+ params.append(f"hotel_dates={self._encode_query(hotel_dates)}")
228
+
229
+ if hotel_occupancy:
230
+ params.append(f"hotel_occupancy={hotel_occupancy}")
231
+
232
+ if params:
233
+ search_url += "&" + "&".join(params)
234
+
235
+ payload = {
236
+ "url": search_url,
237
+ "zone": zone or self._zone,
238
+ "format": "raw",
239
+ "data_format": "markdown" if not return_json else "raw"
240
+ }
241
+
242
+ content = self._make_request(payload)
243
+ return Document(text=content, metadata={
244
+ "query": query,
245
+ "engine": engine,
246
+ "url": search_url
247
+ })
248
+
249
+ def web_data_feed(
250
+ self,
251
+ source_type: str,
252
+ url: str,
253
+ num_of_reviews: Optional[int] = None,
254
+ timeout: int = 600,
255
+ polling_interval: int = 1
256
+ ) -> Dict:
257
+ """
258
+ Retrieve structured web data from various sources like LinkedIn, Amazon, Instagram, etc.
259
+
260
+ Args:
261
+ source_type (str): Type of data source (e.g., 'linkedin_person_profile', 'amazon_product')
262
+ url (str): URL of the web resource to retrieve data from
263
+ num_of_reviews (Optional[int]): Number of reviews to retrieve (only for facebook_company_reviews)
264
+ timeout (int): Maximum time in seconds to wait for data retrieval
265
+ polling_interval (int): Time in seconds between polling attempts
266
+
267
+ Returns:
268
+ Dict: Structured data from the requested source
269
+
270
+ """
271
+ import requests
272
+ import time
273
+
274
+ datasets = {
275
+ "amazon_product": "gd_l7q7dkf244hwjntr0",
276
+ "amazon_product_reviews": "gd_le8e811kzy4ggddlq",
277
+ "linkedin_person_profile": "gd_l1viktl72bvl7bjuj0",
278
+ "linkedin_company_profile": "gd_l1vikfnt1wgvvqz95w",
279
+ "zoominfo_company_profile": "gd_m0ci4a4ivx3j5l6nx",
280
+ "instagram_profiles": "gd_l1vikfch901nx3by4",
281
+ "instagram_posts": "gd_lk5ns7kz21pck8jpis",
282
+ "instagram_reels": "gd_lyclm20il4r5helnj",
283
+ "instagram_comments": "gd_ltppn085pokosxh13",
284
+ "facebook_posts": "gd_lyclm1571iy3mv57zw",
285
+ "facebook_marketplace_listings": "gd_lvt9iwuh6fbcwmx1a",
286
+ "facebook_company_reviews": "gd_m0dtqpiu1mbcyc2g86",
287
+ "x_posts": "gd_lwxkxvnf1cynvib9co",
288
+ "zillow_properties_listing": "gd_lfqkr8wm13ixtbd8f5",
289
+ "booking_hotel_listings": "gd_m5mbdl081229ln6t4a",
290
+ "youtube_videos": "gd_m5mbdl081229ln6t4a"
291
+ }
292
+
293
+ if source_type not in datasets:
294
+ valid_sources = ", ".join(datasets.keys())
295
+ raise ValueError(f"Invalid source_type: {source_type}. Valid options are: {valid_sources}")
296
+
297
+ dataset_id = datasets[source_type]
298
+
299
+ request_data = {"url": url}
300
+ if source_type == "facebook_company_reviews" and num_of_reviews is not None:
301
+ request_data["num_of_reviews"] = str(num_of_reviews)
302
+
303
+ trigger_response = requests.post(
304
+ "https://api.brightdata.com/datasets/v3/trigger",
305
+ params={"dataset_id": dataset_id, "include_errors": True},
306
+ headers=self._headers,
307
+ json=[request_data]
308
+ )
309
+
310
+ trigger_data = trigger_response.json()
311
+ if not trigger_data.get("snapshot_id"):
312
+ raise Exception("No snapshot ID returned from trigger request")
313
+
314
+ snapshot_id = trigger_data["snapshot_id"]
315
+ if self._verbose:
316
+ print(f"[Bright Data] {source_type} triggered with snapshot ID: {snapshot_id}")
317
+
318
+ attempts = 0
319
+ max_attempts = timeout
320
+
321
+ while attempts < max_attempts:
322
+ try:
323
+ snapshot_response = requests.get(
324
+ f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}",
325
+ params={"format": "json"},
326
+ headers=self._headers
327
+ )
328
+
329
+ snapshot_data = snapshot_response.json()
330
+
331
+ if isinstance(snapshot_data, dict) and snapshot_data.get("status") == "running":
332
+ if self._verbose:
333
+ print(f"[Bright Data] Snapshot not ready, polling again (attempt {attempts + 1}/{max_attempts})")
334
+ attempts += 1
335
+ time.sleep(polling_interval)
336
+ continue
337
+
338
+ if self._verbose:
339
+ print(f"[Bright Data] Data received after {attempts + 1} attempts")
340
+
341
+ return snapshot_data
342
+
343
+ except Exception as e:
344
+ if self._verbose:
345
+ print(f"[Bright Data] Polling error: {e!s}")
346
+ attempts += 1
347
+ time.sleep(polling_interval)
348
+
349
+ raise TimeoutError(f"Timeout after {max_attempts} seconds waiting for {source_type} data")
350
+
351
+ @staticmethod
352
+ def _encode_query(query: str) -> str:
353
+ """URL encode a search query."""
354
+ from urllib.parse import quote
355
+ return quote(query)
@@ -0,0 +1,65 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [dependency-groups]
6
+ dev = [
7
+ "ipython==8.10.0",
8
+ "jupyter>=1.0.0,<2",
9
+ "mypy==0.991",
10
+ "pre-commit==3.2.0",
11
+ "pylint==2.15.10",
12
+ "pytest==7.2.1",
13
+ "pytest-mock==3.11.1",
14
+ "ruff==0.0.292",
15
+ "types-Deprecated>=0.1.0",
16
+ "types-PyYAML>=6.0.12.12,<7",
17
+ "types-protobuf>=4.24.0.4,<5",
18
+ "types-redis==4.5.5.0",
19
+ "types-requests==2.28.11.8",
20
+ "types-setuptools==67.1.0.0",
21
+ "black[jupyter]<=23.9.1,>=23.7.0",
22
+ "codespell[toml]>=v2.2.6",
23
+ "diff-cover>=9.2.0",
24
+ "pytest-cov>=6.1.1",
25
+ ]
26
+
27
+ [project]
28
+ name = "llama-index-tools-brightdata"
29
+ version = "0.1.0"
30
+ description = "llama-index tools Bright Data integration"
31
+ authors = [{name = "Bright Data", email = "you@example.com"}]
32
+ requires-python = ">=3.9,<4.0"
33
+ readme = "README.md"
34
+ license = "MIT"
35
+ maintainers = [{name = "brightdata"}]
36
+ dependencies = [
37
+ "llama-index>=0.12.34",
38
+ "llama-index-core>=0.12.0,<0.13",
39
+ ]
40
+
41
+ [tool.codespell]
42
+ check-filenames = true
43
+ check-hidden = true
44
+ skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
45
+
46
+ [tool.hatch.build.targets.sdist]
47
+ include = ["llama_index/"]
48
+ exclude = ["**/BUILD"]
49
+
50
+ [tool.hatch.build.targets.wheel]
51
+ include = ["llama_index/"]
52
+ exclude = ["**/BUILD"]
53
+
54
+ [tool.llamahub]
55
+ contains_example = false
56
+ import_path = "llama_index.tools.brightdata"
57
+
58
+ [tool.llamahub.class_authors]
59
+ BrightDataToolSpec = "brightdata"
60
+
61
+ [tool.mypy]
62
+ disallow_untyped_defs = true
63
+ exclude = ["_static", "build", "examples", "notebooks", "venv"]
64
+ ignore_missing_imports = true
65
+ python_version = "3.8"