hermes-brightdata 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hermes_brightdata-0.1.0/LICENSE +21 -0
- hermes_brightdata-0.1.0/PKG-INFO +132 -0
- hermes_brightdata-0.1.0/README.md +112 -0
- hermes_brightdata-0.1.0/brightdata_plugin/__init__.py +62 -0
- hermes_brightdata-0.1.0/brightdata_plugin/api.py +109 -0
- hermes_brightdata-0.1.0/brightdata_plugin/browser.py +117 -0
- hermes_brightdata-0.1.0/brightdata_plugin/config.py +35 -0
- hermes_brightdata-0.1.0/brightdata_plugin/counter.py +18 -0
- hermes_brightdata-0.1.0/brightdata_plugin/datasets.py +27 -0
- hermes_brightdata-0.1.0/brightdata_plugin/schemas.py +133 -0
- hermes_brightdata-0.1.0/brightdata_plugin/tools.py +168 -0
- hermes_brightdata-0.1.0/hermes_brightdata.egg-info/PKG-INFO +132 -0
- hermes_brightdata-0.1.0/hermes_brightdata.egg-info/SOURCES.txt +30 -0
- hermes_brightdata-0.1.0/hermes_brightdata.egg-info/dependency_links.txt +1 -0
- hermes_brightdata-0.1.0/hermes_brightdata.egg-info/entry_points.txt +2 -0
- hermes_brightdata-0.1.0/hermes_brightdata.egg-info/requires.txt +9 -0
- hermes_brightdata-0.1.0/hermes_brightdata.egg-info/top_level.txt +1 -0
- hermes_brightdata-0.1.0/pyproject.toml +27 -0
- hermes_brightdata-0.1.0/setup.cfg +4 -0
- hermes_brightdata-0.1.0/tests/test_api_dataset.py +87 -0
- hermes_brightdata-0.1.0/tests/test_api_request.py +57 -0
- hermes_brightdata-0.1.0/tests/test_browser.py +141 -0
- hermes_brightdata-0.1.0/tests/test_config.py +60 -0
- hermes_brightdata-0.1.0/tests/test_counter.py +19 -0
- hermes_brightdata-0.1.0/tests/test_datasets.py +26 -0
- hermes_brightdata-0.1.0/tests/test_handler_contract.py +40 -0
- hermes_brightdata-0.1.0/tests/test_live_smoke.py +23 -0
- hermes_brightdata-0.1.0/tests/test_manifest.py +17 -0
- hermes_brightdata-0.1.0/tests/test_register.py +89 -0
- hermes_brightdata-0.1.0/tests/test_schemas.py +30 -0
- hermes_brightdata-0.1.0/tests/test_tools_browser.py +71 -0
- hermes_brightdata-0.1.0/tests/test_tools_core.py +137 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dante Labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hermes-brightdata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bright Data web data tools for Hermes Agent — search, scrape, structured datasets, and browser automation
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/dandacompany/hermes-brightdata-plugin
|
|
7
|
+
Project-URL: Repository, https://github.com/dandacompany/hermes-brightdata-plugin
|
|
8
|
+
Keywords: hermes,bright-data,brightdata,web-scraping,plugin,agent
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: requests>=2.31
|
|
13
|
+
Provides-Extra: browser
|
|
14
|
+
Requires-Dist: playwright>=1.40; extra == "browser"
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
17
|
+
Requires-Dist: responses>=0.25; extra == "dev"
|
|
18
|
+
Requires-Dist: pyyaml>=6; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# hermes-brightdata
|
|
22
|
+
|
|
23
|
+
[](https://github.com/dandacompany/hermes-brightdata-plugin/actions/workflows/ci.yml)
|
|
24
|
+
[](https://pypi.org/project/hermes-brightdata/)
|
|
25
|
+
[](https://pypi.org/project/hermes-brightdata/)
|
|
26
|
+
[](LICENSE)
|
|
27
|
+
|
|
28
|
+
Bright Data web data tools for Hermes Agent — search, scrape, structured datasets, and browser automation.
|
|
29
|
+
|
|
30
|
+
## Overview
|
|
31
|
+
|
|
32
|
+
This plugin integrates Bright Data's web data APIs with Hermes, providing nine powerful tools for web data collection:
|
|
33
|
+
|
|
34
|
+
- **Search & Scraping**: Real-time search, single/batch URL scraping, and structured data extraction
|
|
35
|
+
- **Browser Automation**: Interactive browser sessions with persistent cookies and JavaScript rendering
|
|
36
|
+
- **Bypass Detection**: Automatic handling of bot detection, CAPTCHAs, and JavaScript-heavy sites
|
|
37
|
+
|
|
38
|
+
Bright Data handles all the complexity — proxy rotation, header management, retry logic, and content parsing.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
Install the plugin with base tools:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install hermes-brightdata
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
To enable browser automation tools, install with the `[browser]` extra:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install hermes-brightdata[browser]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The browser extra adds Playwright for interactive session handling (required for `browser_navigate`, `browser_snapshot`, `browser_act`, `browser_get`).
|
|
55
|
+
|
|
56
|
+
## Configuration
|
|
57
|
+
|
|
58
|
+
Before using any Bright Data tool, set your API credentials in environment variables:
|
|
59
|
+
|
|
60
|
+
### Required
|
|
61
|
+
|
|
62
|
+
`BRIGHTDATA_API_TOKEN` (required, secret)
|
|
63
|
+
|
|
64
|
+
- Your Bright Data API token
|
|
65
|
+
- Obtain from Account Settings → API keys in the Bright Data dashboard
|
|
66
|
+
- `BRIGHTDATA_API_KEY` is accepted as an alias if `BRIGHTDATA_API_TOKEN` is unset
|
|
67
|
+
|
|
68
|
+
### Optional
|
|
69
|
+
|
|
70
|
+
`BRIGHTDATA_WEB_UNLOCKER_ZONE` (optional)
|
|
71
|
+
|
|
72
|
+
- Zone name for Web Unlocker API
|
|
73
|
+
- Default: `web_unlocker1`
|
|
74
|
+
- `WEB_UNLOCKER_ZONE` is accepted as an alias if the prefixed name is unset
|
|
75
|
+
|
|
76
|
+
`BRIGHTDATA_SERP_ZONE` (optional)
|
|
77
|
+
|
|
78
|
+
- Zone name for SERP (Search Engine Results Page) API
|
|
79
|
+
- Default: `serp_api1`
|
|
80
|
+
|
|
81
|
+
`BRIGHTDATA_BROWSER_AUTH` (optional, secret)
|
|
82
|
+
|
|
83
|
+
- Authentication credentials for Scraping Browser
|
|
84
|
+
- Format: `brd-customer-<customer-id>-zone-<zone-name>:<password>`
|
|
85
|
+
- Required only if using browser tools with authentication
|
|
86
|
+
|
|
87
|
+
### Example
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
export BRIGHTDATA_API_TOKEN="your-api-token-here"
|
|
91
|
+
export BRIGHTDATA_BROWSER_AUTH="brd-customer-12345-zone-myzone:password123"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Tools
|
|
95
|
+
|
|
96
|
+
The plugin provides nine tools:
|
|
97
|
+
|
|
98
|
+
| Tool | Description |
|
|
99
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
100
|
+
| `search_engine` | Search the web via Bright Data SERP API and return parsed results. Use for current events, fact-checking, finding pages, or any web search. Bypasses bot detection; more reliable than built-in web search. |
|
|
101
|
+
| `scrape` | Fetch a single URL as clean markdown (or html) using Bright Data Web Unlocker. Handles JavaScript, CAPTCHAs, and bot detection. Use to read any webpage, article, doc, or site. |
|
|
102
|
+
| `scrape_batch` | Scrape multiple URLs in one call (max 20). Returns per-URL content or error. Use when you have several pages to read at once. |
|
|
103
|
+
| `web_data` | Get structured JSON data from a supported platform via Bright Data Web Scraper API (e.g. an Amazon product page, a LinkedIn profile). Returns clean structured fields, not raw HTML. May take up to a minute. |
|
|
104
|
+
| `session_stats` | Report Bright Data tool call counts for this session. |
|
|
105
|
+
| `browser_navigate` | Open a URL in a Bright Data Scraping Browser (real browser with a persistent session). Use for interactive flows requiring clicks/typing. Requires the [browser] extra and BRIGHTDATA_BROWSER_AUTH. |
|
|
106
|
+
| `browser_snapshot` | Return an accessibility-tree snapshot of the current browser page, including element refs to use with browser_act. |
|
|
107
|
+
| `browser_act` | Perform an action on the current browser page: click, type, scroll, or wait. Get element refs from browser_snapshot first. |
|
|
108
|
+
| `browser_get` | Read the current browser page: html, text, or a base64 screenshot. |
|
|
109
|
+
|
|
110
|
+
## Activation
|
|
111
|
+
|
|
112
|
+
Enable the plugin in Hermes:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
hermes plugins enable brightdata
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Verify it's loaded:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
hermes plugins list
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
You should see `brightdata` listed with version `0.1.0`.
|
|
125
|
+
|
|
126
|
+
## Pricing
|
|
127
|
+
|
|
128
|
+
Bright Data offers a free tier with up to 5,000 requests per month. For higher volume, refer to Bright Data's pricing page for details on Web Unlocker, SERP API, Web Scraper API, and Scraping Browser plans.
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
For more information on Bright Data services, visit [https://brightdata.com](https://brightdata.com).
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# hermes-brightdata
|
|
2
|
+
|
|
3
|
+
[](https://github.com/dandacompany/hermes-brightdata-plugin/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/hermes-brightdata/)
|
|
5
|
+
[](https://pypi.org/project/hermes-brightdata/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
Bright Data web data tools for Hermes Agent — search, scrape, structured datasets, and browser automation.
|
|
9
|
+
|
|
10
|
+
## Overview
|
|
11
|
+
|
|
12
|
+
This plugin integrates Bright Data's web data APIs with Hermes, providing nine powerful tools for web data collection:
|
|
13
|
+
|
|
14
|
+
- **Search & Scraping**: Real-time search, single/batch URL scraping, and structured data extraction
|
|
15
|
+
- **Browser Automation**: Interactive browser sessions with persistent cookies and JavaScript rendering
|
|
16
|
+
- **Bypass Detection**: Automatic handling of bot detection, CAPTCHAs, and JavaScript-heavy sites
|
|
17
|
+
|
|
18
|
+
Bright Data handles all the complexity — proxy rotation, header management, retry logic, and content parsing.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
Install the plugin with base tools:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install hermes-brightdata
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
To enable browser automation tools, install with the `[browser]` extra:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install hermes-brightdata[browser]
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The browser extra adds Playwright for interactive session handling (required for `browser_navigate`, `browser_snapshot`, `browser_act`, `browser_get`).
|
|
35
|
+
|
|
36
|
+
## Configuration
|
|
37
|
+
|
|
38
|
+
Before using any Bright Data tool, set your API credentials in environment variables:
|
|
39
|
+
|
|
40
|
+
### Required
|
|
41
|
+
|
|
42
|
+
`BRIGHTDATA_API_TOKEN` (required, secret)
|
|
43
|
+
|
|
44
|
+
- Your Bright Data API token
|
|
45
|
+
- Obtain from Account Settings → API keys in the Bright Data dashboard
|
|
46
|
+
- `BRIGHTDATA_API_KEY` is accepted as an alias if `BRIGHTDATA_API_TOKEN` is unset
|
|
47
|
+
|
|
48
|
+
### Optional
|
|
49
|
+
|
|
50
|
+
`BRIGHTDATA_WEB_UNLOCKER_ZONE` (optional)
|
|
51
|
+
|
|
52
|
+
- Zone name for Web Unlocker API
|
|
53
|
+
- Default: `web_unlocker1`
|
|
54
|
+
- `WEB_UNLOCKER_ZONE` is accepted as an alias if the prefixed name is unset
|
|
55
|
+
|
|
56
|
+
`BRIGHTDATA_SERP_ZONE` (optional)
|
|
57
|
+
|
|
58
|
+
- Zone name for SERP (Search Engine Results Page) API
|
|
59
|
+
- Default: `serp_api1`
|
|
60
|
+
|
|
61
|
+
`BRIGHTDATA_BROWSER_AUTH` (optional, secret)
|
|
62
|
+
|
|
63
|
+
- Authentication credentials for Scraping Browser
|
|
64
|
+
- Format: `brd-customer-<customer-id>-zone-<zone-name>:<password>`
|
|
65
|
+
- Required only if using browser tools with authentication
|
|
66
|
+
|
|
67
|
+
### Example
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
export BRIGHTDATA_API_TOKEN="your-api-token-here"
|
|
71
|
+
export BRIGHTDATA_BROWSER_AUTH="brd-customer-12345-zone-myzone:password123"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Tools
|
|
75
|
+
|
|
76
|
+
The plugin provides nine tools:
|
|
77
|
+
|
|
78
|
+
| Tool | Description |
|
|
79
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
80
|
+
| `search_engine` | Search the web via Bright Data SERP API and return parsed results. Use for current events, fact-checking, finding pages, or any web search. Bypasses bot detection; more reliable than built-in web search. |
|
|
81
|
+
| `scrape` | Fetch a single URL as clean markdown (or html) using Bright Data Web Unlocker. Handles JavaScript, CAPTCHAs, and bot detection. Use to read any webpage, article, doc, or site. |
|
|
82
|
+
| `scrape_batch` | Scrape multiple URLs in one call (max 20). Returns per-URL content or error. Use when you have several pages to read at once. |
|
|
83
|
+
| `web_data` | Get structured JSON data from a supported platform via Bright Data Web Scraper API (e.g. an Amazon product page, a LinkedIn profile). Returns clean structured fields, not raw HTML. May take up to a minute. |
|
|
84
|
+
| `session_stats` | Report Bright Data tool call counts for this session. |
|
|
85
|
+
| `browser_navigate` | Open a URL in a Bright Data Scraping Browser (real browser with a persistent session). Use for interactive flows requiring clicks/typing. Requires the [browser] extra and BRIGHTDATA_BROWSER_AUTH. |
|
|
86
|
+
| `browser_snapshot` | Return an accessibility-tree snapshot of the current browser page, including element refs to use with browser_act. |
|
|
87
|
+
| `browser_act` | Perform an action on the current browser page: click, type, scroll, or wait. Get element refs from browser_snapshot first. |
|
|
88
|
+
| `browser_get` | Read the current browser page: html, text, or a base64 screenshot. |
|
|
89
|
+
|
|
90
|
+
## Activation
|
|
91
|
+
|
|
92
|
+
Enable the plugin in Hermes:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
hermes plugins enable brightdata
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Verify it's loaded:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
hermes plugins list
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
You should see `brightdata` listed with version `0.1.0`.
|
|
105
|
+
|
|
106
|
+
## Pricing
|
|
107
|
+
|
|
108
|
+
Bright Data offers a free tier with up to 5,000 requests per month. For higher volume, refer to Bright Data's pricing page for details on Web Unlocker, SERP API, Web Scraper API, and Scraping Browser plans.
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
For more information on Bright Data services, visit [https://brightdata.com](https://brightdata.com).
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from . import schemas
|
|
6
|
+
from .api import BrightDataClient
|
|
7
|
+
from .browser import BrowserSession
|
|
8
|
+
from .config import Config, load_config
|
|
9
|
+
from .counter import SessionCounter
|
|
10
|
+
from .tools import make_browser_handlers, make_core_handlers
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def build_handlers(cfg: Config | None = None) -> dict:
|
|
14
|
+
cfg = cfg or load_config()
|
|
15
|
+
counter = SessionCounter()
|
|
16
|
+
|
|
17
|
+
_client: dict = {}
|
|
18
|
+
def get_client() -> BrightDataClient:
|
|
19
|
+
if "c" not in _client:
|
|
20
|
+
_client["c"] = BrightDataClient(cfg)
|
|
21
|
+
return _client["c"]
|
|
22
|
+
|
|
23
|
+
_session: dict = {}
|
|
24
|
+
def get_session() -> BrowserSession:
|
|
25
|
+
if "s" not in _session:
|
|
26
|
+
_session["s"] = BrowserSession(cfg)
|
|
27
|
+
return _session["s"]
|
|
28
|
+
|
|
29
|
+
handlers = {}
|
|
30
|
+
handlers.update(make_core_handlers(get_client, counter))
|
|
31
|
+
handlers.update(make_browser_handlers(get_session, counter))
|
|
32
|
+
# private keys popped by register(); not real tools
|
|
33
|
+
handlers["_counter"] = counter # type: ignore
|
|
34
|
+
handlers["_get_session"] = get_session # type: ignore
|
|
35
|
+
handlers["_session_holder"] = _session # type: ignore
|
|
36
|
+
return handlers
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def register(ctx) -> None:
|
|
40
|
+
cfg = load_config()
|
|
41
|
+
handlers = build_handlers(cfg)
|
|
42
|
+
counter = handlers.pop("_counter")
|
|
43
|
+
handlers.pop("_get_session")
|
|
44
|
+
session_holder = handlers.pop("_session_holder")
|
|
45
|
+
|
|
46
|
+
for name, handler in handlers.items():
|
|
47
|
+
ctx.register_tool(name, "brightdata", schemas.TOOL_SCHEMAS[name], handler)
|
|
48
|
+
|
|
49
|
+
def brightdata_cmd(args, **kwargs) -> str:
|
|
50
|
+
return json.dumps(counter.stats(), ensure_ascii=False)
|
|
51
|
+
ctx.register_command("brightdata", brightdata_cmd,
|
|
52
|
+
description="Show Bright Data session usage stats")
|
|
53
|
+
|
|
54
|
+
def on_session_end(*args, **kwargs):
|
|
55
|
+
# only close if a browser session was actually created this session
|
|
56
|
+
s = session_holder.get("s")
|
|
57
|
+
if s is not None:
|
|
58
|
+
try:
|
|
59
|
+
s.close()
|
|
60
|
+
except Exception: # noqa: BLE001
|
|
61
|
+
pass
|
|
62
|
+
ctx.register_hook("on_session_end", on_session_end)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from .config import Config
|
|
8
|
+
|
|
9
|
+
BASE_URL = "https://api.brightdata.com"
|
|
10
|
+
REQUEST_ENDPOINT = f"{BASE_URL}/request"
|
|
11
|
+
DATASET_TRIGGER = f"{BASE_URL}/datasets/v3/trigger"
|
|
12
|
+
DATASET_PROGRESS = f"{BASE_URL}/datasets/v3/progress"
|
|
13
|
+
DATASET_SNAPSHOT = f"{BASE_URL}/datasets/v3/snapshot"
|
|
14
|
+
|
|
15
|
+
_HINTS = {
|
|
16
|
+
401: "API token invalid or missing — check BRIGHTDATA_API_TOKEN.",
|
|
17
|
+
402: "Free quota (5,000 req/month) may be exhausted — check billing.",
|
|
18
|
+
429: "Rate limited or quota exhausted — retry later or raise plan limit.",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BrightDataError(Exception):
|
|
23
|
+
def __init__(self, message: str, status: int, hint: str = ""):
|
|
24
|
+
super().__init__(message)
|
|
25
|
+
self.status = status
|
|
26
|
+
self.hint = hint or _HINTS.get(status, "Unexpected Bright Data API error.")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class BrightDataClient:
|
|
30
|
+
def __init__(self, cfg: Config, timeout: int = 60):
|
|
31
|
+
self._cfg = cfg
|
|
32
|
+
self._timeout = timeout
|
|
33
|
+
self._session = requests.Session()
|
|
34
|
+
self._session.headers.update({
|
|
35
|
+
"Authorization": f"Bearer {cfg.token}",
|
|
36
|
+
"Content-Type": "application/json",
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
def _post_request(self, payload: dict) -> requests.Response:
|
|
40
|
+
resp = self._session.post(REQUEST_ENDPOINT, json=payload, timeout=self._timeout)
|
|
41
|
+
if resp.status_code >= 400:
|
|
42
|
+
raise BrightDataError(
|
|
43
|
+
f"Bright Data /request failed: {resp.status_code}",
|
|
44
|
+
status=resp.status_code,
|
|
45
|
+
)
|
|
46
|
+
return resp
|
|
47
|
+
|
|
48
|
+
def unlock(self, url: str, data_format: str = "markdown", render: bool = False) -> str:
|
|
49
|
+
payload = {
|
|
50
|
+
"zone": self._cfg.unlocker_zone,
|
|
51
|
+
"url": url,
|
|
52
|
+
"format": "raw",
|
|
53
|
+
"data_format": data_format,
|
|
54
|
+
}
|
|
55
|
+
if render:
|
|
56
|
+
payload["render"] = True
|
|
57
|
+
return self._post_request(payload).text
|
|
58
|
+
|
|
59
|
+
def serp(self, search_url: str, parse_json: bool = True) -> str:
|
|
60
|
+
url = search_url
|
|
61
|
+
if parse_json and "brd_json=" not in url:
|
|
62
|
+
sep = "&" if "?" in url else "?"
|
|
63
|
+
url = f"{url}{sep}brd_json=1"
|
|
64
|
+
payload = {"zone": self._cfg.serp_zone, "url": url, "format": "raw"}
|
|
65
|
+
return self._post_request(payload).text
|
|
66
|
+
|
|
67
|
+
def trigger_dataset(self, dataset_id: str, urls: list[str]) -> str:
|
|
68
|
+
body = [{"url": u} for u in urls]
|
|
69
|
+
resp = self._session.post(
|
|
70
|
+
DATASET_TRIGGER, params={"dataset_id": dataset_id},
|
|
71
|
+
json=body, timeout=self._timeout,
|
|
72
|
+
)
|
|
73
|
+
if resp.status_code >= 400:
|
|
74
|
+
raise BrightDataError(
|
|
75
|
+
f"trigger failed: {resp.status_code}", status=resp.status_code)
|
|
76
|
+
return resp.json()["snapshot_id"]
|
|
77
|
+
|
|
78
|
+
def poll_snapshot(self, snapshot_id: str) -> str:
|
|
79
|
+
resp = self._session.get(
|
|
80
|
+
f"{DATASET_PROGRESS}/{snapshot_id}", timeout=self._timeout)
|
|
81
|
+
if resp.status_code >= 400:
|
|
82
|
+
raise BrightDataError(
|
|
83
|
+
f"progress failed: {resp.status_code}", status=resp.status_code)
|
|
84
|
+
return resp.json()["status"]
|
|
85
|
+
|
|
86
|
+
def download_snapshot(self, snapshot_id: str, fmt: str = "json") -> str:
|
|
87
|
+
resp = self._session.get(
|
|
88
|
+
f"{DATASET_SNAPSHOT}/{snapshot_id}", params={"format": fmt},
|
|
89
|
+
timeout=self._timeout,
|
|
90
|
+
)
|
|
91
|
+
if resp.status_code >= 400:
|
|
92
|
+
raise BrightDataError(
|
|
93
|
+
f"snapshot failed: {resp.status_code}", status=resp.status_code)
|
|
94
|
+
return resp.text
|
|
95
|
+
|
|
96
|
+
def collect_dataset(self, dataset_id: str, urls: list[str],
|
|
97
|
+
poll_interval: float = 2.0, max_wait: float = 60.0,
|
|
98
|
+
sleep=time.sleep, now=time.monotonic) -> dict:
|
|
99
|
+
snapshot_id = self.trigger_dataset(dataset_id, urls)
|
|
100
|
+
start = now()
|
|
101
|
+
while now() - start <= max_wait:
|
|
102
|
+
status = self.poll_snapshot(snapshot_id)
|
|
103
|
+
if status == "ready":
|
|
104
|
+
return {"status": "ready",
|
|
105
|
+
"data": self.download_snapshot(snapshot_id)}
|
|
106
|
+
if status == "failed":
|
|
107
|
+
return {"status": "failed", "snapshot_id": snapshot_id}
|
|
108
|
+
sleep(poll_interval)
|
|
109
|
+
return {"status": "timeout", "snapshot_id": snapshot_id}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
|
|
5
|
+
from .config import Config
|
|
6
|
+
|
|
7
|
+
CDP_HOST = "brd.superproxy.io:9222"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class BrowserUnavailable(Exception):
|
|
11
|
+
def __init__(self, message: str, hint: str = ""):
|
|
12
|
+
super().__init__(message)
|
|
13
|
+
self.hint = hint
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _default_connector(cfg: Config):
|
|
17
|
+
def connect(cdp_url: str):
|
|
18
|
+
if cfg.browser_auth is None:
|
|
19
|
+
raise BrowserUnavailable(
|
|
20
|
+
"browser auth not configured",
|
|
21
|
+
"set BRIGHTDATA_BROWSER_AUTH to 'brd-customer-<id>-zone-<zone>:<pw>'")
|
|
22
|
+
try:
|
|
23
|
+
from playwright.sync_api import sync_playwright
|
|
24
|
+
except ImportError as e:
|
|
25
|
+
raise BrowserUnavailable(
|
|
26
|
+
"playwright not installed",
|
|
27
|
+
"install with: pip install hermes-brightdata[browser]") from e
|
|
28
|
+
pw = sync_playwright().start()
|
|
29
|
+
try:
|
|
30
|
+
browser_obj = pw.chromium.connect_over_cdp(cdp_url)
|
|
31
|
+
page = browser_obj.new_page()
|
|
32
|
+
except BrowserUnavailable:
|
|
33
|
+
pw.stop()
|
|
34
|
+
raise
|
|
35
|
+
except Exception:
|
|
36
|
+
pw.stop()
|
|
37
|
+
# never surface the raw exception: it can embed the CDP URL,
|
|
38
|
+
# which contains the browser_auth password.
|
|
39
|
+
raise BrowserUnavailable(
|
|
40
|
+
"browser connection failed",
|
|
41
|
+
"check BRIGHTDATA_BROWSER_AUTH and that the Scraping Browser zone is active",
|
|
42
|
+
) from None
|
|
43
|
+
page._bd_pw = pw # keep ref for teardown
|
|
44
|
+
page._bd_browser = browser_obj
|
|
45
|
+
return page
|
|
46
|
+
return connect
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class BrowserSession:
|
|
50
|
+
def __init__(self, cfg: Config, connector=None):
|
|
51
|
+
self._cfg = cfg
|
|
52
|
+
self._connector = connector or _default_connector(cfg)
|
|
53
|
+
self._page = None
|
|
54
|
+
|
|
55
|
+
def _cdp_url(self) -> str:
|
|
56
|
+
return f"wss://{self._cfg.browser_auth}@{CDP_HOST}"
|
|
57
|
+
|
|
58
|
+
def _ensure_page(self):
|
|
59
|
+
if self._page is None:
|
|
60
|
+
self._page = self._connector(self._cdp_url())
|
|
61
|
+
return self._page
|
|
62
|
+
|
|
63
|
+
def navigate(self, url: str) -> dict:
|
|
64
|
+
page = self._ensure_page()
|
|
65
|
+
page.goto(url)
|
|
66
|
+
return {"url": url, "ok": True}
|
|
67
|
+
|
|
68
|
+
def snapshot(self) -> dict:
|
|
69
|
+
page = self._ensure_page()
|
|
70
|
+
return {"snapshot": page.aria_snapshot()}
|
|
71
|
+
|
|
72
|
+
def act(self, action: str, ref: str | None = None, value: str | None = None) -> dict:
|
|
73
|
+
page = self._ensure_page()
|
|
74
|
+
if action == "click":
|
|
75
|
+
page.click(ref)
|
|
76
|
+
elif action == "type":
|
|
77
|
+
if value is None:
|
|
78
|
+
raise ValueError("'value' is required for type action")
|
|
79
|
+
page.fill(ref, value)
|
|
80
|
+
elif action == "scroll":
|
|
81
|
+
delta = int(value) if value else 800
|
|
82
|
+
page.mouse.wheel(0, delta)
|
|
83
|
+
elif action == "wait":
|
|
84
|
+
timeout_ms = float(value) if value else 1000.0
|
|
85
|
+
page.wait_for_timeout(timeout_ms)
|
|
86
|
+
else:
|
|
87
|
+
raise ValueError(f"unknown action '{action}'")
|
|
88
|
+
return {"action": action, "ok": True}
|
|
89
|
+
|
|
90
|
+
def get(self, kind: str) -> dict:
|
|
91
|
+
page = self._ensure_page()
|
|
92
|
+
if kind == "html":
|
|
93
|
+
return {"kind": "html", "data": page.content()}
|
|
94
|
+
if kind == "text":
|
|
95
|
+
return {"kind": "text", "data": page.inner_text("body")}
|
|
96
|
+
if kind == "screenshot":
|
|
97
|
+
png = page.screenshot()
|
|
98
|
+
return {"kind": "screenshot", "encoding": "base64",
|
|
99
|
+
"data": base64.b64encode(png).decode()}
|
|
100
|
+
raise ValueError(f"unknown kind '{kind}'")
|
|
101
|
+
|
|
102
|
+
def close(self) -> None:
|
|
103
|
+
page = self._page
|
|
104
|
+
if page is None:
|
|
105
|
+
return
|
|
106
|
+
for closer in (
|
|
107
|
+
page.close,
|
|
108
|
+
getattr(getattr(page, "_bd_browser", None), "close", None),
|
|
109
|
+
getattr(getattr(page, "_bd_pw", None), "stop", None),
|
|
110
|
+
):
|
|
111
|
+
if closer is None:
|
|
112
|
+
continue
|
|
113
|
+
try:
|
|
114
|
+
closer()
|
|
115
|
+
except Exception: # noqa: BLE001 — best-effort teardown
|
|
116
|
+
pass
|
|
117
|
+
self._page = None
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ConfigError(Exception):
|
|
8
|
+
pass
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True)
|
|
12
|
+
class Config:
|
|
13
|
+
token: str
|
|
14
|
+
unlocker_zone: str
|
|
15
|
+
serp_zone: str
|
|
16
|
+
browser_auth: str | None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def load_config(env: dict | None = None) -> Config:
|
|
20
|
+
env = os.environ if env is None else env
|
|
21
|
+
# BRIGHTDATA_API_KEY is Bright Data's canonical env name; accept it as an alias.
|
|
22
|
+
token = env.get("BRIGHTDATA_API_TOKEN") or env.get("BRIGHTDATA_API_KEY")
|
|
23
|
+
if not token:
|
|
24
|
+
raise ConfigError("BRIGHTDATA_API_TOKEN (or BRIGHTDATA_API_KEY) is not set")
|
|
25
|
+
unlocker_zone = (
|
|
26
|
+
env.get("BRIGHTDATA_WEB_UNLOCKER_ZONE")
|
|
27
|
+
or env.get("WEB_UNLOCKER_ZONE")
|
|
28
|
+
or "web_unlocker1"
|
|
29
|
+
)
|
|
30
|
+
return Config(
|
|
31
|
+
token=token,
|
|
32
|
+
unlocker_zone=unlocker_zone,
|
|
33
|
+
serp_zone=env.get("BRIGHTDATA_SERP_ZONE", "serp_api1"),
|
|
34
|
+
browser_auth=env.get("BRIGHTDATA_BROWSER_AUTH"),
|
|
35
|
+
)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import Counter
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class SessionCounter:
|
|
7
|
+
def __init__(self):
|
|
8
|
+
self._counts: Counter[str] = Counter()
|
|
9
|
+
|
|
10
|
+
def record(self, tool: str) -> None:
|
|
11
|
+
self._counts[tool] += 1
|
|
12
|
+
|
|
13
|
+
def stats(self) -> dict:
|
|
14
|
+
return {"total": sum(self._counts.values()),
|
|
15
|
+
"by_tool": dict(self._counts)}
|
|
16
|
+
|
|
17
|
+
def reset(self) -> None:
|
|
18
|
+
self._counts.clear()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
# 공식 문서 curl 예시에서 직접 확인된 dataset_id만 등록.
|
|
4
|
+
# 추가 platform은 docs.brightdata.com에서 dataset_id 확인 후 등재할 것.
|
|
5
|
+
DATASET_IDS: dict[str, str] = {
|
|
6
|
+
"amazon_product": "gd_l7q7dkf244hwjntr0",
|
|
7
|
+
"linkedin_person": "gd_l1viktl72bvl7bjuj0",
|
|
8
|
+
"instagram_profile": "gd_l1vikfch901nx3by4",
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UnknownPlatform(ValueError):
|
|
13
|
+
def __init__(self, platform: str, available: list[str]):
|
|
14
|
+
super().__init__(f"Unknown platform '{platform}'. Available: {available}")
|
|
15
|
+
self.platform = platform
|
|
16
|
+
self.available = available
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def resolve(platform: str) -> str:
|
|
20
|
+
key = platform.strip().lower()
|
|
21
|
+
if key not in DATASET_IDS:
|
|
22
|
+
raise UnknownPlatform(platform, platforms())
|
|
23
|
+
return DATASET_IDS[key]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def platforms() -> list[str]:
|
|
27
|
+
return sorted(DATASET_IDS)
|