hermes-brightdata 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. hermes_brightdata-0.1.0/LICENSE +21 -0
  2. hermes_brightdata-0.1.0/PKG-INFO +132 -0
  3. hermes_brightdata-0.1.0/README.md +112 -0
  4. hermes_brightdata-0.1.0/brightdata_plugin/__init__.py +62 -0
  5. hermes_brightdata-0.1.0/brightdata_plugin/api.py +109 -0
  6. hermes_brightdata-0.1.0/brightdata_plugin/browser.py +117 -0
  7. hermes_brightdata-0.1.0/brightdata_plugin/config.py +35 -0
  8. hermes_brightdata-0.1.0/brightdata_plugin/counter.py +18 -0
  9. hermes_brightdata-0.1.0/brightdata_plugin/datasets.py +27 -0
  10. hermes_brightdata-0.1.0/brightdata_plugin/schemas.py +133 -0
  11. hermes_brightdata-0.1.0/brightdata_plugin/tools.py +168 -0
  12. hermes_brightdata-0.1.0/hermes_brightdata.egg-info/PKG-INFO +132 -0
  13. hermes_brightdata-0.1.0/hermes_brightdata.egg-info/SOURCES.txt +30 -0
  14. hermes_brightdata-0.1.0/hermes_brightdata.egg-info/dependency_links.txt +1 -0
  15. hermes_brightdata-0.1.0/hermes_brightdata.egg-info/entry_points.txt +2 -0
  16. hermes_brightdata-0.1.0/hermes_brightdata.egg-info/requires.txt +9 -0
  17. hermes_brightdata-0.1.0/hermes_brightdata.egg-info/top_level.txt +1 -0
  18. hermes_brightdata-0.1.0/pyproject.toml +27 -0
  19. hermes_brightdata-0.1.0/setup.cfg +4 -0
  20. hermes_brightdata-0.1.0/tests/test_api_dataset.py +87 -0
  21. hermes_brightdata-0.1.0/tests/test_api_request.py +57 -0
  22. hermes_brightdata-0.1.0/tests/test_browser.py +141 -0
  23. hermes_brightdata-0.1.0/tests/test_config.py +60 -0
  24. hermes_brightdata-0.1.0/tests/test_counter.py +19 -0
  25. hermes_brightdata-0.1.0/tests/test_datasets.py +26 -0
  26. hermes_brightdata-0.1.0/tests/test_handler_contract.py +40 -0
  27. hermes_brightdata-0.1.0/tests/test_live_smoke.py +23 -0
  28. hermes_brightdata-0.1.0/tests/test_manifest.py +17 -0
  29. hermes_brightdata-0.1.0/tests/test_register.py +89 -0
  30. hermes_brightdata-0.1.0/tests/test_schemas.py +30 -0
  31. hermes_brightdata-0.1.0/tests/test_tools_browser.py +71 -0
  32. hermes_brightdata-0.1.0/tests/test_tools_core.py +137 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Dante Labs
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,132 @@
1
+ Metadata-Version: 2.4
2
+ Name: hermes-brightdata
3
+ Version: 0.1.0
4
+ Summary: Bright Data web data tools for Hermes Agent — search, scrape, structured datasets, and browser automation
5
+ License: MIT
6
+ Project-URL: Homepage, https://github.com/dandacompany/hermes-brightdata-plugin
7
+ Project-URL: Repository, https://github.com/dandacompany/hermes-brightdata-plugin
8
+ Keywords: hermes,bright-data,brightdata,web-scraping,plugin,agent
9
+ Requires-Python: >=3.10
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: requests>=2.31
13
+ Provides-Extra: browser
14
+ Requires-Dist: playwright>=1.40; extra == "browser"
15
+ Provides-Extra: dev
16
+ Requires-Dist: pytest>=8; extra == "dev"
17
+ Requires-Dist: responses>=0.25; extra == "dev"
18
+ Requires-Dist: pyyaml>=6; extra == "dev"
19
+ Dynamic: license-file
20
+
21
+ # hermes-brightdata
22
+
23
+ [![CI](https://github.com/dandacompany/hermes-brightdata-plugin/actions/workflows/ci.yml/badge.svg)](https://github.com/dandacompany/hermes-brightdata-plugin/actions/workflows/ci.yml)
24
+ [![PyPI version](https://img.shields.io/pypi/v/hermes-brightdata.svg)](https://pypi.org/project/hermes-brightdata/)
25
+ [![Python](https://img.shields.io/pypi/pyversions/hermes-brightdata.svg)](https://pypi.org/project/hermes-brightdata/)
26
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
27
+
28
+ Bright Data web data tools for Hermes Agent — search, scrape, structured datasets, and browser automation.
29
+
30
+ ## Overview
31
+
32
+ This plugin integrates Bright Data's web data APIs with Hermes, providing nine powerful tools for web data collection:
33
+
34
+ - **Search & Scraping**: Real-time search, single/batch URL scraping, and structured data extraction
35
+ - **Browser Automation**: Interactive browser sessions with persistent cookies and JavaScript rendering
36
+ - **Bypass Detection**: Automatic handling of bot detection, CAPTCHAs, and JavaScript-heavy sites
37
+
38
+ Bright Data handles all the complexity — proxy rotation, header management, retry logic, and content parsing.
39
+
40
+ ## Installation
41
+
42
+ Install the plugin with base tools:
43
+
44
+ ```bash
45
+ pip install hermes-brightdata
46
+ ```
47
+
48
+ To enable browser automation tools, install with the `[browser]` extra:
49
+
50
+ ```bash
51
+ pip install hermes-brightdata[browser]
52
+ ```
53
+
54
+ The browser extra adds Playwright for interactive session handling (required for `browser_navigate`, `browser_snapshot`, `browser_act`, `browser_get`).
55
+
56
+ ## Configuration
57
+
58
+ Before using any Bright Data tool, set your API credentials in environment variables:
59
+
60
+ ### Required
61
+
62
+ `BRIGHTDATA_API_TOKEN` (required, secret)
63
+
64
+ - Your Bright Data API token
65
+ - Obtain from Account Settings → API keys in the Bright Data dashboard
66
+ - `BRIGHTDATA_API_KEY` is accepted as an alias if `BRIGHTDATA_API_TOKEN` is unset
67
+
68
+ ### Optional
69
+
70
+ `BRIGHTDATA_WEB_UNLOCKER_ZONE` (optional)
71
+
72
+ - Zone name for Web Unlocker API
73
+ - Default: `web_unlocker1`
74
+ - `WEB_UNLOCKER_ZONE` is accepted as an alias if the prefixed name is unset
75
+
76
+ `BRIGHTDATA_SERP_ZONE` (optional)
77
+
78
+ - Zone name for SERP (Search Engine Results Page) API
79
+ - Default: `serp_api1`
80
+
81
+ `BRIGHTDATA_BROWSER_AUTH` (optional, secret)
82
+
83
+ - Authentication credentials for Scraping Browser
84
+ - Format: `brd-customer-<customer-id>-zone-<zone-name>:<password>`
85
+ - Required only if using browser tools with authentication
86
+
87
+ ### Example
88
+
89
+ ```bash
90
+ export BRIGHTDATA_API_TOKEN="your-api-token-here"
91
+ export BRIGHTDATA_BROWSER_AUTH="brd-customer-12345-zone-myzone:password123"
92
+ ```
93
+
94
+ ## Tools
95
+
96
+ The plugin provides nine tools:
97
+
98
+ | Tool | Description |
99
+ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
100
+ | `search_engine` | Search the web via Bright Data SERP API and return parsed results. Use for current events, fact-checking, finding pages, or any web search. Bypasses bot detection; more reliable than built-in web search. |
101
+ | `scrape` | Fetch a single URL as clean markdown (or html) using Bright Data Web Unlocker. Handles JavaScript, CAPTCHAs, and bot detection. Use to read any webpage, article, doc, or site. |
102
+ | `scrape_batch` | Scrape multiple URLs in one call (max 20). Returns per-URL content or error. Use when you have several pages to read at once. |
103
+ | `web_data` | Get structured JSON data from a supported platform via Bright Data Web Scraper API (e.g. an Amazon product page, a LinkedIn profile). Returns clean structured fields, not raw HTML. May take up to a minute. |
104
+ | `session_stats` | Report Bright Data tool call counts for this session. |
105
+ | `browser_navigate` | Open a URL in a Bright Data Scraping Browser (real browser with a persistent session). Use for interactive flows requiring clicks/typing. Requires the [browser] extra and BRIGHTDATA_BROWSER_AUTH. |
106
+ | `browser_snapshot` | Return an accessibility-tree snapshot of the current browser page, including element refs to use with browser_act. |
107
+ | `browser_act` | Perform an action on the current browser page: click, type, scroll, or wait. Get element refs from browser_snapshot first. |
108
+ | `browser_get` | Read the current browser page: html, text, or a base64 screenshot. |
109
+
110
+ ## Activation
111
+
112
+ Enable the plugin in Hermes:
113
+
114
+ ```bash
115
+ hermes plugins enable brightdata
116
+ ```
117
+
118
+ Verify it's loaded:
119
+
120
+ ```bash
121
+ hermes plugins list
122
+ ```
123
+
124
+ You should see `brightdata` listed with version `0.1.0`.
125
+
126
+ ## Pricing
127
+
128
+ Bright Data offers a free tier with up to 5,000 requests per month. For higher volume, refer to Bright Data's pricing page for details on Web Unlocker, SERP API, Web Scraper API, and Scraping Browser plans.
129
+
130
+ ---
131
+
132
+ For more information on Bright Data services, visit [https://brightdata.com](https://brightdata.com).
@@ -0,0 +1,112 @@
1
+ # hermes-brightdata
2
+
3
+ [![CI](https://github.com/dandacompany/hermes-brightdata-plugin/actions/workflows/ci.yml/badge.svg)](https://github.com/dandacompany/hermes-brightdata-plugin/actions/workflows/ci.yml)
4
+ [![PyPI version](https://img.shields.io/pypi/v/hermes-brightdata.svg)](https://pypi.org/project/hermes-brightdata/)
5
+ [![Python](https://img.shields.io/pypi/pyversions/hermes-brightdata.svg)](https://pypi.org/project/hermes-brightdata/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
+
8
+ Bright Data web data tools for Hermes Agent — search, scrape, structured datasets, and browser automation.
9
+
10
+ ## Overview
11
+
12
+ This plugin integrates Bright Data's web data APIs with Hermes, providing nine powerful tools for web data collection:
13
+
14
+ - **Search & Scraping**: Real-time search, single/batch URL scraping, and structured data extraction
15
+ - **Browser Automation**: Interactive browser sessions with persistent cookies and JavaScript rendering
16
+ - **Bypass Detection**: Automatic handling of bot detection, CAPTCHAs, and JavaScript-heavy sites
17
+
18
+ Bright Data handles all the complexity — proxy rotation, header management, retry logic, and content parsing.
19
+
20
+ ## Installation
21
+
22
+ Install the plugin with base tools:
23
+
24
+ ```bash
25
+ pip install hermes-brightdata
26
+ ```
27
+
28
+ To enable browser automation tools, install with the `[browser]` extra:
29
+
30
+ ```bash
31
+ pip install hermes-brightdata[browser]
32
+ ```
33
+
34
+ The browser extra adds Playwright for interactive session handling (required for `browser_navigate`, `browser_snapshot`, `browser_act`, `browser_get`).
35
+
36
+ ## Configuration
37
+
38
+ Before using any Bright Data tool, set your API credentials in environment variables:
39
+
40
+ ### Required
41
+
42
+ `BRIGHTDATA_API_TOKEN` (required, secret)
43
+
44
+ - Your Bright Data API token
45
+ - Obtain from Account Settings → API keys in the Bright Data dashboard
46
+ - `BRIGHTDATA_API_KEY` is accepted as an alias if `BRIGHTDATA_API_TOKEN` is unset
47
+
48
+ ### Optional
49
+
50
+ `BRIGHTDATA_WEB_UNLOCKER_ZONE` (optional)
51
+
52
+ - Zone name for Web Unlocker API
53
+ - Default: `web_unlocker1`
54
+ - `WEB_UNLOCKER_ZONE` is accepted as an alias if the prefixed name is unset
55
+
56
+ `BRIGHTDATA_SERP_ZONE` (optional)
57
+
58
+ - Zone name for SERP (Search Engine Results Page) API
59
+ - Default: `serp_api1`
60
+
61
+ `BRIGHTDATA_BROWSER_AUTH` (optional, secret)
62
+
63
+ - Authentication credentials for Scraping Browser
64
+ - Format: `brd-customer-<customer-id>-zone-<zone-name>:<password>`
65
+ - Required only if using browser tools with authentication
66
+
67
+ ### Example
68
+
69
+ ```bash
70
+ export BRIGHTDATA_API_TOKEN="your-api-token-here"
71
+ export BRIGHTDATA_BROWSER_AUTH="brd-customer-12345-zone-myzone:password123"
72
+ ```
73
+
74
+ ## Tools
75
+
76
+ The plugin provides nine tools:
77
+
78
+ | Tool | Description |
79
+ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
80
+ | `search_engine` | Search the web via Bright Data SERP API and return parsed results. Use for current events, fact-checking, finding pages, or any web search. Bypasses bot detection; more reliable than built-in web search. |
81
+ | `scrape` | Fetch a single URL as clean markdown (or html) using Bright Data Web Unlocker. Handles JavaScript, CAPTCHAs, and bot detection. Use to read any webpage, article, doc, or site. |
82
+ | `scrape_batch` | Scrape multiple URLs in one call (max 20). Returns per-URL content or error. Use when you have several pages to read at once. |
83
+ | `web_data` | Get structured JSON data from a supported platform via Bright Data Web Scraper API (e.g. an Amazon product page, a LinkedIn profile). Returns clean structured fields, not raw HTML. May take up to a minute. |
84
+ | `session_stats` | Report Bright Data tool call counts for this session. |
85
+ | `browser_navigate` | Open a URL in a Bright Data Scraping Browser (real browser with a persistent session). Use for interactive flows requiring clicks/typing. Requires the [browser] extra and BRIGHTDATA_BROWSER_AUTH. |
86
+ | `browser_snapshot` | Return an accessibility-tree snapshot of the current browser page, including element refs to use with browser_act. |
87
+ | `browser_act` | Perform an action on the current browser page: click, type, scroll, or wait. Get element refs from browser_snapshot first. |
88
+ | `browser_get` | Read the current browser page: html, text, or a base64 screenshot. |
89
+
90
+ ## Activation
91
+
92
+ Enable the plugin in Hermes:
93
+
94
+ ```bash
95
+ hermes plugins enable brightdata
96
+ ```
97
+
98
+ Verify it's loaded:
99
+
100
+ ```bash
101
+ hermes plugins list
102
+ ```
103
+
104
+ You should see `brightdata` listed with version `0.1.0`.
105
+
106
+ ## Pricing
107
+
108
+ Bright Data offers a free tier with up to 5,000 requests per month. For higher volume, refer to Bright Data's pricing page for details on Web Unlocker, SERP API, Web Scraper API, and Scraping Browser plans.
109
+
110
+ ---
111
+
112
+ For more information on Bright Data services, visit [https://brightdata.com](https://brightdata.com).
@@ -0,0 +1,62 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+
5
+ from . import schemas
6
+ from .api import BrightDataClient
7
+ from .browser import BrowserSession
8
+ from .config import Config, load_config
9
+ from .counter import SessionCounter
10
+ from .tools import make_browser_handlers, make_core_handlers
11
+
12
+
13
+ def build_handlers(cfg: Config | None = None) -> dict:
14
+ cfg = cfg or load_config()
15
+ counter = SessionCounter()
16
+
17
+ _client: dict = {}
18
+ def get_client() -> BrightDataClient:
19
+ if "c" not in _client:
20
+ _client["c"] = BrightDataClient(cfg)
21
+ return _client["c"]
22
+
23
+ _session: dict = {}
24
+ def get_session() -> BrowserSession:
25
+ if "s" not in _session:
26
+ _session["s"] = BrowserSession(cfg)
27
+ return _session["s"]
28
+
29
+ handlers = {}
30
+ handlers.update(make_core_handlers(get_client, counter))
31
+ handlers.update(make_browser_handlers(get_session, counter))
32
+ # private keys popped by register(); not real tools
33
+ handlers["_counter"] = counter # type: ignore
34
+ handlers["_get_session"] = get_session # type: ignore
35
+ handlers["_session_holder"] = _session # type: ignore
36
+ return handlers
37
+
38
+
39
+ def register(ctx) -> None:
40
+ cfg = load_config()
41
+ handlers = build_handlers(cfg)
42
+ counter = handlers.pop("_counter")
43
+ handlers.pop("_get_session")
44
+ session_holder = handlers.pop("_session_holder")
45
+
46
+ for name, handler in handlers.items():
47
+ ctx.register_tool(name, "brightdata", schemas.TOOL_SCHEMAS[name], handler)
48
+
49
+ def brightdata_cmd(args, **kwargs) -> str:
50
+ return json.dumps(counter.stats(), ensure_ascii=False)
51
+ ctx.register_command("brightdata", brightdata_cmd,
52
+ description="Show Bright Data session usage stats")
53
+
54
+ def on_session_end(*args, **kwargs):
55
+ # only close if a browser session was actually created this session
56
+ s = session_holder.get("s")
57
+ if s is not None:
58
+ try:
59
+ s.close()
60
+ except Exception: # noqa: BLE001
61
+ pass
62
+ ctx.register_hook("on_session_end", on_session_end)
@@ -0,0 +1,109 @@
1
+ from __future__ import annotations
2
+
3
+ import time
4
+
5
+ import requests
6
+
7
+ from .config import Config
8
+
9
+ BASE_URL = "https://api.brightdata.com"
10
+ REQUEST_ENDPOINT = f"{BASE_URL}/request"
11
+ DATASET_TRIGGER = f"{BASE_URL}/datasets/v3/trigger"
12
+ DATASET_PROGRESS = f"{BASE_URL}/datasets/v3/progress"
13
+ DATASET_SNAPSHOT = f"{BASE_URL}/datasets/v3/snapshot"
14
+
15
+ _HINTS = {
16
+ 401: "API token invalid or missing — check BRIGHTDATA_API_TOKEN.",
17
+ 402: "Free quota (5,000 req/month) may be exhausted — check billing.",
18
+ 429: "Rate limited or quota exhausted — retry later or raise plan limit.",
19
+ }
20
+
21
+
22
+ class BrightDataError(Exception):
23
+ def __init__(self, message: str, status: int, hint: str = ""):
24
+ super().__init__(message)
25
+ self.status = status
26
+ self.hint = hint or _HINTS.get(status, "Unexpected Bright Data API error.")
27
+
28
+
29
+ class BrightDataClient:
30
+ def __init__(self, cfg: Config, timeout: int = 60):
31
+ self._cfg = cfg
32
+ self._timeout = timeout
33
+ self._session = requests.Session()
34
+ self._session.headers.update({
35
+ "Authorization": f"Bearer {cfg.token}",
36
+ "Content-Type": "application/json",
37
+ })
38
+
39
+ def _post_request(self, payload: dict) -> requests.Response:
40
+ resp = self._session.post(REQUEST_ENDPOINT, json=payload, timeout=self._timeout)
41
+ if resp.status_code >= 400:
42
+ raise BrightDataError(
43
+ f"Bright Data /request failed: {resp.status_code}",
44
+ status=resp.status_code,
45
+ )
46
+ return resp
47
+
48
+ def unlock(self, url: str, data_format: str = "markdown", render: bool = False) -> str:
49
+ payload = {
50
+ "zone": self._cfg.unlocker_zone,
51
+ "url": url,
52
+ "format": "raw",
53
+ "data_format": data_format,
54
+ }
55
+ if render:
56
+ payload["render"] = True
57
+ return self._post_request(payload).text
58
+
59
+ def serp(self, search_url: str, parse_json: bool = True) -> str:
60
+ url = search_url
61
+ if parse_json and "brd_json=" not in url:
62
+ sep = "&" if "?" in url else "?"
63
+ url = f"{url}{sep}brd_json=1"
64
+ payload = {"zone": self._cfg.serp_zone, "url": url, "format": "raw"}
65
+ return self._post_request(payload).text
66
+
67
+ def trigger_dataset(self, dataset_id: str, urls: list[str]) -> str:
68
+ body = [{"url": u} for u in urls]
69
+ resp = self._session.post(
70
+ DATASET_TRIGGER, params={"dataset_id": dataset_id},
71
+ json=body, timeout=self._timeout,
72
+ )
73
+ if resp.status_code >= 400:
74
+ raise BrightDataError(
75
+ f"trigger failed: {resp.status_code}", status=resp.status_code)
76
+ return resp.json()["snapshot_id"]
77
+
78
+ def poll_snapshot(self, snapshot_id: str) -> str:
79
+ resp = self._session.get(
80
+ f"{DATASET_PROGRESS}/{snapshot_id}", timeout=self._timeout)
81
+ if resp.status_code >= 400:
82
+ raise BrightDataError(
83
+ f"progress failed: {resp.status_code}", status=resp.status_code)
84
+ return resp.json()["status"]
85
+
86
+ def download_snapshot(self, snapshot_id: str, fmt: str = "json") -> str:
87
+ resp = self._session.get(
88
+ f"{DATASET_SNAPSHOT}/{snapshot_id}", params={"format": fmt},
89
+ timeout=self._timeout,
90
+ )
91
+ if resp.status_code >= 400:
92
+ raise BrightDataError(
93
+ f"snapshot failed: {resp.status_code}", status=resp.status_code)
94
+ return resp.text
95
+
96
+ def collect_dataset(self, dataset_id: str, urls: list[str],
97
+ poll_interval: float = 2.0, max_wait: float = 60.0,
98
+ sleep=time.sleep, now=time.monotonic) -> dict:
99
+ snapshot_id = self.trigger_dataset(dataset_id, urls)
100
+ start = now()
101
+ while now() - start <= max_wait:
102
+ status = self.poll_snapshot(snapshot_id)
103
+ if status == "ready":
104
+ return {"status": "ready",
105
+ "data": self.download_snapshot(snapshot_id)}
106
+ if status == "failed":
107
+ return {"status": "failed", "snapshot_id": snapshot_id}
108
+ sleep(poll_interval)
109
+ return {"status": "timeout", "snapshot_id": snapshot_id}
@@ -0,0 +1,117 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+
5
+ from .config import Config
6
+
7
+ CDP_HOST = "brd.superproxy.io:9222"
8
+
9
+
10
+ class BrowserUnavailable(Exception):
11
+ def __init__(self, message: str, hint: str = ""):
12
+ super().__init__(message)
13
+ self.hint = hint
14
+
15
+
16
+ def _default_connector(cfg: Config):
17
+ def connect(cdp_url: str):
18
+ if cfg.browser_auth is None:
19
+ raise BrowserUnavailable(
20
+ "browser auth not configured",
21
+ "set BRIGHTDATA_BROWSER_AUTH to 'brd-customer-<id>-zone-<zone>:<pw>'")
22
+ try:
23
+ from playwright.sync_api import sync_playwright
24
+ except ImportError as e:
25
+ raise BrowserUnavailable(
26
+ "playwright not installed",
27
+ "install with: pip install hermes-brightdata[browser]") from e
28
+ pw = sync_playwright().start()
29
+ try:
30
+ browser_obj = pw.chromium.connect_over_cdp(cdp_url)
31
+ page = browser_obj.new_page()
32
+ except BrowserUnavailable:
33
+ pw.stop()
34
+ raise
35
+ except Exception:
36
+ pw.stop()
37
+ # never surface the raw exception: it can embed the CDP URL,
38
+ # which contains the browser_auth password.
39
+ raise BrowserUnavailable(
40
+ "browser connection failed",
41
+ "check BRIGHTDATA_BROWSER_AUTH and that the Scraping Browser zone is active",
42
+ ) from None
43
+ page._bd_pw = pw # keep ref for teardown
44
+ page._bd_browser = browser_obj
45
+ return page
46
+ return connect
47
+
48
+
49
+ class BrowserSession:
50
+ def __init__(self, cfg: Config, connector=None):
51
+ self._cfg = cfg
52
+ self._connector = connector or _default_connector(cfg)
53
+ self._page = None
54
+
55
+ def _cdp_url(self) -> str:
56
+ return f"wss://{self._cfg.browser_auth}@{CDP_HOST}"
57
+
58
+ def _ensure_page(self):
59
+ if self._page is None:
60
+ self._page = self._connector(self._cdp_url())
61
+ return self._page
62
+
63
+ def navigate(self, url: str) -> dict:
64
+ page = self._ensure_page()
65
+ page.goto(url)
66
+ return {"url": url, "ok": True}
67
+
68
+ def snapshot(self) -> dict:
69
+ page = self._ensure_page()
70
+ return {"snapshot": page.aria_snapshot()}
71
+
72
+ def act(self, action: str, ref: str | None = None, value: str | None = None) -> dict:
73
+ page = self._ensure_page()
74
+ if action == "click":
75
+ page.click(ref)
76
+ elif action == "type":
77
+ if value is None:
78
+ raise ValueError("'value' is required for type action")
79
+ page.fill(ref, value)
80
+ elif action == "scroll":
81
+ delta = int(value) if value else 800
82
+ page.mouse.wheel(0, delta)
83
+ elif action == "wait":
84
+ timeout_ms = float(value) if value else 1000.0
85
+ page.wait_for_timeout(timeout_ms)
86
+ else:
87
+ raise ValueError(f"unknown action '{action}'")
88
+ return {"action": action, "ok": True}
89
+
90
+ def get(self, kind: str) -> dict:
91
+ page = self._ensure_page()
92
+ if kind == "html":
93
+ return {"kind": "html", "data": page.content()}
94
+ if kind == "text":
95
+ return {"kind": "text", "data": page.inner_text("body")}
96
+ if kind == "screenshot":
97
+ png = page.screenshot()
98
+ return {"kind": "screenshot", "encoding": "base64",
99
+ "data": base64.b64encode(png).decode()}
100
+ raise ValueError(f"unknown kind '{kind}'")
101
+
102
+ def close(self) -> None:
103
+ page = self._page
104
+ if page is None:
105
+ return
106
+ for closer in (
107
+ page.close,
108
+ getattr(getattr(page, "_bd_browser", None), "close", None),
109
+ getattr(getattr(page, "_bd_pw", None), "stop", None),
110
+ ):
111
+ if closer is None:
112
+ continue
113
+ try:
114
+ closer()
115
+ except Exception: # noqa: BLE001 — best-effort teardown
116
+ pass
117
+ self._page = None
@@ -0,0 +1,35 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+
6
+
7
+ class ConfigError(Exception):
8
+ pass
9
+
10
+
11
+ @dataclass(frozen=True)
12
+ class Config:
13
+ token: str
14
+ unlocker_zone: str
15
+ serp_zone: str
16
+ browser_auth: str | None
17
+
18
+
19
+ def load_config(env: dict | None = None) -> Config:
20
+ env = os.environ if env is None else env
21
+ # BRIGHTDATA_API_KEY is Bright Data's canonical env name; accept it as an alias.
22
+ token = env.get("BRIGHTDATA_API_TOKEN") or env.get("BRIGHTDATA_API_KEY")
23
+ if not token:
24
+ raise ConfigError("BRIGHTDATA_API_TOKEN (or BRIGHTDATA_API_KEY) is not set")
25
+ unlocker_zone = (
26
+ env.get("BRIGHTDATA_WEB_UNLOCKER_ZONE")
27
+ or env.get("WEB_UNLOCKER_ZONE")
28
+ or "web_unlocker1"
29
+ )
30
+ return Config(
31
+ token=token,
32
+ unlocker_zone=unlocker_zone,
33
+ serp_zone=env.get("BRIGHTDATA_SERP_ZONE", "serp_api1"),
34
+ browser_auth=env.get("BRIGHTDATA_BROWSER_AUTH"),
35
+ )
@@ -0,0 +1,18 @@
1
+ from __future__ import annotations
2
+
3
+ from collections import Counter
4
+
5
+
6
+ class SessionCounter:
7
+ def __init__(self):
8
+ self._counts: Counter[str] = Counter()
9
+
10
+ def record(self, tool: str) -> None:
11
+ self._counts[tool] += 1
12
+
13
+ def stats(self) -> dict:
14
+ return {"total": sum(self._counts.values()),
15
+ "by_tool": dict(self._counts)}
16
+
17
+ def reset(self) -> None:
18
+ self._counts.clear()
@@ -0,0 +1,27 @@
1
+ from __future__ import annotations
2
+
3
+ # 공식 문서 curl 예시에서 직접 확인된 dataset_id만 등록.
4
+ # 추가 platform은 docs.brightdata.com에서 dataset_id 확인 후 등재할 것.
5
+ DATASET_IDS: dict[str, str] = {
6
+ "amazon_product": "gd_l7q7dkf244hwjntr0",
7
+ "linkedin_person": "gd_l1viktl72bvl7bjuj0",
8
+ "instagram_profile": "gd_l1vikfch901nx3by4",
9
+ }
10
+
11
+
12
+ class UnknownPlatform(ValueError):
13
+ def __init__(self, platform: str, available: list[str]):
14
+ super().__init__(f"Unknown platform '{platform}'. Available: {available}")
15
+ self.platform = platform
16
+ self.available = available
17
+
18
+
19
+ def resolve(platform: str) -> str:
20
+ key = platform.strip().lower()
21
+ if key not in DATASET_IDS:
22
+ raise UnknownPlatform(platform, platforms())
23
+ return DATASET_IDS[key]
24
+
25
+
26
+ def platforms() -> list[str]:
27
+ return sorted(DATASET_IDS)