hermes-oxylabs-web-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hermes_oxylabs_web_api/__init__.py +14 -0
- hermes_oxylabs_web_api/client.py +196 -0
- hermes_oxylabs_web_api/plugin.yaml +26 -0
- hermes_oxylabs_web_api/provider.py +143 -0
- hermes_oxylabs_web_api/py.typed +0 -0
- hermes_oxylabs_web_api-0.1.0.dist-info/METADATA +133 -0
- hermes_oxylabs_web_api-0.1.0.dist-info/RECORD +11 -0
- hermes_oxylabs_web_api-0.1.0.dist-info/WHEEL +5 -0
- hermes_oxylabs_web_api-0.1.0.dist-info/entry_points.txt +2 -0
- hermes_oxylabs_web_api-0.1.0.dist-info/licenses/LICENSE +21 -0
- hermes_oxylabs_web_api-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Oxylabs Web API search and page extraction for Hermes Agent."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
|
|
7
|
+
__all__ = ["__version__", "register"]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def register(ctx) -> None:
|
|
11
|
+
"""Register the Oxylabs web search and extract provider."""
|
|
12
|
+
from .provider import OxylabsWebSearchProvider
|
|
13
|
+
|
|
14
|
+
ctx.register_web_search_provider(OxylabsWebSearchProvider(settings=ctx.get_config))
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""HTTP client for the Oxylabs Web API (`/v1/search`, `/v1/scrape`, `/v1/async/scrape`)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import platform
|
|
7
|
+
import time
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
|
|
14
|
+
API_BASE_URL = "https://webapi.oxylabs.io"
|
|
15
|
+
API_KEY_ENV = "OXYLABS_WEB_API_KEY"
|
|
16
|
+
MAX_SEARCH_RESULTS = 20
|
|
17
|
+
SEARCH_TIMEOUT_SECONDS = 30.0
|
|
18
|
+
SYNC_SCRAPE_TIMEOUT_SECONDS = 60.0
|
|
19
|
+
POLL_INTERVAL_SECONDS = 3.0
|
|
20
|
+
|
|
21
|
+
_ERROR_DETAIL_MAX_CHARS = 300
|
|
22
|
+
_SCRAPE_TIMEOUT_TITLE = "REQUEST_FAILED_SCRAPE_TIMEOUT"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class OxylabsApiError(RuntimeError):
|
|
26
|
+
"""A Web API error with a message safe to show the agent. It never contains the API key."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, message: str, status: int, request_id: str | None = None) -> None:
|
|
29
|
+
super().__init__(message)
|
|
30
|
+
self.status = status
|
|
31
|
+
self.request_id = request_id
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def sdk_header() -> str:
|
|
35
|
+
"""The `x-oxylabs-sdk` value Oxylabs uses to count usage per integration."""
|
|
36
|
+
return f"oxylabs-web-api-hermes/{__version__} (hermes; python {platform.python_version()})"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _headers(api_key: str) -> dict[str, str]:
|
|
40
|
+
return {
|
|
41
|
+
"Accept": "application/json",
|
|
42
|
+
"Authorization": f"Bearer {api_key}",
|
|
43
|
+
"x-oxylabs-sdk": sdk_header(),
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _json_body(response: httpx.Response) -> dict[str, Any] | None:
|
|
48
|
+
try:
|
|
49
|
+
body = response.json()
|
|
50
|
+
except ValueError:
|
|
51
|
+
return None
|
|
52
|
+
return body if isinstance(body, dict) else None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _request_id(body: dict[str, Any] | None) -> str | None:
|
|
56
|
+
if not body:
|
|
57
|
+
return None
|
|
58
|
+
metadata = body.get("metadata")
|
|
59
|
+
value = metadata.get("request_id") if isinstance(metadata, dict) else None
|
|
60
|
+
value = value if value is not None else body.get("request_id")
|
|
61
|
+
return str(value) if isinstance(value, (str, int)) else None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def describe_api_error(status: int, body: dict[str, Any] | None) -> str:
|
|
65
|
+
"""Turn a failed response into a message the agent (and the user) can act on."""
|
|
66
|
+
request_id = _request_id(body)
|
|
67
|
+
suffix = f" (request_id {request_id})" if request_id else ""
|
|
68
|
+
if status == 401:
|
|
69
|
+
return f"Oxylabs Web API rejected the API key (401). Check {API_KEY_ENV}."
|
|
70
|
+
if status == 403:
|
|
71
|
+
return f"Oxylabs Web API refused the request (403): the account has no access to this feature{suffix}."
|
|
72
|
+
title = body.get("title") if body else None
|
|
73
|
+
if status == 429 and title == "QUOTA_EXCEEDED":
|
|
74
|
+
return "Oxylabs Web API quota is used up (429 QUOTA_EXCEEDED). Top up the plan in the Oxylabs dashboard."
|
|
75
|
+
errors = body.get("errors") if body else None
|
|
76
|
+
field_errors = "; ".join(
|
|
77
|
+
f"{str(entry.get('pointer', '')).removeprefix('#/')}: {entry.get('detail', '')}"
|
|
78
|
+
for entry in (errors if isinstance(errors, list) else [])
|
|
79
|
+
if isinstance(entry, dict)
|
|
80
|
+
)
|
|
81
|
+
detail = field_errors or (body.get("detail") if body and isinstance(body.get("detail"), str) else "")
|
|
82
|
+
message = f"Oxylabs Web API error {status} {title if isinstance(title, str) else 'request failed'}"
|
|
83
|
+
if detail:
|
|
84
|
+
message += f": {detail}"
|
|
85
|
+
if len(message) > _ERROR_DETAIL_MAX_CHARS:
|
|
86
|
+
message = message[:_ERROR_DETAIL_MAX_CHARS] + "…"
|
|
87
|
+
return message + suffix
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _ensure_ok(response: httpx.Response) -> dict[str, Any]:
|
|
91
|
+
body = _json_body(response)
|
|
92
|
+
if 200 <= response.status_code < 300 and body is not None:
|
|
93
|
+
return body
|
|
94
|
+
raise OxylabsApiError(describe_api_error(response.status_code, body), response.status_code, _request_id(body))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def search_web(api_key: str, query: str, limit: int, location: str | None = None) -> list[dict[str, Any]]:
|
|
98
|
+
"""`POST /v1/search`: organic results, each with a short outline of the page."""
|
|
99
|
+
payload: dict[str, Any] = {"query": query, "max_results": limit}
|
|
100
|
+
if location:
|
|
101
|
+
payload["location"] = location
|
|
102
|
+
try:
|
|
103
|
+
response = httpx.post(
|
|
104
|
+
f"{API_BASE_URL}/v1/search", json=payload, headers=_headers(api_key), timeout=SEARCH_TIMEOUT_SECONDS
|
|
105
|
+
)
|
|
106
|
+
except httpx.RequestError as exc:
|
|
107
|
+
raise OxylabsApiError(f"Could not reach the Oxylabs Web API ({type(exc).__name__}).", 0) from None
|
|
108
|
+
data = _ensure_ok(response)
|
|
109
|
+
results = data.get("results")
|
|
110
|
+
hits = []
|
|
111
|
+
for entry in results if isinstance(results, list) else []:
|
|
112
|
+
if not isinstance(entry, dict):
|
|
113
|
+
continue
|
|
114
|
+
outline = entry.get("outline")
|
|
115
|
+
metadata = entry.get("metadata")
|
|
116
|
+
position = metadata.get("position") if isinstance(metadata, dict) else None
|
|
117
|
+
hits.append(
|
|
118
|
+
{
|
|
119
|
+
"title": str(entry.get("title") or ""),
|
|
120
|
+
"url": str(entry.get("url") or ""),
|
|
121
|
+
"description": " … ".join(line for line in outline if isinstance(line, str))
|
|
122
|
+
if isinstance(outline, list)
|
|
123
|
+
else "",
|
|
124
|
+
"position": position if isinstance(position, int) else len(hits) + 1,
|
|
125
|
+
}
|
|
126
|
+
)
|
|
127
|
+
return hits
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _read_page(data: dict[str, Any]) -> tuple[str, int | None]:
|
|
131
|
+
results = data.get("results")
|
|
132
|
+
first = results[0] if isinstance(results, list) and results and isinstance(results[0], dict) else {}
|
|
133
|
+
markdown = first.get("markdown")
|
|
134
|
+
metadata = first.get("metadata")
|
|
135
|
+
statuses = metadata.get("statuses") if isinstance(metadata, dict) else None
|
|
136
|
+
http_request = statuses.get("http_request") if isinstance(statuses, dict) else None
|
|
137
|
+
code = http_request.get("code") if isinstance(http_request, dict) else None
|
|
138
|
+
return (markdown if isinstance(markdown, str) else ""), (code if isinstance(code, int) else None)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
async def scrape_page(
|
|
142
|
+
client: httpx.AsyncClient, api_key: str, url: str, *, render_js: bool, deadline: float
|
|
143
|
+
) -> tuple[str, int | None]:
|
|
144
|
+
"""`POST /v1/scrape` for one page as Markdown; returns ``(markdown, target HTTP status)``.
|
|
145
|
+
|
|
146
|
+
Most pages finish inside the synchronous call. When it times out, the same request is queued on
|
|
147
|
+
`/v1/async/scrape` and polled until ``deadline`` (a ``time.monotonic()`` value), instead of being
|
|
148
|
+
resent synchronously.
|
|
149
|
+
"""
|
|
150
|
+
payload: dict[str, Any] = {"url": url, "output": ["markdown"]}
|
|
151
|
+
if render_js:
|
|
152
|
+
payload["run_js"] = True
|
|
153
|
+
headers = _headers(api_key)
|
|
154
|
+
|
|
155
|
+
def remaining() -> float:
|
|
156
|
+
return max(1.0, deadline - time.monotonic())
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
response = await client.post(
|
|
160
|
+
f"{API_BASE_URL}/v1/scrape",
|
|
161
|
+
json=payload,
|
|
162
|
+
headers=headers,
|
|
163
|
+
timeout=min(SYNC_SCRAPE_TIMEOUT_SECONDS, remaining()),
|
|
164
|
+
)
|
|
165
|
+
body = _json_body(response)
|
|
166
|
+
timed_out = response.status_code == 504 or (body or {}).get("title") == _SCRAPE_TIMEOUT_TITLE
|
|
167
|
+
if not timed_out:
|
|
168
|
+
return _read_page(_ensure_ok(response))
|
|
169
|
+
except httpx.TimeoutException:
|
|
170
|
+
pass
|
|
171
|
+
except httpx.RequestError as exc:
|
|
172
|
+
raise OxylabsApiError(f"Could not reach the Oxylabs Web API ({type(exc).__name__}).", 0) from None
|
|
173
|
+
|
|
174
|
+
try:
|
|
175
|
+
queued = _ensure_ok(
|
|
176
|
+
await client.post(f"{API_BASE_URL}/v1/async/scrape", json=payload, headers=headers, timeout=remaining())
|
|
177
|
+
)
|
|
178
|
+
request_id = _request_id(queued)
|
|
179
|
+
if not request_id:
|
|
180
|
+
raise OxylabsApiError("Oxylabs Web API queued the scrape but returned no request_id.", 502)
|
|
181
|
+
while True:
|
|
182
|
+
if time.monotonic() + POLL_INTERVAL_SECONDS > deadline:
|
|
183
|
+
raise OxylabsApiError(
|
|
184
|
+
f"Oxylabs Web API did not finish scraping {url} in time (request_id {request_id}). Try again.",
|
|
185
|
+
504,
|
|
186
|
+
request_id,
|
|
187
|
+
)
|
|
188
|
+
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
|
189
|
+
polled = await client.get(
|
|
190
|
+
f"{API_BASE_URL}/v1/async/scrape/{request_id}", headers=headers, timeout=remaining()
|
|
191
|
+
)
|
|
192
|
+
if (_json_body(polled) or {}).get("state") == "pending":
|
|
193
|
+
continue
|
|
194
|
+
return _read_page(_ensure_ok(polled))
|
|
195
|
+
except httpx.RequestError as exc:
|
|
196
|
+
raise OxylabsApiError(f"Could not reach the Oxylabs Web API ({type(exc).__name__}).", 0) from None
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: oxylabs-web-api
|
|
2
|
+
version: 0.1.0
|
|
3
|
+
requires_hermes: ">=0.23.1"
|
|
4
|
+
description: "Oxylabs Web API search and page extraction for Hermes Agent: Google results with page outlines, any page as Markdown, including JavaScript-heavy and bot-protected pages."
|
|
5
|
+
author: Oxylabs
|
|
6
|
+
license: MIT
|
|
7
|
+
homepage: https://github.com/oxylabs/hermes-oxylabs-web-api
|
|
8
|
+
kind: backend
|
|
9
|
+
provides_web_providers:
|
|
10
|
+
- oxylabs
|
|
11
|
+
requires_env:
|
|
12
|
+
- name: OXYLABS_WEB_API_KEY
|
|
13
|
+
description: "Oxylabs Web API key"
|
|
14
|
+
url: "https://dashboard.oxylabs.io/"
|
|
15
|
+
secret: true
|
|
16
|
+
config_schema:
|
|
17
|
+
location:
|
|
18
|
+
type: str
|
|
19
|
+
default: ""
|
|
20
|
+
label: "Search country"
|
|
21
|
+
description: "Two-letter country code (for example US or DE) to see search results as users in that country do. Empty means no targeting."
|
|
22
|
+
render_js:
|
|
23
|
+
type: bool
|
|
24
|
+
default: false
|
|
25
|
+
label: "Render JavaScript"
|
|
26
|
+
description: "Load pages in a browser before extracting them. Slower and uses more credits; turn it on if pages come back nearly empty."
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Hermes web search and extraction backed by the Oxylabs Web API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import re
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
from agent.web_search_provider import WebSearchProvider, get_provider_env
|
|
14
|
+
|
|
15
|
+
from .client import API_KEY_ENV, MAX_SEARCH_RESULTS, OxylabsApiError, scrape_page, search_web
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
PROVIDER_NAME = "oxylabs"
|
|
20
|
+
SIGNUP_URL = "https://dashboard.oxylabs.io/"
|
|
21
|
+
# Hermes cancels a whole extract() dispatch after `web.extract_timeout` (120 s by default);
|
|
22
|
+
# finish a little earlier so each page reports its own error instead of a blanket timeout.
|
|
23
|
+
EXTRACT_DEADLINE_SECONDS = 110.0
|
|
24
|
+
_COUNTRY_CODE = re.compile(r"^[A-Za-z]{2}$")
|
|
25
|
+
|
|
26
|
+
# Reads one of this plugin's settings (`plugins.entries.oxylabs-web-api.settings.<key>`).
|
|
27
|
+
SettingReader = Callable[[str, Any], Any]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _no_settings(key: str, default: Any = None) -> Any:
|
|
31
|
+
return default
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _missing_key_error() -> str:
|
|
35
|
+
return (
|
|
36
|
+
f"{API_KEY_ENV} is not set. Get a Web API key at {SIGNUP_URL}, then run `hermes tools` "
|
|
37
|
+
f"(Web Search & Extract → Oxylabs Web API) or add {API_KEY_ENV} to ~/.hermes/.env."
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _title_from_markdown(markdown: str) -> str:
|
|
42
|
+
for line in markdown.splitlines():
|
|
43
|
+
if line.startswith("# "):
|
|
44
|
+
return line[2:].strip()
|
|
45
|
+
return ""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class OxylabsWebSearchProvider(WebSearchProvider):
|
|
49
|
+
"""Search Google and fetch pages as Markdown through the Oxylabs Web API."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, settings: SettingReader = _no_settings) -> None:
|
|
52
|
+
self._settings = settings
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def name(self) -> str:
|
|
56
|
+
return PROVIDER_NAME
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def display_name(self) -> str:
|
|
60
|
+
return "Oxylabs Web API"
|
|
61
|
+
|
|
62
|
+
def is_available(self) -> bool:
|
|
63
|
+
"""True when a key is configured. No network call."""
|
|
64
|
+
return bool(get_provider_env(API_KEY_ENV))
|
|
65
|
+
|
|
66
|
+
def supports_search(self) -> bool:
|
|
67
|
+
return True
|
|
68
|
+
|
|
69
|
+
def supports_extract(self) -> bool:
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
def _location(self) -> tuple[str | None, str | None]:
|
|
73
|
+
"""The configured country as ``(code, None)``, or ``(None, error)`` when it is not a country code."""
|
|
74
|
+
value = self._settings("location", "")
|
|
75
|
+
if not value:
|
|
76
|
+
return None, None
|
|
77
|
+
if not isinstance(value, str) or not _COUNTRY_CODE.match(value.strip()):
|
|
78
|
+
return None, f"Oxylabs location setting must be a two-letter country code such as US, not {value!r}."
|
|
79
|
+
return value.strip().upper(), None
|
|
80
|
+
|
|
81
|
+
def search(self, query: str, limit: int = 5) -> dict[str, Any]:
|
|
82
|
+
query = str(query or "").strip()
|
|
83
|
+
if not query:
|
|
84
|
+
return {"success": False, "error": "Search query must not be empty"}
|
|
85
|
+
api_key = get_provider_env(API_KEY_ENV)
|
|
86
|
+
if not api_key:
|
|
87
|
+
return {"success": False, "error": _missing_key_error()}
|
|
88
|
+
location, location_error = self._location()
|
|
89
|
+
if location_error:
|
|
90
|
+
return {"success": False, "error": location_error}
|
|
91
|
+
try:
|
|
92
|
+
count = max(1, min(int(limit), MAX_SEARCH_RESULTS))
|
|
93
|
+
except (TypeError, ValueError):
|
|
94
|
+
count = 5
|
|
95
|
+
try:
|
|
96
|
+
hits = search_web(api_key, query, count, location)
|
|
97
|
+
except OxylabsApiError as exc:
|
|
98
|
+
logger.warning("Oxylabs search failed (HTTP %s)", exc.status)
|
|
99
|
+
return {"success": False, "error": str(exc)}
|
|
100
|
+
logger.info("Oxylabs search returned %d result(s)", len(hits))
|
|
101
|
+
return {"success": True, "data": {"web": hits}}
|
|
102
|
+
|
|
103
|
+
async def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
|
|
104
|
+
"""Fetch each URL as Markdown, in parallel, keeping the input order and per-URL errors.
|
|
105
|
+
|
|
106
|
+
Hermes has already refused private and internal addresses before calling this.
|
|
107
|
+
"""
|
|
108
|
+
if not urls:
|
|
109
|
+
return []
|
|
110
|
+
api_key = get_provider_env(API_KEY_ENV)
|
|
111
|
+
if not api_key:
|
|
112
|
+
return [{"url": url, "title": "", "content": "", "error": _missing_key_error()} for url in urls]
|
|
113
|
+
render_js = self._settings("render_js", False) is True
|
|
114
|
+
deadline = time.monotonic() + EXTRACT_DEADLINE_SECONDS
|
|
115
|
+
|
|
116
|
+
async def extract_one(client: httpx.AsyncClient, url: str) -> dict[str, Any]:
|
|
117
|
+
try:
|
|
118
|
+
markdown, status = await scrape_page(client, api_key, url, render_js=render_js, deadline=deadline)
|
|
119
|
+
except OxylabsApiError as exc:
|
|
120
|
+
return {"url": url, "title": "", "content": "", "error": str(exc)}
|
|
121
|
+
if status is not None and not 200 <= status < 300:
|
|
122
|
+
return {"url": url, "title": "", "content": "", "error": f"{url} returned HTTP {status}."}
|
|
123
|
+
if not markdown:
|
|
124
|
+
return {"url": url, "title": "", "content": "", "error": f"No content returned for {url}."}
|
|
125
|
+
title = _title_from_markdown(markdown)
|
|
126
|
+
return {
|
|
127
|
+
"url": url,
|
|
128
|
+
"title": title,
|
|
129
|
+
"content": markdown,
|
|
130
|
+
"raw_content": markdown,
|
|
131
|
+
"metadata": {"sourceURL": url, "title": title, "statusCode": status},
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async with httpx.AsyncClient() as client:
|
|
135
|
+
return list(await asyncio.gather(*(extract_one(client, url) for url in urls)))
|
|
136
|
+
|
|
137
|
+
def get_setup_schema(self) -> dict[str, Any]:
|
|
138
|
+
return {
|
|
139
|
+
"name": "Oxylabs Web API",
|
|
140
|
+
"badge": "paid",
|
|
141
|
+
"tag": "Google search and page extraction, including JavaScript-heavy and bot-protected pages.",
|
|
142
|
+
"env_vars": [{"key": API_KEY_ENV, "prompt": "Oxylabs Web API key", "url": SIGNUP_URL}],
|
|
143
|
+
}
|
|
File without changes
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hermes-oxylabs-web-api
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Oxylabs Web API search and page extraction provider for Hermes Agent
|
|
5
|
+
Author-email: Oxylabs <giedrius.stankauskas@oxylabs.io>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/oxylabs/hermes-oxylabs-web-api
|
|
8
|
+
Project-URL: Repository, https://github.com/oxylabs/hermes-oxylabs-web-api
|
|
9
|
+
Project-URL: Issues, https://github.com/oxylabs/hermes-oxylabs-web-api/issues
|
|
10
|
+
Keywords: hermes-agent,plugin,oxylabs,web-search,web-scraping,markdown
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Plugins
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
19
|
+
Requires-Python: <3.16,>=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: httpx<1,>=0.27
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# Oxylabs Web API for Hermes Agent
|
|
26
|
+
|
|
27
|
+
A [Hermes Agent](https://hermes-agent.nousresearch.com/) plugin that adds the [Oxylabs Web API](https://oxylabs.io/products/web-api) as a provider for Hermes's built-in `web_search` and `web_extract` tools.
|
|
28
|
+
|
|
29
|
+
- **web_search**: Google results with titles, URLs and a short outline of each page. Set a country to see results as users in that country do.
|
|
30
|
+
- **web_extract**: any page as Markdown, including JavaScript-heavy pages (turn on JavaScript rendering) and pages that block plain HTTP clients.
|
|
31
|
+
|
|
32
|
+
The agent keeps using the tools Hermes already has; you choose Oxylabs as the backend.
|
|
33
|
+
|
|
34
|
+
## Requirements
|
|
35
|
+
|
|
36
|
+
- Hermes Agent 0.23.1 or later.
|
|
37
|
+
- An Oxylabs Web API key. Get one in the [Oxylabs dashboard](https://dashboard.oxylabs.io/).
|
|
38
|
+
|
|
39
|
+
## Set up
|
|
40
|
+
|
|
41
|
+
1. Install and enable the plugin:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
hermes plugins install oxylabs/hermes-oxylabs-web-api --enable
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Hermes warns that the plugin is not from its reviewed catalog, and asks before installing its one dependency (`httpx`, which Hermes already uses). In scripts, add `--yes-deps`.
|
|
48
|
+
|
|
49
|
+
2. Add your key. The install asks for it; paste it there, or skip and add it later. To add it later, run `hermes tools`, open **Web Search & Extract**, select **Oxylabs Web API** and paste the key. Or add it to your profile's `.env` file (`~/.hermes/.env` for the default profile):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
OXYLABS_WEB_API_KEY=your-web-api-key
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
3. Select Oxylabs for search, extraction, or both:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
hermes config set web.search_backend oxylabs
|
|
59
|
+
hermes config set web.extract_backend oxylabs
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Start a new session after these steps.
|
|
63
|
+
|
|
64
|
+
## Settings
|
|
65
|
+
|
|
66
|
+
Both are optional:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
hermes config set plugins.entries.oxylabs-web-api.settings.location US
|
|
70
|
+
hermes config set plugins.entries.oxylabs-web-api.settings.render_js true
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
| Setting | Default | What it does |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| `location` | none | Two-letter country code for searches, such as `US` or `DE`. |
|
|
76
|
+
| `render_js` | `false` | Load pages in a browser before extracting them. Slower and uses more credits; turn it on if pages come back nearly empty. |
|
|
77
|
+
|
|
78
|
+
The Hermes desktop app shows the same settings on the plugin's page under **Settings → Plugins**.
|
|
79
|
+
|
|
80
|
+
Slow pages are queued on the asynchronous Web API and polled, and each page gets up to 110 seconds, just inside Hermes's 120-second extract timeout.
|
|
81
|
+
|
|
82
|
+
## Your data
|
|
83
|
+
|
|
84
|
+
- The plugin sends your search queries and the URLs you ask Hermes to read to `https://webapi.oxylabs.io`, together with your API key. It contacts no other service. Requests are billed to your Oxylabs account.
|
|
85
|
+
- Oxylabs runs each search and loads each page, so the sites you read receive Oxylabs's request. Before a page is fetched, Hermes checks the address on your machine and blocks private and internal ones.
|
|
86
|
+
- Requests carry an `x-oxylabs-sdk: oxylabs-web-api-hermes/<version>` header so Oxylabs can count usage per integration. It holds the plugin and Python versions only.
|
|
87
|
+
- The plugin reads the key through Hermes and never includes it in results or error messages. It collects no telemetry, runs no shell commands or background processes, and reads none of your files.
|
|
88
|
+
|
|
89
|
+
## Update or remove
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
hermes plugins update oxylabs-web-api
|
|
93
|
+
hermes plugins remove oxylabs-web-api
|
|
94
|
+
hermes config unset web.search_backend
|
|
95
|
+
hermes config unset web.extract_backend
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Troubleshooting
|
|
99
|
+
|
|
100
|
+
| Problem | Fix |
|
|
101
|
+
|---|---|
|
|
102
|
+
| `OXYLABS_WEB_API_KEY is not set` | Add the key as in step 2, then start a new session. |
|
|
103
|
+
| `rejected the API key (401)` | The key is wrong or revoked. Check it in the Oxylabs dashboard. |
|
|
104
|
+
| `quota is used up (429 QUOTA_EXCEEDED)` | Top up the plan in the Oxylabs dashboard. |
|
|
105
|
+
| `web_search` uses another provider | Run the `hermes config set` commands from step 3. Check `hermes plugins list` shows `oxylabs-web-api` as enabled. |
|
|
106
|
+
| A page comes back nearly empty | Turn on `render_js`. |
|
|
107
|
+
|
|
108
|
+
## Install from PyPI
|
|
109
|
+
|
|
110
|
+
If you run Hermes from a Python environment you manage yourself, install the package into the interpreter that runs Hermes and enable it:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
uv pip install --python /path/to/hermes/python hermes-oxylabs-web-api
|
|
114
|
+
hermes plugins enable oxylabs-web-api
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`hermes plugins update` and `remove` don't manage a PyPI install; upgrade or uninstall it with the same tool.
|
|
118
|
+
|
|
119
|
+
## Development
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
uv sync --dev
|
|
123
|
+
uv run ruff check . && uv run ruff format --check .
|
|
124
|
+
uv run pytest # unit tests
|
|
125
|
+
OXYLABS_WEB_API_KEY=... uv run pytest -m live # calls the real API
|
|
126
|
+
hermes plugins validate . # the checks the Hermes catalog runs
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Releases: bump the version in `pyproject.toml`, both `plugin.yaml` files and `src/hermes_oxylabs_web_api/__init__.py`, add a `CHANGELOG.md` entry, then push a `v<version>` tag. The `Publish` workflow publishes to PyPI with trusted publishing.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
MIT
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
hermes_oxylabs_web_api/__init__.py,sha256=6Y8NspZNJJBoKuLaM6_0NrHdVF9Vx2O-5OhuBdTiM1Y,398
|
|
2
|
+
hermes_oxylabs_web_api/client.py,sha256=7iz-5nxnYFzWbZPUHCPtZmjq-cy7pej8UA76LzgwsMM,8017
|
|
3
|
+
hermes_oxylabs_web_api/plugin.yaml,sha256=REt6aUc5vCxdB1vVgRAWIvjETOYNXwNUX3wwrWhlEOM,977
|
|
4
|
+
hermes_oxylabs_web_api/provider.py,sha256=z6O-0sbepbSnRGceFUKUCe5VG1EfwraPQn-92oKAcFI,5672
|
|
5
|
+
hermes_oxylabs_web_api/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
hermes_oxylabs_web_api-0.1.0.dist-info/licenses/LICENSE,sha256=xMb4RQP0tmMUubbQ9Lb9oaTnNHFkUV-NFUviGo0k1wg,1064
|
|
7
|
+
hermes_oxylabs_web_api-0.1.0.dist-info/METADATA,sha256=UK1sF6PAzecd_7h9_q-QNb-W0odCpxzx3pEvoEr1EsQ,5909
|
|
8
|
+
hermes_oxylabs_web_api-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
9
|
+
hermes_oxylabs_web_api-0.1.0.dist-info/entry_points.txt,sha256=Vh-IX3Zz08o-igQUw7aToegTtvYMhY6jeCWn7hKYY1c,64
|
|
10
|
+
hermes_oxylabs_web_api-0.1.0.dist-info/top_level.txt,sha256=azpJht9GuiAxb0bip8xGdswymelYsUWunvZOahq-Y6U,23
|
|
11
|
+
hermes_oxylabs_web_api-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Oxylabs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
hermes_oxylabs_web_api
|