langchain-browsa 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_browsa/__init__.py +27 -0
- langchain_browsa/_common.py +148 -0
- langchain_browsa/document_loaders.py +227 -0
- langchain_browsa/py.typed +0 -0
- langchain_browsa/toolkits.py +89 -0
- langchain_browsa/tools.py +609 -0
- langchain_browsa-0.1.0.dist-info/METADATA +276 -0
- langchain_browsa-0.1.0.dist-info/RECORD +11 -0
- langchain_browsa-0.1.0.dist-info/WHEEL +5 -0
- langchain_browsa-0.1.0.dist-info/licenses/LICENSE +21 -0
- langchain_browsa-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""LangChain integration for Browsa (https://browsa.io): Phantom cloud browsers for AI agents."""
|
|
2
|
+
|
|
3
|
+
from importlib import metadata
|
|
4
|
+
|
|
5
|
+
from langchain_browsa.document_loaders import BrowsaLoader
|
|
6
|
+
from langchain_browsa.toolkits import BrowsaToolkit
|
|
7
|
+
from langchain_browsa.tools import (
|
|
8
|
+
BrowsaExtractTool,
|
|
9
|
+
BrowsaScrapeTool,
|
|
10
|
+
BrowsaScreenshotTool,
|
|
11
|
+
BrowsaTaskTool,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
__version__ = metadata.version("langchain-browsa")
|
|
16
|
+
except metadata.PackageNotFoundError: # running from a source checkout
|
|
17
|
+
__version__ = ""
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"BrowsaExtractTool",
|
|
21
|
+
"BrowsaLoader",
|
|
22
|
+
"BrowsaScrapeTool",
|
|
23
|
+
"BrowsaScreenshotTool",
|
|
24
|
+
"BrowsaTaskTool",
|
|
25
|
+
"BrowsaToolkit",
|
|
26
|
+
"__version__",
|
|
27
|
+
]
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Shared helpers: API key resolution, client construction, error mapping."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import os
|
|
7
|
+
from typing import Any, Optional
|
|
8
|
+
|
|
9
|
+
from browsa import AsyncClient, BrowsaError, Client, Job
|
|
10
|
+
from browsa.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
|
|
11
|
+
from langchain_core.tools import BaseTool, ToolException
|
|
12
|
+
from langchain_core.utils import secret_from_env
|
|
13
|
+
from pydantic import Field, SecretStr, model_validator
|
|
14
|
+
from typing_extensions import Self
|
|
15
|
+
|
|
16
|
+
API_KEY_ENV = "BROWSA_API_KEY"
|
|
17
|
+
|
|
18
|
+
_MISSING_KEY = (
|
|
19
|
+
"No Browsa API key. Set the BROWSA_API_KEY environment variable or pass "
|
|
20
|
+
"api_key=... (keys start with 'agt_'; create one at "
|
|
21
|
+
"https://app.browsa.io/dashboard/keys)."
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def resolve_api_key(api_key: Optional[Any]) -> Optional[str]:
|
|
26
|
+
"""Return the explicit key (str or SecretStr) or BROWSA_API_KEY, or None."""
|
|
27
|
+
if isinstance(api_key, SecretStr):
|
|
28
|
+
api_key = api_key.get_secret_value()
|
|
29
|
+
if api_key:
|
|
30
|
+
return str(api_key)
|
|
31
|
+
return os.environ.get(API_KEY_ENV) or None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_clients(
|
|
35
|
+
api_key: Optional[Any],
|
|
36
|
+
*,
|
|
37
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
38
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
39
|
+
client: Optional[Client] = None,
|
|
40
|
+
async_client: Optional[AsyncClient] = None,
|
|
41
|
+
) -> tuple[Client, Optional[AsyncClient]]:
|
|
42
|
+
"""Build the sync and async SDK clients that are not given.
|
|
43
|
+
|
|
44
|
+
With only ``client`` given and no key available, the async client stays
|
|
45
|
+
None and async calls run the sync client in a worker thread.
|
|
46
|
+
"""
|
|
47
|
+
key = resolve_api_key(api_key)
|
|
48
|
+
if client is None:
|
|
49
|
+
if not key:
|
|
50
|
+
raise ValueError(_MISSING_KEY)
|
|
51
|
+
client = Client(key, base_url=base_url, timeout=timeout)
|
|
52
|
+
if async_client is None and key:
|
|
53
|
+
async_client = AsyncClient(key, base_url=base_url, timeout=timeout)
|
|
54
|
+
return client, async_client
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def call_async(
|
|
58
|
+
async_client: Optional[AsyncClient],
|
|
59
|
+
client: Client,
|
|
60
|
+
method: str,
|
|
61
|
+
*args: Any,
|
|
62
|
+
**kwargs: Any,
|
|
63
|
+
) -> Any:
|
|
64
|
+
"""Call ``method`` on the async client, or on the sync one in a thread."""
|
|
65
|
+
if async_client is not None:
|
|
66
|
+
return await getattr(async_client, method)(*args, **kwargs)
|
|
67
|
+
return await asyncio.to_thread(getattr(client, method), *args, **kwargs)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def to_tool_error(err: BrowsaError) -> ToolException:
|
|
71
|
+
"""Map an SDK error to a ToolException with the API's code and request id."""
|
|
72
|
+
return ToolException(f"Browsa API error: {err}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def envelope_job_id(envelope: Any) -> str:
|
|
76
|
+
"""Job id from a 202 envelope (``job_id``, or ``id`` on some endpoints)."""
|
|
77
|
+
if isinstance(envelope, dict):
|
|
78
|
+
jid = envelope.get("job_id") or envelope.get("id")
|
|
79
|
+
if jid:
|
|
80
|
+
return str(jid)
|
|
81
|
+
raise ToolException(f"Browsa did not return a job id: {envelope!r}")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def job_result(job: Job) -> dict:
|
|
85
|
+
"""The ``result`` object of a finished job (empty dict if there is none)."""
|
|
86
|
+
result = job.raw.get("result") if isinstance(job.raw, dict) else None
|
|
87
|
+
return result if isinstance(result, dict) else {}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def drop_none(d: dict, keep: tuple[str, ...] = ()) -> dict:
|
|
91
|
+
return {k: v for k, v in d.items() if v is not None or k in keep}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class BrowsaBaseTool(BaseTool):
|
|
95
|
+
"""Base class of the Browsa tools: holds the key and the SDK clients."""
|
|
96
|
+
|
|
97
|
+
api_key: Optional[SecretStr] = Field(
|
|
98
|
+
default_factory=secret_from_env(API_KEY_ENV, default=None)
|
|
99
|
+
)
|
|
100
|
+
"""Browsa API key (``agt_live_...``). Read from ``BROWSA_API_KEY`` if not set."""
|
|
101
|
+
|
|
102
|
+
base_url: str = DEFAULT_BASE_URL
|
|
103
|
+
"""API base URL. Leave the default unless Browsa support tells you otherwise."""
|
|
104
|
+
|
|
105
|
+
request_timeout: float = DEFAULT_TIMEOUT
|
|
106
|
+
"""Per-request HTTP timeout of the SDK client, in seconds."""
|
|
107
|
+
|
|
108
|
+
client: Any = Field(default=None, exclude=True)
|
|
109
|
+
"""A ``browsa.Client`` to use instead of building one."""
|
|
110
|
+
|
|
111
|
+
async_client: Any = Field(default=None, exclude=True)
|
|
112
|
+
"""A ``browsa.AsyncClient`` to use instead of building one."""
|
|
113
|
+
|
|
114
|
+
@model_validator(mode="after")
|
|
115
|
+
def _init_clients(self) -> Self:
|
|
116
|
+
self.client, self.async_client = build_clients(
|
|
117
|
+
self.api_key,
|
|
118
|
+
base_url=self.base_url,
|
|
119
|
+
timeout=self.request_timeout,
|
|
120
|
+
client=self.client,
|
|
121
|
+
async_client=self.async_client,
|
|
122
|
+
)
|
|
123
|
+
return self
|
|
124
|
+
|
|
125
|
+
def _call(self, method: str, *args: Any, **kwargs: Any) -> Any:
|
|
126
|
+
try:
|
|
127
|
+
return getattr(self.client, method)(*args, **kwargs)
|
|
128
|
+
except BrowsaError as e:
|
|
129
|
+
raise to_tool_error(e) from e
|
|
130
|
+
|
|
131
|
+
async def _acall(self, method: str, *args: Any, **kwargs: Any) -> Any:
|
|
132
|
+
try:
|
|
133
|
+
return await call_async(self.async_client, self.client, method, *args, **kwargs)
|
|
134
|
+
except BrowsaError as e:
|
|
135
|
+
raise to_tool_error(e) from e
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
__all__ = [
|
|
139
|
+
"API_KEY_ENV",
|
|
140
|
+
"BrowsaBaseTool",
|
|
141
|
+
"build_clients",
|
|
142
|
+
"call_async",
|
|
143
|
+
"drop_none",
|
|
144
|
+
"envelope_job_id",
|
|
145
|
+
"job_result",
|
|
146
|
+
"resolve_api_key",
|
|
147
|
+
"to_tool_error",
|
|
148
|
+
]
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""Document loader: Browsa scrape, batch and crawl results as LangChain Documents."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import AsyncIterator, Iterator, Sequence
|
|
7
|
+
from typing import Any, Literal, Optional, Union
|
|
8
|
+
|
|
9
|
+
from browsa import AsyncClient, BrowsaError, Client, Job
|
|
10
|
+
from browsa.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
|
|
11
|
+
from langchain_core.document_loaders import BaseLoader
|
|
12
|
+
from langchain_core.documents import Document
|
|
13
|
+
|
|
14
|
+
from langchain_browsa._common import (
|
|
15
|
+
build_clients,
|
|
16
|
+
call_async,
|
|
17
|
+
envelope_job_id,
|
|
18
|
+
job_result,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
Mode = Literal["scrape", "batch", "crawl"]
|
|
24
|
+
ContentFormat = Literal["markdown", "text", "html"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class BrowsaLoader(BaseLoader):
|
|
28
|
+
"""Load web pages as Documents, fetched in Phantom, Browsa's cloud browser.
|
|
29
|
+
|
|
30
|
+
Modes:
|
|
31
|
+
|
|
32
|
+
* ``"scrape"``: one ``POST /v1/quick/scrape`` per URL, in order.
|
|
33
|
+
* ``"batch"``: all URLs as one batch job (``POST /v1/quick/batch``, up to
|
|
34
|
+
100 URLs), then wait for it.
|
|
35
|
+
* ``"crawl"``: follow links from one start URL (``POST /v1/quick/crawl``),
|
|
36
|
+
up to ``max_pages``, then wait for it.
|
|
37
|
+
|
|
38
|
+
Each Document's ``page_content`` is the page in ``content_format``
|
|
39
|
+
(Markdown by default) and its metadata has ``source`` (the URL the
|
|
40
|
+
browser landed on), ``requested_url``, ``http_status`` and, for batch
|
|
41
|
+
and crawl, ``job_id``. Pages that failed are skipped and logged.
|
|
42
|
+
|
|
43
|
+
Setup:
|
|
44
|
+
.. code-block:: bash
|
|
45
|
+
|
|
46
|
+
pip install -U langchain-browsa
|
|
47
|
+
export BROWSA_API_KEY="agt_live_..."
|
|
48
|
+
|
|
49
|
+
Instantiate:
|
|
50
|
+
.. code-block:: python
|
|
51
|
+
|
|
52
|
+
from langchain_browsa import BrowsaLoader
|
|
53
|
+
|
|
54
|
+
loader = BrowsaLoader("https://example.com")
|
|
55
|
+
crawl = BrowsaLoader("https://docs.example.com", mode="crawl", max_pages=10)
|
|
56
|
+
|
|
57
|
+
Load:
|
|
58
|
+
.. code-block:: python
|
|
59
|
+
|
|
60
|
+
docs = loader.load()
|
|
61
|
+
docs[0].page_content[:200], docs[0].metadata["source"]
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
url: Union[str, Sequence[str]],
|
|
67
|
+
*,
|
|
68
|
+
mode: Mode = "scrape",
|
|
69
|
+
api_key: Optional[str] = None,
|
|
70
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
71
|
+
content_format: ContentFormat = "markdown",
|
|
72
|
+
formats: Optional[Sequence[str]] = None,
|
|
73
|
+
max_pages: int = 25,
|
|
74
|
+
wait_timeout: Optional[float] = 600,
|
|
75
|
+
request_timeout: float = DEFAULT_TIMEOUT,
|
|
76
|
+
client: Optional[Client] = None,
|
|
77
|
+
async_client: Optional[AsyncClient] = None,
|
|
78
|
+
**params: Any,
|
|
79
|
+
) -> None:
|
|
80
|
+
"""Create the loader.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
url: One URL or a list of URLs. ``"crawl"`` takes exactly one start URL.
|
|
84
|
+
mode: ``"scrape"``, ``"batch"`` or ``"crawl"``.
|
|
85
|
+
api_key: Browsa API key; defaults to the ``BROWSA_API_KEY`` env var.
|
|
86
|
+
base_url: API base URL.
|
|
87
|
+
content_format: Which format becomes ``page_content``.
|
|
88
|
+
formats: Formats to request; defaults to ``[content_format]``.
|
|
89
|
+
``"links"`` adds the page's links to the metadata.
|
|
90
|
+
max_pages: Page cap for ``"crawl"``.
|
|
91
|
+
wait_timeout: Seconds to wait for a batch or crawl job.
|
|
92
|
+
request_timeout: Per-request HTTP timeout of the SDK client.
|
|
93
|
+
client: A ``browsa.Client`` to use instead of building one.
|
|
94
|
+
async_client: A ``browsa.AsyncClient`` to use instead of building one.
|
|
95
|
+
**params: Other endpoint options, e.g. ``wait_ms``, ``max_chars``,
|
|
96
|
+
``country``, ``block``; for crawl ``max_depth``,
|
|
97
|
+
``same_domain``, ``include_regex``, ``exclude_regex``.
|
|
98
|
+
"""
|
|
99
|
+
urls = [url] if isinstance(url, str) else list(url)
|
|
100
|
+
if not urls:
|
|
101
|
+
raise ValueError("BrowsaLoader needs at least one URL.")
|
|
102
|
+
if mode not in ("scrape", "batch", "crawl"):
|
|
103
|
+
raise ValueError(f"mode must be 'scrape', 'batch' or 'crawl', not {mode!r}")
|
|
104
|
+
if mode == "crawl" and len(urls) != 1:
|
|
105
|
+
raise ValueError("mode='crawl' takes exactly one start URL.")
|
|
106
|
+
self.urls = urls
|
|
107
|
+
self.mode: Mode = mode
|
|
108
|
+
self.content_format: ContentFormat = content_format
|
|
109
|
+
fmts = list(formats) if formats else [content_format]
|
|
110
|
+
if content_format not in fmts:
|
|
111
|
+
fmts.append(content_format)
|
|
112
|
+
self.formats = fmts
|
|
113
|
+
self.max_pages = max_pages
|
|
114
|
+
self.wait_timeout = wait_timeout
|
|
115
|
+
self.params = params
|
|
116
|
+
self.client, self.async_client = build_clients(
|
|
117
|
+
api_key,
|
|
118
|
+
base_url=base_url,
|
|
119
|
+
timeout=request_timeout,
|
|
120
|
+
client=client,
|
|
121
|
+
async_client=async_client,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# ── helpers ──────────────────────────────────────────────────────────
|
|
125
|
+
|
|
126
|
+
def _scrape_kwargs(self) -> dict[str, Any]:
|
|
127
|
+
kw = dict(self.params)
|
|
128
|
+
kw["formats"] = list(self.formats)
|
|
129
|
+
return kw
|
|
130
|
+
|
|
131
|
+
def _crawl_kwargs(self) -> dict[str, Any]:
|
|
132
|
+
kw = self._scrape_kwargs()
|
|
133
|
+
kw["max_pages"] = self.max_pages
|
|
134
|
+
return kw
|
|
135
|
+
|
|
136
|
+
def _to_document(
|
|
137
|
+
self, data: dict, requested_url: str, job_id: Optional[str] = None
|
|
138
|
+
) -> Document:
|
|
139
|
+
content = data.get(self.content_format)
|
|
140
|
+
if not isinstance(content, str):
|
|
141
|
+
content = data.get("markdown") or data.get("text") or data.get("html") or ""
|
|
142
|
+
meta: dict[str, Any] = {
|
|
143
|
+
"source": data.get("url") or requested_url,
|
|
144
|
+
"requested_url": requested_url,
|
|
145
|
+
"http_status": data.get("http_status"),
|
|
146
|
+
}
|
|
147
|
+
page_meta = data.get("meta")
|
|
148
|
+
if isinstance(page_meta, dict):
|
|
149
|
+
title = page_meta.get("title")
|
|
150
|
+
if isinstance(title, str) and title:
|
|
151
|
+
meta["title"] = title
|
|
152
|
+
if isinstance(data.get("links"), list):
|
|
153
|
+
meta["links"] = data["links"]
|
|
154
|
+
if job_id:
|
|
155
|
+
meta["job_id"] = job_id
|
|
156
|
+
meta["mode"] = self.mode
|
|
157
|
+
return Document(page_content=content, metadata=meta)
|
|
158
|
+
|
|
159
|
+
def _job_documents(self, job: Job) -> Iterator[Document]:
|
|
160
|
+
if job.status != "completed":
|
|
161
|
+
raise BrowsaError(
|
|
162
|
+
f"Browsa {self.mode} job {job.id} ended {job.status}"
|
|
163
|
+
f" (outcome {job.outcome}): {job.error_code or ''} {job.error_message or ''}".rstrip(),
|
|
164
|
+
code=job.error_code,
|
|
165
|
+
)
|
|
166
|
+
result = job_result(job)
|
|
167
|
+
items = result.get("pages" if self.mode == "crawl" else "items") or []
|
|
168
|
+
for item in items:
|
|
169
|
+
if not isinstance(item, dict):
|
|
170
|
+
continue
|
|
171
|
+
data = item.get("data")
|
|
172
|
+
if item.get("status") != "completed" or not isinstance(data, dict):
|
|
173
|
+
logger.warning(
|
|
174
|
+
"Browsa %s: skipped %s (%s %s)",
|
|
175
|
+
self.mode,
|
|
176
|
+
item.get("url"),
|
|
177
|
+
item.get("error_code"),
|
|
178
|
+
item.get("error_message"),
|
|
179
|
+
)
|
|
180
|
+
continue
|
|
181
|
+
yield self._to_document(data, item.get("url") or "", job.id)
|
|
182
|
+
|
|
183
|
+
def _submit(self) -> dict:
|
|
184
|
+
if self.mode == "crawl":
|
|
185
|
+
return self.client.crawl(self.urls[0], **self._crawl_kwargs())
|
|
186
|
+
return self.client.batch(self.urls, **self._scrape_kwargs())
|
|
187
|
+
|
|
188
|
+
async def _asubmit(self) -> dict:
|
|
189
|
+
if self.mode == "crawl":
|
|
190
|
+
return await call_async(
|
|
191
|
+
self.async_client, self.client, "crawl", self.urls[0], **self._crawl_kwargs()
|
|
192
|
+
)
|
|
193
|
+
return await call_async(
|
|
194
|
+
self.async_client, self.client, "batch", self.urls, **self._scrape_kwargs()
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
# ── BaseLoader ───────────────────────────────────────────────────────
|
|
198
|
+
|
|
199
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
200
|
+
"""Load pages one by one (scrape) or after the job ends (batch, crawl)."""
|
|
201
|
+
if self.mode == "scrape":
|
|
202
|
+
for u in self.urls:
|
|
203
|
+
data = self.client.scrape(u, **self._scrape_kwargs())
|
|
204
|
+
yield self._to_document(data, u)
|
|
205
|
+
return
|
|
206
|
+
job_id = envelope_job_id(self._submit())
|
|
207
|
+
job = self.client.wait_for_job(job_id, timeout=self.wait_timeout)
|
|
208
|
+
yield from self._job_documents(job)
|
|
209
|
+
|
|
210
|
+
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
211
|
+
"""Async version of :meth:`lazy_load`, using ``browsa.AsyncClient``."""
|
|
212
|
+
if self.mode == "scrape":
|
|
213
|
+
for u in self.urls:
|
|
214
|
+
data = await call_async(
|
|
215
|
+
self.async_client, self.client, "scrape", u, **self._scrape_kwargs()
|
|
216
|
+
)
|
|
217
|
+
yield self._to_document(data, u)
|
|
218
|
+
return
|
|
219
|
+
job_id = envelope_job_id(await self._asubmit())
|
|
220
|
+
job = await call_async(
|
|
221
|
+
self.async_client, self.client, "wait_for_job", job_id, timeout=self.wait_timeout
|
|
222
|
+
)
|
|
223
|
+
for doc in self._job_documents(job):
|
|
224
|
+
yield doc
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
__all__ = ["BrowsaLoader"]
|
|
File without changes
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Toolkit with every Browsa tool, sharing one set of SDK clients."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
from browsa.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
|
|
8
|
+
from langchain_core.tools import BaseTool, BaseToolkit
|
|
9
|
+
from langchain_core.utils import secret_from_env
|
|
10
|
+
from pydantic import ConfigDict, Field, SecretStr, model_validator
|
|
11
|
+
from typing_extensions import Self
|
|
12
|
+
|
|
13
|
+
from langchain_browsa._common import API_KEY_ENV, build_clients
|
|
14
|
+
from langchain_browsa.tools import (
|
|
15
|
+
BrowsaExtractTool,
|
|
16
|
+
BrowsaScrapeTool,
|
|
17
|
+
BrowsaScreenshotTool,
|
|
18
|
+
BrowsaTaskTool,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BrowsaToolkit(BaseToolkit):
|
|
23
|
+
"""All Browsa tools: task, scrape, extract and screenshot.
|
|
24
|
+
|
|
25
|
+
Instantiate:
|
|
26
|
+
.. code-block:: python
|
|
27
|
+
|
|
28
|
+
from langchain_browsa import BrowsaToolkit
|
|
29
|
+
|
|
30
|
+
toolkit = BrowsaToolkit() # reads BROWSA_API_KEY
|
|
31
|
+
tools = toolkit.get_tools()
|
|
32
|
+
|
|
33
|
+
Per-tool settings go in ``task_kwargs``, ``scrape_kwargs``,
|
|
34
|
+
``extract_kwargs`` and ``screenshot_kwargs``, e.g.
|
|
35
|
+
``BrowsaToolkit(task_kwargs={"llm": "browsa-smart", "max_credits": 30})``.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
api_key: Optional[SecretStr] = Field(
|
|
39
|
+
default_factory=secret_from_env(API_KEY_ENV, default=None)
|
|
40
|
+
)
|
|
41
|
+
base_url: str = DEFAULT_BASE_URL
|
|
42
|
+
request_timeout: float = DEFAULT_TIMEOUT
|
|
43
|
+
client: Any = Field(default=None, exclude=True)
|
|
44
|
+
async_client: Any = Field(default=None, exclude=True)
|
|
45
|
+
|
|
46
|
+
include_task: bool = True
|
|
47
|
+
include_scrape: bool = True
|
|
48
|
+
include_extract: bool = True
|
|
49
|
+
include_screenshot: bool = True
|
|
50
|
+
|
|
51
|
+
task_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
52
|
+
scrape_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
53
|
+
extract_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
54
|
+
screenshot_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
55
|
+
|
|
56
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
57
|
+
|
|
58
|
+
@model_validator(mode="after")
|
|
59
|
+
def _init_clients(self) -> Self:
|
|
60
|
+
self.client, self.async_client = build_clients(
|
|
61
|
+
self.api_key,
|
|
62
|
+
base_url=self.base_url,
|
|
63
|
+
timeout=self.request_timeout,
|
|
64
|
+
client=self.client,
|
|
65
|
+
async_client=self.async_client,
|
|
66
|
+
)
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
def get_tools(self) -> list[BaseTool]:
|
|
70
|
+
"""Return the tools, all using this toolkit's clients."""
|
|
71
|
+
shared = {
|
|
72
|
+
"api_key": self.api_key,
|
|
73
|
+
"base_url": self.base_url,
|
|
74
|
+
"client": self.client,
|
|
75
|
+
"async_client": self.async_client,
|
|
76
|
+
}
|
|
77
|
+
tools: list[BaseTool] = []
|
|
78
|
+
if self.include_task:
|
|
79
|
+
tools.append(BrowsaTaskTool(**shared, **self.task_kwargs))
|
|
80
|
+
if self.include_scrape:
|
|
81
|
+
tools.append(BrowsaScrapeTool(**shared, **self.scrape_kwargs))
|
|
82
|
+
if self.include_extract:
|
|
83
|
+
tools.append(BrowsaExtractTool(**shared, **self.extract_kwargs))
|
|
84
|
+
if self.include_screenshot:
|
|
85
|
+
tools.append(BrowsaScreenshotTool(**shared, **self.screenshot_kwargs))
|
|
86
|
+
return tools
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
__all__ = ["BrowsaToolkit"]
|