langchain-browsa 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ """LangChain integration for Browsa (https://browsa.io): Phantom cloud browsers for AI agents."""
2
+
3
+ from importlib import metadata
4
+
5
+ from langchain_browsa.document_loaders import BrowsaLoader
6
+ from langchain_browsa.toolkits import BrowsaToolkit
7
+ from langchain_browsa.tools import (
8
+ BrowsaExtractTool,
9
+ BrowsaScrapeTool,
10
+ BrowsaScreenshotTool,
11
+ BrowsaTaskTool,
12
+ )
13
+
14
+ try:
15
+ __version__ = metadata.version("langchain-browsa")
16
+ except metadata.PackageNotFoundError: # running from a source checkout
17
+ __version__ = ""
18
+
19
+ __all__ = [
20
+ "BrowsaExtractTool",
21
+ "BrowsaLoader",
22
+ "BrowsaScrapeTool",
23
+ "BrowsaScreenshotTool",
24
+ "BrowsaTaskTool",
25
+ "BrowsaToolkit",
26
+ "__version__",
27
+ ]
@@ -0,0 +1,148 @@
1
+ """Shared helpers: API key resolution, client construction, error mapping."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import os
7
+ from typing import Any, Optional
8
+
9
+ from browsa import AsyncClient, BrowsaError, Client, Job
10
+ from browsa.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
11
+ from langchain_core.tools import BaseTool, ToolException
12
+ from langchain_core.utils import secret_from_env
13
+ from pydantic import Field, SecretStr, model_validator
14
+ from typing_extensions import Self
15
+
16
+ API_KEY_ENV = "BROWSA_API_KEY"
17
+
18
+ _MISSING_KEY = (
19
+ "No Browsa API key. Set the BROWSA_API_KEY environment variable or pass "
20
+ "api_key=... (keys start with 'agt_'; create one at "
21
+ "https://app.browsa.io/dashboard/keys)."
22
+ )
23
+
24
+
25
+ def resolve_api_key(api_key: Optional[Any]) -> Optional[str]:
26
+ """Return the explicit key (str or SecretStr) or BROWSA_API_KEY, or None."""
27
+ if isinstance(api_key, SecretStr):
28
+ api_key = api_key.get_secret_value()
29
+ if api_key:
30
+ return str(api_key)
31
+ return os.environ.get(API_KEY_ENV) or None
32
+
33
+
34
+ def build_clients(
35
+ api_key: Optional[Any],
36
+ *,
37
+ base_url: str = DEFAULT_BASE_URL,
38
+ timeout: float = DEFAULT_TIMEOUT,
39
+ client: Optional[Client] = None,
40
+ async_client: Optional[AsyncClient] = None,
41
+ ) -> tuple[Client, Optional[AsyncClient]]:
42
+ """Build the sync and async SDK clients that are not given.
43
+
44
+ With only ``client`` given and no key available, the async client stays
45
+ None and async calls run the sync client in a worker thread.
46
+ """
47
+ key = resolve_api_key(api_key)
48
+ if client is None:
49
+ if not key:
50
+ raise ValueError(_MISSING_KEY)
51
+ client = Client(key, base_url=base_url, timeout=timeout)
52
+ if async_client is None and key:
53
+ async_client = AsyncClient(key, base_url=base_url, timeout=timeout)
54
+ return client, async_client
55
+
56
+
57
+ async def call_async(
58
+ async_client: Optional[AsyncClient],
59
+ client: Client,
60
+ method: str,
61
+ *args: Any,
62
+ **kwargs: Any,
63
+ ) -> Any:
64
+ """Call ``method`` on the async client, or on the sync one in a thread."""
65
+ if async_client is not None:
66
+ return await getattr(async_client, method)(*args, **kwargs)
67
+ return await asyncio.to_thread(getattr(client, method), *args, **kwargs)
68
+
69
+
70
+ def to_tool_error(err: BrowsaError) -> ToolException:
71
+ """Map an SDK error to a ToolException with the API's code and request id."""
72
+ return ToolException(f"Browsa API error: {err}")
73
+
74
+
75
+ def envelope_job_id(envelope: Any) -> str:
76
+ """Job id from a 202 envelope (``job_id``, or ``id`` on some endpoints)."""
77
+ if isinstance(envelope, dict):
78
+ jid = envelope.get("job_id") or envelope.get("id")
79
+ if jid:
80
+ return str(jid)
81
+ raise ToolException(f"Browsa did not return a job id: {envelope!r}")
82
+
83
+
84
+ def job_result(job: Job) -> dict:
85
+ """The ``result`` object of a finished job (empty dict if there is none)."""
86
+ result = job.raw.get("result") if isinstance(job.raw, dict) else None
87
+ return result if isinstance(result, dict) else {}
88
+
89
+
90
+ def drop_none(d: dict, keep: tuple[str, ...] = ()) -> dict:
91
+ return {k: v for k, v in d.items() if v is not None or k in keep}
92
+
93
+
94
+ class BrowsaBaseTool(BaseTool):
95
+ """Base class of the Browsa tools: holds the key and the SDK clients."""
96
+
97
+ api_key: Optional[SecretStr] = Field(
98
+ default_factory=secret_from_env(API_KEY_ENV, default=None)
99
+ )
100
+ """Browsa API key (``agt_live_...``). Read from ``BROWSA_API_KEY`` if not set."""
101
+
102
+ base_url: str = DEFAULT_BASE_URL
103
+ """API base URL. Leave the default unless Browsa support tells you otherwise."""
104
+
105
+ request_timeout: float = DEFAULT_TIMEOUT
106
+ """Per-request HTTP timeout of the SDK client, in seconds."""
107
+
108
+ client: Any = Field(default=None, exclude=True)
109
+ """A ``browsa.Client`` to use instead of building one."""
110
+
111
+ async_client: Any = Field(default=None, exclude=True)
112
+ """A ``browsa.AsyncClient`` to use instead of building one."""
113
+
114
+ @model_validator(mode="after")
115
+ def _init_clients(self) -> Self:
116
+ self.client, self.async_client = build_clients(
117
+ self.api_key,
118
+ base_url=self.base_url,
119
+ timeout=self.request_timeout,
120
+ client=self.client,
121
+ async_client=self.async_client,
122
+ )
123
+ return self
124
+
125
+ def _call(self, method: str, *args: Any, **kwargs: Any) -> Any:
126
+ try:
127
+ return getattr(self.client, method)(*args, **kwargs)
128
+ except BrowsaError as e:
129
+ raise to_tool_error(e) from e
130
+
131
+ async def _acall(self, method: str, *args: Any, **kwargs: Any) -> Any:
132
+ try:
133
+ return await call_async(self.async_client, self.client, method, *args, **kwargs)
134
+ except BrowsaError as e:
135
+ raise to_tool_error(e) from e
136
+
137
+
138
+ __all__ = [
139
+ "API_KEY_ENV",
140
+ "BrowsaBaseTool",
141
+ "build_clients",
142
+ "call_async",
143
+ "drop_none",
144
+ "envelope_job_id",
145
+ "job_result",
146
+ "resolve_api_key",
147
+ "to_tool_error",
148
+ ]
@@ -0,0 +1,227 @@
1
+ """Document loader: Browsa scrape, batch and crawl results as LangChain Documents."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from collections.abc import AsyncIterator, Iterator, Sequence
7
+ from typing import Any, Literal, Optional, Union
8
+
9
+ from browsa import AsyncClient, BrowsaError, Client, Job
10
+ from browsa.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
11
+ from langchain_core.document_loaders import BaseLoader
12
+ from langchain_core.documents import Document
13
+
14
+ from langchain_browsa._common import (
15
+ build_clients,
16
+ call_async,
17
+ envelope_job_id,
18
+ job_result,
19
+ )
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ Mode = Literal["scrape", "batch", "crawl"]
24
+ ContentFormat = Literal["markdown", "text", "html"]
25
+
26
+
27
+ class BrowsaLoader(BaseLoader):
28
+ """Load web pages as Documents, fetched in Phantom, Browsa's cloud browser.
29
+
30
+ Modes:
31
+
32
+ * ``"scrape"``: one ``POST /v1/quick/scrape`` per URL, in order.
33
+ * ``"batch"``: all URLs as one batch job (``POST /v1/quick/batch``, up to
34
+ 100 URLs), then wait for it.
35
+ * ``"crawl"``: follow links from one start URL (``POST /v1/quick/crawl``),
36
+ up to ``max_pages``, then wait for it.
37
+
38
+ Each Document's ``page_content`` is the page in ``content_format``
39
+ (Markdown by default) and its metadata has ``source`` (the URL the
40
+ browser landed on), ``requested_url``, ``http_status`` and, for batch
41
+ and crawl, ``job_id``. Pages that failed are skipped and logged.
42
+
43
+ Setup:
44
+ .. code-block:: bash
45
+
46
+ pip install -U langchain-browsa
47
+ export BROWSA_API_KEY="agt_live_..."
48
+
49
+ Instantiate:
50
+ .. code-block:: python
51
+
52
+ from langchain_browsa import BrowsaLoader
53
+
54
+ loader = BrowsaLoader("https://example.com")
55
+ crawl = BrowsaLoader("https://docs.example.com", mode="crawl", max_pages=10)
56
+
57
+ Load:
58
+ .. code-block:: python
59
+
60
+ docs = loader.load()
61
+ docs[0].page_content[:200], docs[0].metadata["source"]
62
+ """
63
+
64
+ def __init__(
65
+ self,
66
+ url: Union[str, Sequence[str]],
67
+ *,
68
+ mode: Mode = "scrape",
69
+ api_key: Optional[str] = None,
70
+ base_url: str = DEFAULT_BASE_URL,
71
+ content_format: ContentFormat = "markdown",
72
+ formats: Optional[Sequence[str]] = None,
73
+ max_pages: int = 25,
74
+ wait_timeout: Optional[float] = 600,
75
+ request_timeout: float = DEFAULT_TIMEOUT,
76
+ client: Optional[Client] = None,
77
+ async_client: Optional[AsyncClient] = None,
78
+ **params: Any,
79
+ ) -> None:
80
+ """Create the loader.
81
+
82
+ Args:
83
+ url: One URL or a list of URLs. ``"crawl"`` takes exactly one start URL.
84
+ mode: ``"scrape"``, ``"batch"`` or ``"crawl"``.
85
+ api_key: Browsa API key; defaults to the ``BROWSA_API_KEY`` env var.
86
+ base_url: API base URL.
87
+ content_format: Which format becomes ``page_content``.
88
+ formats: Formats to request; defaults to ``[content_format]``.
89
+ ``"links"`` adds the page's links to the metadata.
90
+ max_pages: Page cap for ``"crawl"``.
91
+ wait_timeout: Seconds to wait for a batch or crawl job.
92
+ request_timeout: Per-request HTTP timeout of the SDK client.
93
+ client: A ``browsa.Client`` to use instead of building one.
94
+ async_client: A ``browsa.AsyncClient`` to use instead of building one.
95
+ **params: Other endpoint options, e.g. ``wait_ms``, ``max_chars``,
96
+ ``country``, ``block``; for crawl ``max_depth``,
97
+ ``same_domain``, ``include_regex``, ``exclude_regex``.
98
+ """
99
+ urls = [url] if isinstance(url, str) else list(url)
100
+ if not urls:
101
+ raise ValueError("BrowsaLoader needs at least one URL.")
102
+ if mode not in ("scrape", "batch", "crawl"):
103
+ raise ValueError(f"mode must be 'scrape', 'batch' or 'crawl', not {mode!r}")
104
+ if mode == "crawl" and len(urls) != 1:
105
+ raise ValueError("mode='crawl' takes exactly one start URL.")
106
+ self.urls = urls
107
+ self.mode: Mode = mode
108
+ self.content_format: ContentFormat = content_format
109
+ fmts = list(formats) if formats else [content_format]
110
+ if content_format not in fmts:
111
+ fmts.append(content_format)
112
+ self.formats = fmts
113
+ self.max_pages = max_pages
114
+ self.wait_timeout = wait_timeout
115
+ self.params = params
116
+ self.client, self.async_client = build_clients(
117
+ api_key,
118
+ base_url=base_url,
119
+ timeout=request_timeout,
120
+ client=client,
121
+ async_client=async_client,
122
+ )
123
+
124
+ # ── helpers ──────────────────────────────────────────────────────────
125
+
126
+ def _scrape_kwargs(self) -> dict[str, Any]:
127
+ kw = dict(self.params)
128
+ kw["formats"] = list(self.formats)
129
+ return kw
130
+
131
+ def _crawl_kwargs(self) -> dict[str, Any]:
132
+ kw = self._scrape_kwargs()
133
+ kw["max_pages"] = self.max_pages
134
+ return kw
135
+
136
+ def _to_document(
137
+ self, data: dict, requested_url: str, job_id: Optional[str] = None
138
+ ) -> Document:
139
+ content = data.get(self.content_format)
140
+ if not isinstance(content, str):
141
+ content = data.get("markdown") or data.get("text") or data.get("html") or ""
142
+ meta: dict[str, Any] = {
143
+ "source": data.get("url") or requested_url,
144
+ "requested_url": requested_url,
145
+ "http_status": data.get("http_status"),
146
+ }
147
+ page_meta = data.get("meta")
148
+ if isinstance(page_meta, dict):
149
+ title = page_meta.get("title")
150
+ if isinstance(title, str) and title:
151
+ meta["title"] = title
152
+ if isinstance(data.get("links"), list):
153
+ meta["links"] = data["links"]
154
+ if job_id:
155
+ meta["job_id"] = job_id
156
+ meta["mode"] = self.mode
157
+ return Document(page_content=content, metadata=meta)
158
+
159
+ def _job_documents(self, job: Job) -> Iterator[Document]:
160
+ if job.status != "completed":
161
+ raise BrowsaError(
162
+ f"Browsa {self.mode} job {job.id} ended {job.status}"
163
+ f" (outcome {job.outcome}): {job.error_code or ''} {job.error_message or ''}".rstrip(),
164
+ code=job.error_code,
165
+ )
166
+ result = job_result(job)
167
+ items = result.get("pages" if self.mode == "crawl" else "items") or []
168
+ for item in items:
169
+ if not isinstance(item, dict):
170
+ continue
171
+ data = item.get("data")
172
+ if item.get("status") != "completed" or not isinstance(data, dict):
173
+ logger.warning(
174
+ "Browsa %s: skipped %s (%s %s)",
175
+ self.mode,
176
+ item.get("url"),
177
+ item.get("error_code"),
178
+ item.get("error_message"),
179
+ )
180
+ continue
181
+ yield self._to_document(data, item.get("url") or "", job.id)
182
+
183
+ def _submit(self) -> dict:
184
+ if self.mode == "crawl":
185
+ return self.client.crawl(self.urls[0], **self._crawl_kwargs())
186
+ return self.client.batch(self.urls, **self._scrape_kwargs())
187
+
188
+ async def _asubmit(self) -> dict:
189
+ if self.mode == "crawl":
190
+ return await call_async(
191
+ self.async_client, self.client, "crawl", self.urls[0], **self._crawl_kwargs()
192
+ )
193
+ return await call_async(
194
+ self.async_client, self.client, "batch", self.urls, **self._scrape_kwargs()
195
+ )
196
+
197
+ # ── BaseLoader ───────────────────────────────────────────────────────
198
+
199
+ def lazy_load(self) -> Iterator[Document]:
200
+ """Load pages one by one (scrape) or after the job ends (batch, crawl)."""
201
+ if self.mode == "scrape":
202
+ for u in self.urls:
203
+ data = self.client.scrape(u, **self._scrape_kwargs())
204
+ yield self._to_document(data, u)
205
+ return
206
+ job_id = envelope_job_id(self._submit())
207
+ job = self.client.wait_for_job(job_id, timeout=self.wait_timeout)
208
+ yield from self._job_documents(job)
209
+
210
+ async def alazy_load(self) -> AsyncIterator[Document]:
211
+ """Async version of :meth:`lazy_load`, using ``browsa.AsyncClient``."""
212
+ if self.mode == "scrape":
213
+ for u in self.urls:
214
+ data = await call_async(
215
+ self.async_client, self.client, "scrape", u, **self._scrape_kwargs()
216
+ )
217
+ yield self._to_document(data, u)
218
+ return
219
+ job_id = envelope_job_id(await self._asubmit())
220
+ job = await call_async(
221
+ self.async_client, self.client, "wait_for_job", job_id, timeout=self.wait_timeout
222
+ )
223
+ for doc in self._job_documents(job):
224
+ yield doc
225
+
226
+
227
+ __all__ = ["BrowsaLoader"]
File without changes
@@ -0,0 +1,89 @@
1
+ """Toolkit with every Browsa tool, sharing one set of SDK clients."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Optional
6
+
7
+ from browsa.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
8
+ from langchain_core.tools import BaseTool, BaseToolkit
9
+ from langchain_core.utils import secret_from_env
10
+ from pydantic import ConfigDict, Field, SecretStr, model_validator
11
+ from typing_extensions import Self
12
+
13
+ from langchain_browsa._common import API_KEY_ENV, build_clients
14
+ from langchain_browsa.tools import (
15
+ BrowsaExtractTool,
16
+ BrowsaScrapeTool,
17
+ BrowsaScreenshotTool,
18
+ BrowsaTaskTool,
19
+ )
20
+
21
+
22
+ class BrowsaToolkit(BaseToolkit):
23
+ """All Browsa tools: task, scrape, extract and screenshot.
24
+
25
+ Instantiate:
26
+ .. code-block:: python
27
+
28
+ from langchain_browsa import BrowsaToolkit
29
+
30
+ toolkit = BrowsaToolkit() # reads BROWSA_API_KEY
31
+ tools = toolkit.get_tools()
32
+
33
+ Per-tool settings go in ``task_kwargs``, ``scrape_kwargs``,
34
+ ``extract_kwargs`` and ``screenshot_kwargs``, e.g.
35
+ ``BrowsaToolkit(task_kwargs={"llm": "browsa-smart", "max_credits": 30})``.
36
+ """
37
+
38
+ api_key: Optional[SecretStr] = Field(
39
+ default_factory=secret_from_env(API_KEY_ENV, default=None)
40
+ )
41
+ base_url: str = DEFAULT_BASE_URL
42
+ request_timeout: float = DEFAULT_TIMEOUT
43
+ client: Any = Field(default=None, exclude=True)
44
+ async_client: Any = Field(default=None, exclude=True)
45
+
46
+ include_task: bool = True
47
+ include_scrape: bool = True
48
+ include_extract: bool = True
49
+ include_screenshot: bool = True
50
+
51
+ task_kwargs: dict[str, Any] = Field(default_factory=dict)
52
+ scrape_kwargs: dict[str, Any] = Field(default_factory=dict)
53
+ extract_kwargs: dict[str, Any] = Field(default_factory=dict)
54
+ screenshot_kwargs: dict[str, Any] = Field(default_factory=dict)
55
+
56
+ model_config = ConfigDict(arbitrary_types_allowed=True)
57
+
58
+ @model_validator(mode="after")
59
+ def _init_clients(self) -> Self:
60
+ self.client, self.async_client = build_clients(
61
+ self.api_key,
62
+ base_url=self.base_url,
63
+ timeout=self.request_timeout,
64
+ client=self.client,
65
+ async_client=self.async_client,
66
+ )
67
+ return self
68
+
69
+ def get_tools(self) -> list[BaseTool]:
70
+ """Return the tools, all using this toolkit's clients."""
71
+ shared = {
72
+ "api_key": self.api_key,
73
+ "base_url": self.base_url,
74
+ "client": self.client,
75
+ "async_client": self.async_client,
76
+ }
77
+ tools: list[BaseTool] = []
78
+ if self.include_task:
79
+ tools.append(BrowsaTaskTool(**shared, **self.task_kwargs))
80
+ if self.include_scrape:
81
+ tools.append(BrowsaScrapeTool(**shared, **self.scrape_kwargs))
82
+ if self.include_extract:
83
+ tools.append(BrowsaExtractTool(**shared, **self.extract_kwargs))
84
+ if self.include_screenshot:
85
+ tools.append(BrowsaScreenshotTool(**shared, **self.screenshot_kwargs))
86
+ return tools
87
+
88
+
89
+ __all__ = ["BrowsaToolkit"]