scraperack-browsers 0.2.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scraperack_browsers-0.2.0a1/PKG-INFO +25 -0
- scraperack_browsers-0.2.0a1/README.md +15 -0
- scraperack_browsers-0.2.0a1/pyproject.toml +19 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers/__init__.py +14 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers/client.py +274 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers/py.typed +0 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers/types.py +55 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers.egg-info/PKG-INFO +25 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers.egg-info/SOURCES.txt +12 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers.egg-info/dependency_links.txt +1 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers.egg-info/requires.txt +1 -0
- scraperack_browsers-0.2.0a1/scraperack_browsers.egg-info/top_level.txt +1 -0
- scraperack_browsers-0.2.0a1/setup.cfg +4 -0
- scraperack_browsers-0.2.0a1/tests/test_client.py +98 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scraperack-browsers
|
|
3
|
+
Version: 0.2.0a1
|
|
4
|
+
Summary: Browser sessions on a self-hosted Scraperack cluster through its browsers plugin
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
Classifier: Development Status :: 3 - Alpha
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: scraperack>=0.3.0a1
|
|
10
|
+
|
|
11
|
+
# scraperack-browsers
|
|
12
|
+
|
|
13
|
+
Browser sessions on a self-hosted Scraperack cluster through its browsers plugin.
|
|
14
|
+
Each session is a fresh headed Chromium with its own timezone, locale, resolution and
|
|
15
|
+
proxy; connect to it over CDP.
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from scraperack_browsers import Browsers
|
|
19
|
+
|
|
20
|
+
with Browsers("http://controller:7070") as browsers:
|
|
21
|
+
with browsers.session(timezone="Europe/Sofia") as session:
|
|
22
|
+
print(session["cdp_url"])
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
See [docs/browsers-sdk.md](../../docs/browsers-sdk.md).
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# scraperack-browsers
|
|
2
|
+
|
|
3
|
+
Browser sessions on a self-hosted Scraperack cluster through its browsers plugin.
|
|
4
|
+
Each session is a fresh headed Chromium with its own timezone, locale, resolution and
|
|
5
|
+
proxy; connect to it over CDP.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from scraperack_browsers import Browsers
|
|
9
|
+
|
|
10
|
+
with Browsers("http://controller:7070") as browsers:
|
|
11
|
+
with browsers.session(timezone="Europe/Sofia") as session:
|
|
12
|
+
print(session["cdp_url"])
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
See [docs/browsers-sdk.md](../../docs/browsers-sdk.md).
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scraperack-browsers"
|
|
7
|
+
version = "0.2.0a1"
|
|
8
|
+
description = "Browser sessions on a self-hosted Scraperack cluster through its browsers plugin"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = ["scraperack>=0.3.0a1"]
|
|
12
|
+
license = "LicenseRef-Proprietary"
|
|
13
|
+
classifiers = ["Development Status :: 3 - Alpha"]
|
|
14
|
+
|
|
15
|
+
[tool.setuptools]
|
|
16
|
+
packages = ["scraperack_browsers"]
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.package-data]
|
|
19
|
+
scraperack_browsers = ["py.typed"]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Browser sessions on Scraperack through the browsers plugin."""
|
|
2
|
+
from .client import AsyncBrowsers, Browsers, SessionError, proxy_fields
|
|
3
|
+
from .types import Configuration, Session, SessionSettings, SessionStatus
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"AsyncBrowsers",
|
|
7
|
+
"Browsers",
|
|
8
|
+
"Configuration",
|
|
9
|
+
"Session",
|
|
10
|
+
"SessionError",
|
|
11
|
+
"SessionSettings",
|
|
12
|
+
"SessionStatus",
|
|
13
|
+
"proxy_fields",
|
|
14
|
+
]
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Clients for the Scraperack browsers plugin: configurations and sessions.
|
|
2
|
+
|
|
3
|
+
They talk to ``/plugins/browsers/api`` through the core ``scraperack`` client, which
|
|
4
|
+
stays available as ``.cluster``. Drive the browser itself with any CDP client, for
|
|
5
|
+
example ``playwright.chromium.connect_over_cdp(session["cdp_url"])``.
|
|
6
|
+
"""
|
|
7
|
+
import asyncio
|
|
8
|
+
import time
|
|
9
|
+
from contextlib import asynccontextmanager, contextmanager
|
|
10
|
+
from typing import Any, AsyncIterator, Iterator, Sequence
|
|
11
|
+
from urllib.parse import quote, unquote, urlsplit
|
|
12
|
+
|
|
13
|
+
from scraperack import AsyncCluster, Cluster
|
|
14
|
+
|
|
15
|
+
from .types import ENDED, Configuration, Session
|
|
16
|
+
|
|
17
|
+
DEFAULT_URL = "http://127.0.0.1:7070"
|
|
18
|
+
POLL_SECONDS = 0.5
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SessionError(RuntimeError):
|
|
22
|
+
"""The session failed to start or ended before it was ready."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, message: str, session: Session | None = None) -> None:
|
|
25
|
+
super().__init__(message)
|
|
26
|
+
self.session = session
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _key(value: str) -> str:
|
|
30
|
+
return quote(str(value), safe="")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def proxy_fields(proxy: str | None) -> dict[str, Any]:
|
|
34
|
+
"""``http://user:pass@host:port`` → the API's proxy fields."""
|
|
35
|
+
if not proxy:
|
|
36
|
+
return {"proxy_enabled": False}
|
|
37
|
+
url = urlsplit(proxy if "://" in proxy else f"http://{proxy}")
|
|
38
|
+
if not url.hostname or not url.port:
|
|
39
|
+
raise ValueError("proxy must look like http://user:pass@host:port")
|
|
40
|
+
fields = {"proxy_enabled": True, "proxy_host": url.hostname, "proxy_port": str(url.port)}
|
|
41
|
+
if url.username:
|
|
42
|
+
fields["proxy_user"] = unquote(url.username)
|
|
43
|
+
fields["proxy_pass"] = unquote(url.password or "")
|
|
44
|
+
return fields
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def session_body(configuration: str | None, timeout: int, timezone: str | None, lang: str | None,
|
|
48
|
+
lc_all: str | None, resolution: str | None, browser_args: str | Sequence[str] | None,
|
|
49
|
+
proxy: str | None, session_id: str | None, profile_id: str | None,
|
|
50
|
+
queue: bool) -> dict[str, Any]:
|
|
51
|
+
body: dict[str, Any] = {"timeout": timeout, "queue": queue, **proxy_fields(proxy)}
|
|
52
|
+
if not isinstance(browser_args, (str, type(None))):
|
|
53
|
+
browser_args = " ".join(browser_args)
|
|
54
|
+
lc_all = lc_all or lang
|
|
55
|
+
for name, value in (("configuration", configuration), ("timezone", timezone), ("lang", lang),
|
|
56
|
+
("lc_all", lc_all), ("resolution", resolution),
|
|
57
|
+
("custom_browser_args", browser_args), ("session_id", session_id),
|
|
58
|
+
("profile_id", profile_id)):
|
|
59
|
+
if value is not None:
|
|
60
|
+
body[name] = value
|
|
61
|
+
return body
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def configuration_body(name: str | None, image: str | None, cpu: float | None,
|
|
65
|
+
settings: dict[str, Any]) -> dict[str, Any]:
|
|
66
|
+
body = {("max_timeout_secs" if key == "max_timeout" else key): value
|
|
67
|
+
for key, value in settings.items()}
|
|
68
|
+
if name is not None:
|
|
69
|
+
body["name"] = name
|
|
70
|
+
if image is not None:
|
|
71
|
+
body["image"] = image
|
|
72
|
+
if cpu is not None:
|
|
73
|
+
body["cpu_millis"] = round(cpu * 1000)
|
|
74
|
+
return body
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class _Base:
|
|
78
|
+
def _fill(self, session: Session) -> Session:
|
|
79
|
+
"""Nodes without a configured address publish on the controller's host."""
|
|
80
|
+
if session.get("status") not in ENDED and not session.get("host") and session.get("ports"):
|
|
81
|
+
host = urlsplit(self.cluster.url).hostname
|
|
82
|
+
ports = session["ports"]
|
|
83
|
+
session["host"] = host
|
|
84
|
+
if "cdp" in ports:
|
|
85
|
+
session["cdp_url"] = f"http://{host}:{ports['cdp']}"
|
|
86
|
+
if "vnc" in ports:
|
|
87
|
+
session["vnc_url"] = f"http://{host}:{ports['vnc']}/vnc.html?autoconnect=1&resize=scale"
|
|
88
|
+
return session
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def _check_ready(session: Session) -> bool:
|
|
92
|
+
if session["status"] == "ready":
|
|
93
|
+
return True
|
|
94
|
+
if session["status"] in ENDED:
|
|
95
|
+
raise SessionError(session.get("error") or
|
|
96
|
+
f"session {session['status']} ({session.get('end_reason')})", session)
|
|
97
|
+
return False
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Browsers(_Base):
|
|
101
|
+
"""Synchronous client for configurations and sessions."""
|
|
102
|
+
|
|
103
|
+
def __init__(self, cluster: str | Cluster = DEFAULT_URL, *, plugin: str = "browsers") -> None:
|
|
104
|
+
self._owns_cluster = isinstance(cluster, str)
|
|
105
|
+
self.cluster = Cluster(cluster) if isinstance(cluster, str) else cluster
|
|
106
|
+
self.api = f"/plugins/{_key(plugin)}/api"
|
|
107
|
+
|
|
108
|
+
def __enter__(self) -> "Browsers":
|
|
109
|
+
return self
|
|
110
|
+
|
|
111
|
+
def __exit__(self, *_: object) -> None:
|
|
112
|
+
self.close()
|
|
113
|
+
|
|
114
|
+
def close(self) -> None:
|
|
115
|
+
if self._owns_cluster:
|
|
116
|
+
self.cluster.close()
|
|
117
|
+
|
|
118
|
+
def _call(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
119
|
+
return self.cluster.request(method, self.api + path, **kwargs)
|
|
120
|
+
|
|
121
|
+
# Sessions
|
|
122
|
+
def create_session(self, configuration: str | None = None, *, timeout: int = 300,
|
|
123
|
+
timezone: str | None = None, lang: str | None = None,
|
|
124
|
+
lc_all: str | None = None, resolution: str | None = None,
|
|
125
|
+
browser_args: str | Sequence[str] | None = None,
|
|
126
|
+
proxy: str | None = None, session_id: str | None = None,
|
|
127
|
+
profile_id: str | None = None, queue: bool = True,
|
|
128
|
+
wait: bool = True) -> Session:
|
|
129
|
+
"""Start a browser and, with ``wait``, return once it is ready.
|
|
130
|
+
|
|
131
|
+
``configuration`` may be left out when the plugin has only one. ``proxy`` is
|
|
132
|
+
``http://user:pass@host:port``. ``timeout`` is the session's lifetime in seconds.
|
|
133
|
+
"""
|
|
134
|
+
body = session_body(configuration, timeout, timezone, lang, lc_all, resolution, browser_args,
|
|
135
|
+
proxy, session_id, profile_id, queue)
|
|
136
|
+
session = self._fill(self._call("POST", "/sessions", json=body, timeout=None))
|
|
137
|
+
return self.wait_ready(session["id"]) if wait else session
|
|
138
|
+
|
|
139
|
+
def get_session(self, session_id: str) -> Session:
|
|
140
|
+
return self._fill(self._call("GET", f"/sessions/{_key(session_id)}"))
|
|
141
|
+
|
|
142
|
+
def sessions(self, *, active: bool = True) -> list[Session]:
|
|
143
|
+
"""Newest first; ``active=False`` includes ended sessions."""
|
|
144
|
+
params = {"status": "active"} if active else {}
|
|
145
|
+
return [self._fill(s) for s in self._call("GET", "/sessions", params=params)]
|
|
146
|
+
|
|
147
|
+
def wait_ready(self, session_id: str, timeout: float | None = None) -> Session:
|
|
148
|
+
"""Poll until the session is ready; raises ``SessionError`` if it ends first."""
|
|
149
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
150
|
+
while True:
|
|
151
|
+
session = self.get_session(session_id)
|
|
152
|
+
if self._check_ready(session):
|
|
153
|
+
return session
|
|
154
|
+
if deadline is not None and time.monotonic() > deadline:
|
|
155
|
+
raise TimeoutError(f"session {session_id} is still {session['status']}")
|
|
156
|
+
time.sleep(POLL_SECONDS)
|
|
157
|
+
|
|
158
|
+
def stop_session(self, session_id: str) -> Session:
|
|
159
|
+
return self._fill(self._call("DELETE", f"/sessions/{_key(session_id)}", timeout=None))
|
|
160
|
+
|
|
161
|
+
@contextmanager
|
|
162
|
+
def session(self, configuration: str | None = None, **options: Any) -> Iterator[Session]:
|
|
163
|
+
"""``create_session`` that stops the browser when the block exits."""
|
|
164
|
+
session = self.create_session(configuration, wait=False, **options)
|
|
165
|
+
try:
|
|
166
|
+
yield self.wait_ready(session["id"])
|
|
167
|
+
finally:
|
|
168
|
+
self.stop_session(session["id"])
|
|
169
|
+
|
|
170
|
+
# Configurations
|
|
171
|
+
def configurations(self) -> list[Configuration]:
|
|
172
|
+
return self._call("GET", "/configurations")
|
|
173
|
+
|
|
174
|
+
def configuration(self, configuration: str) -> Configuration:
|
|
175
|
+
return self._call("GET", f"/configurations/{_key(configuration)}")
|
|
176
|
+
|
|
177
|
+
def create_configuration(self, name: str, image: str, cpu: float = 1.5,
|
|
178
|
+
**settings: Any) -> Configuration:
|
|
179
|
+
"""Add a configuration: a Scraperack image (by id) with VM size and limits.
|
|
180
|
+
|
|
181
|
+
``settings``: memory_mib, disk_mib, max_sessions, max_timeout (seconds).
|
|
182
|
+
"""
|
|
183
|
+
return self._call("POST", "/configurations",
|
|
184
|
+
json=configuration_body(name, image, cpu, settings))
|
|
185
|
+
|
|
186
|
+
def update_configuration(self, configuration: str, *, cpu: float | None = None,
|
|
187
|
+
**settings: Any) -> Configuration:
|
|
188
|
+
return self._call("PUT", f"/configurations/{_key(configuration)}",
|
|
189
|
+
json=configuration_body(None, None, cpu, settings))
|
|
190
|
+
|
|
191
|
+
def delete_configuration(self, configuration: str) -> None:
|
|
192
|
+
self._call("DELETE", f"/configurations/{_key(configuration)}")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class AsyncBrowsers(_Base):
|
|
196
|
+
"""Asynchronous client for configurations and sessions."""
|
|
197
|
+
|
|
198
|
+
def __init__(self, cluster: str | AsyncCluster = DEFAULT_URL, *,
|
|
199
|
+
plugin: str = "browsers") -> None:
|
|
200
|
+
self._owns_cluster = isinstance(cluster, str)
|
|
201
|
+
self.cluster = AsyncCluster(cluster) if isinstance(cluster, str) else cluster
|
|
202
|
+
self.api = f"/plugins/{_key(plugin)}/api"
|
|
203
|
+
|
|
204
|
+
async def __aenter__(self) -> "AsyncBrowsers":
|
|
205
|
+
return self
|
|
206
|
+
|
|
207
|
+
async def __aexit__(self, *_: object) -> None:
|
|
208
|
+
await self.close()
|
|
209
|
+
|
|
210
|
+
async def close(self) -> None:
|
|
211
|
+
if self._owns_cluster:
|
|
212
|
+
await self.cluster.close()
|
|
213
|
+
|
|
214
|
+
async def _call(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
215
|
+
return await self.cluster.request(method, self.api + path, **kwargs)
|
|
216
|
+
|
|
217
|
+
async def create_session(self, configuration: str | None = None, *, timeout: int = 300,
|
|
218
|
+
timezone: str | None = None, lang: str | None = None,
|
|
219
|
+
lc_all: str | None = None, resolution: str | None = None,
|
|
220
|
+
browser_args: str | Sequence[str] | None = None,
|
|
221
|
+
proxy: str | None = None, session_id: str | None = None,
|
|
222
|
+
profile_id: str | None = None, queue: bool = True,
|
|
223
|
+
wait: bool = True) -> Session:
|
|
224
|
+
body = session_body(configuration, timeout, timezone, lang, lc_all, resolution, browser_args,
|
|
225
|
+
proxy, session_id, profile_id, queue)
|
|
226
|
+
session = self._fill(await self._call("POST", "/sessions", json=body, timeout=None))
|
|
227
|
+
return await self.wait_ready(session["id"]) if wait else session
|
|
228
|
+
|
|
229
|
+
async def get_session(self, session_id: str) -> Session:
|
|
230
|
+
return self._fill(await self._call("GET", f"/sessions/{_key(session_id)}"))
|
|
231
|
+
|
|
232
|
+
async def sessions(self, *, active: bool = True) -> list[Session]:
|
|
233
|
+
params = {"status": "active"} if active else {}
|
|
234
|
+
return [self._fill(s) for s in await self._call("GET", "/sessions", params=params)]
|
|
235
|
+
|
|
236
|
+
async def wait_ready(self, session_id: str, timeout: float | None = None) -> Session:
|
|
237
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
238
|
+
while True:
|
|
239
|
+
session = await self.get_session(session_id)
|
|
240
|
+
if self._check_ready(session):
|
|
241
|
+
return session
|
|
242
|
+
if deadline is not None and time.monotonic() > deadline:
|
|
243
|
+
raise TimeoutError(f"session {session_id} is still {session['status']}")
|
|
244
|
+
await asyncio.sleep(POLL_SECONDS)
|
|
245
|
+
|
|
246
|
+
async def stop_session(self, session_id: str) -> Session:
|
|
247
|
+
return self._fill(await self._call("DELETE", f"/sessions/{_key(session_id)}", timeout=None))
|
|
248
|
+
|
|
249
|
+
@asynccontextmanager
|
|
250
|
+
async def session(self, configuration: str | None = None, **options: Any) -> AsyncIterator[Session]:
|
|
251
|
+
session = await self.create_session(configuration, wait=False, **options)
|
|
252
|
+
try:
|
|
253
|
+
yield await self.wait_ready(session["id"])
|
|
254
|
+
finally:
|
|
255
|
+
await self.stop_session(session["id"])
|
|
256
|
+
|
|
257
|
+
async def configurations(self) -> list[Configuration]:
|
|
258
|
+
return await self._call("GET", "/configurations")
|
|
259
|
+
|
|
260
|
+
async def configuration(self, configuration: str) -> Configuration:
|
|
261
|
+
return await self._call("GET", f"/configurations/{_key(configuration)}")
|
|
262
|
+
|
|
263
|
+
async def create_configuration(self, name: str, image: str, cpu: float = 1.5,
|
|
264
|
+
**settings: Any) -> Configuration:
|
|
265
|
+
return await self._call("POST", "/configurations",
|
|
266
|
+
json=configuration_body(name, image, cpu, settings))
|
|
267
|
+
|
|
268
|
+
async def update_configuration(self, configuration: str, *, cpu: float | None = None,
|
|
269
|
+
**settings: Any) -> Configuration:
|
|
270
|
+
return await self._call("PUT", f"/configurations/{_key(configuration)}",
|
|
271
|
+
json=configuration_body(None, None, cpu, settings))
|
|
272
|
+
|
|
273
|
+
async def delete_configuration(self, configuration: str) -> None:
|
|
274
|
+
await self._call("DELETE", f"/configurations/{_key(configuration)}")
|
|
File without changes
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Shapes returned by the browsers plugin API."""
|
|
2
|
+
from typing import Literal, TypedDict
|
|
3
|
+
|
|
4
|
+
SessionStatus = Literal["queued", "starting", "ready", "stopping", "ended", "failed"]
|
|
5
|
+
ENDED = frozenset({"ended", "failed"})
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SessionSettings(TypedDict):
|
|
9
|
+
timezone: str
|
|
10
|
+
lang: str
|
|
11
|
+
lc_all: str
|
|
12
|
+
resolution: str
|
|
13
|
+
custom_browser_args: str
|
|
14
|
+
proxy_enabled: bool
|
|
15
|
+
proxy_host: str
|
|
16
|
+
proxy_port: str
|
|
17
|
+
proxy_user: str
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Session(TypedDict):
|
|
21
|
+
id: str
|
|
22
|
+
status: SessionStatus
|
|
23
|
+
configuration: str
|
|
24
|
+
configuration_id: str
|
|
25
|
+
image: str | None
|
|
26
|
+
task: str
|
|
27
|
+
settings: SessionSettings
|
|
28
|
+
profile_id: str | None
|
|
29
|
+
timeout: int
|
|
30
|
+
host: str | None
|
|
31
|
+
ports: dict[str, int]
|
|
32
|
+
cdp_url: str | None
|
|
33
|
+
vnc_url: str | None
|
|
34
|
+
node: str | None
|
|
35
|
+
node_name: str | None
|
|
36
|
+
created_at_ms: int
|
|
37
|
+
started_at_ms: int | None
|
|
38
|
+
ready_at_ms: int | None
|
|
39
|
+
expires_at_ms: int | None
|
|
40
|
+
ended_at_ms: int | None
|
|
41
|
+
end_reason: str | None
|
|
42
|
+
error: str | None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Configuration(TypedDict):
|
|
46
|
+
id: str
|
|
47
|
+
name: str
|
|
48
|
+
image: str
|
|
49
|
+
pool: str
|
|
50
|
+
cpu_millis: int
|
|
51
|
+
memory_mib: int
|
|
52
|
+
disk_mib: int
|
|
53
|
+
max_sessions: int | None
|
|
54
|
+
max_timeout_secs: int
|
|
55
|
+
ready: bool
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scraperack-browsers
|
|
3
|
+
Version: 0.2.0a1
|
|
4
|
+
Summary: Browser sessions on a self-hosted Scraperack cluster through its browsers plugin
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
Classifier: Development Status :: 3 - Alpha
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: scraperack>=0.3.0a1
|
|
10
|
+
|
|
11
|
+
# scraperack-browsers
|
|
12
|
+
|
|
13
|
+
Browser sessions on a self-hosted Scraperack cluster through its browsers plugin.
|
|
14
|
+
Each session is a fresh headed Chromium with its own timezone, locale, resolution and
|
|
15
|
+
proxy; connect to it over CDP.
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from scraperack_browsers import Browsers
|
|
19
|
+
|
|
20
|
+
with Browsers("http://controller:7070") as browsers:
|
|
21
|
+
with browsers.session(timezone="Europe/Sofia") as session:
|
|
22
|
+
print(session["cdp_url"])
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
See [docs/browsers-sdk.md](../../docs/browsers-sdk.md).
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
scraperack_browsers/__init__.py
|
|
4
|
+
scraperack_browsers/client.py
|
|
5
|
+
scraperack_browsers/py.typed
|
|
6
|
+
scraperack_browsers/types.py
|
|
7
|
+
scraperack_browsers.egg-info/PKG-INFO
|
|
8
|
+
scraperack_browsers.egg-info/SOURCES.txt
|
|
9
|
+
scraperack_browsers.egg-info/dependency_links.txt
|
|
10
|
+
scraperack_browsers.egg-info/requires.txt
|
|
11
|
+
scraperack_browsers.egg-info/top_level.txt
|
|
12
|
+
tests/test_client.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scraperack>=0.3.0a1
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scraperack_browsers
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import unittest
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from scraperack import AsyncCluster, Cluster
|
|
7
|
+
|
|
8
|
+
from scraperack_browsers import AsyncBrowsers, Browsers, SessionError, proxy_fields
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def session(status="ready", host="10.0.0.5", **extra):
|
|
12
|
+
return {"id": "s1", "status": status, "host": host, "ports": {"cdp": 30001, "vnc": 30002},
|
|
13
|
+
"cdp_url": f"http://{host}:30001" if host else None, "vnc_url": None,
|
|
14
|
+
"error": None, "end_reason": None, **extra}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def mock(handler):
|
|
18
|
+
return Browsers(Cluster("http://controller:7070", client=httpx.Client(
|
|
19
|
+
base_url="http://controller:7070", transport=httpx.MockTransport(handler))))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BrowsersTests(unittest.TestCase):
|
|
23
|
+
def test_proxy_url_becomes_fields(self):
|
|
24
|
+
self.assertEqual(proxy_fields("http://u%40x:p@proxy.example:8080"), {
|
|
25
|
+
"proxy_enabled": True, "proxy_host": "proxy.example", "proxy_port": "8080",
|
|
26
|
+
"proxy_user": "u@x", "proxy_pass": "p"})
|
|
27
|
+
self.assertEqual(proxy_fields(None), {"proxy_enabled": False})
|
|
28
|
+
with self.assertRaises(ValueError):
|
|
29
|
+
proxy_fields("proxy.example")
|
|
30
|
+
|
|
31
|
+
def test_session_context_creates_waits_and_stops(self):
|
|
32
|
+
calls, states = [], iter(["starting", "ready"])
|
|
33
|
+
|
|
34
|
+
def handler(request):
|
|
35
|
+
calls.append((request.method, request.url.path,
|
|
36
|
+
json.loads(request.content) if request.content else None))
|
|
37
|
+
if request.method == "POST":
|
|
38
|
+
return httpx.Response(202, json=session("queued", None, ports={}))
|
|
39
|
+
if request.method == "DELETE":
|
|
40
|
+
return httpx.Response(200, json=session("ended"))
|
|
41
|
+
return httpx.Response(200, json=session(next(states)))
|
|
42
|
+
|
|
43
|
+
with mock(handler) as browsers:
|
|
44
|
+
with browsers.session("chromium", timezone="Europe/Sofia", lang="de_DE.UTF-8",
|
|
45
|
+
browser_args=["--a", "--b"], proxy="http://h:1") as s:
|
|
46
|
+
self.assertEqual(s["cdp_url"], "http://10.0.0.5:30001")
|
|
47
|
+
self.assertEqual([c[:2] for c in calls], [
|
|
48
|
+
("POST", "/plugins/browsers/api/sessions"),
|
|
49
|
+
("GET", "/plugins/browsers/api/sessions/s1"),
|
|
50
|
+
("GET", "/plugins/browsers/api/sessions/s1"),
|
|
51
|
+
("DELETE", "/plugins/browsers/api/sessions/s1")])
|
|
52
|
+
body = calls[0][2]
|
|
53
|
+
self.assertEqual(body["configuration"], "chromium")
|
|
54
|
+
self.assertEqual(body["lc_all"], "de_DE.UTF-8")
|
|
55
|
+
self.assertEqual(body["custom_browser_args"], "--a --b")
|
|
56
|
+
self.assertEqual((body["proxy_host"], body["proxy_port"]), ("h", "1"))
|
|
57
|
+
|
|
58
|
+
def test_missing_node_address_uses_controller_host(self):
|
|
59
|
+
def handler(request):
|
|
60
|
+
return httpx.Response(200, json=session(host=None, cdp_url=None))
|
|
61
|
+
|
|
62
|
+
with mock(handler) as browsers:
|
|
63
|
+
s = browsers.get_session("s1")
|
|
64
|
+
self.assertEqual(s["cdp_url"], "http://controller:30001")
|
|
65
|
+
self.assertTrue(s["vnc_url"].startswith("http://controller:30002/vnc.html"))
|
|
66
|
+
|
|
67
|
+
def test_failed_session_raises(self):
|
|
68
|
+
def handler(request):
|
|
69
|
+
if request.method == "POST":
|
|
70
|
+
return httpx.Response(202, json=session("failed", error="browser did not start"))
|
|
71
|
+
return httpx.Response(200, json=session("failed", error="browser did not start"))
|
|
72
|
+
|
|
73
|
+
with mock(handler) as browsers, self.assertRaises(SessionError) as raised:
|
|
74
|
+
browsers.create_session()
|
|
75
|
+
self.assertIn("did not start", str(raised.exception))
|
|
76
|
+
|
|
77
|
+
def test_async_session(self):
|
|
78
|
+
stopped = []
|
|
79
|
+
|
|
80
|
+
def handler(request):
|
|
81
|
+
if request.method == "DELETE":
|
|
82
|
+
stopped.append(request.url.path)
|
|
83
|
+
return httpx.Response(200, json=session("ended"))
|
|
84
|
+
return httpx.Response(200 if request.method == "GET" else 201, json=session())
|
|
85
|
+
|
|
86
|
+
async def run():
|
|
87
|
+
cluster = AsyncCluster("http://controller:7070", client=httpx.AsyncClient(
|
|
88
|
+
base_url="http://controller:7070", transport=httpx.MockTransport(handler)))
|
|
89
|
+
async with AsyncBrowsers(cluster) as browsers:
|
|
90
|
+
async with browsers.session() as s:
|
|
91
|
+
return s["status"]
|
|
92
|
+
|
|
93
|
+
self.assertEqual(asyncio.run(run()), "ready")
|
|
94
|
+
self.assertEqual(stopped, ["/plugins/browsers/api/sessions/s1"])
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
unittest.main()
|