scraperack-browsers 0.2.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ """Browser sessions on Scraperack through the browsers plugin."""
2
+ from .client import AsyncBrowsers, Browsers, SessionError, proxy_fields
3
+ from .types import Configuration, Session, SessionSettings, SessionStatus
4
+
5
+ __all__ = [
6
+ "AsyncBrowsers",
7
+ "Browsers",
8
+ "Configuration",
9
+ "Session",
10
+ "SessionError",
11
+ "SessionSettings",
12
+ "SessionStatus",
13
+ "proxy_fields",
14
+ ]
@@ -0,0 +1,274 @@
1
+ """Clients for the Scraperack browsers plugin: configurations and sessions.
2
+
3
+ They talk to ``/plugins/browsers/api`` through the core ``scraperack`` client, which
4
+ stays available as ``.cluster``. Drive the browser itself with any CDP client, for
5
+ example ``playwright.chromium.connect_over_cdp(session["cdp_url"])``.
6
+ """
7
+ import asyncio
8
+ import time
9
+ from contextlib import asynccontextmanager, contextmanager
10
+ from typing import Any, AsyncIterator, Iterator, Sequence
11
+ from urllib.parse import quote, unquote, urlsplit
12
+
13
+ from scraperack import AsyncCluster, Cluster
14
+
15
+ from .types import ENDED, Configuration, Session
16
+
17
+ DEFAULT_URL = "http://127.0.0.1:7070"
18
+ POLL_SECONDS = 0.5
19
+
20
+
21
+ class SessionError(RuntimeError):
22
+ """The session failed to start or ended before it was ready."""
23
+
24
+ def __init__(self, message: str, session: Session | None = None) -> None:
25
+ super().__init__(message)
26
+ self.session = session
27
+
28
+
29
+ def _key(value: str) -> str:
30
+ return quote(str(value), safe="")
31
+
32
+
33
+ def proxy_fields(proxy: str | None) -> dict[str, Any]:
34
+ """``http://user:pass@host:port`` → the API's proxy fields."""
35
+ if not proxy:
36
+ return {"proxy_enabled": False}
37
+ url = urlsplit(proxy if "://" in proxy else f"http://{proxy}")
38
+ if not url.hostname or not url.port:
39
+ raise ValueError("proxy must look like http://user:pass@host:port")
40
+ fields = {"proxy_enabled": True, "proxy_host": url.hostname, "proxy_port": str(url.port)}
41
+ if url.username:
42
+ fields["proxy_user"] = unquote(url.username)
43
+ fields["proxy_pass"] = unquote(url.password or "")
44
+ return fields
45
+
46
+
47
+ def session_body(configuration: str | None, timeout: int, timezone: str | None, lang: str | None,
48
+ lc_all: str | None, resolution: str | None, browser_args: str | Sequence[str] | None,
49
+ proxy: str | None, session_id: str | None, profile_id: str | None,
50
+ queue: bool) -> dict[str, Any]:
51
+ body: dict[str, Any] = {"timeout": timeout, "queue": queue, **proxy_fields(proxy)}
52
+ if not isinstance(browser_args, (str, type(None))):
53
+ browser_args = " ".join(browser_args)
54
+ lc_all = lc_all or lang
55
+ for name, value in (("configuration", configuration), ("timezone", timezone), ("lang", lang),
56
+ ("lc_all", lc_all), ("resolution", resolution),
57
+ ("custom_browser_args", browser_args), ("session_id", session_id),
58
+ ("profile_id", profile_id)):
59
+ if value is not None:
60
+ body[name] = value
61
+ return body
62
+
63
+
64
+ def configuration_body(name: str | None, image: str | None, cpu: float | None,
65
+ settings: dict[str, Any]) -> dict[str, Any]:
66
+ body = {("max_timeout_secs" if key == "max_timeout" else key): value
67
+ for key, value in settings.items()}
68
+ if name is not None:
69
+ body["name"] = name
70
+ if image is not None:
71
+ body["image"] = image
72
+ if cpu is not None:
73
+ body["cpu_millis"] = round(cpu * 1000)
74
+ return body
75
+
76
+
77
+ class _Base:
78
+ def _fill(self, session: Session) -> Session:
79
+ """Nodes without a configured address publish on the controller's host."""
80
+ if session.get("status") not in ENDED and not session.get("host") and session.get("ports"):
81
+ host = urlsplit(self.cluster.url).hostname
82
+ ports = session["ports"]
83
+ session["host"] = host
84
+ if "cdp" in ports:
85
+ session["cdp_url"] = f"http://{host}:{ports['cdp']}"
86
+ if "vnc" in ports:
87
+ session["vnc_url"] = f"http://{host}:{ports['vnc']}/vnc.html?autoconnect=1&resize=scale"
88
+ return session
89
+
90
+ @staticmethod
91
+ def _check_ready(session: Session) -> bool:
92
+ if session["status"] == "ready":
93
+ return True
94
+ if session["status"] in ENDED:
95
+ raise SessionError(session.get("error") or
96
+ f"session {session['status']} ({session.get('end_reason')})", session)
97
+ return False
98
+
99
+
100
+ class Browsers(_Base):
101
+ """Synchronous client for configurations and sessions."""
102
+
103
+ def __init__(self, cluster: str | Cluster = DEFAULT_URL, *, plugin: str = "browsers") -> None:
104
+ self._owns_cluster = isinstance(cluster, str)
105
+ self.cluster = Cluster(cluster) if isinstance(cluster, str) else cluster
106
+ self.api = f"/plugins/{_key(plugin)}/api"
107
+
108
+ def __enter__(self) -> "Browsers":
109
+ return self
110
+
111
+ def __exit__(self, *_: object) -> None:
112
+ self.close()
113
+
114
+ def close(self) -> None:
115
+ if self._owns_cluster:
116
+ self.cluster.close()
117
+
118
+ def _call(self, method: str, path: str, **kwargs: Any) -> Any:
119
+ return self.cluster.request(method, self.api + path, **kwargs)
120
+
121
+ # Sessions
122
+ def create_session(self, configuration: str | None = None, *, timeout: int = 300,
123
+ timezone: str | None = None, lang: str | None = None,
124
+ lc_all: str | None = None, resolution: str | None = None,
125
+ browser_args: str | Sequence[str] | None = None,
126
+ proxy: str | None = None, session_id: str | None = None,
127
+ profile_id: str | None = None, queue: bool = True,
128
+ wait: bool = True) -> Session:
129
+ """Start a browser and, with ``wait``, return once it is ready.
130
+
131
+ ``configuration`` may be left out when the plugin has only one. ``proxy`` is
132
+ ``http://user:pass@host:port``. ``timeout`` is the session's lifetime in seconds.
133
+ """
134
+ body = session_body(configuration, timeout, timezone, lang, lc_all, resolution, browser_args,
135
+ proxy, session_id, profile_id, queue)
136
+ session = self._fill(self._call("POST", "/sessions", json=body, timeout=None))
137
+ return self.wait_ready(session["id"]) if wait else session
138
+
139
+ def get_session(self, session_id: str) -> Session:
140
+ return self._fill(self._call("GET", f"/sessions/{_key(session_id)}"))
141
+
142
+ def sessions(self, *, active: bool = True) -> list[Session]:
143
+ """Newest first; ``active=False`` includes ended sessions."""
144
+ params = {"status": "active"} if active else {}
145
+ return [self._fill(s) for s in self._call("GET", "/sessions", params=params)]
146
+
147
+ def wait_ready(self, session_id: str, timeout: float | None = None) -> Session:
148
+ """Poll until the session is ready; raises ``SessionError`` if it ends first."""
149
+ deadline = None if timeout is None else time.monotonic() + timeout
150
+ while True:
151
+ session = self.get_session(session_id)
152
+ if self._check_ready(session):
153
+ return session
154
+ if deadline is not None and time.monotonic() > deadline:
155
+ raise TimeoutError(f"session {session_id} is still {session['status']}")
156
+ time.sleep(POLL_SECONDS)
157
+
158
+ def stop_session(self, session_id: str) -> Session:
159
+ return self._fill(self._call("DELETE", f"/sessions/{_key(session_id)}", timeout=None))
160
+
161
+ @contextmanager
162
+ def session(self, configuration: str | None = None, **options: Any) -> Iterator[Session]:
163
+ """``create_session`` that stops the browser when the block exits."""
164
+ session = self.create_session(configuration, wait=False, **options)
165
+ try:
166
+ yield self.wait_ready(session["id"])
167
+ finally:
168
+ self.stop_session(session["id"])
169
+
170
+ # Configurations
171
+ def configurations(self) -> list[Configuration]:
172
+ return self._call("GET", "/configurations")
173
+
174
+ def configuration(self, configuration: str) -> Configuration:
175
+ return self._call("GET", f"/configurations/{_key(configuration)}")
176
+
177
+ def create_configuration(self, name: str, image: str, cpu: float = 1.5,
178
+ **settings: Any) -> Configuration:
179
+ """Add a configuration: a Scraperack image (by id) with VM size and limits.
180
+
181
+ ``settings``: memory_mib, disk_mib, max_sessions, max_timeout (seconds).
182
+ """
183
+ return self._call("POST", "/configurations",
184
+ json=configuration_body(name, image, cpu, settings))
185
+
186
+ def update_configuration(self, configuration: str, *, cpu: float | None = None,
187
+ **settings: Any) -> Configuration:
188
+ return self._call("PUT", f"/configurations/{_key(configuration)}",
189
+ json=configuration_body(None, None, cpu, settings))
190
+
191
+ def delete_configuration(self, configuration: str) -> None:
192
+ self._call("DELETE", f"/configurations/{_key(configuration)}")
193
+
194
+
195
+ class AsyncBrowsers(_Base):
196
+ """Asynchronous client for configurations and sessions."""
197
+
198
+ def __init__(self, cluster: str | AsyncCluster = DEFAULT_URL, *,
199
+ plugin: str = "browsers") -> None:
200
+ self._owns_cluster = isinstance(cluster, str)
201
+ self.cluster = AsyncCluster(cluster) if isinstance(cluster, str) else cluster
202
+ self.api = f"/plugins/{_key(plugin)}/api"
203
+
204
+ async def __aenter__(self) -> "AsyncBrowsers":
205
+ return self
206
+
207
+ async def __aexit__(self, *_: object) -> None:
208
+ await self.close()
209
+
210
+ async def close(self) -> None:
211
+ if self._owns_cluster:
212
+ await self.cluster.close()
213
+
214
+ async def _call(self, method: str, path: str, **kwargs: Any) -> Any:
215
+ return await self.cluster.request(method, self.api + path, **kwargs)
216
+
217
+ async def create_session(self, configuration: str | None = None, *, timeout: int = 300,
218
+ timezone: str | None = None, lang: str | None = None,
219
+ lc_all: str | None = None, resolution: str | None = None,
220
+ browser_args: str | Sequence[str] | None = None,
221
+ proxy: str | None = None, session_id: str | None = None,
222
+ profile_id: str | None = None, queue: bool = True,
223
+ wait: bool = True) -> Session:
224
+ body = session_body(configuration, timeout, timezone, lang, lc_all, resolution, browser_args,
225
+ proxy, session_id, profile_id, queue)
226
+ session = self._fill(await self._call("POST", "/sessions", json=body, timeout=None))
227
+ return await self.wait_ready(session["id"]) if wait else session
228
+
229
+ async def get_session(self, session_id: str) -> Session:
230
+ return self._fill(await self._call("GET", f"/sessions/{_key(session_id)}"))
231
+
232
+ async def sessions(self, *, active: bool = True) -> list[Session]:
233
+ params = {"status": "active"} if active else {}
234
+ return [self._fill(s) for s in await self._call("GET", "/sessions", params=params)]
235
+
236
+ async def wait_ready(self, session_id: str, timeout: float | None = None) -> Session:
237
+ deadline = None if timeout is None else time.monotonic() + timeout
238
+ while True:
239
+ session = await self.get_session(session_id)
240
+ if self._check_ready(session):
241
+ return session
242
+ if deadline is not None and time.monotonic() > deadline:
243
+ raise TimeoutError(f"session {session_id} is still {session['status']}")
244
+ await asyncio.sleep(POLL_SECONDS)
245
+
246
+ async def stop_session(self, session_id: str) -> Session:
247
+ return self._fill(await self._call("DELETE", f"/sessions/{_key(session_id)}", timeout=None))
248
+
249
+ @asynccontextmanager
250
+ async def session(self, configuration: str | None = None, **options: Any) -> AsyncIterator[Session]:
251
+ session = await self.create_session(configuration, wait=False, **options)
252
+ try:
253
+ yield await self.wait_ready(session["id"])
254
+ finally:
255
+ await self.stop_session(session["id"])
256
+
257
+ async def configurations(self) -> list[Configuration]:
258
+ return await self._call("GET", "/configurations")
259
+
260
+ async def configuration(self, configuration: str) -> Configuration:
261
+ return await self._call("GET", f"/configurations/{_key(configuration)}")
262
+
263
+ async def create_configuration(self, name: str, image: str, cpu: float = 1.5,
264
+ **settings: Any) -> Configuration:
265
+ return await self._call("POST", "/configurations",
266
+ json=configuration_body(name, image, cpu, settings))
267
+
268
+ async def update_configuration(self, configuration: str, *, cpu: float | None = None,
269
+ **settings: Any) -> Configuration:
270
+ return await self._call("PUT", f"/configurations/{_key(configuration)}",
271
+ json=configuration_body(None, None, cpu, settings))
272
+
273
+ async def delete_configuration(self, configuration: str) -> None:
274
+ await self._call("DELETE", f"/configurations/{_key(configuration)}")
File without changes
@@ -0,0 +1,55 @@
1
+ """Shapes returned by the browsers plugin API."""
2
+ from typing import Literal, TypedDict
3
+
4
+ SessionStatus = Literal["queued", "starting", "ready", "stopping", "ended", "failed"]
5
+ ENDED = frozenset({"ended", "failed"})
6
+
7
+
8
+ class SessionSettings(TypedDict):
9
+ timezone: str
10
+ lang: str
11
+ lc_all: str
12
+ resolution: str
13
+ custom_browser_args: str
14
+ proxy_enabled: bool
15
+ proxy_host: str
16
+ proxy_port: str
17
+ proxy_user: str
18
+
19
+
20
+ class Session(TypedDict):
21
+ id: str
22
+ status: SessionStatus
23
+ configuration: str
24
+ configuration_id: str
25
+ image: str | None
26
+ task: str
27
+ settings: SessionSettings
28
+ profile_id: str | None
29
+ timeout: int
30
+ host: str | None
31
+ ports: dict[str, int]
32
+ cdp_url: str | None
33
+ vnc_url: str | None
34
+ node: str | None
35
+ node_name: str | None
36
+ created_at_ms: int
37
+ started_at_ms: int | None
38
+ ready_at_ms: int | None
39
+ expires_at_ms: int | None
40
+ ended_at_ms: int | None
41
+ end_reason: str | None
42
+ error: str | None
43
+
44
+
45
+ class Configuration(TypedDict):
46
+ id: str
47
+ name: str
48
+ image: str
49
+ pool: str
50
+ cpu_millis: int
51
+ memory_mib: int
52
+ disk_mib: int
53
+ max_sessions: int | None
54
+ max_timeout_secs: int
55
+ ready: bool
@@ -0,0 +1,25 @@
1
+ Metadata-Version: 2.4
2
+ Name: scraperack-browsers
3
+ Version: 0.2.0a1
4
+ Summary: Browser sessions on a self-hosted Scraperack cluster through its browsers plugin
5
+ License-Expression: LicenseRef-Proprietary
6
+ Classifier: Development Status :: 3 - Alpha
7
+ Requires-Python: >=3.10
8
+ Description-Content-Type: text/markdown
9
+ Requires-Dist: scraperack>=0.3.0a1
10
+
11
+ # scraperack-browsers
12
+
13
+ Browser sessions on a self-hosted Scraperack cluster through its browsers plugin.
14
+ Each session is a fresh headed Chromium with its own timezone, locale, resolution and
15
+ proxy; connect to it over CDP.
16
+
17
+ ```python
18
+ from scraperack_browsers import Browsers
19
+
20
+ with Browsers("http://controller:7070") as browsers:
21
+ with browsers.session(timezone="Europe/Sofia") as session:
22
+ print(session["cdp_url"])
23
+ ```
24
+
25
+ See [docs/browsers-sdk.md](../../docs/browsers-sdk.md).
@@ -0,0 +1,8 @@
1
+ scraperack_browsers/__init__.py,sha256=8d0eJ2cQFrFd33JlBJIZnbxn6QliWG0XYJS0oagGvcU,384
2
+ scraperack_browsers/client.py,sha256=5EiPX4vFtjshQux082ho14RpbgNB3V5lGXOvcsfcNrY,12690
3
+ scraperack_browsers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
+ scraperack_browsers/types.py,sha256=lqTociXw0GgxCRu-QuKzNOFwrmRDme95Bmckm9TKbIA,1204
5
+ scraperack_browsers-0.2.0a1.dist-info/METADATA,sha256=rSMUdY4LxruwvRc0IQfH4ltsPXsdiqX1ZpSlsC5hzoI,853
6
+ scraperack_browsers-0.2.0a1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
7
+ scraperack_browsers-0.2.0a1.dist-info/top_level.txt,sha256=kbNCDYB9pbKADmsXqp1QIw1OqYOs4ujS-JobmcVF2iU,20
8
+ scraperack_browsers-0.2.0a1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ scraperack_browsers