seleniumbase 4.32.1__py3-none-any.whl → 4.32.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,630 @@
1
+ from __future__ import annotations
2
+ import asyncio
3
+ import collections
4
+ import inspect
5
+ import itertools
6
+ import json
7
+ import logging
8
+ import sys
9
+ import types
10
+ from asyncio import iscoroutine, iscoroutinefunction
11
+ from typing import (
12
+ Optional,
13
+ Generator,
14
+ Union,
15
+ Awaitable,
16
+ Callable,
17
+ Any,
18
+ TypeVar,
19
+ )
20
+ import websockets
21
+ from . import cdp_util as util
22
+ import mycdp as cdp
23
+ import mycdp.network
24
+ import mycdp.page
25
+ import mycdp.storage
26
+ import mycdp.runtime
27
+ import mycdp.target
28
+ import mycdp.util
29
+
30
+ T = TypeVar("T")
31
+ GLOBAL_DELAY = 0.005
32
+ MAX_SIZE: int = 2**28
33
+ PING_TIMEOUT: int = 1800 # 30 minutes
34
+ TargetType = Union[cdp.target.TargetInfo, cdp.target.TargetID]
35
+ logger = logging.getLogger("uc.connection")
36
+
37
+
38
+ class ProtocolException(Exception):
39
+ def __init__(self, *args, **kwargs):
40
+ self.message = None
41
+ self.code = None
42
+ self.args = args
43
+ if isinstance(args[0], dict):
44
+ self.message = args[0].get("message", None) # noqa
45
+ self.code = args[0].get("code", None)
46
+ elif hasattr(args[0], "to_json"):
47
+ def serialize(obj, _d=0):
48
+ res = "\n"
49
+ for k, v in obj.items():
50
+ space = "\t" * _d
51
+ if isinstance(v, dict):
52
+ res += f"{space}{k}: {serialize(v, _d + 1)}\n"
53
+ else:
54
+ res += f"{space}{k}: {v}\n"
55
+ return res
56
+ self.message = serialize(args[0].to_json())
57
+ else:
58
+ self.message = "| ".join(str(x) for x in args)
59
+
60
+ def __str__(self):
61
+ return f"{self.message} [code: {self.code}]" if self.code else f"{self.message}" # noqa
62
+
63
+
64
+ class SettingClassVarNotAllowedException(PermissionError):
65
+ pass
66
+
67
+
68
+ class Transaction(asyncio.Future):
69
+ __cdp_obj__: Generator = None
70
+ method: str = None
71
+ params: dict = None
72
+ id: int = None
73
+
74
+ def __init__(self, cdp_obj: Generator):
75
+ """
76
+ :param cdp_obj:
77
+ """
78
+ super().__init__()
79
+ self.__cdp_obj__ = cdp_obj
80
+ self.connection = None
81
+ self.method, *params = next(self.__cdp_obj__).values()
82
+ if params:
83
+ params = params.pop()
84
+ self.params = params
85
+
86
+ @property
87
+ def message(self):
88
+ return json.dumps(
89
+ {"method": self.method, "params": self.params, "id": self.id}
90
+ )
91
+
92
+ @property
93
+ def has_exception(self):
94
+ try:
95
+ if self.exception():
96
+ return True
97
+ except BaseException:
98
+ return True
99
+ return False
100
+
101
+ def __call__(self, **response: dict):
102
+ """
103
+ Parses the response message and marks the future complete.
104
+ :param response:
105
+ """
106
+ if "error" in response:
107
+ # Set exception and bail out
108
+ return self.set_exception(ProtocolException(response["error"]))
109
+ try:
110
+ # Try to parse the result according to the PyCDP docs.
111
+ self.__cdp_obj__.send(response["result"])
112
+ except StopIteration as e:
113
+ # Exception value holds the parsed response
114
+ return self.set_result(e.value)
115
+ raise ProtocolException(
116
+ "Could not parse the cdp response:\n%s" % response
117
+ )
118
+
119
+ def __repr__(self):
120
+ success = False if (self.done() and self.has_exception) else True
121
+ if self.done():
122
+ status = "finished"
123
+ else:
124
+ status = "pending"
125
+ fmt = (
126
+ f"<{self.__class__.__name__}\n\t"
127
+ f"method: {self.method}\n\t"
128
+ f"status: {status}\n\t"
129
+ f"success: {success}>"
130
+ )
131
+ return fmt
132
+
133
+
134
+ class EventTransaction(Transaction):
135
+ event = None
136
+ value = None
137
+
138
+ def __init__(self, event_object):
139
+ try:
140
+ super().__init__(None)
141
+ except BaseException:
142
+ pass
143
+ self.set_result(event_object)
144
+ self.event = self.value = self.result()
145
+
146
+ def __repr__(self):
147
+ status = "finished"
148
+ success = False if self.exception() else True
149
+ event_object = self.result()
150
+ fmt = (
151
+ f"{self.__class__.__name__}\n\t"
152
+ f"event: {event_object.__class__.__module__}.{event_object.__class__.__name__}\n\t" # noqa
153
+ f"status: {status}\n\t"
154
+ f"success: {success}>"
155
+ )
156
+ return fmt
157
+
158
+
159
+ class CantTouchThis(type):
160
+ def __setattr__(cls, attr, value):
161
+ """:meta private:"""
162
+ if attr == "__annotations__":
163
+ # Fix autodoc
164
+ return super().__setattr__(attr, value)
165
+ raise SettingClassVarNotAllowedException(
166
+ "\n".join(
167
+ (
168
+ "don't set '%s' on the %s class directly, "
169
+ "as those are shared with other objects.",
170
+ "use `my_object.%s = %s` instead",
171
+ )
172
+ )
173
+ % (attr, cls.__name__, attr, value)
174
+ )
175
+
176
+
177
+ class Connection(metaclass=CantTouchThis):
178
+ attached: bool = None
179
+ websocket: websockets.WebSocketClientProtocol
180
+ _target: cdp.target.TargetInfo
181
+
182
+ def __init__(
183
+ self,
184
+ websocket_url=None,
185
+ target=None,
186
+ _owner=None,
187
+ **kwargs,
188
+ ):
189
+ super().__init__()
190
+ self._target = target
191
+ self.__count__ = itertools.count(0)
192
+ self._owner = _owner
193
+ self.websocket_url: str = websocket_url
194
+ self.websocket = None
195
+ self.mapper = {}
196
+ self.handlers = collections.defaultdict(list)
197
+ self.recv_task = None
198
+ self.enabled_domains = []
199
+ self._last_result = []
200
+ self.listener: Listener = None
201
+ self.__dict__.update(**kwargs)
202
+
203
+ @property
204
+ def target(self) -> cdp.target.TargetInfo:
205
+ return self._target
206
+
207
+ @target.setter
208
+ def target(self, target: cdp.target.TargetInfo):
209
+ if not isinstance(target, cdp.target.TargetInfo):
210
+ raise TypeError(
211
+ "target must be set to a '%s' but got '%s"
212
+ % (cdp.target.TargetInfo.__name__, type(target).__name__)
213
+ )
214
+ self._target = target
215
+
216
+ @property
217
+ def closed(self):
218
+ if not self.websocket:
219
+ return True
220
+ return self.websocket.closed
221
+
222
+ def add_handler(
223
+ self,
224
+ event_type_or_domain: Union[type, types.ModuleType],
225
+ handler: Union[Callable, Awaitable],
226
+ ):
227
+ """
228
+ Add a handler for given event.
229
+ If event_type_or_domain is a module instead of a type,
230
+ it will find all available events and add the handler.
231
+ If you want to receive event updates (eg. network traffic),
232
+ you can add handlers for those events.
233
+ Handlers can be regular callback functions
234
+ or async coroutine functions (and also just lambdas).
235
+ For example, if you want to check the network traffic:
236
+ .. code-block::
237
+ page.add_handler(
238
+ cdp.network.RequestWillBeSent, lambda event: print(
239
+ 'network event => %s' % event.request
240
+ )
241
+ )
242
+ Next time there's network traffic, you'll see lots of console output.
243
+ :param event_type_or_domain:
244
+ :param handler:
245
+ """
246
+ if isinstance(event_type_or_domain, types.ModuleType):
247
+ for name, obj in inspect.getmembers_static(event_type_or_domain):
248
+ if name.isupper():
249
+ continue
250
+ if not name[0].isupper():
251
+ continue
252
+ if not isinstance(obj, type):
253
+ continue
254
+ if inspect.isbuiltin(obj):
255
+ continue
256
+ self.handlers[obj].append(handler)
257
+ return
258
+ self.handlers[event_type_or_domain].append(handler)
259
+
260
+ async def aopen(self, **kw):
261
+ """
262
+ Opens the websocket connection. Shouldn't be called manually by users.
263
+ """
264
+ if not self.websocket or self.websocket.closed:
265
+ try:
266
+ self.websocket = await websockets.connect(
267
+ self.websocket_url,
268
+ ping_timeout=PING_TIMEOUT,
269
+ max_size=MAX_SIZE,
270
+ )
271
+ self.listener = Listener(self)
272
+ except (Exception,) as e:
273
+ logger.debug("Exception during opening of websocket: %s", e)
274
+ if self.listener:
275
+ self.listener.cancel()
276
+ raise
277
+ if not self.listener or not self.listener.running:
278
+ self.listener = Listener(self)
279
+ logger.debug(
280
+ "\n✅ Opened websocket connection to %s", self.websocket_url
281
+ )
282
+ # When a websocket connection is closed (either by error or on purpose)
283
+ # and reconnected, the registered event listeners (if any), should be
284
+ # registered again, so the browser sends those events.
285
+ await self._register_handlers()
286
+
287
+ async def aclose(self):
288
+ """
289
+ Closes the websocket connection. Shouldn't be called manually by users.
290
+ """
291
+ if self.websocket and not self.websocket.closed:
292
+ if self.listener and self.listener.running:
293
+ self.listener.cancel()
294
+ self.enabled_domains.clear()
295
+ await self.websocket.close()
296
+ logger.debug(
297
+ "\n❌ Closed websocket connection to %s", self.websocket_url
298
+ )
299
+
300
+ async def sleep(self, t: Union[int, float] = 0.25):
301
+ await self.update_target()
302
+ await asyncio.sleep(t)
303
+
304
+ def feed_cdp(self, cdp_obj):
305
+ """
306
+ Used in specific cases, mostly during cdp.fetch.RequestPaused events,
307
+ in which the browser literally blocks.
308
+ By using feed_cdp, you can issue a response without a blocking "await".
309
+ Note: This method won't cause a response.
310
+ Note: This is not an async method, just a regular method!
311
+ :param cdp_obj:
312
+ """
313
+ asyncio.ensure_future(self.send(cdp_obj))
314
+
315
+ async def wait(self, t: Union[int, float] = None):
316
+ """
317
+ Waits until the event listener reports idle
318
+ (no new events received in certain timespan).
319
+ When `t` is provided, ensures waiting for `t` seconds, no matter what.
320
+ :param t:
321
+ """
322
+ await self.update_target()
323
+ loop = asyncio.get_running_loop()
324
+ start_time = loop.time()
325
+ try:
326
+ if isinstance(t, (int, float)):
327
+ await asyncio.wait_for(self.listener.idle.wait(), timeout=t)
328
+ while (loop.time() - start_time) < t:
329
+ await asyncio.sleep(0.1)
330
+ else:
331
+ await self.listener.idle.wait()
332
+ except asyncio.TimeoutError:
333
+ if isinstance(t, (int, float)):
334
+ # Explicit time is given, which is now passed, so leave now.
335
+ return
336
+ except AttributeError:
337
+ # No listener created yet.
338
+ pass
339
+
340
+ async def set_locale(self, locale: Optional[str] = None):
341
+ """Sets the Language Locale code via set_user_agent_override."""
342
+ await self.send(cdp.network.set_user_agent_override("", locale))
343
+
344
+ def __getattr__(self, item):
345
+ """:meta private:"""
346
+ try:
347
+ return getattr(self.target, item)
348
+ except AttributeError:
349
+ raise
350
+
351
+ async def __aenter__(self):
352
+ """:meta private:"""
353
+ return self
354
+
355
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
356
+ """:meta private:"""
357
+ await self.aclose()
358
+ if exc_type and exc_val:
359
+ raise exc_type(exc_val)
360
+
361
+ def __await__(self):
362
+ """
363
+ Updates targets and wait for event listener to report idle.
364
+ Idle is reported when no new events are received for 1 second.
365
+ """
366
+ return self.wait().__await__()
367
+
368
+ async def update_target(self):
369
+ target_info: cdp.target.TargetInfo = await self.send(
370
+ cdp.target.get_target_info(self.target_id), _is_update=True
371
+ )
372
+ self.target = target_info
373
+
374
+ async def send(
375
+ self,
376
+ cdp_obj: Generator[dict[str, Any], dict[str, Any], Any],
377
+ _is_update=False,
378
+ ) -> Any:
379
+ """
380
+ Send a protocol command.
381
+ The commands are made using any of the cdp.<domain>.<method>()'s
382
+ and is used to send custom cdp commands as well.
383
+ :param cdp_obj: The generator object created by a cdp method
384
+ :param _is_update: Internal flag
385
+ Prevents infinite loop by skipping the registeration of handlers
386
+ when multiple calls to connection.send() are made.
387
+ """
388
+ await self.aopen()
389
+ if not self.websocket or self.closed:
390
+ return
391
+ if self._owner:
392
+ browser = self._owner
393
+ if browser.config:
394
+ if browser.config.expert:
395
+ await self._prepare_expert()
396
+ if browser.config.headless:
397
+ await self._prepare_headless()
398
+ if not self.listener or not self.listener.running:
399
+ self.listener = Listener(self)
400
+ try:
401
+ tx = Transaction(cdp_obj)
402
+ tx.connection = self
403
+ if not self.mapper:
404
+ self.__count__ = itertools.count(0)
405
+ tx.id = next(self.__count__)
406
+ self.mapper.update({tx.id: tx})
407
+ if not _is_update:
408
+ await self._register_handlers()
409
+ await self.websocket.send(tx.message)
410
+ try:
411
+ return await tx
412
+ except ProtocolException as e:
413
+ e.message += f"\ncommand:{tx.method}\nparams:{tx.params}"
414
+ raise e
415
+ except Exception:
416
+ await self.aclose()
417
+
418
+ async def _register_handlers(self):
419
+ """
420
+ Ensure that for current (event) handlers, the corresponding
421
+ domain is enabled in the protocol.
422
+ """
423
+ # Save a copy of current enabled domains in a variable.
424
+ # At the end, this variable will hold the domains that
425
+ # are not represented by handlers, and can be removed.
426
+ enabled_domains = self.enabled_domains.copy()
427
+ for event_type in self.handlers.copy():
428
+ domain_mod = None
429
+ if len(self.handlers[event_type]) == 0:
430
+ self.handlers.pop(event_type)
431
+ continue
432
+ if isinstance(event_type, type):
433
+ domain_mod = util.cdp_get_module(event_type.__module__)
434
+ if domain_mod in self.enabled_domains:
435
+ # At this point, the domain is being used by a handler, so
436
+ # remove that domain from temp variable 'enabled_domains'.
437
+ if domain_mod in enabled_domains:
438
+ enabled_domains.remove(domain_mod)
439
+ continue
440
+ elif domain_mod not in self.enabled_domains:
441
+ if domain_mod in (cdp.target, cdp.storage):
442
+ continue
443
+ try:
444
+ # Prevent infinite loops.
445
+ logger.debug("Registered %s", domain_mod)
446
+ self.enabled_domains.append(domain_mod)
447
+ await self.send(domain_mod.enable(), _is_update=True)
448
+ except BaseException: # Don't error before request is sent
449
+ logger.debug("", exc_info=True)
450
+ try:
451
+ self.enabled_domains.remove(domain_mod)
452
+ except BaseException:
453
+ logger.debug("NOT GOOD", exc_info=True)
454
+ continue
455
+ finally:
456
+ continue
457
+ for ed in enabled_domains:
458
+ # Items still present at this point are unused and need removal.
459
+ self.enabled_domains.remove(ed)
460
+
461
+ async def _prepare_headless(self):
462
+ return # (This functionality has moved to a new location!)
463
+
464
+ async def _prepare_expert(self):
465
+ if getattr(self, "_prep_expert_done", None):
466
+ return
467
+ if self._owner:
468
+ part1 = "Element.prototype._attachShadow = "
469
+ part2 = "Element.prototype.attachShadow"
470
+ parts = part1 + part2
471
+ await self._send_oneshot(
472
+ cdp.page.add_script_to_evaluate_on_new_document(
473
+ """
474
+ %s;
475
+ Element.prototype.attachShadow = function () {
476
+ return this._attachShadow( { mode: "open" } );
477
+ };
478
+ """ % parts
479
+ )
480
+ )
481
+ await self._send_oneshot(cdp.page.enable())
482
+ setattr(self, "_prep_expert_done", True)
483
+
484
+ async def _send_oneshot(self, cdp_obj):
485
+ tx = Transaction(cdp_obj)
486
+ tx.connection = self
487
+ tx.id = -2
488
+ self.mapper.update({tx.id: tx})
489
+ await self.websocket.send(tx.message)
490
+ try:
491
+ # In try/except since if browser connection sends this,
492
+ # then it raises an exception.
493
+ return await tx
494
+ except ProtocolException:
495
+ pass
496
+
497
+
498
+ class Listener:
499
+ def __init__(self, connection: Connection):
500
+ self.connection = connection
501
+ self.history = collections.deque()
502
+ self.max_history = 1000
503
+ self.task: asyncio.Future = None
504
+ is_interactive = getattr(sys, "ps1", sys.flags.interactive)
505
+ self._time_before_considered_idle = 0.10 if not is_interactive else 0.75 # noqa
506
+ self.idle = asyncio.Event()
507
+ self.run()
508
+
509
+ def run(self):
510
+ self.task = asyncio.create_task(self.listener_loop())
511
+
512
+ @property
513
+ def time_before_considered_idle(self):
514
+ return self._time_before_considered_idle
515
+
516
+ @time_before_considered_idle.setter
517
+ def time_before_considered_idle(self, seconds: Union[int, float]):
518
+ self._time_before_considered_idle = seconds
519
+
520
+ def cancel(self):
521
+ if self.task and not self.task.cancelled():
522
+ self.task.cancel()
523
+
524
+ @property
525
+ def running(self):
526
+ if not self.task:
527
+ return False
528
+ if self.task.done():
529
+ return False
530
+ return True
531
+
532
+ async def listener_loop(self):
533
+ while True:
534
+ try:
535
+ msg = await asyncio.wait_for(
536
+ self.connection.websocket.recv(),
537
+ self.time_before_considered_idle,
538
+ )
539
+ except asyncio.TimeoutError:
540
+ self.idle.set()
541
+ # Pause for a moment.
542
+ # await asyncio.sleep(self.time_before_considered_idle / 10)
543
+ continue
544
+ except (Exception,) as e:
545
+ logger.debug(
546
+ "Connection listener exception "
547
+ "while reading websocket:\n%s", e
548
+ )
549
+ break
550
+ if not self.running:
551
+ # If we have been cancelled or otherwise stopped running,
552
+ # then break this loop.
553
+ break
554
+ self.idle.clear() # Not "idle" anymore.
555
+ message = json.loads(msg)
556
+ if "id" in message:
557
+ if message["id"] in self.connection.mapper:
558
+ tx = self.connection.mapper.pop(message["id"])
559
+ logger.debug(
560
+ "Got answer for %s (message_id:%d)", tx, message["id"]
561
+ )
562
+ tx(**message)
563
+ else:
564
+ if message["id"] == -2:
565
+ tx = self.connection.mapper.get(-2)
566
+ if tx:
567
+ tx(**message)
568
+ continue
569
+ else:
570
+ # Probably an event
571
+ try:
572
+ event = cdp.util.parse_json_event(message)
573
+ event_tx = EventTransaction(event)
574
+ if not self.connection.mapper:
575
+ self.connection.__count__ = itertools.count(0)
576
+ event_tx.id = next(self.connection.__count__)
577
+ self.connection.mapper[event_tx.id] = event_tx
578
+ except Exception as e:
579
+ logger.info(
580
+ "%s: %s during parsing of json from event : %s"
581
+ % (type(e).__name__, e.args, message),
582
+ exc_info=True,
583
+ )
584
+ continue
585
+ except KeyError as e:
586
+ logger.info("KeyError: %s" % e, exc_info=True)
587
+ continue
588
+ try:
589
+ if type(event) in self.connection.handlers:
590
+ callbacks = self.connection.handlers[type(event)]
591
+ else:
592
+ continue
593
+ if not len(callbacks):
594
+ continue
595
+ for callback in callbacks:
596
+ try:
597
+ if (
598
+ iscoroutinefunction(callback)
599
+ or iscoroutine(callback)
600
+ ):
601
+ try:
602
+ await callback(event, self.connection)
603
+ except TypeError:
604
+ await callback(event)
605
+ else:
606
+ try:
607
+ callback(event, self.connection)
608
+ except TypeError:
609
+ callback(event)
610
+ except Exception as e:
611
+ logger.warning(
612
+ "Exception in callback %s for event %s => %s",
613
+ callback,
614
+ event.__class__.__name__,
615
+ e,
616
+ exc_info=True,
617
+ )
618
+ raise
619
+ except asyncio.CancelledError:
620
+ break
621
+ except Exception:
622
+ raise
623
+ continue
624
+
625
+ def __repr__(self):
626
+ s_idle = "[idle]" if self.idle.is_set() else "[busy]"
627
+ s_cache_length = f"[cache size: {len(self.history)}]"
628
+ s_running = f"[running: {self.running}]"
629
+ s = f"{self.__class__.__name__} {s_running} {s_idle} {s_cache_length}>"
630
+ return s