vchrome 0.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
vchrome/__init__.py ADDED
@@ -0,0 +1,4088 @@
1
+ __version__ = '0.0.0'
2
+ __author__ = 'v'
3
+ # ----------------------------------------------------------------------------------------------------
4
+ _allowed = {'Chrome'}
5
+ def __getattr__(name):
6
+ if name in _allowed:
7
+ return globals()[name]
8
+ raise AttributeError(f"module {__name__} has no attribute {name}")
9
+ def __dir__():
10
+ return list(_allowed)
11
+ __all__ = __dir__()
12
+ import re
13
+ import json
14
+ import copy
15
+ import types
16
+ import queue
17
+ import time
18
+ import inspect
19
+ import threading
20
+ from time import perf_counter
21
+ from math import factorial, sin, pi
22
+ from random import random
23
+ from collections import deque
24
+ import socket
25
+ import base64
26
+ import traceback
27
+ from json import JSONDecodeError
28
+ from functools import lru_cache
29
+ from threading import Thread, Event, RLock
30
+ from urllib import request
31
+ import builtins
32
+ rl = RLock()
33
+ print_hook = False
34
+ def monkey_print():
35
+ global print_hook
36
+ if not print_hook:
37
+ _original_print = print
38
+ def thread_safe_print(*args, **kwargs):
39
+ with rl:
40
+ _original_print(*args, **kwargs, flush=True)
41
+ builtins.print = thread_safe_print
42
+ print_hook = True
43
+ def cdp_client(hostname, port, debug=False):
44
+ class Logger:
45
+ def _I(self, id):
46
+ return id
47
+ def _L(self, data, key):
48
+ if self.simple:
49
+ return json.dumps(data.get(key),ensure_ascii=False)
50
+ else:
51
+ return json.dumps(data,ensure_ascii=False)
52
+ def _T(self, tp, sessionId=None):
53
+ t = None
54
+ if tp == 'c': t = 'S:{} <->'
55
+ if tp == 'i': t = 'S:{} [*]'
56
+ return t.format(self._I(sessionId or self.id))
57
+ def _C(self, id, msg):
58
+ if self.debug: self.loginfo[id] = msg
59
+ def _G(self, id):
60
+ if self.debug:
61
+ r = self.loginfo.get(id)
62
+ if r:
63
+ del self.loginfo[id]
64
+ return r
65
+ else:
66
+ return {'msg': 'no cache'}
67
+ def __init__(self, id, debug=False):
68
+ self.id = id
69
+ self.debug = debug
70
+ self.simple = True
71
+ self.loginfo = {}
72
+ self.indent = 44
73
+ if self.debug:
74
+ monkey_print()
75
+ def log(*a):
76
+ print(*a)
77
+ def __call__(self, tp, a):
78
+ if self.debug:
79
+ if tp == 'recv':
80
+ id = a.get('id')
81
+ md = a.get('method')
82
+ se = a.get('sessionId')
83
+ if id:
84
+ c = self._G(id)
85
+ cse = c.get('sessionId')
86
+ cmd = c.get('method')
87
+ if self.debug(cmd):
88
+ print(self._T('c', cse), '[<={:3}] {}'.format(id, cmd).ljust(self.indent, ' '), self._L(a, 'result'))
89
+ if md:
90
+ if self.debug(md):
91
+ print(self._T('i', se), (' '*8 + md).ljust(self.indent, ' '), self._L(a, 'params'))
92
+ elif tp == 'req':
93
+ id = a.get('id')
94
+ md = a.get('method')
95
+ se = a.get('sessionId')
96
+ if self.debug(md):
97
+ self._C(id, a)
98
+ print(self._T('c', se), '[->{:3}] {}'.format(id, md).ljust(self.indent, ' '), self._L(a, 'params'))
99
+ else:
100
+ if self.debug.rest:
101
+ print(self._T('i'), (' '*8 + tp).ljust(self.indent, ' '), a)
102
+ class Pool:
103
+ import queue, time, traceback
104
+ from threading import Thread, main_thread
105
+ class KillThreadParams(Exception): pass
106
+ def __init__(self, num):
107
+ if not getattr(Pool.Thread, 'isAlive', None):
108
+ Pool.Thread.isAlive = Pool.Thread.is_alive
109
+ self._pool = Pool.queue.Queue()
110
+ self._monitor_run = Pool.queue.Queue()
111
+ self.main_monitor()
112
+ self.num = num
113
+ self.init()
114
+ self.is_close = True
115
+ def __call__(self,func):
116
+ def _run_threads(*args,**kw):
117
+ if self.is_close:
118
+ self.init()
119
+ self.main_monitor()
120
+ self._pool.put((func,args,kw))
121
+ return _run_threads
122
+ def init(self):
123
+ self.is_close = False
124
+ def _pools_pull():
125
+ while True:
126
+ v = self._pool.get()
127
+ if v == self.KillThreadParams: return
128
+ try:
129
+ func,args,kw = v
130
+ self._monitor_run.put('V')
131
+ func(*args,**kw)
132
+ except BaseException as e:
133
+ with rl:
134
+ print(Pool.traceback.format_exc())
135
+ finally:
136
+ self._monitor_run.get('V')
137
+ for _ in range(self.num):
138
+ Pool.Thread(target=_pools_pull).start()
139
+ def main_monitor(self):
140
+ def _func():
141
+ while True:
142
+ Pool.time.sleep(.12)
143
+ if not Pool.main_thread().isAlive() and self._monitor_run.empty():
144
+ self.close_all()
145
+ self.is_close = True
146
+ break
147
+ Pool.Thread(target=_func,name="MainMonitor").start()
148
+ def close_all(self):
149
+ for i in range(self.num):
150
+ self._pool.put(self.KillThreadParams)
151
+ @lru_cache(maxsize=1024)
152
+ def get_cached_signature(func):
153
+ return inspect.signature(func)
154
+ def to_human_read(size_in_bytes, decimal_places= 2):
155
+ if size_in_bytes < 0:
156
+ raise ValueError("byte size cannot be negative!")
157
+ units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
158
+ scale = 1024
159
+ if size_in_bytes == 0:
160
+ return f"0 B"
161
+ unit_idx = 0
162
+ while size_in_bytes >= scale and unit_idx < len(units) - 1:
163
+ size_in_bytes /= scale
164
+ unit_idx += 1
165
+ return f"{size_in_bytes:.{decimal_places}f} {units[unit_idx]}"
166
+ def myget(url):
167
+ r = request.Request(url, method='GET')
168
+ opener = request.build_opener(request.ProxyHandler(None))
169
+ return json.loads(opener.open(r).read().decode())
170
+ def adj_wsurl(wsurl):
171
+ return re.sub('ws://[^/]+/devtools/', 'ws://{}:{}/devtools/'.format(hostname, port), wsurl)
172
+ def make_dev_page_url(id):
173
+ return "ws://{}:{}/devtools/page/{}".format(hostname, port, id)
174
+ def try_run_result(data):
175
+ is_err = False
176
+ try:
177
+ if data['result'].get('type') == 'undefined':
178
+ return None
179
+ elif data['result'].get('subtype') == 'null':
180
+ return None
181
+ elif data['result'].get('value', None) != None:
182
+ return data['result']['value']
183
+ elif data['result'].get('objectId', None) != None:
184
+ return data['result']
185
+ elif data['result'].get('description'):
186
+ is_err = data['result']['description']
187
+ else:
188
+ raise Exception('err')
189
+ except:
190
+ return data
191
+ if is_err:
192
+ raise Exception(is_err)
193
+ def is_function(obj):
194
+ return type(obj) == types.FunctionType or type(obj) == types.MethodType
195
+ def create_connection_saf(*a, **kw):
196
+ for i in range(50):
197
+ try:
198
+ return create_connection(*a, **kw)
199
+ except WebSocketBadStatusException as e:
200
+ if b'No such target id:' in e.resp_body:
201
+ time.sleep(0.05)
202
+ continue
203
+ raise Exception('connect ws error')
204
+ class Err: pass
205
+ class Waiter:
206
+ def __init__(self, sniff, pattern, is_regex):
207
+ self.s = sniff
208
+ self.pattern = pattern
209
+ self.is_regex = is_regex
210
+ self.count = 0
211
+ self.currc = 0
212
+ self.cache = deque(maxlen=1024)
213
+ self.is_remove = False
214
+ def add(self, r):
215
+ self.count += 1
216
+ self.cache.appendleft(r)
217
+ def wait(self, count=1, timeout=10):
218
+ if self.is_remove:
219
+ raise Exception('listener remove by waiter, cannot use wait.')
220
+ start = perf_counter()
221
+ while True:
222
+ if self._check(count):
223
+ return self.cache.pop()
224
+ time.sleep(0.15)
225
+ if perf_counter() - start > timeout:
226
+ break
227
+ def remove(self):
228
+ f = getattr(self.s, 'remove_listen', None) or getattr(self.s, 'remove_change', None)
229
+ f(self.pattern)
230
+ self.is_remove = True
231
+ def _check(self, count):
232
+ if self.count >= self.currc + count:
233
+ self.currc += count
234
+ return True
235
+ else:
236
+ return False
237
+ def __repr__(self):
238
+ return '<Waiter is_regex:[{}] [{}]>'.format(self.is_regex, json.dumps(self.pattern, ensure_ascii=False))
239
+ class SniffTools:
240
+ def __init__(self, s):
241
+ self.s = s
242
+ self.qlist = queue.Queue()
243
+ self.default_func = lambda a:True
244
+ self.attach()
245
+ def attach(self):
246
+ self.s._match_url = self._match_url
247
+ self.s.qlist = self.qlist
248
+ self.s.default_func = self.default_func
249
+ def _match_str(self, a, b, is_regex):
250
+ if is_regex:
251
+ return re.findall(a, b)
252
+ else:
253
+ return a in b
254
+ def _match_url(self, v, url):
255
+ pattern, is_regex = v
256
+ if type(pattern) == str:
257
+ return self._match_str(pattern, url, is_regex)
258
+ if type(pattern) == list:
259
+ for p in pattern:
260
+ if self._match_str(p, url, is_regex):
261
+ return True
262
+ return False
263
+ class SniffNetwork:
264
+ class NetworkRequest:
265
+ def __init__(self, rinfo, encoding='utf8'):
266
+ self.rinfo = rinfo
267
+ self.encoding = encoding
268
+ self._url = rinfo['request'].get('url')
269
+ self._method = rinfo['request'].get('method')
270
+ self._headers = rinfo['request'].get('headers')
271
+ if rinfo['request'].get('hasPostData'):
272
+ self._content = b''.join(base64.b64decode(e['bytes']) for e in rinfo['request'].get('postDataEntries', []))
273
+ @property
274
+ def url(self): return self._url
275
+ @property
276
+ def method(self): return self._method
277
+ @property
278
+ def headers(self): return self._headers
279
+ @property
280
+ def content(self):
281
+ assert self.method == 'POST', 'must be method: POST'
282
+ return self._content
283
+ @property
284
+ def text(self):
285
+ assert self.method == 'POST', 'must be method: POST'
286
+ return self._content.decode(self.encoding)
287
+ def json(self):
288
+ c = self.text
289
+ return json.loads(c[c.find('{'):c.rfind('}')+1])
290
+ def __repr__(self):
291
+ return '<Request [{}] ReadOnly>'.format(self.method)
292
+ class NetworkResponse:
293
+ def __init__(self, rinfo, encoding='utf8'):
294
+ self.rinfo = rinfo
295
+ self.encoding = encoding
296
+ self.error = None
297
+ if rinfo.get('status') == 'ERROR':
298
+ self.error = rinfo['error']
299
+ self._request = 'ERROR'
300
+ self._url = 'ERROR'
301
+ self._status_code = -1
302
+ self._headers = {}
303
+ self._content = b'ERROR'
304
+ return
305
+ self._request = SniffNetwork.NetworkRequest(rinfo)
306
+ self._url = rinfo['response'].get('url')
307
+ self._status_code = rinfo['response'].get('status')
308
+ self._headers = rinfo['response'].get('headers')
309
+ if rinfo['response_body'].get('base64Encoded'):
310
+ self._content = base64.b64decode(rinfo['response_body']['body'])
311
+ elif 'body' in rinfo['response_body']:
312
+ self._content = rinfo['response_body']['body'].encode(self.encoding)
313
+ else:
314
+ self._content = b'<cannot catch body data. maybe session redirect too fast.>'
315
+ @property
316
+ def url(self): return self._url
317
+ @property
318
+ def status_code(self): return self._status_code
319
+ @property
320
+ def headers(self): return self._headers
321
+ @property
322
+ def content(self): return self._content
323
+ @property
324
+ def text(self): return self._content.decode(self.encoding)
325
+ def json(self):
326
+ c = self.text
327
+ return json.loads(c[c.find('{'):c.rfind('}')+1])
328
+ @property
329
+ def request(self): return self._request
330
+ def __repr__(self):
331
+ if self.error:
332
+ return '<Response [ERROR] reason:{}>'.format(self.error)
333
+ return '<Response [{}] ReadOnly>'.format(self.status_code)
334
+ def __init__(self, f):
335
+ self.f = f
336
+ self.type = self.f.type
337
+ self.is_listen = False
338
+ self.listen_sesslist = [None]
339
+ self.info_listen = {}
340
+ self.call_listen = {}
341
+ self.call_listen_keys = []
342
+ self.call_listen_vals = {}
343
+ self.tools = SniffTools(self)
344
+ def _listen_cdp(self, sessionId):
345
+ self.f.cdp('Network.enable',{
346
+ "maxTotalBufferSize": 100000000,
347
+ "maxResourceBufferSize":50000000,
348
+ "includeNetworkExtraInfo":True,
349
+ }, sessionId=sessionId)
350
+ def add_listen_session(self, sessionId):
351
+ self.listen_sesslist.append(sessionId)
352
+ if self.is_listen:
353
+ self._listen_cdp(sessionId)
354
+ def remove_listen(self, pattern):
355
+ pk = json.dumps(pattern, sort_keys=True)
356
+ self.call_listen.pop(pk, None)
357
+ self.call_listen_keys = self.call_listen.keys()
358
+ self.call_listen_vals.pop(pk, None)
359
+ def listen(self, pattern, on_response=None, on_request=None, is_regex=False):
360
+ if not self.is_listen:
361
+ self._handle_network()
362
+ for sessionId in self.listen_sesslist:
363
+ self._listen_cdp(sessionId)
364
+ self.is_listen = True
365
+ waiter = Waiter(self, pattern, is_regex)
366
+ pk = json.dumps(pattern, sort_keys=True)
367
+ if (not on_request) and (not on_response):
368
+ on_response = self.default_func
369
+ self.call_listen[pk] = [waiter, on_request, on_response]
370
+ self.call_listen_keys = self.call_listen.keys()
371
+ self.call_listen_vals[pk] = [pattern, is_regex]
372
+ return waiter
373
+ def _handle_network(self):
374
+ self.f.set_method_callback('Network.requestWillBeSent', self.Network_requestWillBeSent)
375
+ self.f.set_method_callback('Network.requestWillBeSentExtraInfo', self.Network_requestWillBeSentExtraInfo)
376
+ self.f.set_method_callback('Network.responseReceived', self.Network_responseReceived)
377
+ self.f.set_method_callback('Network.responseReceivedExtraInfo', self.Network_responseReceivedExtraInfo)
378
+ self.f.set_method_callback('Network.loadingFinished', self.Network_loadingFinished)
379
+ self.f.set_method_callback('Network.loadingFailed', self.Network_loadingFailed)
380
+ def Network_requestWillBeSent(self, rdata):
381
+ requestId = rdata['params']['requestId']
382
+ self.info_listen[requestId] = self.info_listen.get(requestId, {})
383
+ self.info_listen[requestId]['request'] = rdata['params']['request']
384
+ if self.info_listen[requestId].get('request_extra'):
385
+ request_extra = self.info_listen[requestId].get('request_extra')
386
+ headers = self.info_listen[requestId]['request']['headers']
387
+ self.info_listen[requestId]['request']['headers'] = {**headers, **request_extra['headers']}
388
+ url = self.info_listen[requestId]['request']['url']
389
+ for k in self.call_listen_keys:
390
+ if self._match_url(self.call_listen_vals[k], url):
391
+ waiter, on_request, on_response = self.call_listen[k]
392
+ if on_request:
393
+ r = SniffNetwork.NetworkRequest(self.info_listen[requestId])
394
+ if on_request(r):
395
+ waiter.add(r)
396
+ def Network_requestWillBeSentExtraInfo(self, rdata):
397
+ request_extra = rdata['params']
398
+ requestId = request_extra['requestId']
399
+ if self.info_listen.get(requestId):
400
+ headers = self.info_listen[requestId]['request']['headers']
401
+ self.info_listen[requestId]['request']['headers'] = {**headers, **request_extra['headers']}
402
+ url = self.info_listen[requestId]['request']['url']
403
+ for k in self.call_listen_keys:
404
+ if self._match_url(self.call_listen_vals[k], url):
405
+ waiter, on_request, on_response = self.call_listen[k]
406
+ if on_request:
407
+ r = SniffNetwork.NetworkRequest(self.info_listen[requestId])
408
+ if on_request(r):
409
+ waiter.add(r)
410
+ else:
411
+ self.info_listen[requestId] = {}
412
+ self.info_listen[requestId]['request_extra'] = request_extra
413
+ def Network_responseReceived(self, rdata):
414
+ requestId = rdata['params']['requestId']
415
+ if requestId not in self.info_listen: return
416
+ self.info_listen[requestId]['response'] = rdata['params']['response']
417
+ response_extra = self.info_listen[requestId].pop('response_extra', None)
418
+ if response_extra:
419
+ _headers = self.info_listen[requestId]['response']['headers']
420
+ _headers = {**response_extra['headers'], **_headers}
421
+ self.info_listen[requestId]['response']['headers'] = _headers
422
+ def Network_responseReceivedExtraInfo(self, rdata):
423
+ response_extra = rdata['params']
424
+ requestId = response_extra['requestId']
425
+ if requestId not in self.info_listen: return
426
+ if self.info_listen[requestId].get('response'):
427
+ _headers = self.info_listen[requestId]['response']['headers']
428
+ _headers = {**response_extra['headers'], **_headers}
429
+ self.info_listen[requestId]['response']['headers'] = _headers
430
+ else:
431
+ self.info_listen[requestId]['response_extra'] = response_extra
432
+ def Network_loadingFinished(self, rdata):
433
+ params = rdata['params']
434
+ requestId = params['requestId']
435
+ if requestId not in self.info_listen: return
436
+ url = self.info_listen[requestId]['request']['url']
437
+ for k in self.call_listen_keys:
438
+ if self._match_url(self.call_listen_vals[k], url):
439
+ waiter, on_request, on_response = self.call_listen[k]
440
+ if on_response:
441
+ self.qlist.put('V')
442
+ rbody = self.f.cdp('Network.getResponseBody', {"requestId": requestId}, sessionId=rdata.get('sessionId'), limit_time=3)
443
+ self.info_listen[requestId]['response_body'] = rbody
444
+ r = SniffNetwork.NetworkResponse(self.info_listen[requestId])
445
+ if on_response(r):
446
+ waiter.add(r)
447
+ self.info_listen.pop(requestId, None)
448
+ self.qlist.get('V')
449
+ def Network_loadingFailed(self, rdata):
450
+ params = rdata['params']
451
+ requestId = params['requestId']
452
+ if requestId not in self.info_listen: return
453
+ url = self.info_listen[requestId]['request']['url']
454
+ self.info_listen[requestId]['status'] = 'ERROR'
455
+ self.info_listen[requestId]['error'] = params['errorText']
456
+ for k in self.call_listen_keys:
457
+ if self._match_url(self.call_listen_vals[k], url):
458
+ waiter, on_request, on_response = self.call_listen[k]
459
+ if on_response:
460
+ self.qlist.put('V')
461
+ r = SniffNetwork.NetworkResponse(self.info_listen[requestId])
462
+ if on_response(r):
463
+ waiter.add(r)
464
+ self.qlist.get('V')
465
+ class SniffFetch:
466
+ class FetchFakeResponse:
467
+ def __init__(self, rinfo, encoding='utf8'):
468
+ self.rinfo = rinfo
469
+ self.encoding = encoding
470
+ self._url = rinfo.get('url')
471
+ self._responseCode = 200
472
+ self._responseHeaders = {"fake-vvv": "fake-vvv"}
473
+ self._body = b''
474
+ self.request = rinfo.get('request')
475
+ @property
476
+ def url(self): return self._url
477
+ @property
478
+ def status_code(self): return self._responseCode
479
+ @status_code.setter
480
+ def status_code(self, value): assert isinstance(value, int), 'must be type: int'; self._responseCode = value
481
+ @property
482
+ def headers(self): return self._responseHeaders
483
+ @headers.setter
484
+ def headers(self, value): assert isinstance(value, dict), 'must be type: dict'; self._responseHeaders = value
485
+ @property
486
+ def content(self): return self._body
487
+ @content.setter
488
+ def content(self, value): assert isinstance(value, bytes), 'must be type: bytes'; self._body = value
489
+ @property
490
+ def text(self): return self._body.decode(self.encoding)
491
+ @text.setter
492
+ def text(self, value): assert isinstance(value, str), 'must be type: str'; self._body = value.encode(self.encoding)
493
+ def json(self):
494
+ c = self.text
495
+ return json.loads(c[c.find('{'):c.rfind('}')+1])
496
+ def __repr__(self): return '<FakeResponse [{}]>'.format(self.status_code)
497
+ def get(self, k):
498
+ return getattr(self, '_'+k, None)
499
+ class FetchRequest:
500
+ def __init__(self, rinfo, encoding='utf8'):
501
+ self.rinfo = rinfo
502
+ self.encoding = encoding
503
+ self._url = rinfo.get('url')
504
+ self._url_copy = self._url
505
+ self._method = rinfo.get('method')
506
+ self._method_copy = self._method
507
+ self._headers = rinfo.get('headers')
508
+ self._headers_copy = copy.deepcopy(self._headers)
509
+ self._postData = rinfo.get('postData')
510
+ self._postData_copy = self._postData
511
+ self._fake = None
512
+ @property
513
+ def url(self): return self._url
514
+ @property
515
+ def method(self): return self._method
516
+ @method.setter
517
+ def method(self, value): assert isinstance(value, str), 'must be type: str'; self._method = value
518
+ @property
519
+ def headers(self): return self._headers
520
+ @headers.setter
521
+ def headers(self, value): assert isinstance(value, dict), 'must be type: dict'; self._headers = value
522
+ @property
523
+ def content(self): return self._postData
524
+ @content.setter
525
+ def content(self, value): assert isinstance(value, bytes), 'must be type: bytes'; self._postData = value
526
+ @property
527
+ def text(self): return self._postData.decode(self.encoding)
528
+ @text.setter
529
+ def text(self, value): assert isinstance(value, str), 'must be type: str'; self._postData = value.encode(self.encoding)
530
+ def json(self):
531
+ c = self.text
532
+ return json.loads(c[c.find('{'):c.rfind('}')+1])
533
+ def __repr__(self): return '<Request [{}]>'.format(self.method)
534
+ def check_change(self):
535
+ return (
536
+ self._url != self._url_copy
537
+ or self._method != self._method_copy
538
+ or self._headers != self._headers_copy
539
+ or json.dumps(self._headers, sort_keys=True) != json.dumps(self._headers_copy, sort_keys=True)
540
+ or (self._method == 'POST' and self._postData != self._postData_copy)
541
+ )
542
+ def get(self, k):
543
+ return getattr(self, '_'+k, None)
544
+ def fake_response(self):
545
+ self._fake = SniffFetch.FetchFakeResponse({"url": self._url, "request": self})
546
+ return self._fake
547
+ class FetchResponse:
548
+ def __init__(self, rinfo, encoding='utf8'):
549
+ self.rinfo = rinfo
550
+ self.encoding = encoding
551
+ self.error = None
552
+ self.request = rinfo.get('request')
553
+ self._url = rinfo.get('url')
554
+ self._url_copy = self._url
555
+ self._responseCode = rinfo.get('responseCode')
556
+ self._responseCode_copy = self._responseCode
557
+ self._responseHeaders = rinfo.get('responseHeaders')
558
+ self._responseHeaders_copy = copy.deepcopy(self._responseHeaders)
559
+ self._body = rinfo.get('body')
560
+ self._body_copy = self._body
561
+ @property
562
+ def url(self): return self._url
563
+ @property
564
+ def status_code(self): return self._responseCode
565
+ @status_code.setter
566
+ def status_code(self, value): assert isinstance(value, int), 'must be type: int'; self._responseCode = value
567
+ @property
568
+ def headers(self): return self._responseHeaders
569
+ @headers.setter
570
+ def headers(self, value): assert isinstance(value, dict), 'must be type: dict'; self._responseHeaders = value
571
+ @property
572
+ def content(self): return self._body
573
+ @content.setter
574
+ def content(self, value): assert isinstance(value, bytes), 'must be type: bytes'; self._body = value
575
+ @property
576
+ def text(self): return self._body.decode(self.encoding)
577
+ @text.setter
578
+ def text(self, value): assert isinstance(value, str), 'must be type: str'; self._body = value.encode(self.encoding)
579
+ def json(self):
580
+ c = self.text
581
+ return json.loads(c[c.find('{'):c.rfind('}')+1])
582
+ def __repr__(self): return '<Response [{}]>'.format(self.status_code)
583
+ def check_change(self):
584
+ return (
585
+ self._url != self._url_copy
586
+ or self._responseCode != self._responseCode_copy
587
+ or self._responseHeaders != self._responseHeaders_copy
588
+ or json.dumps(self._responseHeaders, sort_keys=True) != json.dumps(self._responseHeaders_copy, sort_keys=True)
589
+ or self._body_copy != self._body
590
+ )
591
+ def get(self, k):
592
+ return getattr(self, '_'+k, None)
593
+ def __init__(self, f):
594
+ self.f = f
595
+ self.type = self.f.type
596
+ self.is_change = False
597
+ self.change_sesslist = [None]
598
+ self.info_change = {}
599
+ self.call_change = {}
600
+ self.call_change_keys = []
601
+ self.call_change_vals = {}
602
+ self.tools = SniffTools(self)
603
+ def _change_cdp(self, sessionId):
604
+ self.f.cdp('Fetch.enable',{
605
+ "handleAuthRequests": True,
606
+ "patterns":[
607
+ {"urlPattern":"*","requestStage":"Request"},
608
+ {"urlPattern":"*","requestStage":"Response"},
609
+ ]}, sessionId=sessionId)
610
+ def add_change_session(self, sessionId):
611
+ self.change_sesslist.append(sessionId)
612
+ if self.is_change:
613
+ self._change_cdp(sessionId)
614
+ def remove_change(self, pattern):
615
+ pk = json.dumps(pattern, sort_keys=True)
616
+ self.call_change.pop(pk, None)
617
+ self.call_change_keys = self.call_change.keys()
618
+ self.call_change_vals.pop(pk, None)
619
+ def intercept(self, pattern, on_response=None, on_request=None, is_regex=False):
620
+ if not self.is_change:
621
+ self._handle_fetch()
622
+ for sessionId in self.change_sesslist:
623
+ self._change_cdp(sessionId)
624
+ self.is_change = True
625
+ waiter = Waiter(self, pattern, is_regex)
626
+ pk = json.dumps(pattern, sort_keys=True)
627
+ if (not on_request) and (not on_response):
628
+ on_response = self.default_func
629
+ self.call_change[pk] = [waiter, on_request, on_response]
630
+ self.call_change_keys = self.call_change.keys()
631
+ self.call_change_vals[pk] = [pattern, is_regex]
632
+ return waiter
633
+ def _handle_fetch(self):
634
+ self.f.set_method_callback('Fetch.requestPaused', self.Fetch_requestPaused)
635
+ def _dict_to_list(self, headers_dict):
636
+ return [{"name": k, "value": v} for k, v in headers_dict.items()]
637
+ def _list_to_dict(self, headers_list):
638
+ d = {}
639
+ for kv in headers_list:
640
+ d[kv['name']] = kv['value']
641
+ return d
642
+ def Fetch_requestPaused(self, rdata):
643
+ url = rdata['params']['request']['url']
644
+ method = rdata['params']['request']['method']
645
+ requestId = rdata['params']['requestId']
646
+ if "responseStatusCode" not in rdata['params']:
647
+ for k in self.call_change_keys:
648
+ if self._match_url(self.call_change_vals[k], url):
649
+ waiter, on_request, on_response = self.call_change[k]
650
+ x = SniffFetch.FetchRequest({
651
+ "url": url,
652
+ "method": method,
653
+ "headers": rdata['params']['request']['headers'],
654
+ "postData": b''.join(base64.b64decode(e['bytes']) for e in rdata['params']['request'].get('postDataEntries', [])),
655
+ })
656
+ if on_request:
657
+ try:
658
+ if on_request(x):
659
+ waiter.add(x)
660
+ except:
661
+ self.f.logger.log('[ERROR] in request on_request', traceback.format_exc())
662
+ continue
663
+ try:
664
+ if x._fake:
665
+ fk = x._fake
666
+ d = {
667
+ "requestId": requestId,
668
+ "responseCode": fk.get('responseCode'),
669
+ "responseHeaders": self._dict_to_list(fk.get('responseHeaders')),
670
+ "body": base64.b64encode(fk.get('body')).decode("ascii")
671
+ }
672
+ self.f.cdp('Fetch.fulfillRequest', d, sessionId=rdata.get('sessionId'))
673
+ return
674
+ except:
675
+ self.f.logger.log('[ERROR] in request on_fake continue', traceback.format_exc())
676
+ try:
677
+ if x.check_change():
678
+ d = {
679
+ "requestId": requestId,
680
+ "method": x.get('method') or method,
681
+ "headers": self._dict_to_list(x.get('headers') or rdata['params']['request']['headers']),
682
+ }
683
+ if x.get('method') == 'POST':
684
+ d['postData'] = base64.b64encode(x.get('postData')).decode("ascii")
685
+ self.f.cdp('Fetch.continueRequest', d, sessionId=rdata.get('sessionId'))
686
+ return
687
+ except:
688
+ self.f.logger.log('[ERROR] in request on_request continue', traceback.format_exc())
689
+ self.f.cdp('Fetch.continueRequest', {'requestId': requestId}, sessionId=rdata.get('sessionId'))
690
+ else:
691
+ for k in self.call_change_keys:
692
+ if self._match_url(self.call_change_vals[k], url):
693
+ waiter, on_request, on_response = self.call_change[k]
694
+ if on_response:
695
+ body_info = self.f.cdp("Fetch.getResponseBody", {"requestId": requestId}, sessionId=rdata.get('sessionId'))
696
+ body = base64.b64decode(body_info["body"]) if body_info['base64Encoded'] else body_info["body"].encode()
697
+ x = SniffFetch.FetchResponse({
698
+ "url": url,
699
+ "responseCode": rdata['params']['responseStatusCode'],
700
+ "responseHeaders": self._list_to_dict(rdata['params']['responseHeaders']),
701
+ "body": body,
702
+ "request": SniffFetch.FetchRequest(rdata['params']['request'])
703
+ })
704
+ try:
705
+ if on_response(x):
706
+ waiter.add(x)
707
+ except:
708
+ self.f.logger.log('[ERROR] in response on_response', traceback.format_exc())
709
+ continue
710
+ try:
711
+ if x.check_change():
712
+ d = {
713
+ "requestId": requestId,
714
+ "responseCode": x.get('responseCode'),
715
+ "responseHeaders": self._dict_to_list(x.get('responseHeaders')),
716
+ "body": base64.b64encode(x.get('body') or body).decode("ascii")
717
+ }
718
+ self.f.cdp('Fetch.fulfillRequest', d, sessionId=rdata.get('sessionId'))
719
+ return
720
+ except Exception as e:
721
+ self.f.logger.log('[ERROR] in response on_response continue', traceback.format_exc())
722
+ self.f.cdp('Fetch.continueResponse', {'requestId': requestId}, sessionId=rdata.get('sessionId'))
723
+ class Page:
724
+ def __init__(self, f):
725
+ self.f = f
726
+ self.f.set_method_callback('Page.frameDetached', self.Page_frameDetached)
727
+ self.f.set_method_callback('Page.frameAttached', self.Page_frameAttached)
728
+ self.f.set_method_callback('Page.frameScheduledNavigation', self.Page_frameScheduledNavigation)
729
+ self.f.set_method_callback('Page.frameRequestedNavigation', self.Page_frameRequestedNavigation)
730
+ self.f.set_method_callback('Page.frameStartedNavigating', self.Page_frameStartedNavigating)
731
+ self.f.set_method_callback('Page.frameStartedLoading', self.Page_frameStartedLoading)
732
+ self.f.set_method_callback('Page.frameNavigated', self.Page_frameNavigated)
733
+ self.f.set_method_callback('Page.javascriptDialogOpening', self.Page_javascriptDialogOpening)
734
+ self.f.set_method_callback('Page.javascriptDialogClosed', self.Page_javascriptDialogClosed)
735
+ self.init()
736
+ self.dialog = None
737
+ def init(self, sessionId=None):
738
+ self.f.cdp('Page.enable', sessionId=sessionId)
739
+ def Page_frameNavigated(self, rdata):
740
+ frameId = rdata['params']['frame']['id']
741
+ f = self.f.root.trav_frame(frameId)
742
+ if f:
743
+ f.url = rdata['params']['frame']['url']
744
+ f.iso_contextId = None
745
+ def Page_frameDetached(self, rdata):
746
+ frameId = rdata['params']['frameId']
747
+ f = self.f.root.trav_frame(frameId)
748
+ if f and rdata['params']['reason'] == 'remove':
749
+ f.parent.frames.remove(f)
750
+ f.iso_contextId = None
751
+ def Page_frameAttached(self, rdata):
752
+ frameId = rdata['params'].get('frameId')
753
+ parentFrameId = rdata['params'].get('parentFrameId')
754
+ pf = self.f.root.trav_frame(parentFrameId)
755
+ if not self.f.root.trav_frame(frameId) and pf:
756
+ self.f.root.add_common_frame(self.f, {
757
+ "frameId": frameId,
758
+ "parent": pf,
759
+ })
760
+ def _clear_iso(self, rdata):
761
+ frameId = rdata['params']['frameId']
762
+ f = self.f.root.trav_frame(frameId)
763
+ if f and rdata['params']['reason'] in (
764
+ 'anchorClick', 'formSubmissionGet', 'formSubmissionPost', 'httpHeaderRefresh',
765
+ 'initialFrameNavigation', 'metaTagRefresh', 'other', 'pageBlockInterstitial',
766
+ 'reload', 'scriptInitiated'):
767
+ f.iso_contextId = None
768
+ def Page_frameScheduledNavigation(self, rdata): self._clear_iso(rdata)
769
+ def Page_frameRequestedNavigation(self, rdata): self._clear_iso(rdata)
770
+ def Page_frameStartedNavigating(self, rdata): pass
771
+ def Page_frameStartedLoading(self, rdata): pass
772
+ def Page_javascriptDialogOpening(self, rdata):
773
+ if self.dialog:
774
+ r = self.dialog(rdata['params'])
775
+ if type(r) == dict:
776
+ self.f.cdp('Page.handleJavaScriptDialog', {"accept": r.get('accept'), "promptText": r.get('promptText') or r.get('text')})
777
+ if type(r) in (list, tuple) and len(r) == 2 and type(r[0]) == bool and type(r[1]) == str:
778
+ self.f.cdp('Page.handleJavaScriptDialog', {"accept": r[0], "promptText": r[1]})
779
+ else:
780
+ self.f.cdp('Page.handleJavaScriptDialog', {"accept": bool(r)})
781
+ else:
782
+ time.sleep(0.5)
783
+ self.f.cdp('Page.handleJavaScriptDialog', {"accept": True})
784
+ def Page_javascriptDialogClosed(self, rdata):
785
+ pass
786
+ class Target:
787
+ def __init__(self, f):
788
+ self.f = f
789
+ self.f.set_method_callback('Target.attachedToTarget', self.Target_attachedToTarget)
790
+ self.f.set_method_callback('Target.targetDestroyed', self.Target_targetDestroyed)
791
+ self.f.set_method_callback('Target.targetCreated', self.Target_targetCreated)
792
+ self.f.set_method_callback('Target.targetInfoChanged', self.Target_targetInfoChanged)
793
+ self.f.set_method_callback('Target.detachedFromTarget', self.Target_detachedFromTarget)
794
+ self.init()
795
+ def init(self, sessionId=None):
796
+ self.f.cdp("Target.setAutoAttach", {
797
+ "autoAttach": True,
798
+ "waitForDebuggerOnStart": True,
799
+ "flatten": True,
800
+ }, sessionId=sessionId)
801
+ def Target_attachedToTarget(self, rdata):
802
+ self.f.root._add_init_check()
803
+ tinfo = rdata['params']['targetInfo']
804
+ if tinfo['url'].startswith('chrome-extension'):
805
+ self.f.root._del_init_check()
806
+ return
807
+ if self.f.root.filter_extension(tinfo):
808
+ self.f.root._del_init_check()
809
+ return
810
+ if rdata['params'].get('targetInfo', {}).get('type') == 'service_worker':
811
+ self.f.root._del_init_check()
812
+ return
813
+ frameId = tinfo['targetId']
814
+ sessionId = rdata['params']['sessionId']
815
+ # self.f.cdp("Target.setAutoAttach", {
816
+ # "autoAttach": True,
817
+ # "waitForDebuggerOnStart": True,
818
+ # "flatten": True,
819
+ # }, sessionId=sessionId)
820
+ self.f.cdp('DOM.enable', sessionId=sessionId)
821
+ self.f.cdp('Page.enable', sessionId=sessionId)
822
+ self.f.cdp('Runtime.enable', sessionId=sessionId)
823
+ self.f.add_sniff_session(sessionId)
824
+ self.f.cache.add_cache_session(sessionId)
825
+ self.f.root.add_common_frame(self.f, {
826
+ "frameId": frameId,
827
+ "sessionId": sessionId,
828
+ })
829
+ if self.f.root.is_init and sessionId:
830
+ f = self.f.root.trav_frame(sessionId, 'sessionId')
831
+ if f:
832
+ self.f.root.trav_init_tree(f, self.f, sessionId)
833
+ else:
834
+ # TODO
835
+ # manager worker process.
836
+ # some worker not work in iframe.
837
+ pass
838
+ self.f._page_init_js(sessionId=sessionId)
839
+ self.f.cdp('Runtime.runIfWaitingForDebugger', sessionId=sessionId)
840
+ self.f.root._del_init_check()
841
+ def Target_detachedFromTarget(self, rdata):
842
+ sessionId = rdata['params']['sessionId']
843
+ pass
844
+ def Target_targetCreated(self, rdata):
845
+ pass
846
+ def Target_targetInfoChanged(self, rdata):
847
+ pass
848
+ def Target_targetDestroyed(self, rdata):
849
+ self.f.cdp('Target.closeTarget', {'targetId': rdata['params']['targetId']})
850
+ class JSIterator:
851
+ def __init__(self, jsobj):
852
+ self.idx = 0
853
+ self.jsobj = jsobj
854
+ self.leng = len(self.jsobj)
855
+ def __next__(self):
856
+ if self.idx < self.leng:
857
+ value = self.jsobj[self.idx]
858
+ self.idx += 1
859
+ return value
860
+ else:
861
+ raise StopIteration
862
+ class JSObject:
863
+ def __init__(self, f, einfo, _this=None, iso=False):
864
+ self.f = f
865
+ self.className = einfo.get('className')
866
+ self.objectId = einfo.get('objectId')
867
+ self._this = _this
868
+ self.iso = iso
869
+ self.r_obj = self.f.run_iso_js_obj if self.iso else self.f.run_js_obj
870
+ def __getitem__(self, a):
871
+ einfo = self.r_obj('function(){return this[' + json.dumps(a) + ']}', objectId=self.objectId, returnByValue=False)
872
+ return self.f._parse_js2py(einfo, self, iso=self.iso)
873
+ def __setitem__(self, a, b):
874
+ args = [None,self.f._parse_2arg(a), self.f._parse_2arg(b)]
875
+ einfo = self.r_obj('function(a,b){return this[a]=b}', objectId=self.objectId, arguments=args, returnByValue=False)
876
+ return self.f._parse_js2py(einfo, iso=self.iso)
877
+ def __call__(self, *a):
878
+ args = [self._this]
879
+ for v in a: args.append(self.f._parse_2arg(v))
880
+ scpt = 'function(o,...a){return this.call(o,...a)}' if self._this else 'function(...a){return this(...a)}'
881
+ einfo = self.r_obj(scpt, objectId=self.objectId, arguments=args, returnByValue=False)
882
+ return self.f._parse_js2py(einfo, iso=self.iso)
883
+ def __repr__(self):
884
+ return '<OBJ:[{}] [{}]>'.format(self.className, self.objectId)
885
+ def __add__(self, other):
886
+ args = [None,self.f._parse_2arg(other)]
887
+ einfo = self.r_obj('function(other){return this+other}', objectId=self.objectId, arguments=args, returnByValue=False)
888
+ return self.f._parse_js2py(einfo, iso=self.iso)
889
+ def __iter__(self): return JSIterator(self)
890
+ def __len__(self): return self['length']
891
+ def __bool__(self): return True
892
+ def json(self): return self.r_obj('function(){return this}', objectId=self.objectId, returnByValue=True)
893
+ class Keyboard:
894
+ # from pyppeteer
895
+ def __init__(self, f):
896
+ self.f = f
897
+ self._key_maps = {'0': {'keyCode': 48, 'key': '0', 'code': 'Digit0'},'1': {'keyCode': 49, 'key': '1', 'code': 'Digit1'},'2': {'keyCode': 50, 'key': '2', 'code': 'Digit2'},'3': {'keyCode': 51, 'key': '3', 'code': 'Digit3'},'4': {'keyCode': 52, 'key': '4', 'code': 'Digit4'},'5': {'keyCode': 53, 'key': '5', 'code': 'Digit5'},'6': {'keyCode': 54, 'key': '6', 'code': 'Digit6'},'7': {'keyCode': 55, 'key': '7', 'code': 'Digit7'},'8': {'keyCode': 56, 'key': '8', 'code': 'Digit8'},'9': {'keyCode': 57, 'key': '9', 'code': 'Digit9'},'Power': {'key': 'Power', 'code': 'Power'},'Eject': {'key': 'Eject', 'code': 'Eject'},'Abort': {'keyCode': 3, 'code': 'Abort', 'key': 'Cancel'},'Help': {'keyCode': 6, 'code': 'Help', 'key': 'Help'},'Backspace': {'keyCode': 8, 'code': 'Backspace', 'key': 'Backspace'},'Tab': {'keyCode': 9, 'code': 'Tab', 'key': 'Tab'},'Numpad5': {'keyCode': 12, 'shiftKeyCode': 101, 'key': 'Clear', 'code': 'Numpad5', 'shiftKey': '5', 'location': 3},'NumpadEnter': {'keyCode': 13, 'code': 'NumpadEnter', 'key': 'Enter', 'text': '\r', 'location': 3},'Enter': {'keyCode': 13, 'code': 'Enter', 'key': 'Enter', 'text': '\r'},'\r': {'keyCode': 13, 'code': 'Enter', 'key': 'Enter', 'text': '\r'},'\n': {'keyCode': 13, 'code': 'Enter', 'key': 'Enter', 'text': '\r'},'ShiftLeft': {'keyCode': 16, 'code': 'ShiftLeft', 'key': 'Shift', 'location': 1},'ShiftRight': {'keyCode': 16, 'code': 'ShiftRight', 'key': 'Shift', 'location': 2},'ControlLeft': {'keyCode': 17, 'code': 'ControlLeft', 'key': 'Control', 'location': 1},'ControlRight': {'keyCode': 17, 'code': 'ControlRight', 'key': 'Control', 'location': 2},'AltLeft': {'keyCode': 18, 'code': 'AltLeft', 'key': 'Alt', 'location': 1},'AltRight': {'keyCode': 18, 'code': 'AltRight', 'key': 'Alt', 'location': 2},'Pause': {'keyCode': 19, 'code': 'Pause', 'key': 'Pause'},'CapsLock': {'keyCode': 20, 'code': 'CapsLock', 'key': 'CapsLock'},'Escape': {'keyCode': 27, 'code': 'Escape', 'key': 'Escape'},'Convert': {'keyCode': 28, 'code': 'Convert', 'key': 'Convert'},'NonConvert': {'keyCode': 29, 'code': 'NonConvert', 'key': 'NonConvert'},'Space': {'keyCode': 32, 'code': 'Space', 'key': ' '},'Numpad9': {'keyCode': 33, 'shiftKeyCode': 105, 'key': 'PageUp', 'code': 'Numpad9', 'shiftKey': '9', 'location': 3},'PageUp': {'keyCode': 33, 'code': 'PageUp', 'key': 'PageUp'},'Numpad3': {'keyCode': 34, 'shiftKeyCode': 99, 'key': 'PageDown', 'code': 'Numpad3', 'shiftKey': '3', 'location': 3},'PageDown': {'keyCode': 34, 'code': 'PageDown', 'key': 'PageDown'},'End': {'keyCode': 35, 'code': 'End', 'key': 'End'},'Numpad1': {'keyCode': 35, 'shiftKeyCode': 97, 'key': 'End', 'code': 'Numpad1', 'shiftKey': '1', 'location': 3},'Home': {'keyCode': 36, 'code': 'Home', 'key': 'Home'},'Numpad7': {'keyCode': 36, 'shiftKeyCode': 103, 'key': 'Home', 'code': 'Numpad7', 'shiftKey': '7', 'location': 3},'ArrowLeft': {'keyCode': 37, 'code': 'ArrowLeft', 'key': 'ArrowLeft'},'Numpad4': {'keyCode': 37, 'shiftKeyCode': 100, 'key': 'ArrowLeft', 'code': 'Numpad4', 'shiftKey': '4', 'location': 3},'Numpad8': {'keyCode': 38, 'shiftKeyCode': 104, 'key': 'ArrowUp', 'code': 'Numpad8', 'shiftKey': '8', 'location': 3},'ArrowUp': {'keyCode': 38, 'code': 'ArrowUp', 'key': 'ArrowUp'},'ArrowRight': {'keyCode': 39, 'code': 'ArrowRight', 'key': 'ArrowRight'},'Numpad6': {'keyCode': 39, 'shiftKeyCode': 102, 'key': 'ArrowRight', 'code': 'Numpad6', 'shiftKey': '6', 'location': 3},'Numpad2': {'keyCode': 40, 'shiftKeyCode': 98, 'key': 'ArrowDown', 'code': 'Numpad2', 'shiftKey': '2', 'location': 3},'ArrowDown': {'keyCode': 40, 'code': 'ArrowDown', 'key': 'ArrowDown'},'Select': {'keyCode': 41, 'code': 'Select', 'key': 'Select'},'Open': {'keyCode': 43, 'code': 'Open', 'key': 'Execute'},'PrintScreen': {'keyCode': 44, 'code': 'PrintScreen', 'key': 'PrintScreen'},'Insert': {'keyCode': 45, 'code': 'Insert', 'key': 'Insert'},'Numpad0': {'keyCode': 45, 'shiftKeyCode': 96, 'key': 'Insert', 'code': 'Numpad0', 'shiftKey': '0', 'location': 3},'Delete': {'keyCode': 46, 'code': 'Delete', 'key': 'Delete'},'NumpadDecimal': {'keyCode': 46, 'shiftKeyCode': 110, 'code': 'NumpadDecimal', 'key': '\u0000', 'shiftKey': '.', 'location': 3},'Digit0': {'keyCode': 48, 'code': 'Digit0', 'shiftKey': ')', 'key': '0'},'Digit1': {'keyCode': 49, 'code': 'Digit1', 'shiftKey': '!', 'key': '1'},'Digit2': {'keyCode': 50, 'code': 'Digit2', 'shiftKey': '@', 'key': '2'},'Digit3': {'keyCode': 51, 'code': 'Digit3', 'shiftKey': '#', 'key': '3'},'Digit4': {'keyCode': 52, 'code': 'Digit4', 'shiftKey': '$', 'key': '4'},'Digit5': {'keyCode': 53, 'code': 'Digit5', 'shiftKey': '%', 'key': '5'},'Digit6': {'keyCode': 54, 'code': 'Digit6', 'shiftKey': '^', 'key': '6'},'Digit7': {'keyCode': 55, 'code': 'Digit7', 'shiftKey': '&', 'key': '7'},'Digit8': {'keyCode': 56, 'code': 'Digit8', 'shiftKey': '*', 'key': '8'},'Digit9': {'keyCode': 57, 'code': 'Digit9', 'shiftKey': '(', 'key': '9'},'KeyA': {'keyCode': 65, 'code': 'KeyA', 'shiftKey': 'A', 'key': 'a'},'KeyB': {'keyCode': 66, 'code': 'KeyB', 'shiftKey': 'B', 'key': 'b'},'KeyC': {'keyCode': 67, 'code': 'KeyC', 'shiftKey': 'C', 'key': 'c'},'KeyD': {'keyCode': 68, 'code': 'KeyD', 'shiftKey': 'D', 'key': 'd'},'KeyE': {'keyCode': 69, 'code': 'KeyE', 'shiftKey': 'E', 'key': 'e'},'KeyF': {'keyCode': 70, 'code': 'KeyF', 'shiftKey': 'F', 'key': 'f'},'KeyG': {'keyCode': 71, 'code': 'KeyG', 'shiftKey': 'G', 'key': 'g'},'KeyH': {'keyCode': 72, 'code': 'KeyH', 'shiftKey': 'H', 'key': 'h'},'KeyI': {'keyCode': 73, 'code': 'KeyI', 'shiftKey': 'I', 'key': 'i'},'KeyJ': {'keyCode': 74, 'code': 'KeyJ', 'shiftKey': 'J', 'key': 'j'},'KeyK': {'keyCode': 75, 'code': 'KeyK', 'shiftKey': 'K', 'key': 'k'},'KeyL': {'keyCode': 76, 'code': 'KeyL', 'shiftKey': 'L', 'key': 'l'},'KeyM': {'keyCode': 77, 'code': 'KeyM', 'shiftKey': 'M', 'key': 'm'},'KeyN': {'keyCode': 78, 'code': 'KeyN', 'shiftKey': 'N', 'key': 'n'},'KeyO': {'keyCode': 79, 'code': 'KeyO', 'shiftKey': 'O', 'key': 'o'},'KeyP': {'keyCode': 80, 'code': 'KeyP', 'shiftKey': 'P', 'key': 'p'},'KeyQ': {'keyCode': 81, 'code': 'KeyQ', 'shiftKey': 'Q', 'key': 'q'},'KeyR': {'keyCode': 82, 'code': 'KeyR', 'shiftKey': 'R', 'key': 'r'},'KeyS': {'keyCode': 83, 'code': 'KeyS', 'shiftKey': 'S', 'key': 's'},'KeyT': {'keyCode': 84, 'code': 'KeyT', 'shiftKey': 'T', 'key': 't'},'KeyU': {'keyCode': 85, 'code': 'KeyU', 'shiftKey': 'U', 'key': 'u'},'KeyV': {'keyCode': 86, 'code': 'KeyV', 'shiftKey': 'V', 'key': 'v'},'KeyW': {'keyCode': 87, 'code': 'KeyW', 'shiftKey': 'W', 'key': 'w'},'KeyX': {'keyCode': 88, 'code': 'KeyX', 'shiftKey': 'X', 'key': 'x'},'KeyY': {'keyCode': 89, 'code': 'KeyY', 'shiftKey': 'Y', 'key': 'y'},'KeyZ': {'keyCode': 90, 'code': 'KeyZ', 'shiftKey': 'Z', 'key': 'z'},'MetaLeft': {'keyCode': 91, 'code': 'MetaLeft', 'key': 'Meta'},'MetaRight': {'keyCode': 92, 'code': 'MetaRight', 'key': 'Meta'},'ContextMenu': {'keyCode': 93, 'code': 'ContextMenu', 'key': 'ContextMenu'},'NumpadMultiply': {'keyCode': 106, 'code': 'NumpadMultiply', 'key': '*', 'location': 3},'NumpadAdd': {'keyCode': 107, 'code': 'NumpadAdd', 'key': '+', 'location': 3},'NumpadSubtract': {'keyCode': 109, 'code': 'NumpadSubtract', 'key': '-', 'location': 3},'NumpadDivide': {'keyCode': 111, 'code': 'NumpadDivide', 'key': '/', 'location': 3},'F1': {'keyCode': 112, 'code': 'F1', 'key': 'F1'},'F2': {'keyCode': 113, 'code': 'F2', 'key': 'F2'},'F3': {'keyCode': 114, 'code': 'F3', 'key': 'F3'},'F4': {'keyCode': 115, 'code': 'F4', 'key': 'F4'},'F5': {'keyCode': 116, 'code': 'F5', 'key': 'F5'},'F6': {'keyCode': 117, 'code': 'F6', 'key': 'F6'},'F7': {'keyCode': 118, 'code': 'F7', 'key': 'F7'},'F8': {'keyCode': 119, 'code': 'F8', 'key': 'F8'},'F9': {'keyCode': 120, 'code': 'F9', 'key': 'F9'},'F10': {'keyCode': 121, 'code': 'F10', 'key': 'F10'},'F11': {'keyCode': 122, 'code': 'F11', 'key': 'F11'},'F12': {'keyCode': 123, 'code': 'F12', 'key': 'F12'},'F13': {'keyCode': 124, 'code': 'F13', 'key': 'F13'},'F14': {'keyCode': 125, 'code': 'F14', 'key': 'F14'},'F15': {'keyCode': 126, 'code': 'F15', 'key': 'F15'},'F16': {'keyCode': 127, 'code': 'F16', 'key': 'F16'},'F17': {'keyCode': 128, 'code': 'F17', 'key': 'F17'},'F18': {'keyCode': 129, 'code': 'F18', 'key': 'F18'},'F19': {'keyCode': 130, 'code': 'F19', 'key': 'F19'},'F20': {'keyCode': 131, 'code': 'F20', 'key': 'F20'},'F21': {'keyCode': 132, 'code': 'F21', 'key': 'F21'},'F22': {'keyCode': 133, 'code': 'F22', 'key': 'F22'},'F23': {'keyCode': 134, 'code': 'F23', 'key': 'F23'},'F24': {'keyCode': 135, 'code': 'F24', 'key': 'F24'},'NumLock': {'keyCode': 144, 'code': 'NumLock', 'key': 'NumLock'},'ScrollLock': {'keyCode': 145, 'code': 'ScrollLock', 'key': 'ScrollLock'},'AudioVolumeMute': {'keyCode': 173, 'code': 'AudioVolumeMute', 'key': 'AudioVolumeMute'},'AudioVolumeDown': {'keyCode': 174, 'code': 'AudioVolumeDown', 'key': 'AudioVolumeDown'},'AudioVolumeUp': {'keyCode': 175, 'code': 'AudioVolumeUp', 'key': 'AudioVolumeUp'},'MediaTrackNext': {'keyCode': 176, 'code': 'MediaTrackNext', 'key': 'MediaTrackNext'},'MediaTrackPrevious': {'keyCode': 177, 'code': 'MediaTrackPrevious', 'key': 'MediaTrackPrevious'},'MediaStop': {'keyCode': 178, 'code': 'MediaStop', 'key': 'MediaStop'},'MediaPlayPause': {'keyCode': 179, 'code': 'MediaPlayPause', 'key': 'MediaPlayPause'},'Semicolon': {'keyCode': 186, 'code': 'Semicolon', 'shiftKey': ':', 'key': ';'},'Equal': {'keyCode': 187, 'code': 'Equal', 'shiftKey': '+', 'key': '='},'NumpadEqual': {'keyCode': 187, 'code': 'NumpadEqual', 'key': '=', 'location': 3},'Comma': {'keyCode': 188, 'code': 'Comma', 'shiftKey': '<', 'key': ','},'Minus': {'keyCode': 189, 'code': 'Minus', 'shiftKey': '_', 'key': '-'},'Period': {'keyCode': 190, 'code': 'Period', 'shiftKey': '>', 'key': '.'},'Slash': {'keyCode': 191, 'code': 'Slash', 'shiftKey': '?', 'key': '/'},'Backquote': {'keyCode': 192, 'code': 'Backquote', 'shiftKey': '~', 'key': '`'},'BracketLeft': {'keyCode': 219, 'code': 'BracketLeft', 'shiftKey': '{', 'key': '['},'Backslash': {'keyCode': 220, 'code': 'Backslash', 'shiftKey': '|', 'key': '\\'},'BracketRight': {'keyCode': 221, 'code': 'BracketRight', 'shiftKey': '}', 'key': ']'},'Quote': {'keyCode': 222, 'code': 'Quote', 'shiftKey': '"', 'key': '\''},'AltGraph': {'keyCode': 225, 'code': 'AltGraph', 'key': 'AltGraph'},'Props': {'keyCode': 247, 'code': 'Props', 'key': 'CrSel'},'Cancel': {'keyCode': 3, 'key': 'Cancel', 'code': 'Abort'},'Clear': {'keyCode': 12, 'key': 'Clear', 'code': 'Numpad5', 'location': 3},'Shift': {'keyCode': 16, 'key': 'Shift', 'code': 'ShiftLeft'},'Control': {'keyCode': 17, 'key': 'Control', 'code': 'ControlLeft'},'Alt': {'keyCode': 18, 'key': 'Alt', 'code': 'AltLeft'},'Accept': {'keyCode': 30, 'key': 'Accept'},'ModeChange': {'keyCode': 31, 'key': 'ModeChange'},' ': {'keyCode': 32, 'key': ' ', 'code': 'Space'},'Print': {'keyCode': 42, 'key': 'Print'},'Execute': {'keyCode': 43, 'key': 'Execute', 'code': 'Open'},'\u0000': {'keyCode': 46, 'key': '\u0000', 'code': 'NumpadDecimal', 'location': 3},'a': {'keyCode': 65, 'key': 'a', 'code': 'KeyA'},'b': {'keyCode': 66, 'key': 'b', 'code': 'KeyB'},'c': {'keyCode': 67, 'key': 'c', 'code': 'KeyC'},'d': {'keyCode': 68, 'key': 'd', 'code': 'KeyD'},'e': {'keyCode': 69, 'key': 'e', 'code': 'KeyE'},'f': {'keyCode': 70, 'key': 'f', 'code': 'KeyF'},'g': {'keyCode': 71, 'key': 'g', 'code': 'KeyG'},'h': {'keyCode': 72, 'key': 'h', 'code': 'KeyH'},'i': {'keyCode': 73, 'key': 'i', 'code': 'KeyI'},'j': {'keyCode': 74, 'key': 'j', 'code': 'KeyJ'},'k': {'keyCode': 75, 'key': 'k', 'code': 'KeyK'},'l': {'keyCode': 76, 'key': 'l', 'code': 'KeyL'},'m': {'keyCode': 77, 'key': 'm', 'code': 'KeyM'},'n': {'keyCode': 78, 'key': 'n', 'code': 'KeyN'},'o': {'keyCode': 79, 'key': 'o', 'code': 'KeyO'},'p': {'keyCode': 80, 'key': 'p', 'code': 'KeyP'},'q': {'keyCode': 81, 'key': 'q', 'code': 'KeyQ'},'r': {'keyCode': 82, 'key': 'r', 'code': 'KeyR'},'s': {'keyCode': 83, 'key': 's', 'code': 'KeyS'},'t': {'keyCode': 84, 'key': 't', 'code': 'KeyT'},'u': {'keyCode': 85, 'key': 'u', 'code': 'KeyU'},'v': {'keyCode': 86, 'key': 'v', 'code': 'KeyV'},'w': {'keyCode': 87, 'key': 'w', 'code': 'KeyW'},'x': {'keyCode': 88, 'key': 'x', 'code': 'KeyX'},'y': {'keyCode': 89, 'key': 'y', 'code': 'KeyY'},'z': {'keyCode': 90, 'key': 'z', 'code': 'KeyZ'},'Meta': {'keyCode': 91, 'key': 'Meta', 'code': 'MetaLeft'},'*': {'keyCode': 106, 'key': '*', 'code': 'NumpadMultiply', 'location': 3},'+': {'keyCode': 107, 'key': '+', 'code': 'NumpadAdd', 'location': 3},'-': {'keyCode': 109, 'key': '-', 'code': 'NumpadSubtract', 'location': 3},'/': {'keyCode': 111, 'key': '/', 'code': 'NumpadDivide', 'location': 3},';': {'keyCode': 186, 'key': ';', 'code': 'Semicolon'},'=': {'keyCode': 187, 'key': '=', 'code': 'Equal'},',': {'keyCode': 188, 'key': ',', 'code': 'Comma'},'.': {'keyCode': 190, 'key': '.', 'code': 'Period'},'`': {'keyCode': 192, 'key': '`', 'code': 'Backquote'},'[': {'keyCode': 219, 'key': '[', 'code': 'BracketLeft'},'\\': {'keyCode': 220, 'key': '\\', 'code': 'Backslash'},']': {'keyCode': 221, 'key': ']', 'code': 'BracketRight'},'\'': {'keyCode': 222, 'key': '\'', 'code': 'Quote'},'Attn': {'keyCode': 246, 'key': 'Attn'},'CrSel': {'keyCode': 247, 'key': 'CrSel', 'code': 'Props'},'ExSel': {'keyCode': 248, 'key': 'ExSel'},'EraseEof': {'keyCode': 249, 'key': 'EraseEof'},'Play': {'keyCode': 250, 'key': 'Play'},'ZoomOut': {'keyCode': 251, 'key': 'ZoomOut'},')': {'keyCode': 48, 'key': ')', 'code': 'Digit0'},'!': {'keyCode': 49, 'key': '!', 'code': 'Digit1'},'@': {'keyCode': 50, 'key': '@', 'code': 'Digit2'},'#': {'keyCode': 51, 'key': '#', 'code': 'Digit3'},'$': {'keyCode': 52, 'key': '$', 'code': 'Digit4'},'%': {'keyCode': 53, 'key': '%', 'code': 'Digit5'},'^': {'keyCode': 54, 'key': '^', 'code': 'Digit6'},'&': {'keyCode': 55, 'key': '&', 'code': 'Digit7'},'(': {'keyCode': 57, 'key': '(', 'code': 'Digit9'},'A': {'keyCode': 65, 'key': 'A', 'code': 'KeyA'},'B': {'keyCode': 66, 'key': 'B', 'code': 'KeyB'},'C': {'keyCode': 67, 'key': 'C', 'code': 'KeyC'},'D': {'keyCode': 68, 'key': 'D', 'code': 'KeyD'},'E': {'keyCode': 69, 'key': 'E', 'code': 'KeyE'},'F': {'keyCode': 70, 'key': 'F', 'code': 'KeyF'},'G': {'keyCode': 71, 'key': 'G', 'code': 'KeyG'},'H': {'keyCode': 72, 'key': 'H', 'code': 'KeyH'},'I': {'keyCode': 73, 'key': 'I', 'code': 'KeyI'},'J': {'keyCode': 74, 'key': 'J', 'code': 'KeyJ'},'K': {'keyCode': 75, 'key': 'K', 'code': 'KeyK'},'L': {'keyCode': 76, 'key': 'L', 'code': 'KeyL'},'M': {'keyCode': 77, 'key': 'M', 'code': 'KeyM'},'N': {'keyCode': 78, 'key': 'N', 'code': 'KeyN'},'O': {'keyCode': 79, 'key': 'O', 'code': 'KeyO'},'P': {'keyCode': 80, 'key': 'P', 'code': 'KeyP'},'Q': {'keyCode': 81, 'key': 'Q', 'code': 'KeyQ'},'R': {'keyCode': 82, 'key': 'R', 'code': 'KeyR'},'S': {'keyCode': 83, 'key': 'S', 'code': 'KeyS'},'T': {'keyCode': 84, 'key': 'T', 'code': 'KeyT'},'U': {'keyCode': 85, 'key': 'U', 'code': 'KeyU'},'V': {'keyCode': 86, 'key': 'V', 'code': 'KeyV'},'W': {'keyCode': 87, 'key': 'W', 'code': 'KeyW'},'X': {'keyCode': 88, 'key': 'X', 'code': 'KeyX'},'Y': {'keyCode': 89, 'key': 'Y', 'code': 'KeyY'},'Z': {'keyCode': 90, 'key': 'Z', 'code': 'KeyZ'},':': {'keyCode': 186, 'key': ':', 'code': 'Semicolon'},'<': {'keyCode': 188, 'key': '<', 'code': 'Comma'},'_': {'keyCode': 189, 'key': '_', 'code': 'Minus'},'>': {'keyCode': 190, 'key': '>', 'code': 'Period'},'?': {'keyCode': 191, 'key': '?', 'code': 'Slash'},'~': {'keyCode': 192, 'key': '~', 'code': 'Backquote'},'{': {'keyCode': 219, 'key': '{', 'code': 'BracketLeft'},'|': {'keyCode': 220, 'key': '|', 'code': 'Backslash'},'}': {'keyCode': 221, 'key': '}', 'code': 'BracketRight'},'"': {'keyCode': 222, 'key': '"', 'code': 'Quote'},}
898
+ self._modifiers = 0
899
+ self._pressed_keys = set()
900
+ self.attach(f)
901
+ def attach(self, f):
902
+ f.make_input_events = self.make_input_events
903
+ f.make_clear = self.make_clear
904
+ def make_clear(self):
905
+ charin = []
906
+ charin.append(self._make_down('Control'))
907
+ charin.append(self._make_down('a'))
908
+ charin.append(self._make_up('Control'))
909
+ charin.append(self._make_up('a'))
910
+ charin.append(self._make_down('Delete'))
911
+ charin.append(self._make_up('Delete'))
912
+ return charin
913
+ def make_input_events(self, text):
914
+ return self._make_chain(text)
915
+ def _key_desc(self, keyString): # noqa: C901
916
+ shift = self._modifiers & 8
917
+ desc = {'key': '','keyCode': 0,'code': '','text': '','location': 0}
918
+ defi = self._key_maps.get(keyString)
919
+ if not defi: raise Exception('Unknown key: '+keyString)
920
+ if 'key' in defi: desc['key'] = defi['key']
921
+ if shift and defi.get('shiftKey'): desc['key'] = defi['shiftKey']
922
+ if 'keyCode' in defi: desc['keyCode'] = defi['keyCode']
923
+ if shift and defi.get('shiftKeyCode'): desc['keyCode'] = defi['shiftKeyCode']
924
+ if 'code' in defi: desc['code'] = defi['code']
925
+ if 'location' in defi: desc['location'] = defi['location']
926
+ if len(desc['key']) == 1: desc['text'] = desc['key']
927
+ if 'text' in defi: desc['text'] = defi['text']
928
+ if shift and defi.get('shiftText'): desc['text'] = defi['shiftText']
929
+ if self._modifiers & ~8: desc['text'] = ''
930
+ return desc
931
+ def _modifier_bit(self, key):
932
+ if key == 'Alt': return 1
933
+ if key == 'Control': return 2
934
+ if key == 'Meta': return 4
935
+ if key == 'Shift': return 8
936
+ return 0
937
+ def _make_chain(self, text):
938
+ charin = []
939
+ for char in text:
940
+ if char in self._key_maps:
941
+ charin.append([self._make_down(char), self._make_up(char)])
942
+ else:
943
+ charin.append(['insertText', char])
944
+ return charin
945
+ def _make_down(self, char):
946
+ desc = self._key_desc(char)
947
+ auto_rpt = desc['code'] in self._pressed_keys
948
+ self._pressed_keys.add(desc['code'])
949
+ self._modifiers |= self._modifier_bit(desc['key'])
950
+ text = desc['text']
951
+ return {
952
+ 'type': 'keyDown' if text else 'rawKeyDown','modifiers': self._modifiers,
953
+ 'windowsVirtualKeyCode': desc['keyCode'],
954
+ 'code': desc['code'],'key': desc['key'],'text': text,
955
+ 'unmodifiedText': text,'autoRepeat': auto_rpt,
956
+ 'location': desc['location'], 'isKeypad': desc['location'] == 3, }
957
+ def _make_up(self, char):
958
+ desc = self._key_desc(char)
959
+ self._modifiers &= ~self._modifier_bit(desc['key'])
960
+ if desc['code'] in self._pressed_keys: self._pressed_keys.remove(desc['code'])
961
+ return {
962
+ 'type': 'keyUp', 'modifiers': self._modifiers,
963
+ 'key': desc['key'], 'windowsVirtualKeyCode': desc['keyCode'],
964
+ 'code': desc['code'], 'location': desc['location'], }
965
+ class Screenshot:
966
+ def __init__(self, b64img):
967
+ self.i = b64img
968
+ self.content = base64.b64decode(self.i.encode())
969
+ def test_show(self, timeout=1.5):
970
+ import cv2
971
+ import numpy as np
972
+ img = cv2.imdecode(np.frombuffer(self.content, np.uint8), cv2.IMREAD_UNCHANGED)
973
+ if img is None:
974
+ raise ValueError("decode img fail.")
975
+ h, w = img.shape[:2]
976
+ c1, c2, tile_size = 200, 150, 20
977
+ rows = (np.arange(h) // tile_size) % 2
978
+ cols = (np.arange(w) // tile_size) % 2
979
+ checkerboard = np.where((rows[:, None] ^ cols[None, :]) == 0, c1, c2).astype(np.uint8)
980
+ checkerboard = cv2.merge([checkerboard, checkerboard, checkerboard])
981
+ if img.shape[2] == 4:
982
+ alpha = img[:, :, 3] / 255.0
983
+ bgr = img[:, :, :3]
984
+ blended = (bgr * alpha[..., None] + checkerboard * (1 - alpha[..., None])).astype(np.uint8)
985
+ else:
986
+ blended = img
987
+ cv2.imshow('test', blended)
988
+ cv2.waitKey(int(timeout*1000))
989
+ cv2.destroyAllWindows()
990
+ def __repr__(self):
991
+ return '<Screenshot [size]:{} [prop api]: .content>'.format(to_human_read(len(self.content)))
992
+ class Element:
993
+ def __init__(self, f, einfo, iso=True):
994
+ # {'type': 'object', 'subtype': 'node', 'className': 'HTMLAnchorElement', 'description': 'a', 'objectId': '-1497388350229232364.96.1'}
995
+ self.f = f
996
+ self.className = einfo.get('className')
997
+ self.objectId = einfo.get('objectId')
998
+ self.cache_xy = None
999
+ self.draggable = None
1000
+ self.iso = iso
1001
+ self.r_obj = self.f.run_iso_js_obj if self.iso else self.f.run_js_obj
1002
+ self.r_js = self.f.run_iso_js if self.iso else self.f.run_js
1003
+ @property
1004
+ def backendNodeId(self):
1005
+ return self.f.cdp('DOM.describeNode', {'objectId': self.objectId})['node']['backendNodeId']
1006
+ @property
1007
+ def _box(self):
1008
+ return self.f.cdp('DOM.getBoxModel', {'objectId': self.objectId})
1009
+ def __repr__(self):
1010
+ return '<DOM:[{}] [{}]>'.format(self.className, self.objectId)
1011
+ def clear(self, delay_uptime=0.15, timegap=0.01):
1012
+ self.wait_show()
1013
+ self.f.cdp('DOM.focus', {"objectId": self.objectId})
1014
+ clear_ents = self.f.rootf.make_clear()
1015
+ for kdn in clear_ents[:2]: time.sleep(timegap); self.f.cdp('Input.dispatchKeyEvent', kdn)
1016
+ time.sleep(delay_uptime)
1017
+ for kup in clear_ents[2:4]: time.sleep(timegap); self.f.cdp('Input.dispatchKeyEvent', kup)
1018
+ for delt in clear_ents[4:]: time.sleep(timegap); self.f.cdp('Input.dispatchKeyEvent', delt)
1019
+ def input(self, str, delay_time=0.02, delay_uptime=0.15, random_gap=0.01):
1020
+ self.wait_show()
1021
+ self.f.cdp('DOM.focus', {"objectId": self.objectId})
1022
+ for kdn, kup in self.f.rootf.make_input_events(str):
1023
+ if kdn == 'insertText':
1024
+ self.f.cdp('Input.insertText', {"text": kup})
1025
+ time.sleep(delay_time+(random_gap if random()>0.5 else 0))
1026
+ else:
1027
+ self.f.cdp('Input.dispatchKeyEvent', kdn)
1028
+ time.sleep(delay_time+(random_gap if random()>0.5 else 0))
1029
+ def delay_up(kup, delay_uptime):
1030
+ time.sleep(delay_uptime)
1031
+ self.f.cdp('Input.dispatchKeyEvent', kup)
1032
+ self.f.rootf.root.pool(delay_up)(kup, delay_uptime)
1033
+ def _get_xy(self, x, y, zero='center'):
1034
+ o = self.f.cdp('DOM.getBoxModel', {"objectId": self.objectId})
1035
+ m = o['model']
1036
+ if zero == 'center':
1037
+ _x = (m['content'][0] + m['content'][2]) / 2 + x;
1038
+ _y = (m['content'][1] + m['content'][5]) / 2 + y;
1039
+ elif zero == 'lefttop':
1040
+ _x = m['content'][0] + x
1041
+ _y = m['content'][1] + y
1042
+ elif zero == 'leftbottom':
1043
+ _x = m['content'][6] + x
1044
+ _y = m['content'][7] - y
1045
+ else:
1046
+ raise Exception('zero type must be in (center,lefttop,leftbottom).')
1047
+ return _x, _y
1048
+ def _set_in_view(self):
1049
+ self.f.cdp('DOM.scrollIntoViewIfNeeded', {"objectId": self.objectId})
1050
+ def click(self, x=1, y=2, zero='center', button='left', count=1):
1051
+ self.wait_show()
1052
+ self._set_in_view()
1053
+ _x, _y = self._get_xy(x, y, zero)
1054
+ self.f.cdp('Input.dispatchMouseEvent',{"buttons":0,"type":"mouseMoved","x":_x,"y":_y,"button":"none","clickCount":0})
1055
+ self.f.cdp('Input.dispatchMouseEvent',{"buttons":1,"type":"mousePressed","x":_x,"y":_y,"button":button,"clickCount":count})
1056
+ time.sleep(0.07)
1057
+ self.f.cdp('Input.dispatchMouseEvent',{"buttons":0,"type":"mouseReleased","x":_x,"y":_y,"button":button,"clickCount":count})
1058
+ def _bezier_xxyy(self, x1, y1, x2, y2):
1059
+ def step_len(x1, y1, x2, y2):
1060
+ ln = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5
1061
+ return int(ln / 4)
1062
+ slen = step_len(x1, y1, x2, y2)
1063
+ lp = (random() - 0.5) * 0.5
1064
+ rp = (random() - 0.5) * 0.5 + 1
1065
+ xx1 = int(x1 + (x2 - x1) / 12 * (4-lp*4))
1066
+ yy1 = int(y1 + (y2 - y1) / 12 * (8+lp*4))
1067
+ xx2 = int(x1 + (x2 - x1) / 12 * (8+rp*4))
1068
+ yy2 = int(y1 + (y2 - y1) / 12 * (4-rp*4))
1069
+ points = [[x1, y1], [xx1, yy1], [xx2, yy2], [x2, y2]]
1070
+ N = len(points)
1071
+ n = N - 1
1072
+ r = []
1073
+ for T in range(slen + 1):
1074
+ t = T*(1/slen)
1075
+ t = (sin(t * pi / 2)) ** 18
1076
+ x,y = 0,0
1077
+ for i in range(N):
1078
+ B = factorial(n)*t**i*(1-t)**(n-i)/(factorial(i)*factorial(n-i))
1079
+ x += points[i][0]*B
1080
+ y += points[i][1]*B
1081
+ r.append([x, y])
1082
+ return r
1083
+ def wait_show(self, timeout=5, pretime=0.1):
1084
+ start = perf_counter()
1085
+ count, acount = 0, 0
1086
+ px, py = 0, 0
1087
+ while True:
1088
+ v = self.visible()
1089
+ if v.get('isDisplayed'):
1090
+ x, y = v['clickPoint']['x'], v['clickPoint']['y']
1091
+ if x == px and y == py:
1092
+ count += 1
1093
+ else:
1094
+ count = 0
1095
+ acount += 1
1096
+ px, py = x, y
1097
+ if count > 1 or acount > 5:
1098
+ break
1099
+ if perf_counter() - start > timeout:
1100
+ raise Exception('wait show over time.')
1101
+ time.sleep(0.08)
1102
+ if pretime:
1103
+ time.sleep(pretime) # 组件出现时,可能事件并未挂上去,稍微等待一下
1104
+ def drag(self, x=1, y=2, zero='center', button='left', count=1):
1105
+ self.wait_show()
1106
+ self._set_in_view()
1107
+ x, y = self._get_xy(x, y, zero)
1108
+ self.cache_xy = [x, y]
1109
+ self.f.cdp('Input.dispatchMouseEvent',{"buttons":1,"type":"mousePressed","x":int(x),"y":int(y),"button":button,"clickCount":count})
1110
+ self.draggable = self['draggable']
1111
+ if self.draggable:
1112
+ print('[*] h5 draggable element cannot simulate drag and drop.')
1113
+ # draggable api, CDP not simulate and this api cannot record trajectory
1114
+ # self.f.cdp('Input.dispatchDragEvent',{"type":"dragEnter","x":int(x),"y":int(y),
1115
+ # "data":{"items":[{"type":"text/plain","data":"some text"}]}
1116
+ # })
1117
+ # self.f.cdp('Input.dispatchDragEvent',{"type":"dragOver","x":int(x),"y":int(y)})
1118
+ # self.f.cdp('Input.dispatchDragEvent',{"type":"drop","x":int(x),"y":int(y)})
1119
+ # and <input type=range> cannot simulate sliding through CDP.
1120
+ return self
1121
+ def dragmove(self, shiftx=0, shifty=0, button='left', count=1, costtime=0.8):
1122
+ if self.cache_xy:
1123
+ x, y = self.cache_xy
1124
+ else:
1125
+ cp = self._view_info()['clickPoint']
1126
+ x, y = cp['x'], cp['y']
1127
+ self.cache_xy = x, y
1128
+ c = self._bezier_xxyy(x,y,x+shiftx,y+shifty)
1129
+ g = costtime / len(c)
1130
+ start = perf_counter()
1131
+ ctime = 0
1132
+ gtime = 0
1133
+ for x, y in c:
1134
+ self.f.cdp('Input.dispatchMouseEvent',{"buttons":1,"type":"mouseMoved","x":int(x),"y":int(y),"button":"none","clickCount":0})
1135
+ self.cache_xy = x, y
1136
+ ctime += g
1137
+ gtime = perf_counter() - start
1138
+ if ctime > gtime:
1139
+ time.sleep(ctime - gtime)
1140
+ return self
1141
+ def drop(self, button='left', count=1):
1142
+ if self.cache_xy:
1143
+ x, y = self.cache_xy
1144
+ else:
1145
+ cp = self._view_info()['clickPoint']
1146
+ x, y = cp['x'], cp['y']
1147
+ self.f.cdp('Input.dispatchMouseEvent',{"buttons":0,"type":"mouseReleased","x":int(x),"y":int(y),"button":button,"clickCount":count})
1148
+ self.cache_xy = None
1149
+ return self
1150
+ def drag_to(self, shiftx=0, shifty=0, button='left', x=1, y=2, zero='center', costtime=0.8, count=1):
1151
+ self.drag(x, y, zero, button, count) \
1152
+ .dragmove(shiftx, shifty, button, count, costtime) \
1153
+ .drop(button, count)
1154
+ return self
1155
+ def _is_coverd(self):
1156
+ try:
1157
+ c = self._view_info()['clickPoint']
1158
+ return self.backendNodeId != self.f.cdp('DOM.getNodeForLocation', {'x': c['x'], 'y': c['y']})['backendNodeId']
1159
+ except Exception as e:
1160
+ return True
1161
+ def pic(self, try_use_img=True):
1162
+ self.wait_show()
1163
+ if try_use_img and self.className == 'HTMLImageElement':
1164
+ try:
1165
+ # 如果是 <img> 标签,就尝试通过标签采集,采集不到再走屏幕截图。
1166
+ # 因为 img 标签的图片最好接近原生图片,这样效果会更好,当然也可以配置 try_use_img=False 直接关掉这种图片获取方式,就用截图,那样比较统一。
1167
+ src = self['src']
1168
+ if src:
1169
+ if src.startswith('http'):
1170
+ return Screenshot(self.f.cdp("Page.getResourceContent", {"url": src, "frameId": self.f.frameId})['content'])
1171
+ if src.startswith('data:image/png;base64,'):
1172
+ return Screenshot(src.replace('data:image/png;base64,', ''))
1173
+ except:
1174
+ pass
1175
+ rparentlist = self.f._get_remote_list()
1176
+ if not rparentlist:
1177
+ box = self.f.cdp('DOM.getBoxModel', {'objectId': self.objectId})
1178
+ x, y = box['model']['padding'][0], box['model']['padding'][1]
1179
+ width, height = box['model']['width'], box['model']['height']
1180
+ screenshot = self.f.cdp('Page.captureScreenshot', {"format":'png',
1181
+ "clip":{"x":x,"y":y,"width":width,"height":height,"scale":1} })
1182
+ return Screenshot(screenshot['data'])
1183
+ else:
1184
+ slfele = self.f.element._frame_element(self.f)
1185
+ sftx, sfty = 0, 0
1186
+ f = self.f.element._frame_element(rparentlist[-1])
1187
+ box = f._box['model']['padding']
1188
+ sftx += box[0]
1189
+ sfty += box[1]
1190
+ f_box = self._box
1191
+ x, y = f_box['model']['padding'][0], f_box['model']['padding'][1]
1192
+ width, height = f_box['model']['width'], f_box['model']['height']
1193
+ screenshot = self.f.rootf.cdp('Page.captureScreenshot', {"format":'png',
1194
+ "clip":{"x":x+sftx,"y":y+sfty,"width":width,"height":height,"scale":1} })
1195
+ return Screenshot(screenshot['data'])
1196
+ def _view_info(self):
1197
+ return self.f.element.visibility(self.objectId)
1198
+ def visible(self):
1199
+ d = self._view_info()
1200
+ d['isCoverd'] = self._is_coverd()
1201
+ return d
1202
+ def info(self):
1203
+ d = self.visible()
1204
+ class ViewInfo:
1205
+ def __init__(self, d):
1206
+ self.d = d
1207
+ def __repr__(self):
1208
+ return json.dumps(self.d, indent=4)
1209
+ return ViewInfo(d)
1210
+ def xpath(self, s):
1211
+ return self.f.element.xpath(s, self.objectId)
1212
+ def css(self, s):
1213
+ return self.f.element.css(s, self.objectId)
1214
+ def _ele(self, s, one=False):
1215
+ r = []
1216
+ xf, tp = self.f.element._predict_xpath_or_css(s.strip())
1217
+ if tp == 'xpath': r.extend(self.xpath(xf))
1218
+ if tp == 'css': r.extend(self.css(xf))
1219
+ if one and r: return r
1220
+ nodeId = self.f.cdp('DOM.describeNode', {'objectId': self.objectId})['node']['nodeId']
1221
+ for sr in self.f.element._sr_root(nodeId):
1222
+ if tp == 'xpath': r.extend(sr.xpath(xf))
1223
+ if tp == 'css': r.extend(sr.css(xf))
1224
+ if one and r: return r
1225
+ return r
1226
+ def ele(self, s):
1227
+ rt = self._ele(s, one=True)
1228
+ return rt[0] if rt else None
1229
+ def eles(self, s):
1230
+ return self._ele(s)
1231
+ def id(self, i):
1232
+ return self.ele('#'+i)
1233
+ def ijs(self, s):
1234
+ einfo = self.r_js(s, returnByValue=False)
1235
+ return self.f._parse_js2py(einfo, self, iso=self.iso)
1236
+ def __getitem__(self, a):
1237
+ einfo = self.r_obj('function(){return this[' + json.dumps(a) + ']}', objectId=self.objectId, returnByValue=False)
1238
+ return self.f._parse_js2py(einfo, self, iso=self.iso)
1239
+ def __setitem__(self, a, b):
1240
+ args = [None,self.f._parse_2arg(a), self.f._parse_2arg(b)]
1241
+ einfo = self.r_obj('function(a,b){return this[a]=b}', objectId=self.objectId, arguments=args, returnByValue=False)
1242
+ return self.f._parse_js2py(einfo, iso=self.iso)
1243
+ def __add__(self, other):
1244
+ einfo = self.r_obj('function(){return this["toString"]()}', objectId=self.objectId, returnByValue=False)
1245
+ return self.f._parse_js2py(einfo, self, iso=self.iso) + str(other)
1246
+ text = property(lambda s:s['outerHTML'] or s['textContent'])
1247
+ before = property(lambda s:s['previousElementSibling'])
1248
+ after = property(lambda s:s['nextElementSibling'])
1249
+ parent = property(lambda s:s['parentElement'])
1250
+ children = property(lambda s:s['children'])
1251
+ class ElementTools:
1252
+ def __init__(self, f):
1253
+ self.f = f
1254
+ self.attach(f)
1255
+ def attach(self, f):
1256
+ f.ele = self.ele
1257
+ f.eles = self.eles
1258
+ f.xpath = self.xpath
1259
+ f.css = self.css
1260
+ f._sr_root = self._sr_root
1261
+ def _predict_xpath_or_css(self, x):
1262
+ if x.startswith('x:'):
1263
+ xf, tp = x[2:], 'xpath'
1264
+ elif x.startswith('c:'):
1265
+ xf, tp = x[2:], 'css'
1266
+ else:
1267
+ xf = x
1268
+ x = x.strip()
1269
+ if x == '.' or '/' in x:
1270
+ tp = 'xpath'
1271
+ else:
1272
+ tp = 'css'
1273
+ return xf, tp
1274
+ def _parse_x(self, x, all_frames, one=False, over_v_limit=False):
1275
+ r = []
1276
+ frms = [self.f]
1277
+ xf, tp = self._predict_xpath_or_css(x.strip())
1278
+ if all_frames:
1279
+ frms.extend(self.f.root.flat_child_frames(self.f.frames))
1280
+ for f in frms:
1281
+ if over_v_limit: f.iso_contextId = None
1282
+ if tp == 'xpath':
1283
+ r.extend(f.xpath(xf))
1284
+ if one and r: return r
1285
+ if f.type == 'LocalFrame':continue
1286
+ for sr in f._sr_root():
1287
+ if one and r: return r
1288
+ r.extend(sr.xpath(xf))
1289
+ elif tp == 'css':
1290
+ r.extend(f.css(xf))
1291
+ if one and r: return r
1292
+ if f.type == 'LocalFrame':continue
1293
+ for sr in f._sr_root():
1294
+ if one and r: return r
1295
+ r.extend(sr.css(xf))
1296
+ return r
1297
+ def ele(self, x, timeout=8, no_wait=False, all_frames=True):
1298
+ start = perf_counter()
1299
+ over_v_time = 2
1300
+ over_v_limit = False
1301
+ while True:
1302
+ ctime = perf_counter()
1303
+ if ctime - start > over_v_time: over_v_limit = True
1304
+ r = self._parse_x(x, all_frames, one=True, over_v_limit=over_v_limit)
1305
+ if ctime - start > over_v_time: over_v_limit = False
1306
+ if r: return r[0]
1307
+ if no_wait: return None
1308
+ if ctime - start > timeout:
1309
+ raise Exception('ele selecter timeout.')
1310
+ time.sleep(0.08)
1311
+ def eles(self, x, timeout=8, no_wait=False, all_frames=True):
1312
+ start = perf_counter()
1313
+ over_v_time = 2
1314
+ over_v_limit = False
1315
+ while True:
1316
+ ctime = perf_counter()
1317
+ if ctime - start > over_v_time: over_v_limit = True
1318
+ r = self._parse_x(x, all_frames, over_v_limit=over_v_limit)
1319
+ if ctime - start > over_v_time: over_v_limit = False
1320
+ if r: return r
1321
+ if no_wait: return []
1322
+ if ctime - start > timeout:
1323
+ raise Exception('eles selecter timeout.')
1324
+ time.sleep(0.08)
1325
+ def _parse_array(self, r):
1326
+ if not r.get('objectId'):
1327
+ return []
1328
+ prps = self.f.cdp('Runtime.getProperties', { 'objectId': r['objectId'], 'ownProperties': True })
1329
+ rn = []
1330
+ for prp in prps.get('result', []):
1331
+ if prp['name'].isdigit() and 'objectId' in prp['value']:
1332
+ rn.append(Element(self.f, prp['value']))
1333
+ return rn
1334
+ def _run_iso(self, fscpt, objectId, returnByValue=False):
1335
+ if objectId:
1336
+ r = self.f.run_iso_js_obj(fscpt, returnByValue=returnByValue, objectId=objectId)
1337
+ else:
1338
+ r = self.f.run_iso_js('({})()'.format(fscpt), returnByValue=returnByValue)
1339
+ return r
1340
+ def _trav_node_tree(self, rnode):
1341
+ def collect(n):
1342
+ d = {}
1343
+ d['nodeId'] = n['nodeId']
1344
+ d['backendNodeId'] = n['backendNodeId']
1345
+ d['nodeType'] = n['nodeType']
1346
+ d['nodeName'] = n['nodeName']
1347
+ if n.get('frameId'):
1348
+ d['frameId'] = n['frameId']
1349
+ if n.get('nodeName') == 'IFRAME' and 'contentDocument' in n:
1350
+ _trav(n['contentDocument'])
1351
+ if n.get('shadowRoots'):
1352
+ d['shadowRoots'] = n['shadowRoots']
1353
+ for ssnode in d['shadowRoots']:
1354
+ if ssnode.get('children'):
1355
+ _trav(ssnode)
1356
+ clist.append(d)
1357
+ clist = []
1358
+ def _trav(rnode):
1359
+ collect(rnode)
1360
+ if rnode.get('children'):
1361
+ for snode in rnode['children']:
1362
+ _trav(snode)
1363
+ _trav(rnode)
1364
+ return clist
1365
+ def _get_flattened_node(self, nodeId=None):
1366
+ if nodeId == None:
1367
+ rnode = self.f.cdp('DOM.getDocument', {'depth':-1, 'pierce':True})['root']
1368
+ else:
1369
+ rnode = self.f.cdp('DOM.describeNode', {'nodeId':nodeId, 'depth':-1, 'pierce':True})['node']
1370
+ mnodes = {"nodes": self._trav_node_tree(rnode)}
1371
+ return mnodes
1372
+ def _filter_shadow_root(self, mnodes):
1373
+ r = []
1374
+ if not mnodes.get('nodes'):
1375
+ return []
1376
+ for n in mnodes['nodes']:
1377
+ srlst = n.get('shadowRoots')
1378
+ if srlst:
1379
+ for sr in srlst:
1380
+ if (sr.get('nodeName') == '#document-fragment'
1381
+ and (
1382
+ sr.get('shadowRootType') == 'closed' or
1383
+ sr.get('shadowRootType') == 'open')
1384
+ ):
1385
+ m = self.f.cdp('DOM.resolveNode', {"backendNodeId": sr['backendNodeId']})
1386
+ if m.get('object'):
1387
+ e = Element(self.f, m['object'])
1388
+ r.append(e)
1389
+ return r
1390
+ def _sr_root(self, nodeId=None):
1391
+ mnodes = self._get_flattened_node(nodeId)
1392
+ return self._filter_shadow_root(mnodes)
1393
+ def _make_shadow_root_xpath(self, s):
1394
+ v = re.findall('^([^/]*)(/+)([^/]*)$', s)
1395
+ if v and len(v[0]) == 3:
1396
+ v = v[0]
1397
+ p = v[-1].split('[', 1)
1398
+ tag = p[0]
1399
+ pak = p[1] if len(p) == 2 else ''
1400
+ if pak:
1401
+ pak = "[self::"+tag+' and ' + pak
1402
+ else:
1403
+ pak = "[self::"+tag+']'
1404
+ return '(.|'+v[0]+v[1]+tag+')'+pak
1405
+ else:
1406
+ return s
1407
+ def xpath(self, s, objectId=None):
1408
+ s = s.lstrip()
1409
+ if s[0] != '.' and s[0] != '/': s = './' + s
1410
+ if s[0] == '/': s = '.' + s
1411
+ fscpt = '''
1412
+ function() {
1413
+ var contextNode = window === this ? document : this
1414
+ var node, nodes = [];
1415
+ if (contextNode instanceof ShadowRoot){
1416
+ for (var i = 0; i < contextNode.children.length; i++) {
1417
+ var snode = contextNode.children[i]
1418
+ var iterator = document.evaluate('''+json.dumps(self._make_shadow_root_xpath(s))+''', snode, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
1419
+ while ((node = iterator.iterateNext())) { nodes.push(node); }
1420
+ }
1421
+ }else{
1422
+ var iterator = document.evaluate('''+json.dumps(s)+''', contextNode, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
1423
+ while ((node = iterator.iterateNext())) { nodes.push(node); }
1424
+ }
1425
+ return nodes;
1426
+ }
1427
+ '''
1428
+ r = self._run_iso(fscpt, objectId)
1429
+ return self._parse_array(r)
1430
+ def visibility(self, objectId=None):
1431
+ fscpt = '''
1432
+ function() {
1433
+ const rect = this.getBoundingClientRect();
1434
+ const s = window.getComputedStyle(this);
1435
+ return {
1436
+ inViewport: (
1437
+ rect.top < window.innerHeight &&
1438
+ rect.bottom > 0 &&
1439
+ rect.left < window.innerWidth &&
1440
+ rect.right > 0
1441
+ ),
1442
+ style: {
1443
+ display: s.display,
1444
+ visibility: s.visibility,
1445
+ opacity: s.opacity,
1446
+ position: s.position,
1447
+ zIndex: s.zIndex
1448
+ },
1449
+ isDisplayed: !(
1450
+ s.visibility == 'hidden' ||
1451
+ s.display == 'none' ||
1452
+ this.hidden ||
1453
+ rect.width == 0 ||
1454
+ rect.height == 0
1455
+ ),
1456
+ rect: {
1457
+ x: rect.x,
1458
+ y: rect.y,
1459
+ width: rect.width,
1460
+ height: rect.height,
1461
+ top: rect.top,
1462
+ right: rect.right,
1463
+ bottom: rect.bottom,
1464
+ left: rect.left
1465
+ },
1466
+ clickPoint: {
1467
+ x: (rect.x + rect.width/2)|0,
1468
+ y: (rect.y + rect.height/2)|0,
1469
+ },
1470
+ inDocument: document.contains(this),
1471
+ isClickable: (
1472
+ s.pointerEvents !== 'none' &&
1473
+ s.cursor !== 'not-allowed'
1474
+ )
1475
+ };
1476
+ }
1477
+ '''
1478
+ r = self._run_iso(fscpt, objectId, returnByValue=True)
1479
+ return r
1480
+ def css(self, s, objectId=None):
1481
+ fscpt = '''
1482
+ function(){
1483
+ var contextNode = window === this ? document : this
1484
+ return Array.from(contextNode.querySelectorAll(''' + json.dumps(s) + '''));
1485
+ }
1486
+ '''
1487
+ r = self._run_iso(fscpt, objectId)
1488
+ return self._parse_array(r)
1489
+ @staticmethod
1490
+ def _frame_element(self):
1491
+ nodeId = self.parent.cdp('DOM.getDocument', {"depth": -1})['root']['nodeId']
1492
+ frs = self.parent.cdp('DOM.querySelectorAll', { "nodeId": nodeId, "selector": 'iframe' })
1493
+ node = None
1494
+ for fr in frs['nodeIds']:
1495
+ nd = self.parent.cdp('DOM.describeNode', { "nodeId": fr })
1496
+ if nd['node']['frameId'] == self.frameId:
1497
+ node = nd['node']
1498
+ break
1499
+ return Element(self.parent, self.parent.cdp('DOM.resolveNode', {"backendNodeId": node['backendNodeId']})['object'])
1500
+ class CDPTools:
1501
+ def __init__(self, f, rootf):
1502
+ self.f = f
1503
+ self.rootf = rootf
1504
+ self.attach(f)
1505
+ def attach(self, f):
1506
+ f.run_js = self.run_js
1507
+ f.run_js_obj = self.run_js_obj
1508
+ f.run_iso_js = self.run_iso_js
1509
+ f.run_iso_js_obj = self.run_iso_js_obj
1510
+ f._js2py = self._js2py
1511
+ f._parse_js2py = self._parse_js2py
1512
+ f._parse_2arg = self._parse_2arg
1513
+ def _init_iso(self, d):
1514
+ if not self.f.iso_contextId:
1515
+ x = self.rootf.cdp('Page.createIsolatedWorld', { "frameId": self.f.frameId }, sessionId=self.f.sessionId)
1516
+ if x.get('message') == 'No frame for given id found':
1517
+ return
1518
+ if 'executionContextId' not in x:
1519
+ print('error create iso:'+str(x))
1520
+ return
1521
+ self.f.iso_contextId = x['executionContextId']
1522
+ if self.f.iso_contextId != None: d["contextId"] = self.f.iso_contextId
1523
+ return d
1524
+ def _parse_2arg(self, v):
1525
+ if isinstance(v, (JSObject, Element)):
1526
+ return {"objectId": v.objectId}
1527
+ else:
1528
+ return {"value": v}
1529
+ def _parse_js2py(self, e, _this=None, iso=False):
1530
+ if type(e) in (int, float, str, bool): return e
1531
+ if e == None: return None
1532
+ if iso and type(e) == dict and e.get('subtype') == 'node':
1533
+ # The ISO here is only used to isolate the impact of code on the main context.
1534
+ return Element(self.f, e, iso=iso)
1535
+ return JSObject(self.f, e, _this, iso)
1536
+ def _js2py(self, code, iso=False):
1537
+ return self._parse_js2py((self.run_iso_js if iso else self.run_js)(code, returnByValue=False), iso=iso)
1538
+ def run_js(self, script, awaitPromise=True, returnByValue=True):
1539
+ d = { "expression": script, "awaitPromise": awaitPromise, "returnByValue": returnByValue }
1540
+ if self.f.contextId != None: d["contextId"] = self.f.contextId
1541
+ return self.rootf.cdp('Runtime.evaluate', d, sessionId=self.f.sessionId)
1542
+ def run_js_obj(self, script, awaitPromise=True, returnByValue=True, objectId=None, arguments=[None], includeCommandLineAPI=False):
1543
+ d = { "functionDeclaration": script, "awaitPromise": awaitPromise, "returnByValue": returnByValue, "objectId": objectId, "includeCommandLineAPI": includeCommandLineAPI }
1544
+ if arguments[0]:
1545
+ arguments[0] = { "objectId": arguments[0].objectId }
1546
+ d['arguments'] = arguments
1547
+ else:
1548
+ d['arguments'] = arguments[1:]
1549
+ if self.f.contextId != None: d["contextId"] = self.f.contextId
1550
+ return self.rootf.cdp('Runtime.callFunctionOn', d, sessionId=self.f.sessionId)
1551
+ def run_iso_js(self, script, awaitPromise=True, returnByValue=True, includeCommandLineAPI=False):
1552
+ d = self._init_iso({ "expression": script, "awaitPromise": awaitPromise,"returnByValue": returnByValue, "includeCommandLineAPI": includeCommandLineAPI })
1553
+ return self.rootf.cdp('Runtime.evaluate', d, sessionId=self.f.sessionId) if d else {}
1554
+ def run_iso_js_obj(self, script, awaitPromise=True, returnByValue=True, objectId=None, arguments=[None], includeCommandLineAPI=False):
1555
+ d = self._init_iso({ "functionDeclaration": script, "awaitPromise": awaitPromise,"returnByValue": returnByValue, "objectId": objectId, "includeCommandLineAPI": includeCommandLineAPI })
1556
+ if not d: return {}
1557
+ if arguments[0]:
1558
+ arguments[0] = { "objectId": arguments[0].objectId }
1559
+ d['arguments'] = arguments
1560
+ else:
1561
+ d['arguments'] = arguments[1:]
1562
+ return self.rootf.cdp('Runtime.callFunctionOn', d, sessionId=self.f.sessionId) if d else {}
1563
+ class AbsFrame:
1564
+ def __init__(self, f, finfo):
1565
+ self.rootf = f
1566
+ self.frameId = finfo['frameId']
1567
+ self.frames = []
1568
+ self.parent = finfo.get('parent')
1569
+ self.url = finfo.get('url')
1570
+ # LocalFrame
1571
+ self.contextId = finfo.get('contextId')
1572
+ self.uniqueId = finfo.get('uniqueId')
1573
+ # RemoteFrame
1574
+ self.sessionId = finfo.get('sessionId')
1575
+ self.iso_contextId = None
1576
+ self.tools = CDPTools(self, self.rootf)
1577
+ self.element = ElementTools(self)
1578
+ @property
1579
+ def type(self):
1580
+ if self.sessionId:
1581
+ return 'RemoteFrame'
1582
+ if self.contextId:
1583
+ return 'LocalFrame'
1584
+ return 'UnknownFrame'
1585
+ def __repr__(self):
1586
+ if self.type == 'RemoteFrame':
1587
+ return '<[{}]|{}|F:{}|S:{}>'.format(self.type, self.url, self.frameId, self.sessionId)
1588
+ return '<[{}]|{}|F:{}>'.format(self.type, self.url, self.frameId)
1589
+ def cdp(self, *a, **kw):
1590
+ return self.rootf.cdp(*a, **kw, sessionId=self.sessionId)
1591
+ def _get_remote_list(self):
1592
+ _self = self
1593
+ _rlst = []
1594
+ while _self:
1595
+ if _self.type == 'RemoteFrame':
1596
+ _rlst.append(_self)
1597
+ _self = _self.parent
1598
+ return _rlst
1599
+ def pic(self,):
1600
+ return self._js2py('document.documentElement', iso=True).pic()
1601
+ class Runtime:
1602
+ def __init__(self, f):
1603
+ self.f = f
1604
+ self.f.set_method_callback('Runtime.executionContextCreated', self.Runtime_executionContextCreated)
1605
+ self.f.set_method_callback('Runtime.executionContextDestroyed', self.Runtime_executionContextDestroyed)
1606
+ self.init()
1607
+ def init(self, sessionId=None):
1608
+ self.f.cdp('Runtime.enable', sessionId=sessionId)
1609
+ def Runtime_executionContextCreated(self, rdata):
1610
+ self.f.root._add_init_check()
1611
+ # {"context":
1612
+ # {"id": 10,
1613
+ # "origin": "http://lc1.test:18000",
1614
+ # "name": "",
1615
+ # "uniqueId": "-2899874725492390568.194300135741888500",
1616
+ # "auxData":
1617
+ # {"isDefault": true,
1618
+ # "type": "default",
1619
+ # "frameId": "6D49B34D12C4719D669CE5DC9BED71A4"}}}
1620
+ params = rdata['params']
1621
+ if 'auxData' not in params['context']:
1622
+ # extension context
1623
+ # {"context": {"id": 1, "origin": "chrome-extension://ognihjbdmbjhecdlpjjonacagooanpfa/background.js", "name": "", "uniqueId": "-5795409506706594697.-3458865582039429610"}}
1624
+ # maybe some blob:url
1625
+ # {"context": {"id": 1, "origin": "blob:https://steamdb.info/fd99df19-bda7-4360-8cff-80c782e38697", "name": "", "uniqueId": "-5913474193475720670.7988369434689538446"}}
1626
+ self.f.root._del_init_check()
1627
+ return
1628
+ if params['context']['auxData']['type'] == 'isolated':
1629
+ # DevTools tools isolate context
1630
+ # {"context": {"id": 100, "origin": "http://lc1.test:18000", "name": "DevTools Performance Metrics", "uniqueId": "-4717759096737431074.6555991471704808792", "auxData": {"isDefault": false, "type": "isolated", "frameId": "65764D3FB5953A54725C764602802733"}}}
1631
+ self.f.root._del_init_check()
1632
+ return
1633
+ frameId = params['context']['auxData']['frameId']
1634
+ uniqueId = params['context']['uniqueId']
1635
+ contextId = params['context']['id']
1636
+ if frameId != self.f.frameId:
1637
+ self.f.root.add_common_frame(self.f, {
1638
+ "contextId": contextId,
1639
+ "uniqueId": uniqueId,
1640
+ "frameId": frameId,
1641
+ }, sessionId=rdata.get('sessionId'))
1642
+ self.f.root._del_init_check()
1643
+ def Runtime_executionContextDestroyed(self, rdata):
1644
+ if 'executionContextUniqueId' in rdata['params']:
1645
+ f = self.f.root.trav_frame(rdata['params'].get('executionContextUniqueId'), 'uniqueId')
1646
+ if f:
1647
+ f.parent.frames.remove(f)
1648
+ class Emulation:
1649
+ def __init__(self, f):
1650
+ self.f = f
1651
+ self.init()
1652
+ def init(self, sessionId=None):
1653
+ self.f.cdp('Emulation.setFocusEmulationEnabled', {'enabled': True}, sessionId=sessionId)
1654
+ # chrome 112+: Emulation.setDeviceMetricsOverride connot work in screenX/screenY.
1655
+ # self.f.cdp('Emulation.setDeviceMetricsOverride', {
1656
+ # "positionX": 100,
1657
+ # "positionY": 200,
1658
+ # }, sessionId=sessionId)
1659
+ class CoreCDP:
1660
+ def __init__(self, ws, root, logger=None):
1661
+ self.ws = ws
1662
+ self.root = root
1663
+ self.logger = logger or (lambda *a,**kw:None)
1664
+ self.id = 0
1665
+ self.xid = 0
1666
+ self.qret = {}
1667
+ self.irun = {}
1668
+ self._start()
1669
+ def _start(self):
1670
+ self.is_running = True
1671
+ self.loop_recv = Thread(target=self.start_loop)
1672
+ self.loop_recv.daemon = True
1673
+ self.loop_recv.start()
1674
+ def _handle_method(self, rdata):
1675
+ method = rdata.get('method')
1676
+ if method in self.irun:
1677
+ for xid in self.irun[method]:
1678
+ m = self.irun[method].get(xid, None)
1679
+ if m:
1680
+ if is_function(m):
1681
+ m(rdata)
1682
+ if isinstance(m, queue.Queue):
1683
+ m.put(rdata['params'])
1684
+ def _handle_return(self, rdata):
1685
+ if rdata.get('id') in self.qret:
1686
+ if rdata.get('result', Err) != Err:
1687
+ self.qret[rdata.get('id')].put(rdata['result'])
1688
+ elif rdata.get('error', Err) != Err:
1689
+ self.qret[rdata.get('id')].put(rdata['error'])
1690
+ else:
1691
+ self.logger(rdata, repr(rdata.get('result')))
1692
+ raise Exception('un expect err.' + repr(rdata.get('result')))
1693
+ def attach(self, f):
1694
+ f.cdp = self.cdp
1695
+ f.wait_once_method = self.wait_once_method
1696
+ f.set_method_callback = self.set_method_callback
1697
+ def start_loop(self):
1698
+ while self.is_running:
1699
+ try:
1700
+ recvd = self.ws.recv()
1701
+ if not recvd:
1702
+ continue
1703
+ rdata = json.loads(recvd)
1704
+ self.logger('recv', rdata)
1705
+ except WebSocketTimeoutException:
1706
+ continue
1707
+ except (
1708
+ OSError,
1709
+ JSONDecodeError,
1710
+ WebSocketException,
1711
+ WebSocketConnectionClosedException,
1712
+ ConnectionResetError,
1713
+ ) as e:
1714
+ if not self.is_running:
1715
+ return
1716
+ if (
1717
+ 'Connection to remote host was lost.' in str(e)
1718
+ or 'socket is already closed.' in str(e)
1719
+ or isinstance(e, OSError)
1720
+ ):
1721
+ # Closing the tab will trigger some return message exceptions and possible exception disconnections
1722
+ return
1723
+ raise Exception('[*] maybe exist ws connect from another python exe! pls close it.')
1724
+ except Exception as e:
1725
+ print(e, 'rasie cdp loop 2')
1726
+ raise 2
1727
+ self._handle_method(rdata)
1728
+ self._handle_return(rdata)
1729
+ def get_id(self):
1730
+ self.id += 1
1731
+ return self.id
1732
+ def get_xid(self):
1733
+ self.xid += 1
1734
+ return self.xid
1735
+ def cdp(self, protocal, data={}, sessionId=None, no_wait=False, limit_time=None):
1736
+ rid = self.get_id()
1737
+ cmd = { "id": rid, "method": protocal, "params": data }
1738
+ if sessionId:
1739
+ cmd['sessionId'] = sessionId
1740
+ self.logger('req', cmd)
1741
+ if not no_wait:
1742
+ self.qret[rid] = queue.Queue()
1743
+ try:
1744
+ self.ws.send(json.dumps(cmd))
1745
+ except (
1746
+ OSError,
1747
+ WebSocketConnectionClosedException
1748
+ ) as e:
1749
+ self.logger('[ERROR]', str(e))
1750
+ self.qret.pop(rid, None)
1751
+ return
1752
+ except Exception as e:
1753
+ print(e, 'rasie cdp 1')
1754
+ raise 1
1755
+ if limit_time != None:
1756
+ start = perf_counter()
1757
+ while not no_wait:
1758
+ try:
1759
+ if limit_time != None:
1760
+ if perf_counter() - start > limit_time:
1761
+ self.qret.pop(rid, None)
1762
+ return {'vvv': 'over time'}
1763
+ ret = self.qret[rid].get(timeout=.15)
1764
+ self.qret.pop(rid, None)
1765
+ return try_run_result(ret)
1766
+ except queue.Empty:
1767
+ continue
1768
+ def wait_once_method(self, method, timeout=10, check_ret_func=None, rasie_error=True):
1769
+ self.irun[method] = self.irun.get(method, {})
1770
+ xid = self.get_xid()
1771
+ self.irun[method][xid] = queue.Queue()
1772
+ start = perf_counter()
1773
+ while True:
1774
+ ctime = perf_counter()
1775
+ try:
1776
+ if ctime - start > timeout:
1777
+ self.irun[method].pop(xid, None)
1778
+ return
1779
+ ret = self.irun[method][xid].get(timeout=0.15)
1780
+ if check_ret_func:
1781
+ if not check_ret_func(ret):
1782
+ continue
1783
+ self.irun[method].pop(xid, None)
1784
+ return ret
1785
+ except:
1786
+ if ctime - start > timeout:
1787
+ if rasie_error:
1788
+ raise Exception('wait_once_method {} timeout: {}'.format(method, timeout))
1789
+ continue
1790
+ def set_method_callback(self, method, func):
1791
+ xid = self.get_xid()
1792
+ self.irun[method] = self.irun.get(method, {})
1793
+ self.irun[method][xid] = self.root.pool(func)
1794
+ return xid
1795
+ class Closer:
1796
+ def __init__(self, f):
1797
+ self.f = f
1798
+ self.f.set_method_callback('Inspector.detached', self.close)
1799
+ self.toggle = True
1800
+ def close(self, rdata):
1801
+ self.f.tools.is_running = False
1802
+ self.f.ws.close()
1803
+ if self.f in self.f.root.tabs:
1804
+ self.f.root.tabs.remove(self.f)
1805
+ self.toggle = False
1806
+ class DOM:
1807
+ def __init__(self, f):
1808
+ self.f = f
1809
+ self.f.cdp('DOM.enable')
1810
+ class Cache:
1811
+ def __init__(self, f):
1812
+ self.f = f
1813
+ self.cache_sesslist = [None]
1814
+ self.is_enable = False
1815
+ self.attach(f)
1816
+ def _enable(self):
1817
+ if not self.is_enable:
1818
+ for sessionId in self.cache_sesslist:
1819
+ self._cache_cdp(sessionId)
1820
+ self.is_enable = True
1821
+ def _cache_cdp(self, sessionId):
1822
+ self.f.cdp('Network.enable', sessionId=sessionId)
1823
+ self.f.cdp('Storage.enable', sessionId=sessionId)
1824
+ def attach(self, f):
1825
+ f.clear_cache = self.clear_cache
1826
+ def add_cache_session(self, sessionId):
1827
+ self.cache_sesslist.append(sessionId)
1828
+ if self.is_enable:
1829
+ self._cache_cdp(sessionId)
1830
+ def clear_cache(self):
1831
+ self._enable()
1832
+ for sessionId in self.cache_sesslist:
1833
+ self.f.cdp('Network.clearBrowserCookies',sessionId=sessionId)
1834
+ self.f.cdp('Storage.clearDataForOrigin', {'origin':'*','storageTypes':'local_storage,session_storage,indexeddb'},sessionId=sessionId);
1835
+ class Browser:
1836
+ def __init__(self, f):
1837
+ self.f = f
1838
+ self.attach(f)
1839
+ def attach(self, c):
1840
+ c.get = self.f._go_url
1841
+ c.cdp = self.f.cdp
1842
+ c.init_js = self.f.init_js
1843
+ c.ele = self.f.ele
1844
+ c.eles = self.f.eles
1845
+ c.listen = self.f.listen
1846
+ c.intercept = self.f.intercept
1847
+ c.clear_cache = self.f.clear_cache
1848
+ c.js = self.f.run_js
1849
+ c._js2py = self.f._js2py
1850
+ c.tabs = self.f.root.tabs
1851
+ c.quit = self.f.root.quit
1852
+ c.press = self.f.press
1853
+ c.release = self.f.release
1854
+ c.id = self.f.id
1855
+ c.set_rect = self.f.root.set_rect
1856
+ c.set_fullscreen = self.f.root.set_fullscreen
1857
+ c.set_maxscreen = self.f.root.set_maxscreen
1858
+ c.pic = self.f.pic
1859
+ c.get_dialog = lambda:self.f.page.dialog
1860
+ c.set_dialog = lambda v:setattr(self.f.page, 'dialog', v)
1861
+ c.get_cookies = lambda:self.f._get_cookies()
1862
+ c.set_cookies = lambda v:self.f._set_cookies(v)
1863
+ c.close = self.f.close
1864
+ def __gi(self,a):return self._js2py('window')[a]
1865
+ def __si(self,a,b):self._js2py('window')[a]=b
1866
+ c.__class__.__getitem__ = __gi
1867
+ c.__class__.__setitem__ = __si
1868
+ self.f.root.extension.attach(c)
1869
+ class CookieManager:
1870
+ def __init__(self, f): self.f = f
1871
+ @property
1872
+ def c(self): return self.f.cdp('Network.getCookies', {})['cookies']
1873
+ @property
1874
+ def d(self):
1875
+ dc = {}
1876
+ for d in self.c:
1877
+ dc[d['name']] = d
1878
+ return dc
1879
+ @property
1880
+ def string(self):
1881
+ _r = []
1882
+ for d in self.c:
1883
+ _r.append(d['name'] + '=' + d['value'])
1884
+ return '; '.join(_r)
1885
+ data = property(lambda s:s.c)
1886
+ def __getitem__(self, key): return self.d.get(key, {}).get('value')
1887
+ def __setitem__(self, k, v):
1888
+ s = self.d.get(k)
1889
+ if s:
1890
+ s['value'] = v
1891
+ self.f._set_cookies([s])
1892
+ else:
1893
+ self.f._set_cookies(str(k)+'='+str(v))
1894
+ def __repr__(self): return '<Cookie ['+';'.join(list(self.d.keys()))+']>'
1895
+ class RootFrame:
1896
+ def __init__(self, wsinfo, root, is_auto_create=False):
1897
+ self.root = root
1898
+ self.rootf = self
1899
+ self.wsinfo = wsinfo
1900
+ if not self._check_page(wsinfo): return
1901
+ self.ws = create_connection_saf(wsinfo['webSocketDebuggerUrl'], enable_multithread=True, suppress_origin=True)
1902
+ self.type = wsinfo['type']
1903
+ self.frames = []
1904
+ self._doc_script = []
1905
+ self.url = wsinfo.get('url')
1906
+ self.parent = None
1907
+ self.init_once = False
1908
+ self.logger = Logger(wsinfo['id'], debug)
1909
+ self.frameId = wsinfo['id']
1910
+ self.sessionId = None # Used to maintain consistency with CDPTools objects
1911
+ self.contextId = None # Used to maintain consistency with CDPTools objects
1912
+ self.iso_contextId = None # Used to maintain consistency with CDPTools objects
1913
+ self.cdper = CoreCDP(self.ws, self.root, self.logger)
1914
+ self.cdper.attach(self)
1915
+ if is_auto_create: self._run_init_once()
1916
+ self.root.add_root_frame(self)
1917
+ self.target = Target(self)
1918
+ self.sniff_networt = SniffNetwork(self)
1919
+ self.sniff_fetch = SniffFetch(self)
1920
+ self.cache = Cache(self)
1921
+ self.page = Page(self)
1922
+ self.dom = DOM(self)
1923
+ self.runtime = Runtime(self)
1924
+ self.emulation = Emulation(self)
1925
+ self.tools = CDPTools(self, self)
1926
+ self.closer = Closer(self)
1927
+ self.element = ElementTools(self)
1928
+ self.browser = Browser(self)
1929
+ self.keyboard = Keyboard(self)
1930
+ def id(self, i):
1931
+ return self.ele('#'+ i)
1932
+ def press(self, key): self.cdp('Input.dispatchKeyEvent', self.keyboard._make_down(key))
1933
+ def release(self, key): self.cdp('Input.dispatchKeyEvent', self.keyboard._make_up(key))
1934
+ def close(self):
1935
+ self.cdp('Target.closeTarget', {"targetId": self.frameId})
1936
+ while self.closer.toggle:
1937
+ time.sleep(0.07)
1938
+ def _get_cookies(self, line=False):
1939
+ return CookieManager(self)
1940
+ def _set_cookies(self, v):
1941
+ if type(v) == str:
1942
+ self.tools.run_iso_js('document.cookie='+json.dumps(v))
1943
+ elif type(v) == list:
1944
+ self.cdp('Network.setCookies', {"cookies": v})
1945
+ elif instanceof(v, CookieManager):
1946
+ self.cdp('Network.setCookies', {"cookies": v.data})
1947
+ else:
1948
+ # cookies must be (str/list)
1949
+ # str: document.cookie = 'vvv=123'
1950
+ # list:
1951
+ # [{'domain': '.baidu.com',
1952
+ # 'expires': 1790665876.734538,
1953
+ # 'httpOnly': False,
1954
+ # 'name': 'BIDUPSID',
1955
+ # 'path': '/',
1956
+ # 'priority': 'Medium',
1957
+ # 'sameParty': False,
1958
+ # 'secure': False,
1959
+ # 'session': False,
1960
+ # 'size': 40,
1961
+ # 'sourcePort': 443,
1962
+ # 'sourceScheme': 'Secure',
1963
+ # 'value': '2826F2D60D5E56C841B9E33F82051E8A'},...]
1964
+ # str 作用于当前 url 页面,只能在访问页面后执行。
1965
+ # list 是用于完美还原 cookies 状态,可以在访问页面前执行。
1966
+ raise Exception('cookies must be (str/list/CookieManager)')
1967
+ def _get_remote_list(self):
1968
+ _self = self
1969
+ _rlst = []
1970
+ while _self:
1971
+ if _self.type == 'RemoteFrame':
1972
+ _rlst.append(_self)
1973
+ _self = _self.parent
1974
+ return _rlst
1975
+ def pic(self,):
1976
+ return Screenshot(self.cdp('Page.captureScreenshot', {"format":'png' })['data'])
1977
+ def __repr__(self):
1978
+ return '<[RootFrame]|{}|F:{}>'.format(self.url, self.frameId)
1979
+ def _run_init_once(self):
1980
+ if not self.init_once:
1981
+ self._page_init_js()
1982
+ self.init_once = True
1983
+ def _check_page(self, wsinfo):
1984
+ if (wsinfo['type'] == "service_worker" and wsinfo['url'].startswith('chrome')) \
1985
+ or wsinfo['url'].startswith('devtools') \
1986
+ or wsinfo['type'] != 'page':
1987
+ return False
1988
+ return True
1989
+ def add_sniff_session(self, sessionId):
1990
+ self.sniff_networt.add_listen_session(sessionId)
1991
+ self.sniff_fetch.add_change_session(sessionId)
1992
+ def listen(self, *a,**kw):
1993
+ return self.sniff_networt.listen(*a,**kw)
1994
+ def intercept(self, *a,**kw):
1995
+ return self.sniff_fetch.intercept(*a,**kw)
1996
+ def _go_url(self, url, timeout=None):
1997
+ self.url = url
1998
+ self.iso_contextId = None
1999
+ self._run_init_once()
2000
+ self.cdp("Page.navigate", {"url": url})
2001
+ self.frames = []
2002
+ d_timeout = timeout if timeout != None else 5
2003
+ def check(ret):
2004
+ if self.frameId == ret['frameId']:
2005
+ return True
2006
+ self.wait_once_method("Page.frameStoppedLoading", check_ret_func=check, timeout=d_timeout, rasie_error=False)
2007
+ start = perf_counter()
2008
+ while not timeout:
2009
+ if self.sniff_networt.qlist.empty() and self.sniff_fetch.qlist.empty():
2010
+ break
2011
+ time.sleep(0.1)
2012
+ if perf_counter() - start > 3:
2013
+ break
2014
+ f = self.cdp("Page.getFrameTree")
2015
+ self.url = f['frameTree']['frame']['url']
2016
+ def init_js(self, script):
2017
+ self._doc_script.append(script)
2018
+ def _page_init_js(self, sessionId=None):
2019
+ if self._doc_script:
2020
+ for s in self._doc_script:
2021
+ self.cdp('Page.addScriptToEvaluateOnNewDocument', {"source": s}, sessionId=sessionId)
2022
+ def tree_view(self):
2023
+ r = ''
2024
+ def print_tree(f, level=0, prefix=''):
2025
+ nonlocal r
2026
+ info = str(f)
2027
+ if level == 0:
2028
+ r += info + '\n'
2029
+ else:
2030
+ r += ' ' * (level * 4 - 1) + prefix + info + '\n'
2031
+ for i, item in enumerate(f.frames):
2032
+ if isinstance(item.frames, list):
2033
+ new_prefix = '└──' if i == len(f.frames) - 1 else '├──'
2034
+ print_tree(item, level + 1, new_prefix)
2035
+ print_tree(self)
2036
+ return r.rstrip('\n')
2037
+ class ExtensionConn:
2038
+ def __init__(self, wsinfo, root):
2039
+ self.root = root
2040
+ self.sessionId = None # Used to maintain consistency with CDPTools objects
2041
+ self.contextId = None # Used to maintain consistency with CDPTools objects
2042
+ self.iso_contextId = None # Used to maintain consistency with CDPTools objects
2043
+ self.ws = create_connection_saf(wsinfo['webSocketDebuggerUrl'], enable_multithread=True, suppress_origin=True)
2044
+ self.logger = Logger(wsinfo['id'], debug)
2045
+ self.cdper = CoreCDP(self.ws, self.root, self.logger)
2046
+ self.cdper.attach(self)
2047
+ self.cdp('Runtime.enable')
2048
+ self.tools = CDPTools(self, self)
2049
+ def set_proxy(self, p=None, method='PROXY', script=None):
2050
+ if p:
2051
+ func_script = script or '''
2052
+ function FindProxyForURL(url, host) {
2053
+ return "'''+method+''' '''+p+'''";
2054
+ }'''
2055
+ spt = json.dumps('data:text/plain;base64,' + base64.b64encode((func_script+'''
2056
+ // '''+str(int(time.time()*1000))+'''
2057
+ ''').encode()).decode())
2058
+ cfgscript = '''{
2059
+ mode: 'pac_script',
2060
+ pacScript: { url: '''+spt+''' }
2061
+ }'''
2062
+ else:
2063
+ cfgscript = '{mode: "system"}'
2064
+ finscript = '''
2065
+ new Promise(function(r1,r2){
2066
+ chrome.proxy.settings.set({
2067
+ value: '''+cfgscript+''',
2068
+ scope: 'regular', }, function(e){ r1(e) })
2069
+ })
2070
+ '''
2071
+ err = self.run_js(finscript)
2072
+ if err:
2073
+ raise Exception('set proxy:' + json.dumps(err))
2074
+ class ExtensionManager:
2075
+ def __init__(self, root):
2076
+ self.root = root
2077
+ self.e = None
2078
+ def attach(self, c):
2079
+ c.set_proxy = self.set_proxy
2080
+ def set_proxy(self, p=None, method='PROXY', script=None):
2081
+ # Usually there is no need to configure script, script is actually an emergency backup parameter
2082
+ # Perhaps in some cases, someone may need to customize and configure some bypass lists or handle some special proxy modes.
2083
+ t = self.root.rootchrome.new_tab(False)
2084
+ t.get(wvurl)
2085
+ t.close()
2086
+ while not self.e:
2087
+ time.sleep(0.1)
2088
+ self.e.set_proxy(p=p, method=method, script=script)
2089
+ def set_extension(self, e):
2090
+ if self.e:
2091
+ self.e.tools.is_running = False
2092
+ self.e.ws.close()
2093
+ self.e = e
2094
+ class Root:
2095
+ def __init__(self, verison, iframes):
2096
+ self.pool = Pool(10)
2097
+ self.id = 0
2098
+ self.verison = verison
2099
+ self.iframes = iframes
2100
+ self.active = None
2101
+ self.new_tab_frame = {}
2102
+ self.create_tab_must_be_single = queue.Queue(1)
2103
+ self.tabs = []
2104
+ self.is_init = True
2105
+ self.init_sign = 0
2106
+ self.ws = create_connection_saf(verison['webSocketDebuggerUrl'], enable_multithread=True, suppress_origin=True)
2107
+ self.logger = Logger('[[ ---------- ROOT ---------- ]]', debug)
2108
+ self.cdper = CoreCDP(self.ws, self, self.logger)
2109
+ self.cdper.attach(self)
2110
+ self.extension = ExtensionManager(self)
2111
+ self.extension.set_extension(self._get_extension())
2112
+ self.cdp("Target.setAutoAttach", {
2113
+ "autoAttach": True,
2114
+ "waitForDebuggerOnStart": False,
2115
+ "flatten": True,
2116
+ })
2117
+ self.set_method_callback('Target.attachedToTarget', self.Target_attachedToTarget)
2118
+ # self.cdp("Target.setDiscoverTargets", { "discover": True })
2119
+ def filter_extension(self, tinfo):
2120
+ return tinfo['url'].startswith('chrome-extension://') and tinfo['url'].endswith('/vvv.js')
2121
+ def _wid(self):
2122
+ return self.cdp('Browser.getWindowForTarget', {"targetId":self.tabs[0].frameId})['windowId']
2123
+ def set_rect(self, rect):
2124
+ self.cdp('Browser.setWindowBounds', {"windowId":self._wid(),"bounds":{"left":rect[0],"top":rect[1],"width":rect[2],"height":rect[3]}})
2125
+ def set_fullscreen(self, tg=True):
2126
+ self.cdp('Browser.setWindowBounds', {'windowId':self._wid(),'bounds':{'windowState':'fullscreen' if tg else 'normal'}})
2127
+ def set_maxscreen(self, tg=True):
2128
+ self.cdp('Browser.setWindowBounds', {'windowId':self._wid(),'bounds':{'windowState':'maximized' if tg else 'normal'}})
2129
+ def _new_driver(self, background):
2130
+ tid = self.cdp('Target.createTarget', {"url": "about:blank", "background": background})['targetId']
2131
+ tgap = 0.03
2132
+ while True:
2133
+ if self.new_tab_frame.get(tid):
2134
+ break
2135
+ time.sleep(tgap)
2136
+ tgap = tgap * 1.3
2137
+ return self.new_tab_frame.pop(tid)
2138
+ def quit(self):
2139
+ self.cdp('Browser.close')
2140
+ def _get_extension(self):
2141
+ for tinfo in self.cdp('Target.getTargets')['targetInfos']:
2142
+ if self.filter_extension(tinfo):
2143
+ return ExtensionConn({
2144
+ 'webSocketDebuggerUrl': make_dev_page_url(tinfo['targetId']),
2145
+ 'id': tinfo['targetId'],
2146
+ 'type': tinfo['type'],
2147
+ 'url': tinfo['url'],
2148
+ }, self)
2149
+ def Target_attachedToTarget(self, rdata):
2150
+ params = rdata['params']
2151
+ sessionId = params.get('sessionId')
2152
+ targetInfo = params.get('targetInfo')
2153
+ url = targetInfo.get('url')
2154
+ d = {
2155
+ 'webSocketDebuggerUrl': make_dev_page_url(targetInfo['targetId']),
2156
+ 'id': targetInfo['targetId'],
2157
+ 'type': targetInfo['type'],
2158
+ 'url': url,
2159
+ }
2160
+ if url.startswith('chrome-extension://') and url.endswith('/vvv.js'):
2161
+ self.extension.set_extension(ExtensionConn(d, self))
2162
+ return
2163
+ # TODO
2164
+ # now, Page.addScriptToEvaluateOnNewDocument not fast. cannot work on click open url.
2165
+ f = RootFrame(d, self, is_auto_create=True)
2166
+ if targetInfo.get('openerFrameId'):
2167
+ # TODO
2168
+ # think about how to manager this. bind root? or rootframe?
2169
+ pass
2170
+ else:
2171
+ self.new_tab_frame[targetInfo['targetId']] = f
2172
+ self.cdp('Runtime.runIfWaitingForDebugger', sessionId=sessionId)
2173
+ self.cdp('Target.detachFromTarget', sessionId=sessionId)
2174
+ def _add_init_check(self):
2175
+ if self.is_init:
2176
+ self.init_sign += 1
2177
+ def _del_init_check(self):
2178
+ if self.is_init:
2179
+ self.init_sign -= 1
2180
+ def get_new_id(self):
2181
+ self.id += 1
2182
+ return self.id
2183
+ def add_root_frame(self, f):
2184
+ if not self.active and f.type == 'page':
2185
+ self.active = f
2186
+ self.tabs.append(f)
2187
+ self.trav_init_tree(f)
2188
+ self.trav_remote_tree(f)
2189
+ def add_common_frame(self, rootf, finfo, sessionId=None):
2190
+ f = self.trav_frame(finfo['frameId'])
2191
+ if f:
2192
+ if finfo.get("contextId"): f.contextId = finfo.get("contextId")
2193
+ if finfo.get("uniqueId"): f.uniqueId = finfo.get("uniqueId")
2194
+ if finfo.get('sessionId'): f.sessionId = finfo.get('sessionId')
2195
+ f.frameId = finfo['frameId']
2196
+ else:
2197
+ pf = finfo.get('parent')
2198
+ if not pf:
2199
+ # 初始化阶段,frame 并没有进去对象管理时才可能走的分支
2200
+ return
2201
+ lf = AbsFrame(rootf, {
2202
+ "contextId": finfo.get("contextId"),
2203
+ "uniqueId": finfo.get("uniqueId"),
2204
+ "frameId": finfo['frameId'],
2205
+ "sessionId": getattr(pf, 'sessionId', None),
2206
+ "parent": pf,
2207
+ })
2208
+ pf.frames.append(lf)
2209
+ def trav_remote_tree(self, pf):
2210
+ for f in self.iframes:
2211
+ if f['parentId'] == pf.frameId:
2212
+ lf = AbsFrame(pf, {
2213
+ "frameId": f['id'],
2214
+ "parent": pf,
2215
+ "url": f['url'],
2216
+ })
2217
+ pf.frames.append(lf)
2218
+ def trav_init_tree(self, f, rootf=None, sessionId=None):
2219
+ t = f.cdp('Page.getFrameTree')
2220
+ def _trav(t, pf, rootf):
2221
+ if 'frame' in t:
2222
+ url = t['frame']['url']
2223
+ frameId = t['frame']['id']
2224
+ if frameId != f.frameId:
2225
+ lf = AbsFrame(rootf, {
2226
+ "url": url,
2227
+ "contextId": None,
2228
+ "uniqueId": None,
2229
+ "sessionId": sessionId,
2230
+ "frameId": frameId,
2231
+ "parent": pf,
2232
+ })
2233
+ pf.frames.append(lf)
2234
+ pf = lf
2235
+ if 'childFrames' in t:
2236
+ for i in t['childFrames']:
2237
+ _trav(i, pf, rootf)
2238
+ _trav(t['frameTree'], f, rootf or f)
2239
+ if not rootf and not sessionId:
2240
+ f.url = t['frameTree']['frame']['url']
2241
+ def trav_frame(self, kdata, key='frameId'):
2242
+ def _trav(flist):
2243
+ for f in flist:
2244
+ if getattr(f, key, None) == kdata:
2245
+ return f
2246
+ else:
2247
+ r = _trav(f.frames)
2248
+ if r:
2249
+ return r
2250
+ return _trav(self.tabs)
2251
+ def flat_child_frames(self, flist=None):
2252
+ flst = []
2253
+ def _trav(flist):
2254
+ for f in flist:
2255
+ flst.append(f)
2256
+ _trav(f.frames)
2257
+ _trav(flist)
2258
+ return flst
2259
+ wurl = 'http://{}:{}/json'.format(hostname, port)
2260
+ wvurl = 'http://{}:{}/json/version'.format(hostname, port)
2261
+ wslist = myget(wurl)
2262
+ for idx in range(len(wslist)):
2263
+ wslist[idx]['webSocketDebuggerUrl'] = adj_wsurl(wslist[idx]['webSocketDebuggerUrl'])
2264
+ verison = myget(wvurl)
2265
+ verison['webSocketDebuggerUrl'] = adj_wsurl(verison['webSocketDebuggerUrl'])
2266
+ iframes = [i for i in wslist if i['type'] == 'iframe']
2267
+ root = Root(verison, iframes)
2268
+ for wsinfo in wslist: RootFrame(wsinfo, root)
2269
+ start = perf_counter()
2270
+ while root.init_sign:
2271
+ if perf_counter() - start > 2.5:
2272
+ break
2273
+ time.sleep(0.1)
2274
+ root.is_init = False
2275
+ return root
2276
+ # ----------------------------------------------------------------------------------------------------
2277
+ import sys
2278
+ import re
2279
+ import time
2280
+ import json
2281
+ import shutil
2282
+ import socket
2283
+ import hashlib
2284
+ import platform
2285
+ from os import environ, path, cpu_count, getenv
2286
+ from platform import system
2287
+ from tempfile import gettempdir
2288
+ from pathlib import Path
2289
+ from shutil import rmtree
2290
+ from subprocess import Popen, DEVNULL
2291
+ def find_chrome(path):
2292
+ def get_win_chrome_path():
2293
+ d = Path('C:\\Program Files\\Google\\Chrome\\Application')
2294
+ if d.exists() and (d / 'chrome.exe').exists():
2295
+ return d
2296
+ import os, winreg
2297
+ sub_key = [
2298
+ 'SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall',
2299
+ 'SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall'
2300
+ ]
2301
+ def get_install_list(key, root):
2302
+ try:
2303
+ _key = winreg.OpenKey(root, key, 0, winreg.KEY_ALL_ACCESS)
2304
+ for j in range(0, winreg.QueryInfoKey(_key)[0]-1):
2305
+ try:
2306
+ each_key = winreg.OpenKey(root, key + '\\' + winreg.EnumKey(_key, j), 0, winreg.KEY_ALL_ACCESS)
2307
+ displayname, REG_SZ = winreg.QueryValueEx(each_key, 'DisplayName')
2308
+ install_loc, REG_SZ = winreg.QueryValueEx(each_key, 'InstallLocation')
2309
+ display_var, REG_SZ = winreg.QueryValueEx(each_key, 'DisplayVersion')
2310
+ yield displayname, install_loc, display_var
2311
+ except WindowsError:
2312
+ pass
2313
+ except:
2314
+ pass
2315
+ for key in sub_key:
2316
+ for root in [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]:
2317
+ for name, local, var in get_install_list(key, root):
2318
+ if name == 'Google Chrome':
2319
+ return Path(local)
2320
+ if path:
2321
+ if Path(path).exists():
2322
+ return Path(path)
2323
+ return None
2324
+ sys = system().lower()
2325
+ if sys in ('macos', 'darwin', 'linux'):
2326
+ for p in (
2327
+ '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
2328
+ '/usr/bin/google-chrome',
2329
+ '/opt/google/chrome/google-chrome',
2330
+ '/user/lib/chromium-browser/chromium-browser'
2331
+ ):
2332
+ if Path(p).exists():
2333
+ return p
2334
+ return None
2335
+ elif sys != 'windows':
2336
+ return None
2337
+ return get_win_chrome_path()
2338
+ def try_some(func, times=7, timegap=0.15):
2339
+ for i in range(times):
2340
+ try:
2341
+ func()
2342
+ return False
2343
+ except:
2344
+ time.sleep(timegap)
2345
+ continue
2346
+ return True
2347
+ def find_free_port(start_port=9233, end_port=60000):
2348
+ class EnvPRNG:
2349
+ def __init__(self):
2350
+ self.seed = int(self.get_env_fingerprint(), 16)
2351
+ self.state = self.seed
2352
+ def get_env_fingerprint(self):
2353
+ try:
2354
+ system = platform.system()
2355
+ release = platform.release()
2356
+ version = platform.version()
2357
+ arch = platform.machine()
2358
+ python_version = platform.python_version()
2359
+ cpu_n = cpu_count()
2360
+ raw = f"{system}-{release}-{version}-{arch}-{python_version}-{cpu_n}-{getenv('USERNAME','')}-{getenv('USER','')}"
2361
+ except:
2362
+ raw = "vvv"
2363
+ return hashlib.sha256(raw.encode('utf-8')).hexdigest()
2364
+ def rand(self):
2365
+ self.state = (1664525 * self.state + 1013904223) % (2**32)
2366
+ return self.state
2367
+ def randint(self, a, b):
2368
+ return a + self.rand() % (b - a + 1)
2369
+ erdn = EnvPRNG()
2370
+ for _ in range(200):
2371
+ try:
2372
+ port = erdn.randint(start_port, end_port)
2373
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
2374
+ s.bind(('0.0.0.0', port))
2375
+ return port
2376
+ except OSError:
2377
+ continue
2378
+ raise ValueError("no free port ({}-{})".format(start_port, end_port))
2379
+ def is_port_open(port, host="127.0.0.1", timeout=0.01):
2380
+ try:
2381
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
2382
+ s.settimeout(timeout)
2383
+ return s.connect_ex((host, port)) == 0
2384
+ except socket.timeout:
2385
+ return False
2386
+ def get_default_user_data_dir():
2387
+ home = path.expanduser("~")
2388
+ if sys.platform.startswith("win"):
2389
+ return path.join(home, "AppData", "Local", "Google", "Chrome", "User Data")
2390
+ elif sys.platform == "darwin":
2391
+ return path.join(home, "Library", "Application Support", "Google", "Chrome")
2392
+ elif sys.platform.startswith("linux"):
2393
+ return path.join(home, ".config", "google-chrome")
2394
+ else:
2395
+ raise Exception("Unsupported platform")
2396
+ class Debug:
2397
+ def __init__(self, debug):
2398
+ self.all = False
2399
+ self.plist = [
2400
+ 'env','rest','mouse','proxy',
2401
+ 'Network','Fetch','Page','DOM','Target','Runtime','Input','Debugger','DOMDebugger',
2402
+ ]
2403
+ if type(debug) == str: self.parse_str(debug)
2404
+ if type(debug) == bool: self.parse_bool(debug)
2405
+ self._bool = self.has_Debug()
2406
+ def __call__(self, tp):
2407
+ if self.all:
2408
+ return True
2409
+ if type(tp) == str:
2410
+ return getattr(self, tp.split('.')[0], None)
2411
+ def __bool__(self):
2412
+ return self._bool
2413
+ def parse_str(self, debug):
2414
+ self.parse_bool(False)
2415
+ dbgls = re.split(r'[,;|/]', debug)
2416
+ for p in dbgls:
2417
+ if p in self.plist:
2418
+ setattr(self, p, True)
2419
+ def parse_bool(self, debug):
2420
+ for p in self.plist: setattr(self, p, debug)
2421
+ self.all = debug
2422
+ def has_Debug(self):
2423
+ tg = False
2424
+ for p in self.plist:
2425
+ if getattr(self, p):
2426
+ tg = True
2427
+ return tg
2428
+ class Chrome:
2429
+ def __init__(self,
2430
+ debug = False,
2431
+ path = None,
2432
+ port = None,
2433
+ hostname = '127.0.0.1',
2434
+ user_path = None,
2435
+ use_system_user_path = False,
2436
+ verison_check = True,
2437
+ headless = False,
2438
+ incognito = False,
2439
+ user_agent = None,
2440
+ arguments = [],
2441
+ proxy = None,
2442
+ extension = None,
2443
+ ):
2444
+ self.debug = Debug(debug)
2445
+ self.path = find_chrome(path)
2446
+ self.verison_check = verison_check
2447
+ self.headless = headless
2448
+ self.user_agent = user_agent
2449
+ self.incognito = incognito
2450
+ self.arguments = arguments
2451
+ self.proxy = proxy
2452
+ self.extension = extension
2453
+ self._user_cmd = self._make_user_cmd()
2454
+ if not self.path:
2455
+ raise Exception('chrome path not find.')
2456
+ self.port = port or find_free_port()
2457
+ self.hostname = hostname
2458
+ self.user_path_mode = None
2459
+ # chrome 136+
2460
+ # disable default user path to remote contral
2461
+ if use_system_user_path:
2462
+ self.user_path_mode = 'sys'
2463
+ user_path = Path(get_default_user_data_dir())
2464
+ if user_path:
2465
+ self.user_path_mode = 'user'
2466
+ user_path = Path(user_path)
2467
+ if not user_path.exists() or not user_path.is_dir():
2468
+ raise Exception('user path not exist.')
2469
+ else:
2470
+ self.user_path_mode = 'default'
2471
+ if self.debug.env:
2472
+ print('[*]', 'user_path_mode', self.user_path_mode)
2473
+ self.user_path = user_path or Path(gettempdir()) / 'vchrome' / 'cache_user_temp' / 'userData' / str(self.port)
2474
+ if self.debug.env:
2475
+ print('[*]', self.user_path)
2476
+ self._connect()
2477
+ def _cmd_check_in(self, ls, s):
2478
+ for i in ls:
2479
+ if i.startswith(s.split('=')[0]):
2480
+ return i, s
2481
+ def _make_user_cmd(self):
2482
+ _user_cmd = self.arguments
2483
+ if self.headless and not self._cmd_check_in(_user_cmd, '--headless'):
2484
+ # TODO
2485
+ # think about CDP:Input Event compatible.
2486
+ _user_cmd.append('--headless')
2487
+ if self.incognito and not self._cmd_check_in(_user_cmd, '--incognito'):
2488
+ _user_cmd.append('--incognito')
2489
+ if self.user_agent and not self._cmd_check_in(_user_cmd, '--user-agent'):
2490
+ _user_cmd.append('--user-agent=' + json.dumps(self.user_agent))
2491
+ if self.proxy and not self._cmd_check_in(_user_cmd, '--proxy-server'):
2492
+ _user_cmd.append('--proxy-server=' + self.proxy)
2493
+ if self.extension:
2494
+ if type(self.extension) == str:
2495
+ ep = self.extension
2496
+ _user_cmd.append('--disable-extensions-except=' + ep)
2497
+ _user_cmd.append('--load-extension=' + ep)
2498
+ if type(self.extension) == list:
2499
+ for ep in self.extension:
2500
+ _user_cmd.append('--disable-extensions-except=' + ep)
2501
+ _user_cmd.append('--load-extension=' + ep)
2502
+ return _user_cmd
2503
+ def new_tab(self, foreground=True):
2504
+ class Chrome:
2505
+ def __init__(self, root, dr):
2506
+ self.root = root
2507
+ self.dr = dr
2508
+ self.dr.browser.attach(self)
2509
+ def __str__(self):
2510
+ return str(self.dr.tree_view())
2511
+ def new_tab(self, foreground=True):
2512
+ return Chrome(self.root, self.dr.root._new_driver(not foreground))
2513
+ frames = property(lambda s:s.dr.frames)
2514
+ cookies = property(lambda s:s.get_cookies(), lambda s,v:s.set_cookies(v))
2515
+ dialog = property(lambda s:s.get_dialog(), lambda s,v:s.set_dialog(v))
2516
+ return Chrome(self.root, self.dr.root._new_driver(not foreground))
2517
+ frames = property(lambda s:s.dr.frames)
2518
+ cookies = property(lambda s:s.get_cookies(), lambda s,v:s.set_cookies(v))
2519
+ dialog = property(lambda s:s.get_dialog(), lambda s,v:s.set_dialog(v))
2520
+ def _check_lower_version(self):
2521
+ try:
2522
+ verison = self.dr.root.verison
2523
+ if verison and verison.get('Browser'):
2524
+ ver = int(verison.get('Browser').split('/')[-1].split('.')[0])
2525
+ # Restricting Chrome versions to prevent abnormal states
2526
+ if ver < 100:
2527
+ return True
2528
+ except: pass
2529
+ return False
2530
+ def _connect(self):
2531
+ if not is_port_open(self.port):
2532
+ self._init()
2533
+ self.root = cdp_client(self.hostname, port=self.port, debug=self.debug)
2534
+ self.root.rootchrome = self
2535
+ self.dr = self.root.active
2536
+ if not self.dr:
2537
+ raise Exception('maybe single devtools not close.')
2538
+ self.dr.browser.attach(self)
2539
+ if self.verison_check and self._check_lower_version():
2540
+ raise Exception('chrome verison is less then 100. not reliable. you can set (verison_check=False) for ignore this alert.')
2541
+ if self.debug.mouse:
2542
+ jscode = r'''
2543
+ function f(e){
2544
+ var nDiv = document.createElement('div')
2545
+ var e = e || window.event
2546
+ Object.assign(nDiv.style, {
2547
+ position: 'fixed',
2548
+ left: `${e.clientX+1}px`,
2549
+ top: `${e.clientY+1}px`,
2550
+ width: '5px',
2551
+ height: '5px',
2552
+ backgroundColor: 'red',
2553
+ borderRadius: '50%',
2554
+ pointerEvents: 'none', // 禁止鼠标交互
2555
+ userSelect: 'none', // 禁止选中
2556
+ webkitUserSelect: 'none', // Safari 兼容
2557
+ mozUserSelect: 'none', // Firefox 兼容
2558
+ msUserSelect: 'none', // IE/Edge 兼容
2559
+ zIndex: '2147483647', // 最大 z-index 确保置顶
2560
+ willChange: 'transform', // 优化渲染性能
2561
+ });
2562
+ document.body.appendChild(nDiv)
2563
+ setTimeout(function(){ nDiv.remove(); },1000)
2564
+ }
2565
+ document.addEventListener('mousemove', f, true)
2566
+ document.addEventListener('mousedown', f, true)
2567
+ document.addEventListener('mouseup', f, true)
2568
+ '''
2569
+ self.init_js(jscode)
2570
+ def _merge_config(self, cmd):
2571
+ # The Chrome command line is sequence sensitive and comes with conflict resolution. I just need to add my new command at the end. (Tail instruction priority)
2572
+ return cmd[:-1] + self._user_cmd + cmd[-1:]
2573
+ def _make_files_extension(self, path):
2574
+ def writeF(p, s):
2575
+ with open(p, 'w', encoding='utf8') as f:
2576
+ f.write(s)
2577
+ e = path / 'v_extension'
2578
+ e.mkdir(parents=True, exist_ok=True)
2579
+ d = {
2580
+ "name": "vvv", "version": "0.0.0", "description": "vvv",
2581
+ "permissions": [ "proxy" ],
2582
+ "background": { "service_worker": "vvv.js" },
2583
+ "content_scripts": [{
2584
+ "matches": ["<all_urls>"],
2585
+ "js": ["content_script.js"],
2586
+ "run_at": "document_idle"
2587
+ }],
2588
+ "host_permissions": [ "<all_urls>" ],
2589
+ "manifest_version": 3
2590
+ }
2591
+ writeF(e / 'manifest.json', json.dumps(d))
2592
+ writeF(e / 'content_script.js', '''
2593
+ var port = chrome.runtime.connect({ name: 'keepAlive' });
2594
+ port.postMessage({ ping: Date.now() })
2595
+ ''')
2596
+ writeF(e / 'vvv.js', '''
2597
+ var ctime;
2598
+ chrome.runtime.onConnect.addListener((port) => {
2599
+ if (port.name === 'keepAlive') { port.onMessage.addListener((msg) => { ctime = msg }); }
2600
+ });
2601
+ ''')
2602
+ return e
2603
+ def _prepare_cmd(self):
2604
+ # TODO
2605
+ # maybe need check writable compatible.
2606
+ path = self.path
2607
+ user_path = self.user_path
2608
+ port = self.port
2609
+ user_path_mode = self.user_path_mode
2610
+ p = Path(path)
2611
+ p = str(p / 'chrome') if p.is_dir() else str(path)
2612
+ u = '--user-data-dir='
2613
+ args = []
2614
+ if user_path_mode == 'sys':
2615
+ args.append(u + str(user_path))
2616
+ if user_path_mode == 'user':
2617
+ args.append(u + str(user_path))
2618
+ if user_path_mode == 'default':
2619
+ if user_path.exists():
2620
+ if try_some(lambda:shutil.rmtree(user_path)):
2621
+ print('[*] error remove cache', user_path)
2622
+ user_path.mkdir(parents=True, exist_ok=True)
2623
+ args.append(u + str(user_path))
2624
+ ep = self._make_files_extension(user_path)
2625
+ # args.append('--disable-extensions-except=' + str(ep))
2626
+ args.append('--load-extension=' + str(ep))
2627
+ return [
2628
+ p,
2629
+ '--remote-debugging-port=' + str(port),
2630
+ '--no-default-browser-check',
2631
+ '--disable-suggestions-ui',
2632
+ '--no-first-run',
2633
+ '--disable-infobars',
2634
+ '--disable-popup-blocking',
2635
+ '--hide-crash-restore-bubble',
2636
+ '--remote-allow-origins=*',
2637
+ '--enable-features=NetworkService',
2638
+ '--enable-features=NetworkServiceInProcess',
2639
+ '--disable-features=PaymentRequest',
2640
+ '--disable-features=DigitalGoodsApi',
2641
+ '--disable-features=PrivacySandboxSettings4',
2642
+ '--disable-features=DisableLoadExtensionCommandLineSwitch', # Starting from a higher version, the command line to load plugins has been disabled. This line needs to be added
2643
+ '--disable-component-extensions-with-background-pages',
2644
+ # # Disable automatic password saving prompt, the new version is invalid
2645
+ # '--password-store=basic',
2646
+ # # Disable automatic password saving prompt, the new version is invalid
2647
+ # '--disable-features=AutofillEnableSavePasswordBubble',
2648
+ # '--disable-features=PasswordManagerOnboarding',
2649
+ # '--disable-features=PasswordImport',
2650
+ # '--disable-features=PasswordManagerRedesign',
2651
+ # # Disable automatic password saving prompt, the new version is invalid
2652
+ # '--disable-save-password-bubble',
2653
+ # '--disable-autofill',
2654
+ # '--disable-password-manager-reauthentication',
2655
+ # '--disable-autofill-keyboard-accessory-view',
2656
+ # from pyppeteer
2657
+ '--disable-background-networking',
2658
+ '--disable-background-timer-throttling',
2659
+ '--disable-breakpad',
2660
+ '--disable-browser-side-navigation',
2661
+ '--disable-client-side-phishing-detection',
2662
+ '--disable-default-apps',
2663
+ '--disable-dev-shm-usage',
2664
+ '--disable-hang-monitor',
2665
+ '--disable-prompt-on-repost',
2666
+ '--disable-sync',
2667
+ '--disable-translate',
2668
+ '--metrics-recording-only',
2669
+ '--no-first-run',
2670
+ '--safebrowsing-disable-auto-update',
2671
+ # '--disable-component-update',
2672
+ # '--site-per-process',
2673
+ # '--disable-extensions',
2674
+ # '--disable-web-security',
2675
+ *args,
2676
+ 'about:blank',
2677
+ # # This part may be related to performance
2678
+ # '--disable-extensions',
2679
+ # '--disable-gpu',
2680
+ # '--no-sandbox',
2681
+ # '--disable-dev-shm-usage',
2682
+ ]
2683
+ def _init(self):
2684
+ def _start_browser(cmd):
2685
+ try:
2686
+ return Popen(cmd, shell=False, stdout=DEVNULL, stderr=DEVNULL)
2687
+ except FileNotFoundError:
2688
+ raise FileNotFoundError('browser not find.')
2689
+ cmd = self._prepare_cmd()
2690
+ cmd = self._merge_config(cmd)
2691
+ if self.debug.env:
2692
+ print('[*]', json.dumps(cmd, indent=4))
2693
+ _start_browser(cmd)
2694
+ def __str__(self):
2695
+ return str(self.dr.tree_view())
2696
+ # ----------------------------------------------------------------------------------------------------
2697
+ """
2698
+ websocket - WebSocket client library for Python
2699
+
2700
+ Copyright 2024 engn33r
2701
+
2702
+ Licensed under the Apache License, Version 2.0 (the "License");
2703
+ you may not use this file except in compliance with the License.
2704
+ You may obtain a copy of the License at
2705
+
2706
+ http://www.apache.org/licenses/LICENSE-2.0
2707
+
2708
+ Unless required by applicable law or agreed to in writing, software
2709
+ distributed under the License is distributed on an "AS IS" BASIS,
2710
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2711
+ See the License for the specific language governing permissions and
2712
+ limitations under the License.
2713
+ """
2714
+ # websocket-client:1.8.0
2715
+ import array
2716
+ import os
2717
+ import struct
2718
+ import sys
2719
+ from threading import Lock
2720
+ from typing import Callable, Optional, Union
2721
+ class WebSocketException(Exception):pass
2722
+ class WebSocketProtocolException(WebSocketException):pass
2723
+ class WebSocketPayloadException(WebSocketException):pass
2724
+ class WebSocketConnectionClosedException(WebSocketException):pass
2725
+ class WebSocketTimeoutException(WebSocketException):pass
2726
+ class WebSocketProxyException(WebSocketException):pass
2727
+ class WebSocketBadStatusException(WebSocketException):
2728
+ def __init__(self, message: str, status_code: int, status_message=None, resp_headers=None, resp_body=None):
2729
+ super().__init__(message)
2730
+ self.status_code = status_code
2731
+ self.resp_headers = resp_headers
2732
+ self.resp_body = resp_body
2733
+ class WebSocketAddressException(WebSocketException):pass
2734
+ from typing import Union
2735
+ class NoLock:
2736
+ def __enter__(self) -> None:pass
2737
+ def __exit__(self, exc_type, exc_value, traceback) -> None:pass
2738
+ try:
2739
+ from wsaccel.utf8validator import Utf8Validator
2740
+ def _validate_utf8(utfbytes: Union[str, bytes]) -> bool:
2741
+ result: bool = Utf8Validator().validate(utfbytes)[0]
2742
+ return result
2743
+ except ImportError:
2744
+ _UTF8_ACCEPT = 0
2745
+ _UTF8_REJECT = 12
2746
+ _UTF8D = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]
2747
+ def _decode(state: int, codep: int, ch: int) -> tuple:
2748
+ tp = _UTF8D[ch]
2749
+ codep = ch & 63 | codep << 6 if state != _UTF8_ACCEPT else 255 >> tp & ch
2750
+ state = _UTF8D[256 + state + tp]
2751
+ return (state, codep)
2752
+ def _validate_utf8(utfbytes: Union[str, bytes]) -> bool:
2753
+ state = _UTF8_ACCEPT
2754
+ codep = 0
2755
+ for i in utfbytes:
2756
+ (state, codep) = _decode(state, codep, int(i))
2757
+ if state == _UTF8_REJECT:
2758
+ return False
2759
+ return True
2760
+ def validate_utf8(utfbytes: Union[str, bytes]) -> bool:
2761
+ return _validate_utf8(utfbytes)
2762
+ def extract_err_message(exception: Exception) -> Union[str, None]:
2763
+ if exception.args:
2764
+ exception_message: str = exception.args[0]
2765
+ return exception_message
2766
+ else:
2767
+ return None
2768
+ def extract_error_code(exception: Exception) -> Union[int, None]:
2769
+ if exception.args and len(exception.args) > 1:
2770
+ return exception.args[0] if isinstance(exception.args[0], int) else None
2771
+ try:
2772
+ from wsaccel.xormask import XorMaskerSimple
2773
+ def _mask(mask_value: array.array, data_value: array.array) -> bytes:
2774
+ mask_result: bytes = XorMaskerSimple(mask_value).process(data_value)
2775
+ return mask_result
2776
+ except ImportError:
2777
+ native_byteorder = sys.byteorder
2778
+ def _mask(mask_value: array.array, data_value: array.array) -> bytes:
2779
+ datalen = len(data_value)
2780
+ int_data_value = int.from_bytes(data_value, native_byteorder)
2781
+ int_mask_value = int.from_bytes(mask_value * (datalen // 4) + mask_value[:datalen % 4], native_byteorder)
2782
+ return (int_data_value ^ int_mask_value).to_bytes(datalen, native_byteorder)
2783
+ STATUS_NORMAL = 1000
2784
+ STATUS_GOING_AWAY = 1001
2785
+ STATUS_PROTOCOL_ERROR = 1002
2786
+ STATUS_UNSUPPORTED_DATA_TYPE = 1003
2787
+ STATUS_STATUS_NOT_AVAILABLE = 1005
2788
+ STATUS_ABNORMAL_CLOSED = 1006
2789
+ STATUS_INVALID_PAYLOAD = 1007
2790
+ STATUS_POLICY_VIOLATION = 1008
2791
+ STATUS_MESSAGE_TOO_BIG = 1009
2792
+ STATUS_INVALID_EXTENSION = 1010
2793
+ STATUS_UNEXPECTED_CONDITION = 1011
2794
+ STATUS_SERVICE_RESTART = 1012
2795
+ STATUS_TRY_AGAIN_LATER = 1013
2796
+ STATUS_BAD_GATEWAY = 1014
2797
+ STATUS_TLS_HANDSHAKE_ERROR = 1015
2798
+ VALID_CLOSE_STATUS = (STATUS_NORMAL, STATUS_GOING_AWAY, STATUS_PROTOCOL_ERROR, STATUS_UNSUPPORTED_DATA_TYPE, STATUS_INVALID_PAYLOAD, STATUS_POLICY_VIOLATION, STATUS_MESSAGE_TOO_BIG, STATUS_INVALID_EXTENSION, STATUS_UNEXPECTED_CONDITION, STATUS_SERVICE_RESTART, STATUS_TRY_AGAIN_LATER, STATUS_BAD_GATEWAY)
2799
+ class ABNF:
2800
+ OPCODE_CONT = 0
2801
+ OPCODE_TEXT = 1
2802
+ OPCODE_BINARY = 2
2803
+ OPCODE_CLOSE = 8
2804
+ OPCODE_PING = 9
2805
+ OPCODE_PONG = 10
2806
+ OPCODES = (OPCODE_CONT, OPCODE_TEXT, OPCODE_BINARY, OPCODE_CLOSE, OPCODE_PING, OPCODE_PONG)
2807
+ OPCODE_MAP = {OPCODE_CONT: 'cont', OPCODE_TEXT: 'text', OPCODE_BINARY: 'binary', OPCODE_CLOSE: 'close', OPCODE_PING: 'ping', OPCODE_PONG: 'pong'}
2808
+ LENGTH_7 = 126
2809
+ LENGTH_16 = 1 << 16
2810
+ LENGTH_63 = 1 << 63
2811
+ def __init__(self, fin: int=0, rsv1: int=0, rsv2: int=0, rsv3: int=0, opcode: int=OPCODE_TEXT, mask_value: int=1, data: Union[str, bytes, None]='') -> None:
2812
+ self.fin = fin
2813
+ self.rsv1 = rsv1
2814
+ self.rsv2 = rsv2
2815
+ self.rsv3 = rsv3
2816
+ self.opcode = opcode
2817
+ self.mask_value = mask_value
2818
+ if data is None:
2819
+ data = ''
2820
+ self.data = data
2821
+ self.get_mask_key = os.urandom
2822
+ def validate(self, skip_utf8_validation: bool=False) -> None:
2823
+ if self.rsv1 or self.rsv2 or self.rsv3:
2824
+ raise WebSocketProtocolException('rsv is not implemented, yet')
2825
+ if self.opcode not in ABNF.OPCODES:
2826
+ raise WebSocketProtocolException('Invalid opcode %r', self.opcode)
2827
+ if self.opcode == ABNF.OPCODE_PING and (not self.fin):
2828
+ raise WebSocketProtocolException('Invalid ping frame.')
2829
+ if self.opcode == ABNF.OPCODE_CLOSE:
2830
+ l = len(self.data)
2831
+ if not l:
2832
+ return
2833
+ if l == 1 or l >= 126:
2834
+ raise WebSocketProtocolException('Invalid close frame.')
2835
+ if l > 2 and (not skip_utf8_validation) and (not validate_utf8(self.data[2:])):
2836
+ raise WebSocketProtocolException('Invalid close frame.')
2837
+ code = 256 * int(self.data[0]) + int(self.data[1])
2838
+ if not self._is_valid_close_status(code):
2839
+ raise WebSocketProtocolException('Invalid close opcode %r', code)
2840
+ @staticmethod
2841
+ def _is_valid_close_status(code: int) -> bool:
2842
+ return code in VALID_CLOSE_STATUS or 3000 <= code < 5000
2843
+ def __str__(self) -> str:
2844
+ return f'fin={self.fin} opcode={self.opcode} data={self.data}'
2845
+ @staticmethod
2846
+ def create_frame(data: Union[bytes, str], opcode: int, fin: int=1) -> 'ABNF':
2847
+ if opcode == ABNF.OPCODE_TEXT and isinstance(data, str):
2848
+ data = data.encode('utf-8')
2849
+ return ABNF(fin, 0, 0, 0, opcode, 1, data)
2850
+ def format(self) -> bytes:
2851
+ if any((x not in (0, 1) for x in [self.fin, self.rsv1, self.rsv2, self.rsv3])):
2852
+ raise ValueError('not 0 or 1')
2853
+ if self.opcode not in ABNF.OPCODES:
2854
+ raise ValueError('Invalid OPCODE')
2855
+ length = len(self.data)
2856
+ if length >= ABNF.LENGTH_63:
2857
+ raise ValueError('data is too long')
2858
+ frame_header = chr(self.fin << 7 | self.rsv1 << 6 | self.rsv2 << 5 | self.rsv3 << 4 | self.opcode).encode('latin-1')
2859
+ if length < ABNF.LENGTH_7:
2860
+ frame_header += chr(self.mask_value << 7 | length).encode('latin-1')
2861
+ elif length < ABNF.LENGTH_16:
2862
+ frame_header += chr(self.mask_value << 7 | 126).encode('latin-1')
2863
+ frame_header += struct.pack('!H', length)
2864
+ else:
2865
+ frame_header += chr(self.mask_value << 7 | 127).encode('latin-1')
2866
+ frame_header += struct.pack('!Q', length)
2867
+ if not self.mask_value:
2868
+ if isinstance(self.data, str):
2869
+ self.data = self.data.encode('utf-8')
2870
+ return frame_header + self.data
2871
+ mask_key = self.get_mask_key(4)
2872
+ return frame_header + self._get_masked(mask_key)
2873
+ def _get_masked(self, mask_key: Union[str, bytes]) -> bytes:
2874
+ s = ABNF.mask(mask_key, self.data)
2875
+ if isinstance(mask_key, str):
2876
+ mask_key = mask_key.encode('utf-8')
2877
+ return mask_key + s
2878
+ @staticmethod
2879
+ def mask(mask_key: Union[str, bytes], data: Union[str, bytes]) -> bytes:
2880
+ if data is None:
2881
+ data = ''
2882
+ if isinstance(mask_key, str):
2883
+ mask_key = mask_key.encode('latin-1')
2884
+ if isinstance(data, str):
2885
+ data = data.encode('latin-1')
2886
+ return _mask(array.array('B', mask_key), array.array('B', data))
2887
+ class frame_buffer:
2888
+ _HEADER_MASK_INDEX = 5
2889
+ _HEADER_LENGTH_INDEX = 6
2890
+ def __init__(self, recv_fn: Callable[[int], int], skip_utf8_validation: bool) -> None:
2891
+ self.recv = recv_fn
2892
+ self.skip_utf8_validation = skip_utf8_validation
2893
+ self.recv_buffer: list = []
2894
+ self.clear()
2895
+ self.lock = Lock()
2896
+ def clear(self) -> None:
2897
+ self.header: Optional[tuple] = None
2898
+ self.length: Optional[int] = None
2899
+ self.mask_value: Union[bytes, str, None] = None
2900
+ def has_received_header(self) -> bool:
2901
+ return self.header is None
2902
+ def recv_header(self) -> None:
2903
+ header = self.recv_strict(2)
2904
+ b1 = header[0]
2905
+ fin = b1 >> 7 & 1
2906
+ rsv1 = b1 >> 6 & 1
2907
+ rsv2 = b1 >> 5 & 1
2908
+ rsv3 = b1 >> 4 & 1
2909
+ opcode = b1 & 15
2910
+ b2 = header[1]
2911
+ has_mask = b2 >> 7 & 1
2912
+ length_bits = b2 & 127
2913
+ self.header = (fin, rsv1, rsv2, rsv3, opcode, has_mask, length_bits)
2914
+ def has_mask(self) -> Union[bool, int]:
2915
+ if not self.header:
2916
+ return False
2917
+ header_val: int = self.header[frame_buffer._HEADER_MASK_INDEX]
2918
+ return header_val
2919
+ def has_received_length(self) -> bool:
2920
+ return self.length is None
2921
+ def recv_length(self) -> None:
2922
+ bits = self.header[frame_buffer._HEADER_LENGTH_INDEX]
2923
+ length_bits = bits & 127
2924
+ if length_bits == 126:
2925
+ v = self.recv_strict(2)
2926
+ self.length = struct.unpack('!H', v)[0]
2927
+ elif length_bits == 127:
2928
+ v = self.recv_strict(8)
2929
+ self.length = struct.unpack('!Q', v)[0]
2930
+ else:
2931
+ self.length = length_bits
2932
+ def has_received_mask(self) -> bool:
2933
+ return self.mask_value is None
2934
+ def recv_mask(self) -> None:
2935
+ self.mask_value = self.recv_strict(4) if self.has_mask() else ''
2936
+ def recv_frame(self) -> ABNF:
2937
+ with self.lock:
2938
+ if self.has_received_header():
2939
+ self.recv_header()
2940
+ (fin, rsv1, rsv2, rsv3, opcode, has_mask, _) = self.header
2941
+ if self.has_received_length():
2942
+ self.recv_length()
2943
+ length = self.length
2944
+ if self.has_received_mask():
2945
+ self.recv_mask()
2946
+ mask_value = self.mask_value
2947
+ payload = self.recv_strict(length)
2948
+ if has_mask:
2949
+ payload = ABNF.mask(mask_value, payload)
2950
+ self.clear()
2951
+ frame = ABNF(fin, rsv1, rsv2, rsv3, opcode, has_mask, payload)
2952
+ frame.validate(self.skip_utf8_validation)
2953
+ return frame
2954
+ def recv_strict(self, bufsize: int) -> bytes:
2955
+ shortage = bufsize - sum(map(len, self.recv_buffer))
2956
+ while shortage > 0:
2957
+ bytes_ = self.recv(min(16384, shortage))
2958
+ self.recv_buffer.append(bytes_)
2959
+ shortage -= len(bytes_)
2960
+ unified = b''.join(self.recv_buffer)
2961
+ if shortage == 0:
2962
+ self.recv_buffer = []
2963
+ return unified
2964
+ else:
2965
+ self.recv_buffer = [unified[bufsize:]]
2966
+ return unified[:bufsize]
2967
+ class continuous_frame:
2968
+ def __init__(self, fire_cont_frame: bool, skip_utf8_validation: bool) -> None:
2969
+ self.fire_cont_frame = fire_cont_frame
2970
+ self.skip_utf8_validation = skip_utf8_validation
2971
+ self.cont_data: Optional[list] = None
2972
+ self.recving_frames: Optional[int] = None
2973
+ def validate(self, frame: ABNF) -> None:
2974
+ if not self.recving_frames and frame.opcode == ABNF.OPCODE_CONT:
2975
+ raise WebSocketProtocolException('Illegal frame')
2976
+ if self.recving_frames and frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
2977
+ raise WebSocketProtocolException('Illegal frame')
2978
+ def add(self, frame: ABNF) -> None:
2979
+ if self.cont_data:
2980
+ self.cont_data[1] += frame.data
2981
+ else:
2982
+ if frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
2983
+ self.recving_frames = frame.opcode
2984
+ self.cont_data = [frame.opcode, frame.data]
2985
+ if frame.fin:
2986
+ self.recving_frames = None
2987
+ def is_fire(self, frame: ABNF) -> Union[bool, int]:
2988
+ return frame.fin or self.fire_cont_frame
2989
+ def extract(self, frame: ABNF) -> tuple:
2990
+ data = self.cont_data
2991
+ self.cont_data = None
2992
+ frame.data = data[1]
2993
+ if not self.fire_cont_frame and data[0] == ABNF.OPCODE_TEXT and (not self.skip_utf8_validation) and (not validate_utf8(frame.data)):
2994
+ raise WebSocketPayloadException(f'cannot decode: {repr(frame.data)}')
2995
+ return (data[0], frame)
2996
+ import inspect
2997
+ import selectors
2998
+ import socket
2999
+ import threading
3000
+ import time
3001
+ from typing import Any, Callable, Optional, Union
3002
+ import logging
3003
+ _logger = logging.getLogger('websocket')
3004
+ try:
3005
+ from logging import NullHandler
3006
+ except ImportError:
3007
+ class NullHandler(logging.Handler):
3008
+ def emit(self, record) -> None:pass
3009
+ _logger.addHandler(NullHandler())
3010
+ _traceEnabled = False
3011
+ def enableTrace(traceable: bool, handler: logging.StreamHandler=logging.StreamHandler(), level: str='DEBUG') -> None:
3012
+ global _traceEnabled
3013
+ _traceEnabled = traceable
3014
+ if traceable:
3015
+ _logger.addHandler(handler)
3016
+ _logger.setLevel(getattr(logging, level))
3017
+ def dump(title: str, message: str) -> None:
3018
+ if _traceEnabled:
3019
+ _logger.debug(f'--- {title} ---')
3020
+ _logger.debug(message)
3021
+ _logger.debug('-----------------------')
3022
+ def error(msg: str) -> None:
3023
+ _logger.error(msg)
3024
+ def warning(msg: str) -> None:
3025
+ _logger.warning(msg)
3026
+ def debug(msg: str) -> None:
3027
+ _logger.debug(msg)
3028
+ def info(msg: str) -> None:
3029
+ _logger.info(msg)
3030
+ def trace(msg: str) -> None:
3031
+ if _traceEnabled:
3032
+ _logger.debug(msg)
3033
+ def isEnabledForError() -> bool:
3034
+ return _logger.isEnabledFor(logging.ERROR)
3035
+ def isEnabledForDebug() -> bool:
3036
+ return _logger.isEnabledFor(logging.DEBUG)
3037
+ def isEnabledForTrace() -> bool:
3038
+ return _traceEnabled
3039
+ import socket
3040
+ import struct
3041
+ import threading
3042
+ import time
3043
+ from typing import Optional, Union
3044
+ import hashlib
3045
+ import hmac
3046
+ import os
3047
+ from base64 import encodebytes as base64encode
3048
+ from http import HTTPStatus
3049
+ import http.cookies
3050
+ from typing import Optional
3051
+ class SimpleCookieJar:
3052
+ def __init__(self) -> None:
3053
+ self.jar: dict = {}
3054
+ def add(self, set_cookie: Optional[str]) -> None:
3055
+ if set_cookie:
3056
+ simple_cookie = http.cookies.SimpleCookie(set_cookie)
3057
+ for v in simple_cookie.values():
3058
+ if (domain := v.get('domain')):
3059
+ if not domain.startswith('.'):
3060
+ domain = f'.{domain}'
3061
+ cookie = self.jar.get(domain) if self.jar.get(domain) else http.cookies.SimpleCookie()
3062
+ cookie.update(simple_cookie)
3063
+ self.jar[domain.lower()] = cookie
3064
+ def set(self, set_cookie: str) -> None:
3065
+ if set_cookie:
3066
+ simple_cookie = http.cookies.SimpleCookie(set_cookie)
3067
+ for v in simple_cookie.values():
3068
+ if (domain := v.get('domain')):
3069
+ if not domain.startswith('.'):
3070
+ domain = f'.{domain}'
3071
+ self.jar[domain.lower()] = simple_cookie
3072
+ def get(self, host: str) -> str:
3073
+ if not host:
3074
+ return ''
3075
+ cookies = []
3076
+ for (domain, _) in self.jar.items():
3077
+ host = host.lower()
3078
+ if host.endswith(domain) or host == domain[1:]:
3079
+ cookies.append(self.jar.get(domain))
3080
+ return '; '.join(filter(None, sorted([f'{k}={v.value}' for cookie in filter(None, cookies) for (k, v) in cookie.items()])))
3081
+ import errno
3082
+ import os
3083
+ import socket
3084
+ from base64 import encodebytes as base64encode
3085
+ import errno
3086
+ import selectors
3087
+ import socket
3088
+ from typing import Union
3089
+ try:
3090
+ import ssl
3091
+ from ssl import SSLError, SSLEOFError, SSLWantReadError, SSLWantWriteError
3092
+ HAVE_SSL = True
3093
+ except ImportError:
3094
+ class SSLError(Exception):pass
3095
+ class SSLEOFError(Exception):pass
3096
+ class SSLWantReadError(Exception):pass
3097
+ class SSLWantWriteError(Exception):pass
3098
+ ssl = None
3099
+ HAVE_SSL = False
3100
+ DEFAULT_SOCKET_OPTION = [(socket.SOL_TCP, socket.TCP_NODELAY, 1)]
3101
+ if hasattr(socket, 'SO_KEEPALIVE'):
3102
+ DEFAULT_SOCKET_OPTION.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
3103
+ if hasattr(socket, 'TCP_KEEPIDLE'):
3104
+ DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPIDLE, 30))
3105
+ if hasattr(socket, 'TCP_KEEPINTVL'):
3106
+ DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPINTVL, 10))
3107
+ if hasattr(socket, 'TCP_KEEPCNT'):
3108
+ DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPCNT, 3))
3109
+ _default_timeout = None
3110
+ class sock_opt:
3111
+ def __init__(self, sockopt: list, sslopt: dict) -> None:
3112
+ if sockopt is None:
3113
+ sockopt = []
3114
+ if sslopt is None:
3115
+ sslopt = {}
3116
+ self.sockopt = sockopt
3117
+ self.sslopt = sslopt
3118
+ self.timeout = None
3119
+ def setdefaulttimeout(timeout: Union[int, float, None]) -> None:
3120
+ global _default_timeout
3121
+ _default_timeout = timeout
3122
+ def getdefaulttimeout() -> Union[int, float, None]:
3123
+ return _default_timeout
3124
+ def recv(sock: socket.socket, bufsize: int) -> bytes:
3125
+ if not sock:
3126
+ raise WebSocketConnectionClosedException('socket is already closed.')
3127
+ def _recv():
3128
+ try:
3129
+ return sock.recv(bufsize)
3130
+ except SSLWantReadError:pass
3131
+ except socket.error as exc:
3132
+ error_code = extract_error_code(exc)
3133
+ if error_code not in [errno.EAGAIN, errno.EWOULDBLOCK]:
3134
+ raise
3135
+ sel = selectors.DefaultSelector()
3136
+ sel.register(sock, selectors.EVENT_READ)
3137
+ r = sel.select(sock.gettimeout())
3138
+ sel.close()
3139
+ if r:
3140
+ return sock.recv(bufsize)
3141
+ try:
3142
+ if sock.gettimeout() == 0:
3143
+ bytes_ = sock.recv(bufsize)
3144
+ else:
3145
+ bytes_ = _recv()
3146
+ except TimeoutError:
3147
+ raise WebSocketTimeoutException('Connection timed out')
3148
+ except socket.timeout as e:
3149
+ message = extract_err_message(e)
3150
+ raise WebSocketTimeoutException(message)
3151
+ except SSLError as e:
3152
+ message = extract_err_message(e)
3153
+ if isinstance(message, str) and 'timed out' in message:
3154
+ raise WebSocketTimeoutException(message)
3155
+ else:
3156
+ raise
3157
+ if not bytes_:
3158
+ raise WebSocketConnectionClosedException('Connection to remote host was lost.')
3159
+ return bytes_
3160
+ def recv_line(sock: socket.socket) -> bytes:
3161
+ line = []
3162
+ while True:
3163
+ c = recv(sock, 1)
3164
+ line.append(c)
3165
+ if c == b'\n':
3166
+ break
3167
+ return b''.join(line)
3168
+ def send(sock: socket.socket, data: Union[bytes, str]) -> int:
3169
+ if isinstance(data, str):
3170
+ data = data.encode('utf-8')
3171
+ if not sock:
3172
+ raise WebSocketConnectionClosedException('socket is already closed.')
3173
+ def _send():
3174
+ try:
3175
+ return sock.send(data)
3176
+ except SSLWantWriteError:pass
3177
+ except socket.error as exc:
3178
+ error_code = extract_error_code(exc)
3179
+ if error_code is None:
3180
+ raise
3181
+ if error_code not in [errno.EAGAIN, errno.EWOULDBLOCK]:
3182
+ raise
3183
+ sel = selectors.DefaultSelector()
3184
+ sel.register(sock, selectors.EVENT_WRITE)
3185
+ w = sel.select(sock.gettimeout())
3186
+ sel.close()
3187
+ if w:
3188
+ return sock.send(data)
3189
+ try:
3190
+ if sock.gettimeout() == 0:
3191
+ return sock.send(data)
3192
+ else:
3193
+ return _send()
3194
+ except socket.timeout as e:
3195
+ message = extract_err_message(e)
3196
+ raise WebSocketTimeoutException(message)
3197
+ except Exception as e:
3198
+ message = extract_err_message(e)
3199
+ if isinstance(message, str) and 'timed out' in message:
3200
+ raise WebSocketTimeoutException(message)
3201
+ else:
3202
+ raise
3203
+ import os
3204
+ import socket
3205
+ import struct
3206
+ from typing import Optional
3207
+ from urllib.parse import unquote, urlparse
3208
+ def parse_url(url: str) -> tuple:
3209
+ if ':' not in url:
3210
+ raise ValueError('url is invalid')
3211
+ (scheme, url) = url.split(':', 1)
3212
+ parsed = urlparse(url, scheme='http')
3213
+ if parsed.hostname:
3214
+ hostname = parsed.hostname
3215
+ else:
3216
+ raise ValueError('hostname is invalid')
3217
+ port = 0
3218
+ if parsed.port:
3219
+ port = parsed.port
3220
+ is_secure = False
3221
+ if scheme == 'ws':
3222
+ if not port:
3223
+ port = 80
3224
+ elif scheme == 'wss':
3225
+ is_secure = True
3226
+ if not port:
3227
+ port = 443
3228
+ else:
3229
+ raise ValueError('scheme %s is invalid' % scheme)
3230
+ if parsed.path:
3231
+ resource = parsed.path
3232
+ else:
3233
+ resource = '/'
3234
+ if parsed.query:
3235
+ resource += f'?{parsed.query}'
3236
+ return (hostname, port, resource, is_secure)
3237
+ DEFAULT_NO_PROXY_HOST = ['localhost', '127.0.0.1']
3238
+ def _is_ip_address(addr: str) -> bool:
3239
+ try:
3240
+ socket.inet_aton(addr)
3241
+ except socket.error:
3242
+ return False
3243
+ else:
3244
+ return True
3245
+ def _is_subnet_address(hostname: str) -> bool:
3246
+ try:
3247
+ (addr, netmask) = hostname.split('/')
3248
+ return _is_ip_address(addr) and 0 <= int(netmask) < 32
3249
+ except ValueError:
3250
+ return False
3251
+ def _is_address_in_network(ip: str, net: str) -> bool:
3252
+ ipaddr: int = struct.unpack('!I', socket.inet_aton(ip))[0]
3253
+ (netaddr, netmask) = net.split('/')
3254
+ netaddr: int = struct.unpack('!I', socket.inet_aton(netaddr))[0]
3255
+ netmask = 4294967295 << 32 - int(netmask) & 4294967295
3256
+ return ipaddr & netmask == netaddr
3257
+ def _is_no_proxy_host(hostname: str, no_proxy: Optional[list]) -> bool:
3258
+ if not no_proxy:
3259
+ if (v := os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')).replace(' ', '')):
3260
+ no_proxy = v.split(',')
3261
+ if not no_proxy:
3262
+ no_proxy = DEFAULT_NO_PROXY_HOST
3263
+ if '*' in no_proxy:
3264
+ return True
3265
+ if hostname in no_proxy:
3266
+ return True
3267
+ if _is_ip_address(hostname):
3268
+ return any([_is_address_in_network(hostname, subnet) for subnet in no_proxy if _is_subnet_address(subnet)])
3269
+ for domain in [domain for domain in no_proxy if domain.startswith('.')]:
3270
+ if hostname.endswith(domain):
3271
+ return True
3272
+ return False
3273
+ def get_proxy_info(hostname: str, is_secure: bool, proxy_host: Optional[str]=None, proxy_port: int=0, proxy_auth: Optional[tuple]=None, no_proxy: Optional[list]=None, proxy_type: str='http') -> tuple:
3274
+ if _is_no_proxy_host(hostname, no_proxy):
3275
+ return (None, 0, None)
3276
+ if proxy_host:
3277
+ if not proxy_port:
3278
+ raise WebSocketProxyException('Cannot use port 0 when proxy_host specified')
3279
+ port = proxy_port
3280
+ auth = proxy_auth
3281
+ return (proxy_host, port, auth)
3282
+ env_key = 'https_proxy' if is_secure else 'http_proxy'
3283
+ value = os.environ.get(env_key, os.environ.get(env_key.upper(), '')).replace(' ', '')
3284
+ if value:
3285
+ proxy = urlparse(value)
3286
+ auth = (unquote(proxy.username), unquote(proxy.password)) if proxy.username else None
3287
+ return (proxy.hostname, proxy.port, auth)
3288
+ return (None, 0, None)
3289
+ try:
3290
+ from python_socks._errors import *
3291
+ from python_socks._types import ProxyType
3292
+ from python_socks.sync import Proxy
3293
+ HAVE_PYTHON_SOCKS = True
3294
+ except:
3295
+ HAVE_PYTHON_SOCKS = False
3296
+ class ProxyError(Exception):pass
3297
+ class ProxyTimeoutError(Exception):pass
3298
+ class ProxyConnectionError(Exception):pass
3299
+ class proxy_info:
3300
+ def __init__(self, **options):
3301
+ self.proxy_host = options.get('http_proxy_host', None)
3302
+ if self.proxy_host:
3303
+ self.proxy_port = options.get('http_proxy_port', 0)
3304
+ self.auth = options.get('http_proxy_auth', None)
3305
+ self.no_proxy = options.get('http_no_proxy', None)
3306
+ self.proxy_protocol = options.get('proxy_type', 'http')
3307
+ self.proxy_timeout = options.get('http_proxy_timeout', None)
3308
+ if self.proxy_protocol not in ['http', 'socks4', 'socks4a', 'socks5', 'socks5h']:
3309
+ raise ProxyError('Only http, socks4, socks5 proxy protocols are supported')
3310
+ else:
3311
+ self.proxy_port = 0
3312
+ self.auth = None
3313
+ self.no_proxy = None
3314
+ self.proxy_protocol = 'http'
3315
+ def _start_proxied_socket(url: str, options, proxy) -> tuple:
3316
+ if not HAVE_PYTHON_SOCKS:
3317
+ raise WebSocketException('Python Socks is needed for SOCKS proxying but is not available')
3318
+ (hostname, port, resource, is_secure) = parse_url(url)
3319
+ if proxy.proxy_protocol == 'socks4':
3320
+ rdns = False
3321
+ proxy_type = ProxyType.SOCKS4
3322
+ elif proxy.proxy_protocol == 'socks4a':
3323
+ rdns = True
3324
+ proxy_type = ProxyType.SOCKS4
3325
+ elif proxy.proxy_protocol == 'socks5':
3326
+ rdns = False
3327
+ proxy_type = ProxyType.SOCKS5
3328
+ elif proxy.proxy_protocol == 'socks5h':
3329
+ rdns = True
3330
+ proxy_type = ProxyType.SOCKS5
3331
+ ws_proxy = Proxy.create(proxy_type=proxy_type, host=proxy.proxy_host, port=int(proxy.proxy_port), username=proxy.auth[0] if proxy.auth else None, password=proxy.auth[1] if proxy.auth else None, rdns=rdns)
3332
+ sock = ws_proxy.connect(hostname, port, timeout=proxy.proxy_timeout)
3333
+ if is_secure:
3334
+ if HAVE_SSL:
3335
+ sock = _ssl_socket(sock, options.sslopt, hostname)
3336
+ else:
3337
+ raise WebSocketException('SSL not available.')
3338
+ return (sock, (hostname, port, resource))
3339
+ def connect(url: str, options, proxy, socket):
3340
+ if proxy.proxy_host and (not socket) and (proxy.proxy_protocol != 'http'):
3341
+ return _start_proxied_socket(url, options, proxy)
3342
+ (hostname, port_from_url, resource, is_secure) = parse_url(url)
3343
+ if socket:
3344
+ return (socket, (hostname, port_from_url, resource))
3345
+ (addrinfo_list, need_tunnel, auth) = _get_addrinfo_list(hostname, port_from_url, is_secure, proxy)
3346
+ if not addrinfo_list:
3347
+ raise WebSocketException(f'Host not found.: {hostname}:{port_from_url}')
3348
+ sock = None
3349
+ try:
3350
+ sock = _open_socket(addrinfo_list, options.sockopt, options.timeout)
3351
+ if need_tunnel:
3352
+ sock = _tunnel(sock, hostname, port_from_url, auth)
3353
+ if is_secure:
3354
+ if HAVE_SSL:
3355
+ sock = _ssl_socket(sock, options.sslopt, hostname)
3356
+ else:
3357
+ raise WebSocketException('SSL not available.')
3358
+ return (sock, (hostname, port_from_url, resource))
3359
+ except:
3360
+ if sock:
3361
+ sock.close()
3362
+ raise
3363
+ def _get_addrinfo_list(hostname, port: int, is_secure: bool, proxy) -> tuple:
3364
+ (phost, pport, pauth) = get_proxy_info(hostname, is_secure, proxy.proxy_host, proxy.proxy_port, proxy.auth, proxy.no_proxy)
3365
+ try:
3366
+ if not phost:
3367
+ addrinfo_list = socket.getaddrinfo(hostname, port, 0, socket.SOCK_STREAM, socket.SOL_TCP)
3368
+ return (addrinfo_list, False, None)
3369
+ else:
3370
+ pport = pport and pport or 80
3371
+ addrinfo_list = socket.getaddrinfo(phost, pport, 0, socket.SOCK_STREAM, socket.SOL_TCP)
3372
+ return (addrinfo_list, True, pauth)
3373
+ except socket.gaierror as e:
3374
+ raise WebSocketAddressException(e)
3375
+ def _open_socket(addrinfo_list, sockopt, timeout):
3376
+ err = None
3377
+ for addrinfo in addrinfo_list:
3378
+ (family, socktype, proto) = addrinfo[:3]
3379
+ sock = socket.socket(family, socktype, proto)
3380
+ sock.settimeout(timeout)
3381
+ for opts in DEFAULT_SOCKET_OPTION:
3382
+ sock.setsockopt(*opts)
3383
+ for opts in sockopt:
3384
+ sock.setsockopt(*opts)
3385
+ address = addrinfo[4]
3386
+ err = None
3387
+ while not err:
3388
+ try:
3389
+ sock.connect(address)
3390
+ except socket.error as error:
3391
+ sock.close()
3392
+ error.remote_ip = str(address[0])
3393
+ try:
3394
+ eConnRefused = (errno.ECONNREFUSED, errno.WSAECONNREFUSED, errno.ENETUNREACH)
3395
+ except AttributeError:
3396
+ eConnRefused = (errno.ECONNREFUSED, errno.ENETUNREACH)
3397
+ if error.errno not in eConnRefused:
3398
+ raise error
3399
+ err = error
3400
+ continue
3401
+ else:
3402
+ break
3403
+ else:
3404
+ continue
3405
+ break
3406
+ else:
3407
+ if err:
3408
+ raise err
3409
+ return sock
3410
+ def _wrap_sni_socket(sock: socket.socket, sslopt: dict, hostname, check_hostname):
3411
+ context = sslopt.get('context', None)
3412
+ if not context:
3413
+ context = ssl.SSLContext(sslopt.get('ssl_version', ssl.PROTOCOL_TLS_CLIENT))
3414
+ context.keylog_filename = os.environ.get('SSLKEYLOGFILE', None)
3415
+ if sslopt.get('cert_reqs', ssl.CERT_NONE) != ssl.CERT_NONE:
3416
+ cafile = sslopt.get('ca_certs', None)
3417
+ capath = sslopt.get('ca_cert_path', None)
3418
+ if cafile or capath:
3419
+ context.load_verify_locations(cafile=cafile, capath=capath)
3420
+ elif hasattr(context, 'load_default_certs'):
3421
+ context.load_default_certs(ssl.Purpose.SERVER_AUTH)
3422
+ if sslopt.get('certfile', None):
3423
+ context.load_cert_chain(sslopt['certfile'], sslopt.get('keyfile', None), sslopt.get('password', None))
3424
+ if sslopt.get('cert_reqs', ssl.CERT_NONE) == ssl.CERT_NONE and (not sslopt.get('check_hostname', False)):
3425
+ context.check_hostname = False
3426
+ context.verify_mode = ssl.CERT_NONE
3427
+ else:
3428
+ context.check_hostname = sslopt.get('check_hostname', True)
3429
+ context.verify_mode = sslopt.get('cert_reqs', ssl.CERT_REQUIRED)
3430
+ if 'ciphers' in sslopt:
3431
+ context.set_ciphers(sslopt['ciphers'])
3432
+ if 'cert_chain' in sslopt:
3433
+ (certfile, keyfile, password) = sslopt['cert_chain']
3434
+ context.load_cert_chain(certfile, keyfile, password)
3435
+ if 'ecdh_curve' in sslopt:
3436
+ context.set_ecdh_curve(sslopt['ecdh_curve'])
3437
+ return context.wrap_socket(sock, do_handshake_on_connect=sslopt.get('do_handshake_on_connect', True), suppress_ragged_eofs=sslopt.get('suppress_ragged_eofs', True), server_hostname=hostname)
3438
+ def _ssl_socket(sock: socket.socket, user_sslopt: dict, hostname):
3439
+ sslopt: dict = {'cert_reqs': ssl.CERT_REQUIRED}
3440
+ sslopt.update(user_sslopt)
3441
+ cert_path = os.environ.get('WEBSOCKET_CLIENT_CA_BUNDLE')
3442
+ if cert_path and os.path.isfile(cert_path) and (user_sslopt.get('ca_certs', None) is None):
3443
+ sslopt['ca_certs'] = cert_path
3444
+ elif cert_path and os.path.isdir(cert_path) and (user_sslopt.get('ca_cert_path', None) is None):
3445
+ sslopt['ca_cert_path'] = cert_path
3446
+ if sslopt.get('server_hostname', None):
3447
+ hostname = sslopt['server_hostname']
3448
+ check_hostname = sslopt.get('check_hostname', True)
3449
+ sock = _wrap_sni_socket(sock, sslopt, hostname, check_hostname)
3450
+ return sock
3451
+ def _tunnel(sock: socket.socket, host, port: int, auth) -> socket.socket:
3452
+ debug('Connecting proxy...')
3453
+ connect_header = f'CONNECT {host}:{port} HTTP/1.1\r\n'
3454
+ connect_header += f'Host: {host}:{port}\r\n'
3455
+ if auth and auth[0]:
3456
+ auth_str = auth[0]
3457
+ if auth[1]:
3458
+ auth_str += f':{auth[1]}'
3459
+ encoded_str = base64encode(auth_str.encode()).strip().decode().replace('\n', '')
3460
+ connect_header += f'Proxy-Authorization: Basic {encoded_str}\r\n'
3461
+ connect_header += '\r\n'
3462
+ dump('request header', connect_header)
3463
+ send(sock, connect_header)
3464
+ try:
3465
+ (status, _, _) = read_headers(sock)
3466
+ except Exception as e:
3467
+ raise WebSocketProxyException(str(e))
3468
+ if status != 200:
3469
+ raise WebSocketProxyException(f'failed CONNECT via proxy status: {status}')
3470
+ return sock
3471
+ def read_headers(sock: socket.socket) -> tuple:
3472
+ status = None
3473
+ status_message = None
3474
+ headers: dict = {}
3475
+ trace('--- response header ---')
3476
+ while True:
3477
+ line = recv_line(sock)
3478
+ line = line.decode('utf-8').strip()
3479
+ if not line:
3480
+ break
3481
+ trace(line)
3482
+ if not status:
3483
+ status_info = line.split(' ', 2)
3484
+ status = int(status_info[1])
3485
+ if len(status_info) > 2:
3486
+ status_message = status_info[2]
3487
+ else:
3488
+ kv = line.split(':', 1)
3489
+ if len(kv) != 2:
3490
+ raise WebSocketException('Invalid header')
3491
+ (key, value) = kv
3492
+ if key.lower() == 'set-cookie' and headers.get('set-cookie'):
3493
+ headers['set-cookie'] = headers.get('set-cookie') + '; ' + value.strip()
3494
+ else:
3495
+ headers[key.lower()] = value.strip()
3496
+ trace('-----------------------')
3497
+ return (status, headers, status_message)
3498
+ VERSION = 13
3499
+ SUPPORTED_REDIRECT_STATUSES = (HTTPStatus.MOVED_PERMANENTLY, HTTPStatus.FOUND, HTTPStatus.SEE_OTHER, HTTPStatus.TEMPORARY_REDIRECT, HTTPStatus.PERMANENT_REDIRECT)
3500
+ SUCCESS_STATUSES = SUPPORTED_REDIRECT_STATUSES + (HTTPStatus.SWITCHING_PROTOCOLS,)
3501
+ CookieJar = SimpleCookieJar()
3502
+ class handshake_response:
3503
+ def __init__(self, status: int, headers: dict, subprotocol):
3504
+ self.status = status
3505
+ self.headers = headers
3506
+ self.subprotocol = subprotocol
3507
+ CookieJar.add(headers.get('set-cookie'))
3508
+ def handshake(sock, url: str, hostname: str, port: int, resource: str, **options) -> handshake_response:
3509
+ (headers, key) = _get_handshake_headers(resource, url, hostname, port, options)
3510
+ header_str = '\r\n'.join(headers)
3511
+ send(sock, header_str)
3512
+ dump('request header', header_str)
3513
+ (status, resp) = _get_resp_headers(sock)
3514
+ if status in SUPPORTED_REDIRECT_STATUSES:
3515
+ return handshake_response(status, resp, None)
3516
+ (success, subproto) = _validate(resp, key, options.get('subprotocols'))
3517
+ if not success:
3518
+ raise WebSocketException('Invalid WebSocket Header')
3519
+ return handshake_response(status, resp, subproto)
3520
+ def _pack_hostname(hostname: str) -> str:
3521
+ if ':' in hostname:
3522
+ return f'[{hostname}]'
3523
+ return hostname
3524
+ def _get_handshake_headers(resource: str, url: str, host: str, port: int, options: dict) -> tuple:
3525
+ headers = [f'GET {resource} HTTP/1.1', 'Upgrade: websocket']
3526
+ if port in [80, 443]:
3527
+ hostport = _pack_hostname(host)
3528
+ else:
3529
+ hostport = f'{_pack_hostname(host)}:{port}'
3530
+ if options.get('host'):
3531
+ headers.append(f"Host: {options['host']}")
3532
+ else:
3533
+ headers.append(f'Host: {hostport}')
3534
+ (scheme, url) = url.split(':', 1)
3535
+ if not options.get('suppress_origin'):
3536
+ if 'origin' in options and options['origin'] is not None:
3537
+ headers.append(f"Origin: {options['origin']}")
3538
+ elif scheme == 'wss':
3539
+ headers.append(f'Origin: https://{hostport}')
3540
+ else:
3541
+ headers.append(f'Origin: http://{hostport}')
3542
+ key = _create_sec_websocket_key()
3543
+ if not options.get('header') or 'Sec-WebSocket-Key' not in options['header']:
3544
+ headers.append(f'Sec-WebSocket-Key: {key}')
3545
+ else:
3546
+ key = options['header']['Sec-WebSocket-Key']
3547
+ if not options.get('header') or 'Sec-WebSocket-Version' not in options['header']:
3548
+ headers.append(f'Sec-WebSocket-Version: {VERSION}')
3549
+ if not options.get('connection'):
3550
+ headers.append('Connection: Upgrade')
3551
+ else:
3552
+ headers.append(options['connection'])
3553
+ if (subprotocols := options.get('subprotocols')):
3554
+ headers.append(f"Sec-WebSocket-Protocol: {','.join(subprotocols)}")
3555
+ if (header := options.get('header')):
3556
+ if isinstance(header, dict):
3557
+ header = [': '.join([k, v]) for (k, v) in header.items() if v is not None]
3558
+ headers.extend(header)
3559
+ server_cookie = CookieJar.get(host)
3560
+ client_cookie = options.get('cookie', None)
3561
+ if (cookie := '; '.join(filter(None, [server_cookie, client_cookie]))):
3562
+ headers.append(f'Cookie: {cookie}')
3563
+ headers.extend(('', ''))
3564
+ return (headers, key)
3565
+ def _get_resp_headers(sock, success_statuses: tuple=SUCCESS_STATUSES) -> tuple:
3566
+ (status, resp_headers, status_message) = read_headers(sock)
3567
+ if status not in success_statuses:
3568
+ content_len = resp_headers.get('content-length')
3569
+ if content_len:
3570
+ response_body = sock.recv(int(content_len))
3571
+ else:
3572
+ response_body = None
3573
+ raise WebSocketBadStatusException(f'Handshake status {status} {status_message} -+-+- {resp_headers} -+-+- {response_body}', status, status_message, resp_headers, response_body)
3574
+ return (status, resp_headers)
3575
+ _HEADERS_TO_CHECK = {'upgrade': 'websocket', 'connection': 'upgrade'}
3576
+ def _validate(headers, key: str, subprotocols) -> tuple:
3577
+ subproto = None
3578
+ for (k, v) in _HEADERS_TO_CHECK.items():
3579
+ r = headers.get(k, None)
3580
+ if not r:
3581
+ return (False, None)
3582
+ r = [x.strip().lower() for x in r.split(',')]
3583
+ if v not in r:
3584
+ return (False, None)
3585
+ if subprotocols:
3586
+ subproto = headers.get('sec-websocket-protocol', None)
3587
+ if not subproto or subproto.lower() not in [s.lower() for s in subprotocols]:
3588
+ error(f'Invalid subprotocol: {subprotocols}')
3589
+ return (False, None)
3590
+ subproto = subproto.lower()
3591
+ result = headers.get('sec-websocket-accept', None)
3592
+ if not result:
3593
+ return (False, None)
3594
+ result = result.lower()
3595
+ if isinstance(result, str):
3596
+ result = result.encode('utf-8')
3597
+ value = f'{key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11'.encode('utf-8')
3598
+ hashed = base64encode(hashlib.sha1(value).digest()).strip().lower()
3599
+ if hmac.compare_digest(hashed, result):
3600
+ return (True, subproto)
3601
+ else:
3602
+ return (False, None)
3603
+ def _create_sec_websocket_key() -> str:
3604
+ randomness = os.urandom(16)
3605
+ return base64encode(randomness).decode('utf-8').strip()
3606
+ class WebSocket:
3607
+ def __init__(self, get_mask_key=None, sockopt=None, sslopt=None, fire_cont_frame: bool=False, enable_multithread: bool=True, skip_utf8_validation: bool=False, **_):
3608
+ self.sock_opt = sock_opt(sockopt, sslopt)
3609
+ self.handshake_response = None
3610
+ self.sock: Optional[socket.socket] = None
3611
+ self.connected = False
3612
+ self.get_mask_key = get_mask_key
3613
+ self.frame_buffer = frame_buffer(self._recv, skip_utf8_validation)
3614
+ self.cont_frame = continuous_frame(fire_cont_frame, skip_utf8_validation)
3615
+ if enable_multithread:
3616
+ self.lock = threading.Lock()
3617
+ self.readlock = threading.Lock()
3618
+ else:
3619
+ self.lock = NoLock()
3620
+ self.readlock = NoLock()
3621
+ def __iter__(self):
3622
+ while True:
3623
+ yield self.recv()
3624
+ def __next__(self):
3625
+ return self.recv()
3626
+ def next(self):
3627
+ return self.__next__()
3628
+ def fileno(self):
3629
+ return self.sock.fileno()
3630
+ def set_mask_key(self, func):
3631
+ self.get_mask_key = func
3632
+ def gettimeout(self) -> Union[float, int, None]:
3633
+ return self.sock_opt.timeout
3634
+ def settimeout(self, timeout: Union[float, int, None]):
3635
+ self.sock_opt.timeout = timeout
3636
+ if self.sock:
3637
+ self.sock.settimeout(timeout)
3638
+ timeout = property(gettimeout, settimeout)
3639
+ def getsubprotocol(self):
3640
+ if self.handshake_response:
3641
+ return self.handshake_response.subprotocol
3642
+ else:
3643
+ return None
3644
+ subprotocol = property(getsubprotocol)
3645
+ def getstatus(self):
3646
+ if self.handshake_response:
3647
+ return self.handshake_response.status
3648
+ else:
3649
+ return None
3650
+ status = property(getstatus)
3651
+ def getheaders(self):
3652
+ if self.handshake_response:
3653
+ return self.handshake_response.headers
3654
+ else:
3655
+ return None
3656
+ def is_ssl(self):
3657
+ try:
3658
+ return isinstance(self.sock, ssl.SSLSocket)
3659
+ except:
3660
+ return False
3661
+ headers = property(getheaders)
3662
+ def connect(self, url, **options):
3663
+ self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
3664
+ (self.sock, addrs) = connect(url, self.sock_opt, proxy_info(**options), options.pop('socket', None))
3665
+ try:
3666
+ self.handshake_response = handshake(self.sock, url, *addrs, **options)
3667
+ for _ in range(options.pop('redirect_limit', 3)):
3668
+ if self.handshake_response.status in SUPPORTED_REDIRECT_STATUSES:
3669
+ url = self.handshake_response.headers['location']
3670
+ self.sock.close()
3671
+ (self.sock, addrs) = connect(url, self.sock_opt, proxy_info(**options), options.pop('socket', None))
3672
+ self.handshake_response = handshake(self.sock, url, *addrs, **options)
3673
+ self.connected = True
3674
+ except:
3675
+ if self.sock:
3676
+ self.sock.close()
3677
+ self.sock = None
3678
+ raise
3679
+ def send(self, payload: Union[bytes, str], opcode: int=ABNF.OPCODE_TEXT) -> int:
3680
+ frame = ABNF.create_frame(payload, opcode)
3681
+ return self.send_frame(frame)
3682
+ def send_text(self, text_data: str) -> int:
3683
+ return self.send(text_data, ABNF.OPCODE_TEXT)
3684
+ def send_bytes(self, data: Union[bytes, bytearray]) -> int:
3685
+ return self.send(data, ABNF.OPCODE_BINARY)
3686
+ def send_frame(self, frame) -> int:
3687
+ if self.get_mask_key:
3688
+ frame.get_mask_key = self.get_mask_key
3689
+ data = frame.format()
3690
+ length = len(data)
3691
+ if isEnabledForTrace():
3692
+ trace(f'++Sent raw: {repr(data)}')
3693
+ trace(f'++Sent decoded: {frame.__str__()}')
3694
+ with self.lock:
3695
+ while data:
3696
+ l = self._send(data)
3697
+ data = data[l:]
3698
+ return length
3699
+ def send_binary(self, payload: bytes) -> int:
3700
+ return self.send(payload, ABNF.OPCODE_BINARY)
3701
+ def ping(self, payload: Union[str, bytes]=''):
3702
+ if isinstance(payload, str):
3703
+ payload = payload.encode('utf-8')
3704
+ self.send(payload, ABNF.OPCODE_PING)
3705
+ def pong(self, payload: Union[str, bytes]=''):
3706
+ if isinstance(payload, str):
3707
+ payload = payload.encode('utf-8')
3708
+ self.send(payload, ABNF.OPCODE_PONG)
3709
+ def recv(self) -> Union[str, bytes]:
3710
+ with self.readlock:
3711
+ (opcode, data) = self.recv_data()
3712
+ if opcode == ABNF.OPCODE_TEXT:
3713
+ data_received: Union[bytes, str] = data
3714
+ if isinstance(data_received, bytes):
3715
+ return data_received.decode('utf-8')
3716
+ elif isinstance(data_received, str):
3717
+ return data_received
3718
+ elif opcode == ABNF.OPCODE_BINARY:
3719
+ data_binary: bytes = data
3720
+ return data_binary
3721
+ else:
3722
+ return ''
3723
+ def recv_data(self, control_frame: bool=False) -> tuple:
3724
+ (opcode, frame) = self.recv_data_frame(control_frame)
3725
+ return (opcode, frame.data)
3726
+ def recv_data_frame(self, control_frame: bool=False) -> tuple:
3727
+ while True:
3728
+ frame = self.recv_frame()
3729
+ if isEnabledForTrace():
3730
+ trace(f'++Rcv raw: {repr(frame.format())}')
3731
+ trace(f'++Rcv decoded: {frame.__str__()}')
3732
+ if not frame:
3733
+ raise WebSocketProtocolException(f'Not a valid frame {frame}')
3734
+ elif frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY, ABNF.OPCODE_CONT):
3735
+ self.cont_frame.validate(frame)
3736
+ self.cont_frame.add(frame)
3737
+ if self.cont_frame.is_fire(frame):
3738
+ return self.cont_frame.extract(frame)
3739
+ elif frame.opcode == ABNF.OPCODE_CLOSE:
3740
+ self.send_close()
3741
+ return (frame.opcode, frame)
3742
+ elif frame.opcode == ABNF.OPCODE_PING:
3743
+ if len(frame.data) < 126:
3744
+ self.pong(frame.data)
3745
+ else:
3746
+ raise WebSocketProtocolException('Ping message is too long')
3747
+ if control_frame:
3748
+ return (frame.opcode, frame)
3749
+ elif frame.opcode == ABNF.OPCODE_PONG:
3750
+ if control_frame:
3751
+ return (frame.opcode, frame)
3752
+ def recv_frame(self):
3753
+ return self.frame_buffer.recv_frame()
3754
+ def send_close(self, status: int=STATUS_NORMAL, reason: bytes=b''):
3755
+ if status < 0 or status >= ABNF.LENGTH_16:
3756
+ raise ValueError('code is invalid range')
3757
+ self.connected = False
3758
+ self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE)
3759
+ def close(self, status: int=STATUS_NORMAL, reason: bytes=b'', timeout: int=3):
3760
+ if not self.connected:
3761
+ return
3762
+ if status < 0 or status >= ABNF.LENGTH_16:
3763
+ raise ValueError('code is invalid range')
3764
+ try:
3765
+ self.connected = False
3766
+ self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE)
3767
+ sock_timeout = self.sock.gettimeout()
3768
+ self.sock.settimeout(timeout)
3769
+ start_time = time.time()
3770
+ while timeout is None or time.time() - start_time < timeout:
3771
+ try:
3772
+ frame = self.recv_frame()
3773
+ if frame.opcode != ABNF.OPCODE_CLOSE:
3774
+ continue
3775
+ if isEnabledForError():
3776
+ recv_status = struct.unpack('!H', frame.data[0:2])[0]
3777
+ if recv_status >= 3000 and recv_status <= 4999:
3778
+ debug(f'close status: {repr(recv_status)}')
3779
+ elif recv_status != STATUS_NORMAL:
3780
+ error(f'close status: {repr(recv_status)}')
3781
+ break
3782
+ except:
3783
+ break
3784
+ self.sock.settimeout(sock_timeout)
3785
+ self.sock.shutdown(socket.SHUT_RDWR)
3786
+ except:pass
3787
+ self.shutdown()
3788
+ def abort(self):
3789
+ if self.connected:
3790
+ self.sock.shutdown(socket.SHUT_RDWR)
3791
+ def shutdown(self):
3792
+ if self.sock:
3793
+ self.sock.close()
3794
+ self.sock = None
3795
+ self.connected = False
3796
+ def _send(self, data: Union[str, bytes]):
3797
+ return send(self.sock, data)
3798
+ def _recv(self, bufsize):
3799
+ try:
3800
+ return recv(self.sock, bufsize)
3801
+ except WebSocketConnectionClosedException:
3802
+ if self.sock:
3803
+ self.sock.close()
3804
+ self.sock = None
3805
+ self.connected = False
3806
+ raise
3807
+ def create_connection(url: str, timeout=None, class_=WebSocket, **options):
3808
+ sockopt = options.pop('sockopt', [])
3809
+ sslopt = options.pop('sslopt', {})
3810
+ fire_cont_frame = options.pop('fire_cont_frame', False)
3811
+ enable_multithread = options.pop('enable_multithread', True)
3812
+ skip_utf8_validation = options.pop('skip_utf8_validation', False)
3813
+ websock = class_(sockopt=sockopt, sslopt=sslopt, fire_cont_frame=fire_cont_frame, enable_multithread=enable_multithread, skip_utf8_validation=skip_utf8_validation, **options)
3814
+ websock.settimeout(timeout if timeout is not None else getdefaulttimeout())
3815
+ websock.connect(url, **options)
3816
+ return websock
3817
+ RECONNECT = 0
3818
+ def setReconnect(reconnectInterval: int) -> None:
3819
+ global RECONNECT
3820
+ RECONNECT = reconnectInterval
3821
+ class DispatcherBase:
3822
+ def __init__(self, app: Any, ping_timeout: Union[float, int, None]) -> None:
3823
+ self.app = app
3824
+ self.ping_timeout = ping_timeout
3825
+ def timeout(self, seconds: Union[float, int, None], callback: Callable) -> None:
3826
+ time.sleep(seconds)
3827
+ callback()
3828
+ def reconnect(self, seconds: int, reconnector: Callable) -> None:
3829
+ try:
3830
+ _logging.info(f'reconnect() - retrying in {seconds} seconds [{len(inspect.stack())} frames in stack]')
3831
+ time.sleep(seconds)
3832
+ reconnector(reconnecting=True)
3833
+ except KeyboardInterrupt as e:
3834
+ _logging.info(f'User exited {e}')
3835
+ raise e
3836
+ class Dispatcher(DispatcherBase):
3837
+ def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
3838
+ sel = selectors.DefaultSelector()
3839
+ sel.register(self.app.sock.sock, selectors.EVENT_READ)
3840
+ try:
3841
+ while self.app.keep_running:
3842
+ if sel.select(self.ping_timeout):
3843
+ if not read_callback():
3844
+ break
3845
+ check_callback()
3846
+ finally:
3847
+ sel.close()
3848
+ class SSLDispatcher(DispatcherBase):
3849
+ def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
3850
+ sock = self.app.sock.sock
3851
+ sel = selectors.DefaultSelector()
3852
+ sel.register(sock, selectors.EVENT_READ)
3853
+ try:
3854
+ while self.app.keep_running:
3855
+ if self.select(sock, sel):
3856
+ if not read_callback():
3857
+ break
3858
+ check_callback()
3859
+ finally:
3860
+ sel.close()
3861
+ def select(self, sock, sel: selectors.DefaultSelector):
3862
+ sock = self.app.sock.sock
3863
+ if sock.pending():
3864
+ return [sock]
3865
+ r = sel.select(self.ping_timeout)
3866
+ if len(r) > 0:
3867
+ return r[0][0]
3868
+ class WrappedDispatcher:
3869
+ def __init__(self, app, ping_timeout: Union[float, int, None], dispatcher) -> None:
3870
+ self.app = app
3871
+ self.ping_timeout = ping_timeout
3872
+ self.dispatcher = dispatcher
3873
+ dispatcher.signal(2, dispatcher.abort)
3874
+ def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
3875
+ self.dispatcher.read(sock, read_callback)
3876
+ self.ping_timeout and self.timeout(self.ping_timeout, check_callback)
3877
+ def timeout(self, seconds: float, callback: Callable) -> None:
3878
+ self.dispatcher.timeout(seconds, callback)
3879
+ def reconnect(self, seconds: int, reconnector: Callable) -> None:
3880
+ self.timeout(seconds, reconnector)
3881
+ class WebSocketApp:
3882
+ def __init__(self, url: str, header: Union[list, dict, Callable, None]=None, on_open: Optional[Callable[[WebSocket], None]]=None, on_reconnect: Optional[Callable[[WebSocket], None]]=None, on_message: Optional[Callable[[WebSocket, Any], None]]=None, on_error: Optional[Callable[[WebSocket, Any], None]]=None, on_close: Optional[Callable[[WebSocket, Any, Any], None]]=None, on_ping: Optional[Callable]=None, on_pong: Optional[Callable]=None, on_cont_message: Optional[Callable]=None, keep_running: bool=True, get_mask_key: Optional[Callable]=None, cookie: Optional[str]=None, subprotocols: Optional[list]=None, on_data: Optional[Callable]=None, socket: Optional[socket.socket]=None) -> None:
3883
+ self.url = url
3884
+ self.header = header if header is not None else []
3885
+ self.cookie = cookie
3886
+ self.on_open = on_open
3887
+ self.on_reconnect = on_reconnect
3888
+ self.on_message = on_message
3889
+ self.on_data = on_data
3890
+ self.on_error = on_error
3891
+ self.on_close = on_close
3892
+ self.on_ping = on_ping
3893
+ self.on_pong = on_pong
3894
+ self.on_cont_message = on_cont_message
3895
+ self.keep_running = False
3896
+ self.get_mask_key = get_mask_key
3897
+ self.sock: Optional[WebSocket] = None
3898
+ self.last_ping_tm = float(0)
3899
+ self.last_pong_tm = float(0)
3900
+ self.ping_thread: Optional[threading.Thread] = None
3901
+ self.stop_ping: Optional[threading.Event] = None
3902
+ self.ping_interval = float(0)
3903
+ self.ping_timeout: Union[float, int, None] = None
3904
+ self.ping_payload = ''
3905
+ self.subprotocols = subprotocols
3906
+ self.prepared_socket = socket
3907
+ self.has_errored = False
3908
+ self.has_done_teardown = False
3909
+ self.has_done_teardown_lock = threading.Lock()
3910
+ def send(self, data: Union[bytes, str], opcode: int=ABNF.OPCODE_TEXT) -> None:
3911
+ if not self.sock or self.sock.send(data, opcode) == 0:
3912
+ raise WebSocketConnectionClosedException('Connection is already closed.')
3913
+ def send_text(self, text_data: str) -> None:
3914
+ if not self.sock or self.sock.send(text_data, ABNF.OPCODE_TEXT) == 0:
3915
+ raise WebSocketConnectionClosedException('Connection is already closed.')
3916
+ def send_bytes(self, data: Union[bytes, bytearray]) -> None:
3917
+ if not self.sock or self.sock.send(data, ABNF.OPCODE_BINARY) == 0:
3918
+ raise WebSocketConnectionClosedException('Connection is already closed.')
3919
+ def close(self, **kwargs) -> None:
3920
+ self.keep_running = False
3921
+ if self.sock:
3922
+ self.sock.close(**kwargs)
3923
+ self.sock = None
3924
+ def _start_ping_thread(self) -> None:
3925
+ self.last_ping_tm = self.last_pong_tm = float(0)
3926
+ self.stop_ping = threading.Event()
3927
+ self.ping_thread = threading.Thread(target=self._send_ping)
3928
+ self.ping_thread.daemon = True
3929
+ self.ping_thread.start()
3930
+ def _stop_ping_thread(self) -> None:
3931
+ if self.stop_ping:
3932
+ self.stop_ping.set()
3933
+ if self.ping_thread and self.ping_thread.is_alive():
3934
+ self.ping_thread.join(3)
3935
+ self.last_ping_tm = self.last_pong_tm = float(0)
3936
+ def _send_ping(self) -> None:
3937
+ if self.stop_ping.wait(self.ping_interval) or self.keep_running is False:
3938
+ return
3939
+ while not self.stop_ping.wait(self.ping_interval) and self.keep_running is True:
3940
+ if self.sock:
3941
+ self.last_ping_tm = time.time()
3942
+ try:
3943
+ _logging.debug('Sending ping')
3944
+ self.sock.ping(self.ping_payload)
3945
+ except Exception as e:
3946
+ _logging.debug(f'Failed to send ping: {e}')
3947
+ def run_forever(self, sockopt: tuple=None, sslopt: dict=None, ping_interval: Union[float, int]=0, ping_timeout: Union[float, int, None]=None, ping_payload: str='', http_proxy_host: str=None, http_proxy_port: Union[int, str]=None, http_no_proxy: list=None, http_proxy_auth: tuple=None, http_proxy_timeout: Optional[float]=None, skip_utf8_validation: bool=False, host: str=None, origin: str=None, dispatcher=None, suppress_origin: bool=False, proxy_type: str=None, reconnect: int=None) -> bool:
3948
+ if reconnect is None:
3949
+ reconnect = RECONNECT
3950
+ if ping_timeout is not None and ping_timeout <= 0:
3951
+ raise WebSocketException('Ensure ping_timeout > 0')
3952
+ if ping_interval is not None and ping_interval < 0:
3953
+ raise WebSocketException('Ensure ping_interval >= 0')
3954
+ if ping_timeout and ping_interval and (ping_interval <= ping_timeout):
3955
+ raise WebSocketException('Ensure ping_interval > ping_timeout')
3956
+ if not sockopt:
3957
+ sockopt = ()
3958
+ if not sslopt:
3959
+ sslopt = {}
3960
+ if self.sock:
3961
+ raise WebSocketException('socket is already opened')
3962
+ self.ping_interval = ping_interval
3963
+ self.ping_timeout = ping_timeout
3964
+ self.ping_payload = ping_payload
3965
+ self.has_done_teardown = False
3966
+ self.keep_running = True
3967
+ def teardown(close_frame: ABNF=None):
3968
+ with self.has_done_teardown_lock:
3969
+ if self.has_done_teardown:
3970
+ return
3971
+ self.has_done_teardown = True
3972
+ self._stop_ping_thread()
3973
+ self.keep_running = False
3974
+ if self.sock:
3975
+ self.sock.close()
3976
+ (close_status_code, close_reason) = self._get_close_args(close_frame if close_frame else None)
3977
+ self.sock = None
3978
+ self._callback(self.on_close, close_status_code, close_reason)
3979
+ def setSock(reconnecting: bool=False) -> None:
3980
+ if reconnecting and self.sock:
3981
+ self.sock.shutdown()
3982
+ self.sock = WebSocket(self.get_mask_key, sockopt=sockopt, sslopt=sslopt, fire_cont_frame=self.on_cont_message is not None, skip_utf8_validation=skip_utf8_validation, enable_multithread=True)
3983
+ self.sock.settimeout(getdefaulttimeout())
3984
+ try:
3985
+ header = self.header() if callable(self.header) else self.header
3986
+ self.sock.connect(self.url, header=header, cookie=self.cookie, http_proxy_host=http_proxy_host, http_proxy_port=http_proxy_port, http_no_proxy=http_no_proxy, http_proxy_auth=http_proxy_auth, http_proxy_timeout=http_proxy_timeout, subprotocols=self.subprotocols, host=host, origin=origin, suppress_origin=suppress_origin, proxy_type=proxy_type, socket=self.prepared_socket)
3987
+ _logging.info('Websocket connected')
3988
+ if self.ping_interval:
3989
+ self._start_ping_thread()
3990
+ if reconnecting and self.on_reconnect:
3991
+ self._callback(self.on_reconnect)
3992
+ else:
3993
+ self._callback(self.on_open)
3994
+ dispatcher.read(self.sock.sock, read, check)
3995
+ except (WebSocketConnectionClosedException, ConnectionRefusedError, KeyboardInterrupt, SystemExit, Exception) as e:
3996
+ handleDisconnect(e, reconnecting)
3997
+ def read() -> bool:
3998
+ if not self.keep_running:
3999
+ return teardown()
4000
+ try:
4001
+ (op_code, frame) = self.sock.recv_data_frame(True)
4002
+ except (WebSocketConnectionClosedException, KeyboardInterrupt, SSLEOFError) as e:
4003
+ if custom_dispatcher:
4004
+ return handleDisconnect(e, bool(reconnect))
4005
+ else:
4006
+ raise e
4007
+ if op_code == ABNF.OPCODE_CLOSE:
4008
+ return teardown(frame)
4009
+ elif op_code == ABNF.OPCODE_PING:
4010
+ self._callback(self.on_ping, frame.data)
4011
+ elif op_code == ABNF.OPCODE_PONG:
4012
+ self.last_pong_tm = time.time()
4013
+ self._callback(self.on_pong, frame.data)
4014
+ elif op_code == ABNF.OPCODE_CONT and self.on_cont_message:
4015
+ self._callback(self.on_data, frame.data, frame.opcode, frame.fin)
4016
+ self._callback(self.on_cont_message, frame.data, frame.fin)
4017
+ else:
4018
+ data = frame.data
4019
+ if op_code == ABNF.OPCODE_TEXT and (not skip_utf8_validation):
4020
+ data = data.decode('utf-8')
4021
+ self._callback(self.on_data, data, frame.opcode, True)
4022
+ self._callback(self.on_message, data)
4023
+ return True
4024
+ def check() -> bool:
4025
+ if self.ping_timeout:
4026
+ has_timeout_expired = time.time() - self.last_ping_tm > self.ping_timeout
4027
+ has_pong_not_arrived_after_last_ping = self.last_pong_tm - self.last_ping_tm < 0
4028
+ has_pong_arrived_too_late = self.last_pong_tm - self.last_ping_tm > self.ping_timeout
4029
+ if self.last_ping_tm and has_timeout_expired and (has_pong_not_arrived_after_last_ping or has_pong_arrived_too_late):
4030
+ raise WebSocketTimeoutException('ping/pong timed out')
4031
+ return True
4032
+ def handleDisconnect(e: Union[WebSocketConnectionClosedException, ConnectionRefusedError, KeyboardInterrupt, SystemExit, Exception], reconnecting: bool=False) -> bool:
4033
+ self.has_errored = True
4034
+ self._stop_ping_thread()
4035
+ if not reconnecting:
4036
+ self._callback(self.on_error, e)
4037
+ if isinstance(e, (KeyboardInterrupt, SystemExit)):
4038
+ teardown()
4039
+ raise
4040
+ if reconnect:
4041
+ _logging.info(f'{e} - reconnect')
4042
+ if custom_dispatcher:
4043
+ _logging.debug(f'Calling custom dispatcher reconnect [{len(inspect.stack())} frames in stack]')
4044
+ dispatcher.reconnect(reconnect, setSock)
4045
+ else:
4046
+ _logging.error(f'{e} - goodbye')
4047
+ teardown()
4048
+ custom_dispatcher = bool(dispatcher)
4049
+ dispatcher = self.create_dispatcher(ping_timeout, dispatcher, parse_url(self.url)[3])
4050
+ try:
4051
+ setSock()
4052
+ if not custom_dispatcher and reconnect:
4053
+ while self.keep_running:
4054
+ _logging.debug(f'Calling dispatcher reconnect [{len(inspect.stack())} frames in stack]')
4055
+ dispatcher.reconnect(reconnect, setSock)
4056
+ except (KeyboardInterrupt, Exception) as e:
4057
+ _logging.info(f'tearing down on exception {e}')
4058
+ teardown()
4059
+ finally:
4060
+ if not custom_dispatcher:
4061
+ teardown()
4062
+ return self.has_errored
4063
+ def create_dispatcher(self, ping_timeout: Union[float, int, None], dispatcher: Optional[DispatcherBase]=None, is_ssl: bool=False) -> Union[Dispatcher, SSLDispatcher, WrappedDispatcher]:
4064
+ if dispatcher:
4065
+ return WrappedDispatcher(self, ping_timeout, dispatcher)
4066
+ timeout = ping_timeout or 10
4067
+ if is_ssl:
4068
+ return SSLDispatcher(self, timeout)
4069
+ return Dispatcher(self, timeout)
4070
+ def _get_close_args(self, close_frame: ABNF) -> list:
4071
+ if not self.on_close or not close_frame:
4072
+ return [None, None]
4073
+ if close_frame.data and len(close_frame.data) >= 2:
4074
+ close_status_code = 256 * int(close_frame.data[0]) + int(close_frame.data[1])
4075
+ reason = close_frame.data[2:]
4076
+ if isinstance(reason, bytes):
4077
+ reason = reason.decode('utf-8')
4078
+ return [close_status_code, reason]
4079
+ else:
4080
+ return [None, None]
4081
+ def _callback(self, callback, *args) -> None:
4082
+ if callback:
4083
+ try:
4084
+ callback(self, *args)
4085
+ except Exception as e:
4086
+ _logging.error(f'error from callback {callback}: {e}')
4087
+ if self.on_error:
4088
+ self.on_error(self, e)