vchrome 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vchrome/__init__.py +4088 -0
- vchrome-0.0.0.dist-info/METADATA +3 -0
- vchrome-0.0.0.dist-info/RECORD +5 -0
- vchrome-0.0.0.dist-info/WHEEL +5 -0
- vchrome-0.0.0.dist-info/top_level.txt +1 -0
vchrome/__init__.py
ADDED
|
@@ -0,0 +1,4088 @@
|
|
|
1
|
+
__version__ = '0.0.0'
|
|
2
|
+
__author__ = 'v'
|
|
3
|
+
# ----------------------------------------------------------------------------------------------------
|
|
4
|
+
_allowed = {'Chrome'}
|
|
5
|
+
def __getattr__(name):
|
|
6
|
+
if name in _allowed:
|
|
7
|
+
return globals()[name]
|
|
8
|
+
raise AttributeError(f"module {__name__} has no attribute {name}")
|
|
9
|
+
def __dir__():
|
|
10
|
+
return list(_allowed)
|
|
11
|
+
__all__ = __dir__()
|
|
12
|
+
import re
|
|
13
|
+
import json
|
|
14
|
+
import copy
|
|
15
|
+
import types
|
|
16
|
+
import queue
|
|
17
|
+
import time
|
|
18
|
+
import inspect
|
|
19
|
+
import threading
|
|
20
|
+
from time import perf_counter
|
|
21
|
+
from math import factorial, sin, pi
|
|
22
|
+
from random import random
|
|
23
|
+
from collections import deque
|
|
24
|
+
import socket
|
|
25
|
+
import base64
|
|
26
|
+
import traceback
|
|
27
|
+
from json import JSONDecodeError
|
|
28
|
+
from functools import lru_cache
|
|
29
|
+
from threading import Thread, Event, RLock
|
|
30
|
+
from urllib import request
|
|
31
|
+
import builtins
|
|
32
|
+
rl = RLock()
|
|
33
|
+
print_hook = False
|
|
34
|
+
def monkey_print():
|
|
35
|
+
global print_hook
|
|
36
|
+
if not print_hook:
|
|
37
|
+
_original_print = print
|
|
38
|
+
def thread_safe_print(*args, **kwargs):
|
|
39
|
+
with rl:
|
|
40
|
+
_original_print(*args, **kwargs, flush=True)
|
|
41
|
+
builtins.print = thread_safe_print
|
|
42
|
+
print_hook = True
|
|
43
|
+
def cdp_client(hostname, port, debug=False):
|
|
44
|
+
class Logger:
|
|
45
|
+
def _I(self, id):
|
|
46
|
+
return id
|
|
47
|
+
def _L(self, data, key):
|
|
48
|
+
if self.simple:
|
|
49
|
+
return json.dumps(data.get(key),ensure_ascii=False)
|
|
50
|
+
else:
|
|
51
|
+
return json.dumps(data,ensure_ascii=False)
|
|
52
|
+
def _T(self, tp, sessionId=None):
|
|
53
|
+
t = None
|
|
54
|
+
if tp == 'c': t = 'S:{} <->'
|
|
55
|
+
if tp == 'i': t = 'S:{} [*]'
|
|
56
|
+
return t.format(self._I(sessionId or self.id))
|
|
57
|
+
def _C(self, id, msg):
|
|
58
|
+
if self.debug: self.loginfo[id] = msg
|
|
59
|
+
def _G(self, id):
|
|
60
|
+
if self.debug:
|
|
61
|
+
r = self.loginfo.get(id)
|
|
62
|
+
if r:
|
|
63
|
+
del self.loginfo[id]
|
|
64
|
+
return r
|
|
65
|
+
else:
|
|
66
|
+
return {'msg': 'no cache'}
|
|
67
|
+
def __init__(self, id, debug=False):
|
|
68
|
+
self.id = id
|
|
69
|
+
self.debug = debug
|
|
70
|
+
self.simple = True
|
|
71
|
+
self.loginfo = {}
|
|
72
|
+
self.indent = 44
|
|
73
|
+
if self.debug:
|
|
74
|
+
monkey_print()
|
|
75
|
+
def log(*a):
|
|
76
|
+
print(*a)
|
|
77
|
+
def __call__(self, tp, a):
|
|
78
|
+
if self.debug:
|
|
79
|
+
if tp == 'recv':
|
|
80
|
+
id = a.get('id')
|
|
81
|
+
md = a.get('method')
|
|
82
|
+
se = a.get('sessionId')
|
|
83
|
+
if id:
|
|
84
|
+
c = self._G(id)
|
|
85
|
+
cse = c.get('sessionId')
|
|
86
|
+
cmd = c.get('method')
|
|
87
|
+
if self.debug(cmd):
|
|
88
|
+
print(self._T('c', cse), '[<={:3}] {}'.format(id, cmd).ljust(self.indent, ' '), self._L(a, 'result'))
|
|
89
|
+
if md:
|
|
90
|
+
if self.debug(md):
|
|
91
|
+
print(self._T('i', se), (' '*8 + md).ljust(self.indent, ' '), self._L(a, 'params'))
|
|
92
|
+
elif tp == 'req':
|
|
93
|
+
id = a.get('id')
|
|
94
|
+
md = a.get('method')
|
|
95
|
+
se = a.get('sessionId')
|
|
96
|
+
if self.debug(md):
|
|
97
|
+
self._C(id, a)
|
|
98
|
+
print(self._T('c', se), '[->{:3}] {}'.format(id, md).ljust(self.indent, ' '), self._L(a, 'params'))
|
|
99
|
+
else:
|
|
100
|
+
if self.debug.rest:
|
|
101
|
+
print(self._T('i'), (' '*8 + tp).ljust(self.indent, ' '), a)
|
|
102
|
+
class Pool:
|
|
103
|
+
import queue, time, traceback
|
|
104
|
+
from threading import Thread, main_thread
|
|
105
|
+
class KillThreadParams(Exception): pass
|
|
106
|
+
def __init__(self, num):
|
|
107
|
+
if not getattr(Pool.Thread, 'isAlive', None):
|
|
108
|
+
Pool.Thread.isAlive = Pool.Thread.is_alive
|
|
109
|
+
self._pool = Pool.queue.Queue()
|
|
110
|
+
self._monitor_run = Pool.queue.Queue()
|
|
111
|
+
self.main_monitor()
|
|
112
|
+
self.num = num
|
|
113
|
+
self.init()
|
|
114
|
+
self.is_close = True
|
|
115
|
+
def __call__(self,func):
|
|
116
|
+
def _run_threads(*args,**kw):
|
|
117
|
+
if self.is_close:
|
|
118
|
+
self.init()
|
|
119
|
+
self.main_monitor()
|
|
120
|
+
self._pool.put((func,args,kw))
|
|
121
|
+
return _run_threads
|
|
122
|
+
def init(self):
|
|
123
|
+
self.is_close = False
|
|
124
|
+
def _pools_pull():
|
|
125
|
+
while True:
|
|
126
|
+
v = self._pool.get()
|
|
127
|
+
if v == self.KillThreadParams: return
|
|
128
|
+
try:
|
|
129
|
+
func,args,kw = v
|
|
130
|
+
self._monitor_run.put('V')
|
|
131
|
+
func(*args,**kw)
|
|
132
|
+
except BaseException as e:
|
|
133
|
+
with rl:
|
|
134
|
+
print(Pool.traceback.format_exc())
|
|
135
|
+
finally:
|
|
136
|
+
self._monitor_run.get('V')
|
|
137
|
+
for _ in range(self.num):
|
|
138
|
+
Pool.Thread(target=_pools_pull).start()
|
|
139
|
+
def main_monitor(self):
|
|
140
|
+
def _func():
|
|
141
|
+
while True:
|
|
142
|
+
Pool.time.sleep(.12)
|
|
143
|
+
if not Pool.main_thread().isAlive() and self._monitor_run.empty():
|
|
144
|
+
self.close_all()
|
|
145
|
+
self.is_close = True
|
|
146
|
+
break
|
|
147
|
+
Pool.Thread(target=_func,name="MainMonitor").start()
|
|
148
|
+
def close_all(self):
|
|
149
|
+
for i in range(self.num):
|
|
150
|
+
self._pool.put(self.KillThreadParams)
|
|
151
|
+
@lru_cache(maxsize=1024)
|
|
152
|
+
def get_cached_signature(func):
|
|
153
|
+
return inspect.signature(func)
|
|
154
|
+
def to_human_read(size_in_bytes, decimal_places= 2):
|
|
155
|
+
if size_in_bytes < 0:
|
|
156
|
+
raise ValueError("byte size cannot be negative!")
|
|
157
|
+
units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
|
|
158
|
+
scale = 1024
|
|
159
|
+
if size_in_bytes == 0:
|
|
160
|
+
return f"0 B"
|
|
161
|
+
unit_idx = 0
|
|
162
|
+
while size_in_bytes >= scale and unit_idx < len(units) - 1:
|
|
163
|
+
size_in_bytes /= scale
|
|
164
|
+
unit_idx += 1
|
|
165
|
+
return f"{size_in_bytes:.{decimal_places}f} {units[unit_idx]}"
|
|
166
|
+
def myget(url):
|
|
167
|
+
r = request.Request(url, method='GET')
|
|
168
|
+
opener = request.build_opener(request.ProxyHandler(None))
|
|
169
|
+
return json.loads(opener.open(r).read().decode())
|
|
170
|
+
def adj_wsurl(wsurl):
|
|
171
|
+
return re.sub('ws://[^/]+/devtools/', 'ws://{}:{}/devtools/'.format(hostname, port), wsurl)
|
|
172
|
+
def make_dev_page_url(id):
|
|
173
|
+
return "ws://{}:{}/devtools/page/{}".format(hostname, port, id)
|
|
174
|
+
def try_run_result(data):
|
|
175
|
+
is_err = False
|
|
176
|
+
try:
|
|
177
|
+
if data['result'].get('type') == 'undefined':
|
|
178
|
+
return None
|
|
179
|
+
elif data['result'].get('subtype') == 'null':
|
|
180
|
+
return None
|
|
181
|
+
elif data['result'].get('value', None) != None:
|
|
182
|
+
return data['result']['value']
|
|
183
|
+
elif data['result'].get('objectId', None) != None:
|
|
184
|
+
return data['result']
|
|
185
|
+
elif data['result'].get('description'):
|
|
186
|
+
is_err = data['result']['description']
|
|
187
|
+
else:
|
|
188
|
+
raise Exception('err')
|
|
189
|
+
except:
|
|
190
|
+
return data
|
|
191
|
+
if is_err:
|
|
192
|
+
raise Exception(is_err)
|
|
193
|
+
def is_function(obj):
|
|
194
|
+
return type(obj) == types.FunctionType or type(obj) == types.MethodType
|
|
195
|
+
def create_connection_saf(*a, **kw):
|
|
196
|
+
for i in range(50):
|
|
197
|
+
try:
|
|
198
|
+
return create_connection(*a, **kw)
|
|
199
|
+
except WebSocketBadStatusException as e:
|
|
200
|
+
if b'No such target id:' in e.resp_body:
|
|
201
|
+
time.sleep(0.05)
|
|
202
|
+
continue
|
|
203
|
+
raise Exception('connect ws error')
|
|
204
|
+
class Err: pass
|
|
205
|
+
class Waiter:
|
|
206
|
+
def __init__(self, sniff, pattern, is_regex):
|
|
207
|
+
self.s = sniff
|
|
208
|
+
self.pattern = pattern
|
|
209
|
+
self.is_regex = is_regex
|
|
210
|
+
self.count = 0
|
|
211
|
+
self.currc = 0
|
|
212
|
+
self.cache = deque(maxlen=1024)
|
|
213
|
+
self.is_remove = False
|
|
214
|
+
def add(self, r):
|
|
215
|
+
self.count += 1
|
|
216
|
+
self.cache.appendleft(r)
|
|
217
|
+
def wait(self, count=1, timeout=10):
|
|
218
|
+
if self.is_remove:
|
|
219
|
+
raise Exception('listener remove by waiter, cannot use wait.')
|
|
220
|
+
start = perf_counter()
|
|
221
|
+
while True:
|
|
222
|
+
if self._check(count):
|
|
223
|
+
return self.cache.pop()
|
|
224
|
+
time.sleep(0.15)
|
|
225
|
+
if perf_counter() - start > timeout:
|
|
226
|
+
break
|
|
227
|
+
def remove(self):
|
|
228
|
+
f = getattr(self.s, 'remove_listen', None) or getattr(self.s, 'remove_change', None)
|
|
229
|
+
f(self.pattern)
|
|
230
|
+
self.is_remove = True
|
|
231
|
+
def _check(self, count):
|
|
232
|
+
if self.count >= self.currc + count:
|
|
233
|
+
self.currc += count
|
|
234
|
+
return True
|
|
235
|
+
else:
|
|
236
|
+
return False
|
|
237
|
+
def __repr__(self):
|
|
238
|
+
return '<Waiter is_regex:[{}] [{}]>'.format(self.is_regex, json.dumps(self.pattern, ensure_ascii=False))
|
|
239
|
+
class SniffTools:
|
|
240
|
+
def __init__(self, s):
|
|
241
|
+
self.s = s
|
|
242
|
+
self.qlist = queue.Queue()
|
|
243
|
+
self.default_func = lambda a:True
|
|
244
|
+
self.attach()
|
|
245
|
+
def attach(self):
|
|
246
|
+
self.s._match_url = self._match_url
|
|
247
|
+
self.s.qlist = self.qlist
|
|
248
|
+
self.s.default_func = self.default_func
|
|
249
|
+
def _match_str(self, a, b, is_regex):
|
|
250
|
+
if is_regex:
|
|
251
|
+
return re.findall(a, b)
|
|
252
|
+
else:
|
|
253
|
+
return a in b
|
|
254
|
+
def _match_url(self, v, url):
|
|
255
|
+
pattern, is_regex = v
|
|
256
|
+
if type(pattern) == str:
|
|
257
|
+
return self._match_str(pattern, url, is_regex)
|
|
258
|
+
if type(pattern) == list:
|
|
259
|
+
for p in pattern:
|
|
260
|
+
if self._match_str(p, url, is_regex):
|
|
261
|
+
return True
|
|
262
|
+
return False
|
|
263
|
+
class SniffNetwork:
|
|
264
|
+
class NetworkRequest:
|
|
265
|
+
def __init__(self, rinfo, encoding='utf8'):
|
|
266
|
+
self.rinfo = rinfo
|
|
267
|
+
self.encoding = encoding
|
|
268
|
+
self._url = rinfo['request'].get('url')
|
|
269
|
+
self._method = rinfo['request'].get('method')
|
|
270
|
+
self._headers = rinfo['request'].get('headers')
|
|
271
|
+
if rinfo['request'].get('hasPostData'):
|
|
272
|
+
self._content = b''.join(base64.b64decode(e['bytes']) for e in rinfo['request'].get('postDataEntries', []))
|
|
273
|
+
@property
|
|
274
|
+
def url(self): return self._url
|
|
275
|
+
@property
|
|
276
|
+
def method(self): return self._method
|
|
277
|
+
@property
|
|
278
|
+
def headers(self): return self._headers
|
|
279
|
+
@property
|
|
280
|
+
def content(self):
|
|
281
|
+
assert self.method == 'POST', 'must be method: POST'
|
|
282
|
+
return self._content
|
|
283
|
+
@property
|
|
284
|
+
def text(self):
|
|
285
|
+
assert self.method == 'POST', 'must be method: POST'
|
|
286
|
+
return self._content.decode(self.encoding)
|
|
287
|
+
def json(self):
|
|
288
|
+
c = self.text
|
|
289
|
+
return json.loads(c[c.find('{'):c.rfind('}')+1])
|
|
290
|
+
def __repr__(self):
|
|
291
|
+
return '<Request [{}] ReadOnly>'.format(self.method)
|
|
292
|
+
class NetworkResponse:
|
|
293
|
+
def __init__(self, rinfo, encoding='utf8'):
|
|
294
|
+
self.rinfo = rinfo
|
|
295
|
+
self.encoding = encoding
|
|
296
|
+
self.error = None
|
|
297
|
+
if rinfo.get('status') == 'ERROR':
|
|
298
|
+
self.error = rinfo['error']
|
|
299
|
+
self._request = 'ERROR'
|
|
300
|
+
self._url = 'ERROR'
|
|
301
|
+
self._status_code = -1
|
|
302
|
+
self._headers = {}
|
|
303
|
+
self._content = b'ERROR'
|
|
304
|
+
return
|
|
305
|
+
self._request = SniffNetwork.NetworkRequest(rinfo)
|
|
306
|
+
self._url = rinfo['response'].get('url')
|
|
307
|
+
self._status_code = rinfo['response'].get('status')
|
|
308
|
+
self._headers = rinfo['response'].get('headers')
|
|
309
|
+
if rinfo['response_body'].get('base64Encoded'):
|
|
310
|
+
self._content = base64.b64decode(rinfo['response_body']['body'])
|
|
311
|
+
elif 'body' in rinfo['response_body']:
|
|
312
|
+
self._content = rinfo['response_body']['body'].encode(self.encoding)
|
|
313
|
+
else:
|
|
314
|
+
self._content = b'<cannot catch body data. maybe session redirect too fast.>'
|
|
315
|
+
@property
|
|
316
|
+
def url(self): return self._url
|
|
317
|
+
@property
|
|
318
|
+
def status_code(self): return self._status_code
|
|
319
|
+
@property
|
|
320
|
+
def headers(self): return self._headers
|
|
321
|
+
@property
|
|
322
|
+
def content(self): return self._content
|
|
323
|
+
@property
|
|
324
|
+
def text(self): return self._content.decode(self.encoding)
|
|
325
|
+
def json(self):
|
|
326
|
+
c = self.text
|
|
327
|
+
return json.loads(c[c.find('{'):c.rfind('}')+1])
|
|
328
|
+
@property
|
|
329
|
+
def request(self): return self._request
|
|
330
|
+
def __repr__(self):
|
|
331
|
+
if self.error:
|
|
332
|
+
return '<Response [ERROR] reason:{}>'.format(self.error)
|
|
333
|
+
return '<Response [{}] ReadOnly>'.format(self.status_code)
|
|
334
|
+
def __init__(self, f):
|
|
335
|
+
self.f = f
|
|
336
|
+
self.type = self.f.type
|
|
337
|
+
self.is_listen = False
|
|
338
|
+
self.listen_sesslist = [None]
|
|
339
|
+
self.info_listen = {}
|
|
340
|
+
self.call_listen = {}
|
|
341
|
+
self.call_listen_keys = []
|
|
342
|
+
self.call_listen_vals = {}
|
|
343
|
+
self.tools = SniffTools(self)
|
|
344
|
+
def _listen_cdp(self, sessionId):
|
|
345
|
+
self.f.cdp('Network.enable',{
|
|
346
|
+
"maxTotalBufferSize": 100000000,
|
|
347
|
+
"maxResourceBufferSize":50000000,
|
|
348
|
+
"includeNetworkExtraInfo":True,
|
|
349
|
+
}, sessionId=sessionId)
|
|
350
|
+
def add_listen_session(self, sessionId):
|
|
351
|
+
self.listen_sesslist.append(sessionId)
|
|
352
|
+
if self.is_listen:
|
|
353
|
+
self._listen_cdp(sessionId)
|
|
354
|
+
def remove_listen(self, pattern):
|
|
355
|
+
pk = json.dumps(pattern, sort_keys=True)
|
|
356
|
+
self.call_listen.pop(pk, None)
|
|
357
|
+
self.call_listen_keys = self.call_listen.keys()
|
|
358
|
+
self.call_listen_vals.pop(pk, None)
|
|
359
|
+
def listen(self, pattern, on_response=None, on_request=None, is_regex=False):
|
|
360
|
+
if not self.is_listen:
|
|
361
|
+
self._handle_network()
|
|
362
|
+
for sessionId in self.listen_sesslist:
|
|
363
|
+
self._listen_cdp(sessionId)
|
|
364
|
+
self.is_listen = True
|
|
365
|
+
waiter = Waiter(self, pattern, is_regex)
|
|
366
|
+
pk = json.dumps(pattern, sort_keys=True)
|
|
367
|
+
if (not on_request) and (not on_response):
|
|
368
|
+
on_response = self.default_func
|
|
369
|
+
self.call_listen[pk] = [waiter, on_request, on_response]
|
|
370
|
+
self.call_listen_keys = self.call_listen.keys()
|
|
371
|
+
self.call_listen_vals[pk] = [pattern, is_regex]
|
|
372
|
+
return waiter
|
|
373
|
+
def _handle_network(self):
|
|
374
|
+
self.f.set_method_callback('Network.requestWillBeSent', self.Network_requestWillBeSent)
|
|
375
|
+
self.f.set_method_callback('Network.requestWillBeSentExtraInfo', self.Network_requestWillBeSentExtraInfo)
|
|
376
|
+
self.f.set_method_callback('Network.responseReceived', self.Network_responseReceived)
|
|
377
|
+
self.f.set_method_callback('Network.responseReceivedExtraInfo', self.Network_responseReceivedExtraInfo)
|
|
378
|
+
self.f.set_method_callback('Network.loadingFinished', self.Network_loadingFinished)
|
|
379
|
+
self.f.set_method_callback('Network.loadingFailed', self.Network_loadingFailed)
|
|
380
|
+
def Network_requestWillBeSent(self, rdata):
|
|
381
|
+
requestId = rdata['params']['requestId']
|
|
382
|
+
self.info_listen[requestId] = self.info_listen.get(requestId, {})
|
|
383
|
+
self.info_listen[requestId]['request'] = rdata['params']['request']
|
|
384
|
+
if self.info_listen[requestId].get('request_extra'):
|
|
385
|
+
request_extra = self.info_listen[requestId].get('request_extra')
|
|
386
|
+
headers = self.info_listen[requestId]['request']['headers']
|
|
387
|
+
self.info_listen[requestId]['request']['headers'] = {**headers, **request_extra['headers']}
|
|
388
|
+
url = self.info_listen[requestId]['request']['url']
|
|
389
|
+
for k in self.call_listen_keys:
|
|
390
|
+
if self._match_url(self.call_listen_vals[k], url):
|
|
391
|
+
waiter, on_request, on_response = self.call_listen[k]
|
|
392
|
+
if on_request:
|
|
393
|
+
r = SniffNetwork.NetworkRequest(self.info_listen[requestId])
|
|
394
|
+
if on_request(r):
|
|
395
|
+
waiter.add(r)
|
|
396
|
+
def Network_requestWillBeSentExtraInfo(self, rdata):
|
|
397
|
+
request_extra = rdata['params']
|
|
398
|
+
requestId = request_extra['requestId']
|
|
399
|
+
if self.info_listen.get(requestId):
|
|
400
|
+
headers = self.info_listen[requestId]['request']['headers']
|
|
401
|
+
self.info_listen[requestId]['request']['headers'] = {**headers, **request_extra['headers']}
|
|
402
|
+
url = self.info_listen[requestId]['request']['url']
|
|
403
|
+
for k in self.call_listen_keys:
|
|
404
|
+
if self._match_url(self.call_listen_vals[k], url):
|
|
405
|
+
waiter, on_request, on_response = self.call_listen[k]
|
|
406
|
+
if on_request:
|
|
407
|
+
r = SniffNetwork.NetworkRequest(self.info_listen[requestId])
|
|
408
|
+
if on_request(r):
|
|
409
|
+
waiter.add(r)
|
|
410
|
+
else:
|
|
411
|
+
self.info_listen[requestId] = {}
|
|
412
|
+
self.info_listen[requestId]['request_extra'] = request_extra
|
|
413
|
+
def Network_responseReceived(self, rdata):
|
|
414
|
+
requestId = rdata['params']['requestId']
|
|
415
|
+
if requestId not in self.info_listen: return
|
|
416
|
+
self.info_listen[requestId]['response'] = rdata['params']['response']
|
|
417
|
+
response_extra = self.info_listen[requestId].pop('response_extra', None)
|
|
418
|
+
if response_extra:
|
|
419
|
+
_headers = self.info_listen[requestId]['response']['headers']
|
|
420
|
+
_headers = {**response_extra['headers'], **_headers}
|
|
421
|
+
self.info_listen[requestId]['response']['headers'] = _headers
|
|
422
|
+
def Network_responseReceivedExtraInfo(self, rdata):
|
|
423
|
+
response_extra = rdata['params']
|
|
424
|
+
requestId = response_extra['requestId']
|
|
425
|
+
if requestId not in self.info_listen: return
|
|
426
|
+
if self.info_listen[requestId].get('response'):
|
|
427
|
+
_headers = self.info_listen[requestId]['response']['headers']
|
|
428
|
+
_headers = {**response_extra['headers'], **_headers}
|
|
429
|
+
self.info_listen[requestId]['response']['headers'] = _headers
|
|
430
|
+
else:
|
|
431
|
+
self.info_listen[requestId]['response_extra'] = response_extra
|
|
432
|
+
def Network_loadingFinished(self, rdata):
|
|
433
|
+
params = rdata['params']
|
|
434
|
+
requestId = params['requestId']
|
|
435
|
+
if requestId not in self.info_listen: return
|
|
436
|
+
url = self.info_listen[requestId]['request']['url']
|
|
437
|
+
for k in self.call_listen_keys:
|
|
438
|
+
if self._match_url(self.call_listen_vals[k], url):
|
|
439
|
+
waiter, on_request, on_response = self.call_listen[k]
|
|
440
|
+
if on_response:
|
|
441
|
+
self.qlist.put('V')
|
|
442
|
+
rbody = self.f.cdp('Network.getResponseBody', {"requestId": requestId}, sessionId=rdata.get('sessionId'), limit_time=3)
|
|
443
|
+
self.info_listen[requestId]['response_body'] = rbody
|
|
444
|
+
r = SniffNetwork.NetworkResponse(self.info_listen[requestId])
|
|
445
|
+
if on_response(r):
|
|
446
|
+
waiter.add(r)
|
|
447
|
+
self.info_listen.pop(requestId, None)
|
|
448
|
+
self.qlist.get('V')
|
|
449
|
+
def Network_loadingFailed(self, rdata):
|
|
450
|
+
params = rdata['params']
|
|
451
|
+
requestId = params['requestId']
|
|
452
|
+
if requestId not in self.info_listen: return
|
|
453
|
+
url = self.info_listen[requestId]['request']['url']
|
|
454
|
+
self.info_listen[requestId]['status'] = 'ERROR'
|
|
455
|
+
self.info_listen[requestId]['error'] = params['errorText']
|
|
456
|
+
for k in self.call_listen_keys:
|
|
457
|
+
if self._match_url(self.call_listen_vals[k], url):
|
|
458
|
+
waiter, on_request, on_response = self.call_listen[k]
|
|
459
|
+
if on_response:
|
|
460
|
+
self.qlist.put('V')
|
|
461
|
+
r = SniffNetwork.NetworkResponse(self.info_listen[requestId])
|
|
462
|
+
if on_response(r):
|
|
463
|
+
waiter.add(r)
|
|
464
|
+
self.qlist.get('V')
|
|
465
|
+
class SniffFetch:
|
|
466
|
+
class FetchFakeResponse:
|
|
467
|
+
def __init__(self, rinfo, encoding='utf8'):
|
|
468
|
+
self.rinfo = rinfo
|
|
469
|
+
self.encoding = encoding
|
|
470
|
+
self._url = rinfo.get('url')
|
|
471
|
+
self._responseCode = 200
|
|
472
|
+
self._responseHeaders = {"fake-vvv": "fake-vvv"}
|
|
473
|
+
self._body = b''
|
|
474
|
+
self.request = rinfo.get('request')
|
|
475
|
+
@property
|
|
476
|
+
def url(self): return self._url
|
|
477
|
+
@property
|
|
478
|
+
def status_code(self): return self._responseCode
|
|
479
|
+
@status_code.setter
|
|
480
|
+
def status_code(self, value): assert isinstance(value, int), 'must be type: int'; self._responseCode = value
|
|
481
|
+
@property
|
|
482
|
+
def headers(self): return self._responseHeaders
|
|
483
|
+
@headers.setter
|
|
484
|
+
def headers(self, value): assert isinstance(value, dict), 'must be type: dict'; self._responseHeaders = value
|
|
485
|
+
@property
|
|
486
|
+
def content(self): return self._body
|
|
487
|
+
@content.setter
|
|
488
|
+
def content(self, value): assert isinstance(value, bytes), 'must be type: bytes'; self._body = value
|
|
489
|
+
@property
|
|
490
|
+
def text(self): return self._body.decode(self.encoding)
|
|
491
|
+
@text.setter
|
|
492
|
+
def text(self, value): assert isinstance(value, str), 'must be type: str'; self._body = value.encode(self.encoding)
|
|
493
|
+
def json(self):
|
|
494
|
+
c = self.text
|
|
495
|
+
return json.loads(c[c.find('{'):c.rfind('}')+1])
|
|
496
|
+
def __repr__(self): return '<FakeResponse [{}]>'.format(self.status_code)
|
|
497
|
+
def get(self, k):
|
|
498
|
+
return getattr(self, '_'+k, None)
|
|
499
|
+
class FetchRequest:
|
|
500
|
+
def __init__(self, rinfo, encoding='utf8'):
|
|
501
|
+
self.rinfo = rinfo
|
|
502
|
+
self.encoding = encoding
|
|
503
|
+
self._url = rinfo.get('url')
|
|
504
|
+
self._url_copy = self._url
|
|
505
|
+
self._method = rinfo.get('method')
|
|
506
|
+
self._method_copy = self._method
|
|
507
|
+
self._headers = rinfo.get('headers')
|
|
508
|
+
self._headers_copy = copy.deepcopy(self._headers)
|
|
509
|
+
self._postData = rinfo.get('postData')
|
|
510
|
+
self._postData_copy = self._postData
|
|
511
|
+
self._fake = None
|
|
512
|
+
@property
|
|
513
|
+
def url(self): return self._url
|
|
514
|
+
@property
|
|
515
|
+
def method(self): return self._method
|
|
516
|
+
@method.setter
|
|
517
|
+
def method(self, value): assert isinstance(value, str), 'must be type: str'; self._method = value
|
|
518
|
+
@property
|
|
519
|
+
def headers(self): return self._headers
|
|
520
|
+
@headers.setter
|
|
521
|
+
def headers(self, value): assert isinstance(value, dict), 'must be type: dict'; self._headers = value
|
|
522
|
+
@property
|
|
523
|
+
def content(self): return self._postData
|
|
524
|
+
@content.setter
|
|
525
|
+
def content(self, value): assert isinstance(value, bytes), 'must be type: bytes'; self._postData = value
|
|
526
|
+
@property
|
|
527
|
+
def text(self): return self._postData.decode(self.encoding)
|
|
528
|
+
@text.setter
|
|
529
|
+
def text(self, value): assert isinstance(value, str), 'must be type: str'; self._postData = value.encode(self.encoding)
|
|
530
|
+
def json(self):
|
|
531
|
+
c = self.text
|
|
532
|
+
return json.loads(c[c.find('{'):c.rfind('}')+1])
|
|
533
|
+
def __repr__(self): return '<Request [{}]>'.format(self.method)
|
|
534
|
+
def check_change(self):
|
|
535
|
+
return (
|
|
536
|
+
self._url != self._url_copy
|
|
537
|
+
or self._method != self._method_copy
|
|
538
|
+
or self._headers != self._headers_copy
|
|
539
|
+
or json.dumps(self._headers, sort_keys=True) != json.dumps(self._headers_copy, sort_keys=True)
|
|
540
|
+
or (self._method == 'POST' and self._postData != self._postData_copy)
|
|
541
|
+
)
|
|
542
|
+
def get(self, k):
|
|
543
|
+
return getattr(self, '_'+k, None)
|
|
544
|
+
def fake_response(self):
|
|
545
|
+
self._fake = SniffFetch.FetchFakeResponse({"url": self._url, "request": self})
|
|
546
|
+
return self._fake
|
|
547
|
+
class FetchResponse:
|
|
548
|
+
def __init__(self, rinfo, encoding='utf8'):
|
|
549
|
+
self.rinfo = rinfo
|
|
550
|
+
self.encoding = encoding
|
|
551
|
+
self.error = None
|
|
552
|
+
self.request = rinfo.get('request')
|
|
553
|
+
self._url = rinfo.get('url')
|
|
554
|
+
self._url_copy = self._url
|
|
555
|
+
self._responseCode = rinfo.get('responseCode')
|
|
556
|
+
self._responseCode_copy = self._responseCode
|
|
557
|
+
self._responseHeaders = rinfo.get('responseHeaders')
|
|
558
|
+
self._responseHeaders_copy = copy.deepcopy(self._responseHeaders)
|
|
559
|
+
self._body = rinfo.get('body')
|
|
560
|
+
self._body_copy = self._body
|
|
561
|
+
@property
|
|
562
|
+
def url(self): return self._url
|
|
563
|
+
@property
|
|
564
|
+
def status_code(self): return self._responseCode
|
|
565
|
+
@status_code.setter
|
|
566
|
+
def status_code(self, value): assert isinstance(value, int), 'must be type: int'; self._responseCode = value
|
|
567
|
+
@property
|
|
568
|
+
def headers(self): return self._responseHeaders
|
|
569
|
+
@headers.setter
|
|
570
|
+
def headers(self, value): assert isinstance(value, dict), 'must be type: dict'; self._responseHeaders = value
|
|
571
|
+
@property
|
|
572
|
+
def content(self): return self._body
|
|
573
|
+
@content.setter
|
|
574
|
+
def content(self, value): assert isinstance(value, bytes), 'must be type: bytes'; self._body = value
|
|
575
|
+
@property
|
|
576
|
+
def text(self): return self._body.decode(self.encoding)
|
|
577
|
+
@text.setter
|
|
578
|
+
def text(self, value): assert isinstance(value, str), 'must be type: str'; self._body = value.encode(self.encoding)
|
|
579
|
+
def json(self):
|
|
580
|
+
c = self.text
|
|
581
|
+
return json.loads(c[c.find('{'):c.rfind('}')+1])
|
|
582
|
+
def __repr__(self): return '<Response [{}]>'.format(self.status_code)
|
|
583
|
+
def check_change(self):
|
|
584
|
+
return (
|
|
585
|
+
self._url != self._url_copy
|
|
586
|
+
or self._responseCode != self._responseCode_copy
|
|
587
|
+
or self._responseHeaders != self._responseHeaders_copy
|
|
588
|
+
or json.dumps(self._responseHeaders, sort_keys=True) != json.dumps(self._responseHeaders_copy, sort_keys=True)
|
|
589
|
+
or self._body_copy != self._body
|
|
590
|
+
)
|
|
591
|
+
def get(self, k):
|
|
592
|
+
return getattr(self, '_'+k, None)
|
|
593
|
+
def __init__(self, f):
|
|
594
|
+
self.f = f
|
|
595
|
+
self.type = self.f.type
|
|
596
|
+
self.is_change = False
|
|
597
|
+
self.change_sesslist = [None]
|
|
598
|
+
self.info_change = {}
|
|
599
|
+
self.call_change = {}
|
|
600
|
+
self.call_change_keys = []
|
|
601
|
+
self.call_change_vals = {}
|
|
602
|
+
self.tools = SniffTools(self)
|
|
603
|
+
def _change_cdp(self, sessionId):
|
|
604
|
+
self.f.cdp('Fetch.enable',{
|
|
605
|
+
"handleAuthRequests": True,
|
|
606
|
+
"patterns":[
|
|
607
|
+
{"urlPattern":"*","requestStage":"Request"},
|
|
608
|
+
{"urlPattern":"*","requestStage":"Response"},
|
|
609
|
+
]}, sessionId=sessionId)
|
|
610
|
+
def add_change_session(self, sessionId):
|
|
611
|
+
self.change_sesslist.append(sessionId)
|
|
612
|
+
if self.is_change:
|
|
613
|
+
self._change_cdp(sessionId)
|
|
614
|
+
def remove_change(self, pattern):
|
|
615
|
+
pk = json.dumps(pattern, sort_keys=True)
|
|
616
|
+
self.call_change.pop(pk, None)
|
|
617
|
+
self.call_change_keys = self.call_change.keys()
|
|
618
|
+
self.call_change_vals.pop(pk, None)
|
|
619
|
+
def intercept(self, pattern, on_response=None, on_request=None, is_regex=False):
|
|
620
|
+
if not self.is_change:
|
|
621
|
+
self._handle_fetch()
|
|
622
|
+
for sessionId in self.change_sesslist:
|
|
623
|
+
self._change_cdp(sessionId)
|
|
624
|
+
self.is_change = True
|
|
625
|
+
waiter = Waiter(self, pattern, is_regex)
|
|
626
|
+
pk = json.dumps(pattern, sort_keys=True)
|
|
627
|
+
if (not on_request) and (not on_response):
|
|
628
|
+
on_response = self.default_func
|
|
629
|
+
self.call_change[pk] = [waiter, on_request, on_response]
|
|
630
|
+
self.call_change_keys = self.call_change.keys()
|
|
631
|
+
self.call_change_vals[pk] = [pattern, is_regex]
|
|
632
|
+
return waiter
|
|
633
|
+
def _handle_fetch(self):
|
|
634
|
+
self.f.set_method_callback('Fetch.requestPaused', self.Fetch_requestPaused)
|
|
635
|
+
def _dict_to_list(self, headers_dict):
|
|
636
|
+
return [{"name": k, "value": v} for k, v in headers_dict.items()]
|
|
637
|
+
def _list_to_dict(self, headers_list):
|
|
638
|
+
d = {}
|
|
639
|
+
for kv in headers_list:
|
|
640
|
+
d[kv['name']] = kv['value']
|
|
641
|
+
return d
|
|
642
|
+
def Fetch_requestPaused(self, rdata):
|
|
643
|
+
url = rdata['params']['request']['url']
|
|
644
|
+
method = rdata['params']['request']['method']
|
|
645
|
+
requestId = rdata['params']['requestId']
|
|
646
|
+
if "responseStatusCode" not in rdata['params']:
|
|
647
|
+
for k in self.call_change_keys:
|
|
648
|
+
if self._match_url(self.call_change_vals[k], url):
|
|
649
|
+
waiter, on_request, on_response = self.call_change[k]
|
|
650
|
+
x = SniffFetch.FetchRequest({
|
|
651
|
+
"url": url,
|
|
652
|
+
"method": method,
|
|
653
|
+
"headers": rdata['params']['request']['headers'],
|
|
654
|
+
"postData": b''.join(base64.b64decode(e['bytes']) for e in rdata['params']['request'].get('postDataEntries', [])),
|
|
655
|
+
})
|
|
656
|
+
if on_request:
|
|
657
|
+
try:
|
|
658
|
+
if on_request(x):
|
|
659
|
+
waiter.add(x)
|
|
660
|
+
except:
|
|
661
|
+
self.f.logger.log('[ERROR] in request on_request', traceback.format_exc())
|
|
662
|
+
continue
|
|
663
|
+
try:
|
|
664
|
+
if x._fake:
|
|
665
|
+
fk = x._fake
|
|
666
|
+
d = {
|
|
667
|
+
"requestId": requestId,
|
|
668
|
+
"responseCode": fk.get('responseCode'),
|
|
669
|
+
"responseHeaders": self._dict_to_list(fk.get('responseHeaders')),
|
|
670
|
+
"body": base64.b64encode(fk.get('body')).decode("ascii")
|
|
671
|
+
}
|
|
672
|
+
self.f.cdp('Fetch.fulfillRequest', d, sessionId=rdata.get('sessionId'))
|
|
673
|
+
return
|
|
674
|
+
except:
|
|
675
|
+
self.f.logger.log('[ERROR] in request on_fake continue', traceback.format_exc())
|
|
676
|
+
try:
|
|
677
|
+
if x.check_change():
|
|
678
|
+
d = {
|
|
679
|
+
"requestId": requestId,
|
|
680
|
+
"method": x.get('method') or method,
|
|
681
|
+
"headers": self._dict_to_list(x.get('headers') or rdata['params']['request']['headers']),
|
|
682
|
+
}
|
|
683
|
+
if x.get('method') == 'POST':
|
|
684
|
+
d['postData'] = base64.b64encode(x.get('postData')).decode("ascii")
|
|
685
|
+
self.f.cdp('Fetch.continueRequest', d, sessionId=rdata.get('sessionId'))
|
|
686
|
+
return
|
|
687
|
+
except:
|
|
688
|
+
self.f.logger.log('[ERROR] in request on_request continue', traceback.format_exc())
|
|
689
|
+
self.f.cdp('Fetch.continueRequest', {'requestId': requestId}, sessionId=rdata.get('sessionId'))
|
|
690
|
+
else:
|
|
691
|
+
for k in self.call_change_keys:
|
|
692
|
+
if self._match_url(self.call_change_vals[k], url):
|
|
693
|
+
waiter, on_request, on_response = self.call_change[k]
|
|
694
|
+
if on_response:
|
|
695
|
+
body_info = self.f.cdp("Fetch.getResponseBody", {"requestId": requestId}, sessionId=rdata.get('sessionId'))
|
|
696
|
+
body = base64.b64decode(body_info["body"]) if body_info['base64Encoded'] else body_info["body"].encode()
|
|
697
|
+
x = SniffFetch.FetchResponse({
|
|
698
|
+
"url": url,
|
|
699
|
+
"responseCode": rdata['params']['responseStatusCode'],
|
|
700
|
+
"responseHeaders": self._list_to_dict(rdata['params']['responseHeaders']),
|
|
701
|
+
"body": body,
|
|
702
|
+
"request": SniffFetch.FetchRequest(rdata['params']['request'])
|
|
703
|
+
})
|
|
704
|
+
try:
|
|
705
|
+
if on_response(x):
|
|
706
|
+
waiter.add(x)
|
|
707
|
+
except:
|
|
708
|
+
self.f.logger.log('[ERROR] in response on_response', traceback.format_exc())
|
|
709
|
+
continue
|
|
710
|
+
try:
|
|
711
|
+
if x.check_change():
|
|
712
|
+
d = {
|
|
713
|
+
"requestId": requestId,
|
|
714
|
+
"responseCode": x.get('responseCode'),
|
|
715
|
+
"responseHeaders": self._dict_to_list(x.get('responseHeaders')),
|
|
716
|
+
"body": base64.b64encode(x.get('body') or body).decode("ascii")
|
|
717
|
+
}
|
|
718
|
+
self.f.cdp('Fetch.fulfillRequest', d, sessionId=rdata.get('sessionId'))
|
|
719
|
+
return
|
|
720
|
+
except Exception as e:
|
|
721
|
+
self.f.logger.log('[ERROR] in response on_response continue', traceback.format_exc())
|
|
722
|
+
self.f.cdp('Fetch.continueResponse', {'requestId': requestId}, sessionId=rdata.get('sessionId'))
|
|
723
|
+
class Page:
|
|
724
|
+
def __init__(self, f):
|
|
725
|
+
self.f = f
|
|
726
|
+
self.f.set_method_callback('Page.frameDetached', self.Page_frameDetached)
|
|
727
|
+
self.f.set_method_callback('Page.frameAttached', self.Page_frameAttached)
|
|
728
|
+
self.f.set_method_callback('Page.frameScheduledNavigation', self.Page_frameScheduledNavigation)
|
|
729
|
+
self.f.set_method_callback('Page.frameRequestedNavigation', self.Page_frameRequestedNavigation)
|
|
730
|
+
self.f.set_method_callback('Page.frameStartedNavigating', self.Page_frameStartedNavigating)
|
|
731
|
+
self.f.set_method_callback('Page.frameStartedLoading', self.Page_frameStartedLoading)
|
|
732
|
+
self.f.set_method_callback('Page.frameNavigated', self.Page_frameNavigated)
|
|
733
|
+
self.f.set_method_callback('Page.javascriptDialogOpening', self.Page_javascriptDialogOpening)
|
|
734
|
+
self.f.set_method_callback('Page.javascriptDialogClosed', self.Page_javascriptDialogClosed)
|
|
735
|
+
self.init()
|
|
736
|
+
self.dialog = None
|
|
737
|
+
def init(self, sessionId=None):
|
|
738
|
+
self.f.cdp('Page.enable', sessionId=sessionId)
|
|
739
|
+
def Page_frameNavigated(self, rdata):
|
|
740
|
+
frameId = rdata['params']['frame']['id']
|
|
741
|
+
f = self.f.root.trav_frame(frameId)
|
|
742
|
+
if f:
|
|
743
|
+
f.url = rdata['params']['frame']['url']
|
|
744
|
+
f.iso_contextId = None
|
|
745
|
+
def Page_frameDetached(self, rdata):
|
|
746
|
+
frameId = rdata['params']['frameId']
|
|
747
|
+
f = self.f.root.trav_frame(frameId)
|
|
748
|
+
if f and rdata['params']['reason'] == 'remove':
|
|
749
|
+
f.parent.frames.remove(f)
|
|
750
|
+
f.iso_contextId = None
|
|
751
|
+
def Page_frameAttached(self, rdata):
|
|
752
|
+
frameId = rdata['params'].get('frameId')
|
|
753
|
+
parentFrameId = rdata['params'].get('parentFrameId')
|
|
754
|
+
pf = self.f.root.trav_frame(parentFrameId)
|
|
755
|
+
if not self.f.root.trav_frame(frameId) and pf:
|
|
756
|
+
self.f.root.add_common_frame(self.f, {
|
|
757
|
+
"frameId": frameId,
|
|
758
|
+
"parent": pf,
|
|
759
|
+
})
|
|
760
|
+
def _clear_iso(self, rdata):
|
|
761
|
+
frameId = rdata['params']['frameId']
|
|
762
|
+
f = self.f.root.trav_frame(frameId)
|
|
763
|
+
if f and rdata['params']['reason'] in (
|
|
764
|
+
'anchorClick', 'formSubmissionGet', 'formSubmissionPost', 'httpHeaderRefresh',
|
|
765
|
+
'initialFrameNavigation', 'metaTagRefresh', 'other', 'pageBlockInterstitial',
|
|
766
|
+
'reload', 'scriptInitiated'):
|
|
767
|
+
f.iso_contextId = None
|
|
768
|
+
def Page_frameScheduledNavigation(self, rdata): self._clear_iso(rdata)
|
|
769
|
+
def Page_frameRequestedNavigation(self, rdata): self._clear_iso(rdata)
|
|
770
|
+
def Page_frameStartedNavigating(self, rdata): pass
|
|
771
|
+
def Page_frameStartedLoading(self, rdata): pass
|
|
772
|
+
def Page_javascriptDialogOpening(self, rdata):
|
|
773
|
+
if self.dialog:
|
|
774
|
+
r = self.dialog(rdata['params'])
|
|
775
|
+
if type(r) == dict:
|
|
776
|
+
self.f.cdp('Page.handleJavaScriptDialog', {"accept": r.get('accept'), "promptText": r.get('promptText') or r.get('text')})
|
|
777
|
+
if type(r) in (list, tuple) and len(r) == 2 and type(r[0]) == bool and type(r[1]) == str:
|
|
778
|
+
self.f.cdp('Page.handleJavaScriptDialog', {"accept": r[0], "promptText": r[1]})
|
|
779
|
+
else:
|
|
780
|
+
self.f.cdp('Page.handleJavaScriptDialog', {"accept": bool(r)})
|
|
781
|
+
else:
|
|
782
|
+
time.sleep(0.5)
|
|
783
|
+
self.f.cdp('Page.handleJavaScriptDialog', {"accept": True})
|
|
784
|
+
def Page_javascriptDialogClosed(self, rdata):
|
|
785
|
+
pass
|
|
786
|
+
class Target:
|
|
787
|
+
def __init__(self, f):
|
|
788
|
+
self.f = f
|
|
789
|
+
self.f.set_method_callback('Target.attachedToTarget', self.Target_attachedToTarget)
|
|
790
|
+
self.f.set_method_callback('Target.targetDestroyed', self.Target_targetDestroyed)
|
|
791
|
+
self.f.set_method_callback('Target.targetCreated', self.Target_targetCreated)
|
|
792
|
+
self.f.set_method_callback('Target.targetInfoChanged', self.Target_targetInfoChanged)
|
|
793
|
+
self.f.set_method_callback('Target.detachedFromTarget', self.Target_detachedFromTarget)
|
|
794
|
+
self.init()
|
|
795
|
+
def init(self, sessionId=None):
|
|
796
|
+
self.f.cdp("Target.setAutoAttach", {
|
|
797
|
+
"autoAttach": True,
|
|
798
|
+
"waitForDebuggerOnStart": True,
|
|
799
|
+
"flatten": True,
|
|
800
|
+
}, sessionId=sessionId)
|
|
801
|
+
def Target_attachedToTarget(self, rdata):
|
|
802
|
+
self.f.root._add_init_check()
|
|
803
|
+
tinfo = rdata['params']['targetInfo']
|
|
804
|
+
if tinfo['url'].startswith('chrome-extension'):
|
|
805
|
+
self.f.root._del_init_check()
|
|
806
|
+
return
|
|
807
|
+
if self.f.root.filter_extension(tinfo):
|
|
808
|
+
self.f.root._del_init_check()
|
|
809
|
+
return
|
|
810
|
+
if rdata['params'].get('targetInfo', {}).get('type') == 'service_worker':
|
|
811
|
+
self.f.root._del_init_check()
|
|
812
|
+
return
|
|
813
|
+
frameId = tinfo['targetId']
|
|
814
|
+
sessionId = rdata['params']['sessionId']
|
|
815
|
+
# self.f.cdp("Target.setAutoAttach", {
|
|
816
|
+
# "autoAttach": True,
|
|
817
|
+
# "waitForDebuggerOnStart": True,
|
|
818
|
+
# "flatten": True,
|
|
819
|
+
# }, sessionId=sessionId)
|
|
820
|
+
self.f.cdp('DOM.enable', sessionId=sessionId)
|
|
821
|
+
self.f.cdp('Page.enable', sessionId=sessionId)
|
|
822
|
+
self.f.cdp('Runtime.enable', sessionId=sessionId)
|
|
823
|
+
self.f.add_sniff_session(sessionId)
|
|
824
|
+
self.f.cache.add_cache_session(sessionId)
|
|
825
|
+
self.f.root.add_common_frame(self.f, {
|
|
826
|
+
"frameId": frameId,
|
|
827
|
+
"sessionId": sessionId,
|
|
828
|
+
})
|
|
829
|
+
if self.f.root.is_init and sessionId:
|
|
830
|
+
f = self.f.root.trav_frame(sessionId, 'sessionId')
|
|
831
|
+
if f:
|
|
832
|
+
self.f.root.trav_init_tree(f, self.f, sessionId)
|
|
833
|
+
else:
|
|
834
|
+
# TODO
|
|
835
|
+
# manager worker process.
|
|
836
|
+
# some worker not work in iframe.
|
|
837
|
+
pass
|
|
838
|
+
self.f._page_init_js(sessionId=sessionId)
|
|
839
|
+
self.f.cdp('Runtime.runIfWaitingForDebugger', sessionId=sessionId)
|
|
840
|
+
self.f.root._del_init_check()
|
|
841
|
+
def Target_detachedFromTarget(self, rdata):
|
|
842
|
+
sessionId = rdata['params']['sessionId']
|
|
843
|
+
pass
|
|
844
|
+
def Target_targetCreated(self, rdata):
|
|
845
|
+
pass
|
|
846
|
+
def Target_targetInfoChanged(self, rdata):
|
|
847
|
+
pass
|
|
848
|
+
def Target_targetDestroyed(self, rdata):
|
|
849
|
+
self.f.cdp('Target.closeTarget', {'targetId': rdata['params']['targetId']})
|
|
850
|
+
class JSIterator:
|
|
851
|
+
def __init__(self, jsobj):
|
|
852
|
+
self.idx = 0
|
|
853
|
+
self.jsobj = jsobj
|
|
854
|
+
self.leng = len(self.jsobj)
|
|
855
|
+
def __next__(self):
|
|
856
|
+
if self.idx < self.leng:
|
|
857
|
+
value = self.jsobj[self.idx]
|
|
858
|
+
self.idx += 1
|
|
859
|
+
return value
|
|
860
|
+
else:
|
|
861
|
+
raise StopIteration
|
|
862
|
+
class JSObject:
|
|
863
|
+
def __init__(self, f, einfo, _this=None, iso=False):
|
|
864
|
+
self.f = f
|
|
865
|
+
self.className = einfo.get('className')
|
|
866
|
+
self.objectId = einfo.get('objectId')
|
|
867
|
+
self._this = _this
|
|
868
|
+
self.iso = iso
|
|
869
|
+
self.r_obj = self.f.run_iso_js_obj if self.iso else self.f.run_js_obj
|
|
870
|
+
def __getitem__(self, a):
|
|
871
|
+
einfo = self.r_obj('function(){return this[' + json.dumps(a) + ']}', objectId=self.objectId, returnByValue=False)
|
|
872
|
+
return self.f._parse_js2py(einfo, self, iso=self.iso)
|
|
873
|
+
def __setitem__(self, a, b):
|
|
874
|
+
args = [None,self.f._parse_2arg(a), self.f._parse_2arg(b)]
|
|
875
|
+
einfo = self.r_obj('function(a,b){return this[a]=b}', objectId=self.objectId, arguments=args, returnByValue=False)
|
|
876
|
+
return self.f._parse_js2py(einfo, iso=self.iso)
|
|
877
|
+
def __call__(self, *a):
|
|
878
|
+
args = [self._this]
|
|
879
|
+
for v in a: args.append(self.f._parse_2arg(v))
|
|
880
|
+
scpt = 'function(o,...a){return this.call(o,...a)}' if self._this else 'function(...a){return this(...a)}'
|
|
881
|
+
einfo = self.r_obj(scpt, objectId=self.objectId, arguments=args, returnByValue=False)
|
|
882
|
+
return self.f._parse_js2py(einfo, iso=self.iso)
|
|
883
|
+
def __repr__(self):
|
|
884
|
+
return '<OBJ:[{}] [{}]>'.format(self.className, self.objectId)
|
|
885
|
+
def __add__(self, other):
|
|
886
|
+
args = [None,self.f._parse_2arg(other)]
|
|
887
|
+
einfo = self.r_obj('function(other){return this+other}', objectId=self.objectId, arguments=args, returnByValue=False)
|
|
888
|
+
return self.f._parse_js2py(einfo, iso=self.iso)
|
|
889
|
+
def __iter__(self): return JSIterator(self)
|
|
890
|
+
def __len__(self): return self['length']
|
|
891
|
+
def __bool__(self): return True
|
|
892
|
+
def json(self): return self.r_obj('function(){return this}', objectId=self.objectId, returnByValue=True)
|
|
893
|
+
class Keyboard:
|
|
894
|
+
# from pyppeteer
|
|
895
|
+
def __init__(self, f):
|
|
896
|
+
self.f = f
|
|
897
|
+
self._key_maps = {'0': {'keyCode': 48, 'key': '0', 'code': 'Digit0'},'1': {'keyCode': 49, 'key': '1', 'code': 'Digit1'},'2': {'keyCode': 50, 'key': '2', 'code': 'Digit2'},'3': {'keyCode': 51, 'key': '3', 'code': 'Digit3'},'4': {'keyCode': 52, 'key': '4', 'code': 'Digit4'},'5': {'keyCode': 53, 'key': '5', 'code': 'Digit5'},'6': {'keyCode': 54, 'key': '6', 'code': 'Digit6'},'7': {'keyCode': 55, 'key': '7', 'code': 'Digit7'},'8': {'keyCode': 56, 'key': '8', 'code': 'Digit8'},'9': {'keyCode': 57, 'key': '9', 'code': 'Digit9'},'Power': {'key': 'Power', 'code': 'Power'},'Eject': {'key': 'Eject', 'code': 'Eject'},'Abort': {'keyCode': 3, 'code': 'Abort', 'key': 'Cancel'},'Help': {'keyCode': 6, 'code': 'Help', 'key': 'Help'},'Backspace': {'keyCode': 8, 'code': 'Backspace', 'key': 'Backspace'},'Tab': {'keyCode': 9, 'code': 'Tab', 'key': 'Tab'},'Numpad5': {'keyCode': 12, 'shiftKeyCode': 101, 'key': 'Clear', 'code': 'Numpad5', 'shiftKey': '5', 'location': 3},'NumpadEnter': {'keyCode': 13, 'code': 'NumpadEnter', 'key': 'Enter', 'text': '\r', 'location': 3},'Enter': {'keyCode': 13, 'code': 'Enter', 'key': 'Enter', 'text': '\r'},'\r': {'keyCode': 13, 'code': 'Enter', 'key': 'Enter', 'text': '\r'},'\n': {'keyCode': 13, 'code': 'Enter', 'key': 'Enter', 'text': '\r'},'ShiftLeft': {'keyCode': 16, 'code': 'ShiftLeft', 'key': 'Shift', 'location': 1},'ShiftRight': {'keyCode': 16, 'code': 'ShiftRight', 'key': 'Shift', 'location': 2},'ControlLeft': {'keyCode': 17, 'code': 'ControlLeft', 'key': 'Control', 'location': 1},'ControlRight': {'keyCode': 17, 'code': 'ControlRight', 'key': 'Control', 'location': 2},'AltLeft': {'keyCode': 18, 'code': 'AltLeft', 'key': 'Alt', 'location': 1},'AltRight': {'keyCode': 18, 'code': 'AltRight', 'key': 'Alt', 'location': 2},'Pause': {'keyCode': 19, 'code': 'Pause', 'key': 'Pause'},'CapsLock': {'keyCode': 20, 'code': 'CapsLock', 'key': 'CapsLock'},'Escape': {'keyCode': 27, 'code': 'Escape', 'key': 'Escape'},'Convert': {'keyCode': 28, 'code': 'Convert', 'key': 'Convert'},'NonConvert': {'keyCode': 29, 'code': 'NonConvert', 'key': 'NonConvert'},'Space': {'keyCode': 32, 'code': 'Space', 'key': ' '},'Numpad9': {'keyCode': 33, 'shiftKeyCode': 105, 'key': 'PageUp', 'code': 'Numpad9', 'shiftKey': '9', 'location': 3},'PageUp': {'keyCode': 33, 'code': 'PageUp', 'key': 'PageUp'},'Numpad3': {'keyCode': 34, 'shiftKeyCode': 99, 'key': 'PageDown', 'code': 'Numpad3', 'shiftKey': '3', 'location': 3},'PageDown': {'keyCode': 34, 'code': 'PageDown', 'key': 'PageDown'},'End': {'keyCode': 35, 'code': 'End', 'key': 'End'},'Numpad1': {'keyCode': 35, 'shiftKeyCode': 97, 'key': 'End', 'code': 'Numpad1', 'shiftKey': '1', 'location': 3},'Home': {'keyCode': 36, 'code': 'Home', 'key': 'Home'},'Numpad7': {'keyCode': 36, 'shiftKeyCode': 103, 'key': 'Home', 'code': 'Numpad7', 'shiftKey': '7', 'location': 3},'ArrowLeft': {'keyCode': 37, 'code': 'ArrowLeft', 'key': 'ArrowLeft'},'Numpad4': {'keyCode': 37, 'shiftKeyCode': 100, 'key': 'ArrowLeft', 'code': 'Numpad4', 'shiftKey': '4', 'location': 3},'Numpad8': {'keyCode': 38, 'shiftKeyCode': 104, 'key': 'ArrowUp', 'code': 'Numpad8', 'shiftKey': '8', 'location': 3},'ArrowUp': {'keyCode': 38, 'code': 'ArrowUp', 'key': 'ArrowUp'},'ArrowRight': {'keyCode': 39, 'code': 'ArrowRight', 'key': 'ArrowRight'},'Numpad6': {'keyCode': 39, 'shiftKeyCode': 102, 'key': 'ArrowRight', 'code': 'Numpad6', 'shiftKey': '6', 'location': 3},'Numpad2': {'keyCode': 40, 'shiftKeyCode': 98, 'key': 'ArrowDown', 'code': 'Numpad2', 'shiftKey': '2', 'location': 3},'ArrowDown': {'keyCode': 40, 'code': 'ArrowDown', 'key': 'ArrowDown'},'Select': {'keyCode': 41, 'code': 'Select', 'key': 'Select'},'Open': {'keyCode': 43, 'code': 'Open', 'key': 'Execute'},'PrintScreen': {'keyCode': 44, 'code': 'PrintScreen', 'key': 'PrintScreen'},'Insert': {'keyCode': 45, 'code': 'Insert', 'key': 'Insert'},'Numpad0': {'keyCode': 45, 'shiftKeyCode': 96, 'key': 'Insert', 'code': 'Numpad0', 'shiftKey': '0', 'location': 3},'Delete': {'keyCode': 46, 'code': 'Delete', 'key': 'Delete'},'NumpadDecimal': {'keyCode': 46, 'shiftKeyCode': 110, 'code': 'NumpadDecimal', 'key': '\u0000', 'shiftKey': '.', 'location': 3},'Digit0': {'keyCode': 48, 'code': 'Digit0', 'shiftKey': ')', 'key': '0'},'Digit1': {'keyCode': 49, 'code': 'Digit1', 'shiftKey': '!', 'key': '1'},'Digit2': {'keyCode': 50, 'code': 'Digit2', 'shiftKey': '@', 'key': '2'},'Digit3': {'keyCode': 51, 'code': 'Digit3', 'shiftKey': '#', 'key': '3'},'Digit4': {'keyCode': 52, 'code': 'Digit4', 'shiftKey': '$', 'key': '4'},'Digit5': {'keyCode': 53, 'code': 'Digit5', 'shiftKey': '%', 'key': '5'},'Digit6': {'keyCode': 54, 'code': 'Digit6', 'shiftKey': '^', 'key': '6'},'Digit7': {'keyCode': 55, 'code': 'Digit7', 'shiftKey': '&', 'key': '7'},'Digit8': {'keyCode': 56, 'code': 'Digit8', 'shiftKey': '*', 'key': '8'},'Digit9': {'keyCode': 57, 'code': 'Digit9', 'shiftKey': '(', 'key': '9'},'KeyA': {'keyCode': 65, 'code': 'KeyA', 'shiftKey': 'A', 'key': 'a'},'KeyB': {'keyCode': 66, 'code': 'KeyB', 'shiftKey': 'B', 'key': 'b'},'KeyC': {'keyCode': 67, 'code': 'KeyC', 'shiftKey': 'C', 'key': 'c'},'KeyD': {'keyCode': 68, 'code': 'KeyD', 'shiftKey': 'D', 'key': 'd'},'KeyE': {'keyCode': 69, 'code': 'KeyE', 'shiftKey': 'E', 'key': 'e'},'KeyF': {'keyCode': 70, 'code': 'KeyF', 'shiftKey': 'F', 'key': 'f'},'KeyG': {'keyCode': 71, 'code': 'KeyG', 'shiftKey': 'G', 'key': 'g'},'KeyH': {'keyCode': 72, 'code': 'KeyH', 'shiftKey': 'H', 'key': 'h'},'KeyI': {'keyCode': 73, 'code': 'KeyI', 'shiftKey': 'I', 'key': 'i'},'KeyJ': {'keyCode': 74, 'code': 'KeyJ', 'shiftKey': 'J', 'key': 'j'},'KeyK': {'keyCode': 75, 'code': 'KeyK', 'shiftKey': 'K', 'key': 'k'},'KeyL': {'keyCode': 76, 'code': 'KeyL', 'shiftKey': 'L', 'key': 'l'},'KeyM': {'keyCode': 77, 'code': 'KeyM', 'shiftKey': 'M', 'key': 'm'},'KeyN': {'keyCode': 78, 'code': 'KeyN', 'shiftKey': 'N', 'key': 'n'},'KeyO': {'keyCode': 79, 'code': 'KeyO', 'shiftKey': 'O', 'key': 'o'},'KeyP': {'keyCode': 80, 'code': 'KeyP', 'shiftKey': 'P', 'key': 'p'},'KeyQ': {'keyCode': 81, 'code': 'KeyQ', 'shiftKey': 'Q', 'key': 'q'},'KeyR': {'keyCode': 82, 'code': 'KeyR', 'shiftKey': 'R', 'key': 'r'},'KeyS': {'keyCode': 83, 'code': 'KeyS', 'shiftKey': 'S', 'key': 's'},'KeyT': {'keyCode': 84, 'code': 'KeyT', 'shiftKey': 'T', 'key': 't'},'KeyU': {'keyCode': 85, 'code': 'KeyU', 'shiftKey': 'U', 'key': 'u'},'KeyV': {'keyCode': 86, 'code': 'KeyV', 'shiftKey': 'V', 'key': 'v'},'KeyW': {'keyCode': 87, 'code': 'KeyW', 'shiftKey': 'W', 'key': 'w'},'KeyX': {'keyCode': 88, 'code': 'KeyX', 'shiftKey': 'X', 'key': 'x'},'KeyY': {'keyCode': 89, 'code': 'KeyY', 'shiftKey': 'Y', 'key': 'y'},'KeyZ': {'keyCode': 90, 'code': 'KeyZ', 'shiftKey': 'Z', 'key': 'z'},'MetaLeft': {'keyCode': 91, 'code': 'MetaLeft', 'key': 'Meta'},'MetaRight': {'keyCode': 92, 'code': 'MetaRight', 'key': 'Meta'},'ContextMenu': {'keyCode': 93, 'code': 'ContextMenu', 'key': 'ContextMenu'},'NumpadMultiply': {'keyCode': 106, 'code': 'NumpadMultiply', 'key': '*', 'location': 3},'NumpadAdd': {'keyCode': 107, 'code': 'NumpadAdd', 'key': '+', 'location': 3},'NumpadSubtract': {'keyCode': 109, 'code': 'NumpadSubtract', 'key': '-', 'location': 3},'NumpadDivide': {'keyCode': 111, 'code': 'NumpadDivide', 'key': '/', 'location': 3},'F1': {'keyCode': 112, 'code': 'F1', 'key': 'F1'},'F2': {'keyCode': 113, 'code': 'F2', 'key': 'F2'},'F3': {'keyCode': 114, 'code': 'F3', 'key': 'F3'},'F4': {'keyCode': 115, 'code': 'F4', 'key': 'F4'},'F5': {'keyCode': 116, 'code': 'F5', 'key': 'F5'},'F6': {'keyCode': 117, 'code': 'F6', 'key': 'F6'},'F7': {'keyCode': 118, 'code': 'F7', 'key': 'F7'},'F8': {'keyCode': 119, 'code': 'F8', 'key': 'F8'},'F9': {'keyCode': 120, 'code': 'F9', 'key': 'F9'},'F10': {'keyCode': 121, 'code': 'F10', 'key': 'F10'},'F11': {'keyCode': 122, 'code': 'F11', 'key': 'F11'},'F12': {'keyCode': 123, 'code': 'F12', 'key': 'F12'},'F13': {'keyCode': 124, 'code': 'F13', 'key': 'F13'},'F14': {'keyCode': 125, 'code': 'F14', 'key': 'F14'},'F15': {'keyCode': 126, 'code': 'F15', 'key': 'F15'},'F16': {'keyCode': 127, 'code': 'F16', 'key': 'F16'},'F17': {'keyCode': 128, 'code': 'F17', 'key': 'F17'},'F18': {'keyCode': 129, 'code': 'F18', 'key': 'F18'},'F19': {'keyCode': 130, 'code': 'F19', 'key': 'F19'},'F20': {'keyCode': 131, 'code': 'F20', 'key': 'F20'},'F21': {'keyCode': 132, 'code': 'F21', 'key': 'F21'},'F22': {'keyCode': 133, 'code': 'F22', 'key': 'F22'},'F23': {'keyCode': 134, 'code': 'F23', 'key': 'F23'},'F24': {'keyCode': 135, 'code': 'F24', 'key': 'F24'},'NumLock': {'keyCode': 144, 'code': 'NumLock', 'key': 'NumLock'},'ScrollLock': {'keyCode': 145, 'code': 'ScrollLock', 'key': 'ScrollLock'},'AudioVolumeMute': {'keyCode': 173, 'code': 'AudioVolumeMute', 'key': 'AudioVolumeMute'},'AudioVolumeDown': {'keyCode': 174, 'code': 'AudioVolumeDown', 'key': 'AudioVolumeDown'},'AudioVolumeUp': {'keyCode': 175, 'code': 'AudioVolumeUp', 'key': 'AudioVolumeUp'},'MediaTrackNext': {'keyCode': 176, 'code': 'MediaTrackNext', 'key': 'MediaTrackNext'},'MediaTrackPrevious': {'keyCode': 177, 'code': 'MediaTrackPrevious', 'key': 'MediaTrackPrevious'},'MediaStop': {'keyCode': 178, 'code': 'MediaStop', 'key': 'MediaStop'},'MediaPlayPause': {'keyCode': 179, 'code': 'MediaPlayPause', 'key': 'MediaPlayPause'},'Semicolon': {'keyCode': 186, 'code': 'Semicolon', 'shiftKey': ':', 'key': ';'},'Equal': {'keyCode': 187, 'code': 'Equal', 'shiftKey': '+', 'key': '='},'NumpadEqual': {'keyCode': 187, 'code': 'NumpadEqual', 'key': '=', 'location': 3},'Comma': {'keyCode': 188, 'code': 'Comma', 'shiftKey': '<', 'key': ','},'Minus': {'keyCode': 189, 'code': 'Minus', 'shiftKey': '_', 'key': '-'},'Period': {'keyCode': 190, 'code': 'Period', 'shiftKey': '>', 'key': '.'},'Slash': {'keyCode': 191, 'code': 'Slash', 'shiftKey': '?', 'key': '/'},'Backquote': {'keyCode': 192, 'code': 'Backquote', 'shiftKey': '~', 'key': '`'},'BracketLeft': {'keyCode': 219, 'code': 'BracketLeft', 'shiftKey': '{', 'key': '['},'Backslash': {'keyCode': 220, 'code': 'Backslash', 'shiftKey': '|', 'key': '\\'},'BracketRight': {'keyCode': 221, 'code': 'BracketRight', 'shiftKey': '}', 'key': ']'},'Quote': {'keyCode': 222, 'code': 'Quote', 'shiftKey': '"', 'key': '\''},'AltGraph': {'keyCode': 225, 'code': 'AltGraph', 'key': 'AltGraph'},'Props': {'keyCode': 247, 'code': 'Props', 'key': 'CrSel'},'Cancel': {'keyCode': 3, 'key': 'Cancel', 'code': 'Abort'},'Clear': {'keyCode': 12, 'key': 'Clear', 'code': 'Numpad5', 'location': 3},'Shift': {'keyCode': 16, 'key': 'Shift', 'code': 'ShiftLeft'},'Control': {'keyCode': 17, 'key': 'Control', 'code': 'ControlLeft'},'Alt': {'keyCode': 18, 'key': 'Alt', 'code': 'AltLeft'},'Accept': {'keyCode': 30, 'key': 'Accept'},'ModeChange': {'keyCode': 31, 'key': 'ModeChange'},' ': {'keyCode': 32, 'key': ' ', 'code': 'Space'},'Print': {'keyCode': 42, 'key': 'Print'},'Execute': {'keyCode': 43, 'key': 'Execute', 'code': 'Open'},'\u0000': {'keyCode': 46, 'key': '\u0000', 'code': 'NumpadDecimal', 'location': 3},'a': {'keyCode': 65, 'key': 'a', 'code': 'KeyA'},'b': {'keyCode': 66, 'key': 'b', 'code': 'KeyB'},'c': {'keyCode': 67, 'key': 'c', 'code': 'KeyC'},'d': {'keyCode': 68, 'key': 'd', 'code': 'KeyD'},'e': {'keyCode': 69, 'key': 'e', 'code': 'KeyE'},'f': {'keyCode': 70, 'key': 'f', 'code': 'KeyF'},'g': {'keyCode': 71, 'key': 'g', 'code': 'KeyG'},'h': {'keyCode': 72, 'key': 'h', 'code': 'KeyH'},'i': {'keyCode': 73, 'key': 'i', 'code': 'KeyI'},'j': {'keyCode': 74, 'key': 'j', 'code': 'KeyJ'},'k': {'keyCode': 75, 'key': 'k', 'code': 'KeyK'},'l': {'keyCode': 76, 'key': 'l', 'code': 'KeyL'},'m': {'keyCode': 77, 'key': 'm', 'code': 'KeyM'},'n': {'keyCode': 78, 'key': 'n', 'code': 'KeyN'},'o': {'keyCode': 79, 'key': 'o', 'code': 'KeyO'},'p': {'keyCode': 80, 'key': 'p', 'code': 'KeyP'},'q': {'keyCode': 81, 'key': 'q', 'code': 'KeyQ'},'r': {'keyCode': 82, 'key': 'r', 'code': 'KeyR'},'s': {'keyCode': 83, 'key': 's', 'code': 'KeyS'},'t': {'keyCode': 84, 'key': 't', 'code': 'KeyT'},'u': {'keyCode': 85, 'key': 'u', 'code': 'KeyU'},'v': {'keyCode': 86, 'key': 'v', 'code': 'KeyV'},'w': {'keyCode': 87, 'key': 'w', 'code': 'KeyW'},'x': {'keyCode': 88, 'key': 'x', 'code': 'KeyX'},'y': {'keyCode': 89, 'key': 'y', 'code': 'KeyY'},'z': {'keyCode': 90, 'key': 'z', 'code': 'KeyZ'},'Meta': {'keyCode': 91, 'key': 'Meta', 'code': 'MetaLeft'},'*': {'keyCode': 106, 'key': '*', 'code': 'NumpadMultiply', 'location': 3},'+': {'keyCode': 107, 'key': '+', 'code': 'NumpadAdd', 'location': 3},'-': {'keyCode': 109, 'key': '-', 'code': 'NumpadSubtract', 'location': 3},'/': {'keyCode': 111, 'key': '/', 'code': 'NumpadDivide', 'location': 3},';': {'keyCode': 186, 'key': ';', 'code': 'Semicolon'},'=': {'keyCode': 187, 'key': '=', 'code': 'Equal'},',': {'keyCode': 188, 'key': ',', 'code': 'Comma'},'.': {'keyCode': 190, 'key': '.', 'code': 'Period'},'`': {'keyCode': 192, 'key': '`', 'code': 'Backquote'},'[': {'keyCode': 219, 'key': '[', 'code': 'BracketLeft'},'\\': {'keyCode': 220, 'key': '\\', 'code': 'Backslash'},']': {'keyCode': 221, 'key': ']', 'code': 'BracketRight'},'\'': {'keyCode': 222, 'key': '\'', 'code': 'Quote'},'Attn': {'keyCode': 246, 'key': 'Attn'},'CrSel': {'keyCode': 247, 'key': 'CrSel', 'code': 'Props'},'ExSel': {'keyCode': 248, 'key': 'ExSel'},'EraseEof': {'keyCode': 249, 'key': 'EraseEof'},'Play': {'keyCode': 250, 'key': 'Play'},'ZoomOut': {'keyCode': 251, 'key': 'ZoomOut'},')': {'keyCode': 48, 'key': ')', 'code': 'Digit0'},'!': {'keyCode': 49, 'key': '!', 'code': 'Digit1'},'@': {'keyCode': 50, 'key': '@', 'code': 'Digit2'},'#': {'keyCode': 51, 'key': '#', 'code': 'Digit3'},'$': {'keyCode': 52, 'key': '$', 'code': 'Digit4'},'%': {'keyCode': 53, 'key': '%', 'code': 'Digit5'},'^': {'keyCode': 54, 'key': '^', 'code': 'Digit6'},'&': {'keyCode': 55, 'key': '&', 'code': 'Digit7'},'(': {'keyCode': 57, 'key': '(', 'code': 'Digit9'},'A': {'keyCode': 65, 'key': 'A', 'code': 'KeyA'},'B': {'keyCode': 66, 'key': 'B', 'code': 'KeyB'},'C': {'keyCode': 67, 'key': 'C', 'code': 'KeyC'},'D': {'keyCode': 68, 'key': 'D', 'code': 'KeyD'},'E': {'keyCode': 69, 'key': 'E', 'code': 'KeyE'},'F': {'keyCode': 70, 'key': 'F', 'code': 'KeyF'},'G': {'keyCode': 71, 'key': 'G', 'code': 'KeyG'},'H': {'keyCode': 72, 'key': 'H', 'code': 'KeyH'},'I': {'keyCode': 73, 'key': 'I', 'code': 'KeyI'},'J': {'keyCode': 74, 'key': 'J', 'code': 'KeyJ'},'K': {'keyCode': 75, 'key': 'K', 'code': 'KeyK'},'L': {'keyCode': 76, 'key': 'L', 'code': 'KeyL'},'M': {'keyCode': 77, 'key': 'M', 'code': 'KeyM'},'N': {'keyCode': 78, 'key': 'N', 'code': 'KeyN'},'O': {'keyCode': 79, 'key': 'O', 'code': 'KeyO'},'P': {'keyCode': 80, 'key': 'P', 'code': 'KeyP'},'Q': {'keyCode': 81, 'key': 'Q', 'code': 'KeyQ'},'R': {'keyCode': 82, 'key': 'R', 'code': 'KeyR'},'S': {'keyCode': 83, 'key': 'S', 'code': 'KeyS'},'T': {'keyCode': 84, 'key': 'T', 'code': 'KeyT'},'U': {'keyCode': 85, 'key': 'U', 'code': 'KeyU'},'V': {'keyCode': 86, 'key': 'V', 'code': 'KeyV'},'W': {'keyCode': 87, 'key': 'W', 'code': 'KeyW'},'X': {'keyCode': 88, 'key': 'X', 'code': 'KeyX'},'Y': {'keyCode': 89, 'key': 'Y', 'code': 'KeyY'},'Z': {'keyCode': 90, 'key': 'Z', 'code': 'KeyZ'},':': {'keyCode': 186, 'key': ':', 'code': 'Semicolon'},'<': {'keyCode': 188, 'key': '<', 'code': 'Comma'},'_': {'keyCode': 189, 'key': '_', 'code': 'Minus'},'>': {'keyCode': 190, 'key': '>', 'code': 'Period'},'?': {'keyCode': 191, 'key': '?', 'code': 'Slash'},'~': {'keyCode': 192, 'key': '~', 'code': 'Backquote'},'{': {'keyCode': 219, 'key': '{', 'code': 'BracketLeft'},'|': {'keyCode': 220, 'key': '|', 'code': 'Backslash'},'}': {'keyCode': 221, 'key': '}', 'code': 'BracketRight'},'"': {'keyCode': 222, 'key': '"', 'code': 'Quote'},}
|
|
898
|
+
self._modifiers = 0
|
|
899
|
+
self._pressed_keys = set()
|
|
900
|
+
self.attach(f)
|
|
901
|
+
def attach(self, f):
|
|
902
|
+
f.make_input_events = self.make_input_events
|
|
903
|
+
f.make_clear = self.make_clear
|
|
904
|
+
def make_clear(self):
|
|
905
|
+
charin = []
|
|
906
|
+
charin.append(self._make_down('Control'))
|
|
907
|
+
charin.append(self._make_down('a'))
|
|
908
|
+
charin.append(self._make_up('Control'))
|
|
909
|
+
charin.append(self._make_up('a'))
|
|
910
|
+
charin.append(self._make_down('Delete'))
|
|
911
|
+
charin.append(self._make_up('Delete'))
|
|
912
|
+
return charin
|
|
913
|
+
def make_input_events(self, text):
|
|
914
|
+
return self._make_chain(text)
|
|
915
|
+
def _key_desc(self, keyString): # noqa: C901
|
|
916
|
+
shift = self._modifiers & 8
|
|
917
|
+
desc = {'key': '','keyCode': 0,'code': '','text': '','location': 0}
|
|
918
|
+
defi = self._key_maps.get(keyString)
|
|
919
|
+
if not defi: raise Exception('Unknown key: '+keyString)
|
|
920
|
+
if 'key' in defi: desc['key'] = defi['key']
|
|
921
|
+
if shift and defi.get('shiftKey'): desc['key'] = defi['shiftKey']
|
|
922
|
+
if 'keyCode' in defi: desc['keyCode'] = defi['keyCode']
|
|
923
|
+
if shift and defi.get('shiftKeyCode'): desc['keyCode'] = defi['shiftKeyCode']
|
|
924
|
+
if 'code' in defi: desc['code'] = defi['code']
|
|
925
|
+
if 'location' in defi: desc['location'] = defi['location']
|
|
926
|
+
if len(desc['key']) == 1: desc['text'] = desc['key']
|
|
927
|
+
if 'text' in defi: desc['text'] = defi['text']
|
|
928
|
+
if shift and defi.get('shiftText'): desc['text'] = defi['shiftText']
|
|
929
|
+
if self._modifiers & ~8: desc['text'] = ''
|
|
930
|
+
return desc
|
|
931
|
+
def _modifier_bit(self, key):
|
|
932
|
+
if key == 'Alt': return 1
|
|
933
|
+
if key == 'Control': return 2
|
|
934
|
+
if key == 'Meta': return 4
|
|
935
|
+
if key == 'Shift': return 8
|
|
936
|
+
return 0
|
|
937
|
+
def _make_chain(self, text):
|
|
938
|
+
charin = []
|
|
939
|
+
for char in text:
|
|
940
|
+
if char in self._key_maps:
|
|
941
|
+
charin.append([self._make_down(char), self._make_up(char)])
|
|
942
|
+
else:
|
|
943
|
+
charin.append(['insertText', char])
|
|
944
|
+
return charin
|
|
945
|
+
def _make_down(self, char):
|
|
946
|
+
desc = self._key_desc(char)
|
|
947
|
+
auto_rpt = desc['code'] in self._pressed_keys
|
|
948
|
+
self._pressed_keys.add(desc['code'])
|
|
949
|
+
self._modifiers |= self._modifier_bit(desc['key'])
|
|
950
|
+
text = desc['text']
|
|
951
|
+
return {
|
|
952
|
+
'type': 'keyDown' if text else 'rawKeyDown','modifiers': self._modifiers,
|
|
953
|
+
'windowsVirtualKeyCode': desc['keyCode'],
|
|
954
|
+
'code': desc['code'],'key': desc['key'],'text': text,
|
|
955
|
+
'unmodifiedText': text,'autoRepeat': auto_rpt,
|
|
956
|
+
'location': desc['location'], 'isKeypad': desc['location'] == 3, }
|
|
957
|
+
def _make_up(self, char):
|
|
958
|
+
desc = self._key_desc(char)
|
|
959
|
+
self._modifiers &= ~self._modifier_bit(desc['key'])
|
|
960
|
+
if desc['code'] in self._pressed_keys: self._pressed_keys.remove(desc['code'])
|
|
961
|
+
return {
|
|
962
|
+
'type': 'keyUp', 'modifiers': self._modifiers,
|
|
963
|
+
'key': desc['key'], 'windowsVirtualKeyCode': desc['keyCode'],
|
|
964
|
+
'code': desc['code'], 'location': desc['location'], }
|
|
965
|
+
class Screenshot:
|
|
966
|
+
def __init__(self, b64img):
|
|
967
|
+
self.i = b64img
|
|
968
|
+
self.content = base64.b64decode(self.i.encode())
|
|
969
|
+
def test_show(self, timeout=1.5):
|
|
970
|
+
import cv2
|
|
971
|
+
import numpy as np
|
|
972
|
+
img = cv2.imdecode(np.frombuffer(self.content, np.uint8), cv2.IMREAD_UNCHANGED)
|
|
973
|
+
if img is None:
|
|
974
|
+
raise ValueError("decode img fail.")
|
|
975
|
+
h, w = img.shape[:2]
|
|
976
|
+
c1, c2, tile_size = 200, 150, 20
|
|
977
|
+
rows = (np.arange(h) // tile_size) % 2
|
|
978
|
+
cols = (np.arange(w) // tile_size) % 2
|
|
979
|
+
checkerboard = np.where((rows[:, None] ^ cols[None, :]) == 0, c1, c2).astype(np.uint8)
|
|
980
|
+
checkerboard = cv2.merge([checkerboard, checkerboard, checkerboard])
|
|
981
|
+
if img.shape[2] == 4:
|
|
982
|
+
alpha = img[:, :, 3] / 255.0
|
|
983
|
+
bgr = img[:, :, :3]
|
|
984
|
+
blended = (bgr * alpha[..., None] + checkerboard * (1 - alpha[..., None])).astype(np.uint8)
|
|
985
|
+
else:
|
|
986
|
+
blended = img
|
|
987
|
+
cv2.imshow('test', blended)
|
|
988
|
+
cv2.waitKey(int(timeout*1000))
|
|
989
|
+
cv2.destroyAllWindows()
|
|
990
|
+
def __repr__(self):
|
|
991
|
+
return '<Screenshot [size]:{} [prop api]: .content>'.format(to_human_read(len(self.content)))
|
|
992
|
+
class Element:
|
|
993
|
+
def __init__(self, f, einfo, iso=True):
|
|
994
|
+
# {'type': 'object', 'subtype': 'node', 'className': 'HTMLAnchorElement', 'description': 'a', 'objectId': '-1497388350229232364.96.1'}
|
|
995
|
+
self.f = f
|
|
996
|
+
self.className = einfo.get('className')
|
|
997
|
+
self.objectId = einfo.get('objectId')
|
|
998
|
+
self.cache_xy = None
|
|
999
|
+
self.draggable = None
|
|
1000
|
+
self.iso = iso
|
|
1001
|
+
self.r_obj = self.f.run_iso_js_obj if self.iso else self.f.run_js_obj
|
|
1002
|
+
self.r_js = self.f.run_iso_js if self.iso else self.f.run_js
|
|
1003
|
+
@property
|
|
1004
|
+
def backendNodeId(self):
|
|
1005
|
+
return self.f.cdp('DOM.describeNode', {'objectId': self.objectId})['node']['backendNodeId']
|
|
1006
|
+
@property
|
|
1007
|
+
def _box(self):
|
|
1008
|
+
return self.f.cdp('DOM.getBoxModel', {'objectId': self.objectId})
|
|
1009
|
+
def __repr__(self):
|
|
1010
|
+
return '<DOM:[{}] [{}]>'.format(self.className, self.objectId)
|
|
1011
|
+
def clear(self, delay_uptime=0.15, timegap=0.01):
|
|
1012
|
+
self.wait_show()
|
|
1013
|
+
self.f.cdp('DOM.focus', {"objectId": self.objectId})
|
|
1014
|
+
clear_ents = self.f.rootf.make_clear()
|
|
1015
|
+
for kdn in clear_ents[:2]: time.sleep(timegap); self.f.cdp('Input.dispatchKeyEvent', kdn)
|
|
1016
|
+
time.sleep(delay_uptime)
|
|
1017
|
+
for kup in clear_ents[2:4]: time.sleep(timegap); self.f.cdp('Input.dispatchKeyEvent', kup)
|
|
1018
|
+
for delt in clear_ents[4:]: time.sleep(timegap); self.f.cdp('Input.dispatchKeyEvent', delt)
|
|
1019
|
+
def input(self, str, delay_time=0.02, delay_uptime=0.15, random_gap=0.01):
|
|
1020
|
+
self.wait_show()
|
|
1021
|
+
self.f.cdp('DOM.focus', {"objectId": self.objectId})
|
|
1022
|
+
for kdn, kup in self.f.rootf.make_input_events(str):
|
|
1023
|
+
if kdn == 'insertText':
|
|
1024
|
+
self.f.cdp('Input.insertText', {"text": kup})
|
|
1025
|
+
time.sleep(delay_time+(random_gap if random()>0.5 else 0))
|
|
1026
|
+
else:
|
|
1027
|
+
self.f.cdp('Input.dispatchKeyEvent', kdn)
|
|
1028
|
+
time.sleep(delay_time+(random_gap if random()>0.5 else 0))
|
|
1029
|
+
def delay_up(kup, delay_uptime):
|
|
1030
|
+
time.sleep(delay_uptime)
|
|
1031
|
+
self.f.cdp('Input.dispatchKeyEvent', kup)
|
|
1032
|
+
self.f.rootf.root.pool(delay_up)(kup, delay_uptime)
|
|
1033
|
+
def _get_xy(self, x, y, zero='center'):
|
|
1034
|
+
o = self.f.cdp('DOM.getBoxModel', {"objectId": self.objectId})
|
|
1035
|
+
m = o['model']
|
|
1036
|
+
if zero == 'center':
|
|
1037
|
+
_x = (m['content'][0] + m['content'][2]) / 2 + x;
|
|
1038
|
+
_y = (m['content'][1] + m['content'][5]) / 2 + y;
|
|
1039
|
+
elif zero == 'lefttop':
|
|
1040
|
+
_x = m['content'][0] + x
|
|
1041
|
+
_y = m['content'][1] + y
|
|
1042
|
+
elif zero == 'leftbottom':
|
|
1043
|
+
_x = m['content'][6] + x
|
|
1044
|
+
_y = m['content'][7] - y
|
|
1045
|
+
else:
|
|
1046
|
+
raise Exception('zero type must be in (center,lefttop,leftbottom).')
|
|
1047
|
+
return _x, _y
|
|
1048
|
+
def _set_in_view(self):
|
|
1049
|
+
self.f.cdp('DOM.scrollIntoViewIfNeeded', {"objectId": self.objectId})
|
|
1050
|
+
def click(self, x=1, y=2, zero='center', button='left', count=1):
|
|
1051
|
+
self.wait_show()
|
|
1052
|
+
self._set_in_view()
|
|
1053
|
+
_x, _y = self._get_xy(x, y, zero)
|
|
1054
|
+
self.f.cdp('Input.dispatchMouseEvent',{"buttons":0,"type":"mouseMoved","x":_x,"y":_y,"button":"none","clickCount":0})
|
|
1055
|
+
self.f.cdp('Input.dispatchMouseEvent',{"buttons":1,"type":"mousePressed","x":_x,"y":_y,"button":button,"clickCount":count})
|
|
1056
|
+
time.sleep(0.07)
|
|
1057
|
+
self.f.cdp('Input.dispatchMouseEvent',{"buttons":0,"type":"mouseReleased","x":_x,"y":_y,"button":button,"clickCount":count})
|
|
1058
|
+
def _bezier_xxyy(self, x1, y1, x2, y2):
|
|
1059
|
+
def step_len(x1, y1, x2, y2):
|
|
1060
|
+
ln = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5
|
|
1061
|
+
return int(ln / 4)
|
|
1062
|
+
slen = step_len(x1, y1, x2, y2)
|
|
1063
|
+
lp = (random() - 0.5) * 0.5
|
|
1064
|
+
rp = (random() - 0.5) * 0.5 + 1
|
|
1065
|
+
xx1 = int(x1 + (x2 - x1) / 12 * (4-lp*4))
|
|
1066
|
+
yy1 = int(y1 + (y2 - y1) / 12 * (8+lp*4))
|
|
1067
|
+
xx2 = int(x1 + (x2 - x1) / 12 * (8+rp*4))
|
|
1068
|
+
yy2 = int(y1 + (y2 - y1) / 12 * (4-rp*4))
|
|
1069
|
+
points = [[x1, y1], [xx1, yy1], [xx2, yy2], [x2, y2]]
|
|
1070
|
+
N = len(points)
|
|
1071
|
+
n = N - 1
|
|
1072
|
+
r = []
|
|
1073
|
+
for T in range(slen + 1):
|
|
1074
|
+
t = T*(1/slen)
|
|
1075
|
+
t = (sin(t * pi / 2)) ** 18
|
|
1076
|
+
x,y = 0,0
|
|
1077
|
+
for i in range(N):
|
|
1078
|
+
B = factorial(n)*t**i*(1-t)**(n-i)/(factorial(i)*factorial(n-i))
|
|
1079
|
+
x += points[i][0]*B
|
|
1080
|
+
y += points[i][1]*B
|
|
1081
|
+
r.append([x, y])
|
|
1082
|
+
return r
|
|
1083
|
+
def wait_show(self, timeout=5, pretime=0.1):
|
|
1084
|
+
start = perf_counter()
|
|
1085
|
+
count, acount = 0, 0
|
|
1086
|
+
px, py = 0, 0
|
|
1087
|
+
while True:
|
|
1088
|
+
v = self.visible()
|
|
1089
|
+
if v.get('isDisplayed'):
|
|
1090
|
+
x, y = v['clickPoint']['x'], v['clickPoint']['y']
|
|
1091
|
+
if x == px and y == py:
|
|
1092
|
+
count += 1
|
|
1093
|
+
else:
|
|
1094
|
+
count = 0
|
|
1095
|
+
acount += 1
|
|
1096
|
+
px, py = x, y
|
|
1097
|
+
if count > 1 or acount > 5:
|
|
1098
|
+
break
|
|
1099
|
+
if perf_counter() - start > timeout:
|
|
1100
|
+
raise Exception('wait show over time.')
|
|
1101
|
+
time.sleep(0.08)
|
|
1102
|
+
if pretime:
|
|
1103
|
+
time.sleep(pretime) # 组件出现时,可能事件并未挂上去,稍微等待一下
|
|
1104
|
+
def drag(self, x=1, y=2, zero='center', button='left', count=1):
|
|
1105
|
+
self.wait_show()
|
|
1106
|
+
self._set_in_view()
|
|
1107
|
+
x, y = self._get_xy(x, y, zero)
|
|
1108
|
+
self.cache_xy = [x, y]
|
|
1109
|
+
self.f.cdp('Input.dispatchMouseEvent',{"buttons":1,"type":"mousePressed","x":int(x),"y":int(y),"button":button,"clickCount":count})
|
|
1110
|
+
self.draggable = self['draggable']
|
|
1111
|
+
if self.draggable:
|
|
1112
|
+
print('[*] h5 draggable element cannot simulate drag and drop.')
|
|
1113
|
+
# draggable api, CDP not simulate and this api cannot record trajectory
|
|
1114
|
+
# self.f.cdp('Input.dispatchDragEvent',{"type":"dragEnter","x":int(x),"y":int(y),
|
|
1115
|
+
# "data":{"items":[{"type":"text/plain","data":"some text"}]}
|
|
1116
|
+
# })
|
|
1117
|
+
# self.f.cdp('Input.dispatchDragEvent',{"type":"dragOver","x":int(x),"y":int(y)})
|
|
1118
|
+
# self.f.cdp('Input.dispatchDragEvent',{"type":"drop","x":int(x),"y":int(y)})
|
|
1119
|
+
# and <input type=range> cannot simulate sliding through CDP.
|
|
1120
|
+
return self
|
|
1121
|
+
def dragmove(self, shiftx=0, shifty=0, button='left', count=1, costtime=0.8):
|
|
1122
|
+
if self.cache_xy:
|
|
1123
|
+
x, y = self.cache_xy
|
|
1124
|
+
else:
|
|
1125
|
+
cp = self._view_info()['clickPoint']
|
|
1126
|
+
x, y = cp['x'], cp['y']
|
|
1127
|
+
self.cache_xy = x, y
|
|
1128
|
+
c = self._bezier_xxyy(x,y,x+shiftx,y+shifty)
|
|
1129
|
+
g = costtime / len(c)
|
|
1130
|
+
start = perf_counter()
|
|
1131
|
+
ctime = 0
|
|
1132
|
+
gtime = 0
|
|
1133
|
+
for x, y in c:
|
|
1134
|
+
self.f.cdp('Input.dispatchMouseEvent',{"buttons":1,"type":"mouseMoved","x":int(x),"y":int(y),"button":"none","clickCount":0})
|
|
1135
|
+
self.cache_xy = x, y
|
|
1136
|
+
ctime += g
|
|
1137
|
+
gtime = perf_counter() - start
|
|
1138
|
+
if ctime > gtime:
|
|
1139
|
+
time.sleep(ctime - gtime)
|
|
1140
|
+
return self
|
|
1141
|
+
def drop(self, button='left', count=1):
|
|
1142
|
+
if self.cache_xy:
|
|
1143
|
+
x, y = self.cache_xy
|
|
1144
|
+
else:
|
|
1145
|
+
cp = self._view_info()['clickPoint']
|
|
1146
|
+
x, y = cp['x'], cp['y']
|
|
1147
|
+
self.f.cdp('Input.dispatchMouseEvent',{"buttons":0,"type":"mouseReleased","x":int(x),"y":int(y),"button":button,"clickCount":count})
|
|
1148
|
+
self.cache_xy = None
|
|
1149
|
+
return self
|
|
1150
|
+
def drag_to(self, shiftx=0, shifty=0, button='left', x=1, y=2, zero='center', costtime=0.8, count=1):
|
|
1151
|
+
self.drag(x, y, zero, button, count) \
|
|
1152
|
+
.dragmove(shiftx, shifty, button, count, costtime) \
|
|
1153
|
+
.drop(button, count)
|
|
1154
|
+
return self
|
|
1155
|
+
def _is_coverd(self):
|
|
1156
|
+
try:
|
|
1157
|
+
c = self._view_info()['clickPoint']
|
|
1158
|
+
return self.backendNodeId != self.f.cdp('DOM.getNodeForLocation', {'x': c['x'], 'y': c['y']})['backendNodeId']
|
|
1159
|
+
except Exception as e:
|
|
1160
|
+
return True
|
|
1161
|
+
def pic(self, try_use_img=True):
|
|
1162
|
+
self.wait_show()
|
|
1163
|
+
if try_use_img and self.className == 'HTMLImageElement':
|
|
1164
|
+
try:
|
|
1165
|
+
# 如果是 <img> 标签,就尝试通过标签采集,采集不到再走屏幕截图。
|
|
1166
|
+
# 因为 img 标签的图片最好接近原生图片,这样效果会更好,当然也可以配置 try_use_img=False 直接关掉这种图片获取方式,就用截图,那样比较统一。
|
|
1167
|
+
src = self['src']
|
|
1168
|
+
if src:
|
|
1169
|
+
if src.startswith('http'):
|
|
1170
|
+
return Screenshot(self.f.cdp("Page.getResourceContent", {"url": src, "frameId": self.f.frameId})['content'])
|
|
1171
|
+
if src.startswith('data:image/png;base64,'):
|
|
1172
|
+
return Screenshot(src.replace('data:image/png;base64,', ''))
|
|
1173
|
+
except:
|
|
1174
|
+
pass
|
|
1175
|
+
rparentlist = self.f._get_remote_list()
|
|
1176
|
+
if not rparentlist:
|
|
1177
|
+
box = self.f.cdp('DOM.getBoxModel', {'objectId': self.objectId})
|
|
1178
|
+
x, y = box['model']['padding'][0], box['model']['padding'][1]
|
|
1179
|
+
width, height = box['model']['width'], box['model']['height']
|
|
1180
|
+
screenshot = self.f.cdp('Page.captureScreenshot', {"format":'png',
|
|
1181
|
+
"clip":{"x":x,"y":y,"width":width,"height":height,"scale":1} })
|
|
1182
|
+
return Screenshot(screenshot['data'])
|
|
1183
|
+
else:
|
|
1184
|
+
slfele = self.f.element._frame_element(self.f)
|
|
1185
|
+
sftx, sfty = 0, 0
|
|
1186
|
+
f = self.f.element._frame_element(rparentlist[-1])
|
|
1187
|
+
box = f._box['model']['padding']
|
|
1188
|
+
sftx += box[0]
|
|
1189
|
+
sfty += box[1]
|
|
1190
|
+
f_box = self._box
|
|
1191
|
+
x, y = f_box['model']['padding'][0], f_box['model']['padding'][1]
|
|
1192
|
+
width, height = f_box['model']['width'], f_box['model']['height']
|
|
1193
|
+
screenshot = self.f.rootf.cdp('Page.captureScreenshot', {"format":'png',
|
|
1194
|
+
"clip":{"x":x+sftx,"y":y+sfty,"width":width,"height":height,"scale":1} })
|
|
1195
|
+
return Screenshot(screenshot['data'])
|
|
1196
|
+
def _view_info(self):
|
|
1197
|
+
return self.f.element.visibility(self.objectId)
|
|
1198
|
+
def visible(self):
|
|
1199
|
+
d = self._view_info()
|
|
1200
|
+
d['isCoverd'] = self._is_coverd()
|
|
1201
|
+
return d
|
|
1202
|
+
def info(self):
|
|
1203
|
+
d = self.visible()
|
|
1204
|
+
class ViewInfo:
|
|
1205
|
+
def __init__(self, d):
|
|
1206
|
+
self.d = d
|
|
1207
|
+
def __repr__(self):
|
|
1208
|
+
return json.dumps(self.d, indent=4)
|
|
1209
|
+
return ViewInfo(d)
|
|
1210
|
+
def xpath(self, s):
|
|
1211
|
+
return self.f.element.xpath(s, self.objectId)
|
|
1212
|
+
def css(self, s):
|
|
1213
|
+
return self.f.element.css(s, self.objectId)
|
|
1214
|
+
def _ele(self, s, one=False):
|
|
1215
|
+
r = []
|
|
1216
|
+
xf, tp = self.f.element._predict_xpath_or_css(s.strip())
|
|
1217
|
+
if tp == 'xpath': r.extend(self.xpath(xf))
|
|
1218
|
+
if tp == 'css': r.extend(self.css(xf))
|
|
1219
|
+
if one and r: return r
|
|
1220
|
+
nodeId = self.f.cdp('DOM.describeNode', {'objectId': self.objectId})['node']['nodeId']
|
|
1221
|
+
for sr in self.f.element._sr_root(nodeId):
|
|
1222
|
+
if tp == 'xpath': r.extend(sr.xpath(xf))
|
|
1223
|
+
if tp == 'css': r.extend(sr.css(xf))
|
|
1224
|
+
if one and r: return r
|
|
1225
|
+
return r
|
|
1226
|
+
def ele(self, s):
|
|
1227
|
+
rt = self._ele(s, one=True)
|
|
1228
|
+
return rt[0] if rt else None
|
|
1229
|
+
def eles(self, s):
|
|
1230
|
+
return self._ele(s)
|
|
1231
|
+
def id(self, i):
|
|
1232
|
+
return self.ele('#'+i)
|
|
1233
|
+
def ijs(self, s):
|
|
1234
|
+
einfo = self.r_js(s, returnByValue=False)
|
|
1235
|
+
return self.f._parse_js2py(einfo, self, iso=self.iso)
|
|
1236
|
+
def __getitem__(self, a):
|
|
1237
|
+
einfo = self.r_obj('function(){return this[' + json.dumps(a) + ']}', objectId=self.objectId, returnByValue=False)
|
|
1238
|
+
return self.f._parse_js2py(einfo, self, iso=self.iso)
|
|
1239
|
+
def __setitem__(self, a, b):
|
|
1240
|
+
args = [None,self.f._parse_2arg(a), self.f._parse_2arg(b)]
|
|
1241
|
+
einfo = self.r_obj('function(a,b){return this[a]=b}', objectId=self.objectId, arguments=args, returnByValue=False)
|
|
1242
|
+
return self.f._parse_js2py(einfo, iso=self.iso)
|
|
1243
|
+
def __add__(self, other):
|
|
1244
|
+
einfo = self.r_obj('function(){return this["toString"]()}', objectId=self.objectId, returnByValue=False)
|
|
1245
|
+
return self.f._parse_js2py(einfo, self, iso=self.iso) + str(other)
|
|
1246
|
+
text = property(lambda s:s['outerHTML'] or s['textContent'])
|
|
1247
|
+
before = property(lambda s:s['previousElementSibling'])
|
|
1248
|
+
after = property(lambda s:s['nextElementSibling'])
|
|
1249
|
+
parent = property(lambda s:s['parentElement'])
|
|
1250
|
+
children = property(lambda s:s['children'])
|
|
1251
|
+
class ElementTools:
|
|
1252
|
+
def __init__(self, f):
|
|
1253
|
+
self.f = f
|
|
1254
|
+
self.attach(f)
|
|
1255
|
+
def attach(self, f):
|
|
1256
|
+
f.ele = self.ele
|
|
1257
|
+
f.eles = self.eles
|
|
1258
|
+
f.xpath = self.xpath
|
|
1259
|
+
f.css = self.css
|
|
1260
|
+
f._sr_root = self._sr_root
|
|
1261
|
+
def _predict_xpath_or_css(self, x):
|
|
1262
|
+
if x.startswith('x:'):
|
|
1263
|
+
xf, tp = x[2:], 'xpath'
|
|
1264
|
+
elif x.startswith('c:'):
|
|
1265
|
+
xf, tp = x[2:], 'css'
|
|
1266
|
+
else:
|
|
1267
|
+
xf = x
|
|
1268
|
+
x = x.strip()
|
|
1269
|
+
if x == '.' or '/' in x:
|
|
1270
|
+
tp = 'xpath'
|
|
1271
|
+
else:
|
|
1272
|
+
tp = 'css'
|
|
1273
|
+
return xf, tp
|
|
1274
|
+
def _parse_x(self, x, all_frames, one=False, over_v_limit=False):
|
|
1275
|
+
r = []
|
|
1276
|
+
frms = [self.f]
|
|
1277
|
+
xf, tp = self._predict_xpath_or_css(x.strip())
|
|
1278
|
+
if all_frames:
|
|
1279
|
+
frms.extend(self.f.root.flat_child_frames(self.f.frames))
|
|
1280
|
+
for f in frms:
|
|
1281
|
+
if over_v_limit: f.iso_contextId = None
|
|
1282
|
+
if tp == 'xpath':
|
|
1283
|
+
r.extend(f.xpath(xf))
|
|
1284
|
+
if one and r: return r
|
|
1285
|
+
if f.type == 'LocalFrame':continue
|
|
1286
|
+
for sr in f._sr_root():
|
|
1287
|
+
if one and r: return r
|
|
1288
|
+
r.extend(sr.xpath(xf))
|
|
1289
|
+
elif tp == 'css':
|
|
1290
|
+
r.extend(f.css(xf))
|
|
1291
|
+
if one and r: return r
|
|
1292
|
+
if f.type == 'LocalFrame':continue
|
|
1293
|
+
for sr in f._sr_root():
|
|
1294
|
+
if one and r: return r
|
|
1295
|
+
r.extend(sr.css(xf))
|
|
1296
|
+
return r
|
|
1297
|
+
def ele(self, x, timeout=8, no_wait=False, all_frames=True):
|
|
1298
|
+
start = perf_counter()
|
|
1299
|
+
over_v_time = 2
|
|
1300
|
+
over_v_limit = False
|
|
1301
|
+
while True:
|
|
1302
|
+
ctime = perf_counter()
|
|
1303
|
+
if ctime - start > over_v_time: over_v_limit = True
|
|
1304
|
+
r = self._parse_x(x, all_frames, one=True, over_v_limit=over_v_limit)
|
|
1305
|
+
if ctime - start > over_v_time: over_v_limit = False
|
|
1306
|
+
if r: return r[0]
|
|
1307
|
+
if no_wait: return None
|
|
1308
|
+
if ctime - start > timeout:
|
|
1309
|
+
raise Exception('ele selecter timeout.')
|
|
1310
|
+
time.sleep(0.08)
|
|
1311
|
+
def eles(self, x, timeout=8, no_wait=False, all_frames=True):
|
|
1312
|
+
start = perf_counter()
|
|
1313
|
+
over_v_time = 2
|
|
1314
|
+
over_v_limit = False
|
|
1315
|
+
while True:
|
|
1316
|
+
ctime = perf_counter()
|
|
1317
|
+
if ctime - start > over_v_time: over_v_limit = True
|
|
1318
|
+
r = self._parse_x(x, all_frames, over_v_limit=over_v_limit)
|
|
1319
|
+
if ctime - start > over_v_time: over_v_limit = False
|
|
1320
|
+
if r: return r
|
|
1321
|
+
if no_wait: return []
|
|
1322
|
+
if ctime - start > timeout:
|
|
1323
|
+
raise Exception('eles selecter timeout.')
|
|
1324
|
+
time.sleep(0.08)
|
|
1325
|
+
def _parse_array(self, r):
|
|
1326
|
+
if not r.get('objectId'):
|
|
1327
|
+
return []
|
|
1328
|
+
prps = self.f.cdp('Runtime.getProperties', { 'objectId': r['objectId'], 'ownProperties': True })
|
|
1329
|
+
rn = []
|
|
1330
|
+
for prp in prps.get('result', []):
|
|
1331
|
+
if prp['name'].isdigit() and 'objectId' in prp['value']:
|
|
1332
|
+
rn.append(Element(self.f, prp['value']))
|
|
1333
|
+
return rn
|
|
1334
|
+
def _run_iso(self, fscpt, objectId, returnByValue=False):
|
|
1335
|
+
if objectId:
|
|
1336
|
+
r = self.f.run_iso_js_obj(fscpt, returnByValue=returnByValue, objectId=objectId)
|
|
1337
|
+
else:
|
|
1338
|
+
r = self.f.run_iso_js('({})()'.format(fscpt), returnByValue=returnByValue)
|
|
1339
|
+
return r
|
|
1340
|
+
def _trav_node_tree(self, rnode):
|
|
1341
|
+
def collect(n):
|
|
1342
|
+
d = {}
|
|
1343
|
+
d['nodeId'] = n['nodeId']
|
|
1344
|
+
d['backendNodeId'] = n['backendNodeId']
|
|
1345
|
+
d['nodeType'] = n['nodeType']
|
|
1346
|
+
d['nodeName'] = n['nodeName']
|
|
1347
|
+
if n.get('frameId'):
|
|
1348
|
+
d['frameId'] = n['frameId']
|
|
1349
|
+
if n.get('nodeName') == 'IFRAME' and 'contentDocument' in n:
|
|
1350
|
+
_trav(n['contentDocument'])
|
|
1351
|
+
if n.get('shadowRoots'):
|
|
1352
|
+
d['shadowRoots'] = n['shadowRoots']
|
|
1353
|
+
for ssnode in d['shadowRoots']:
|
|
1354
|
+
if ssnode.get('children'):
|
|
1355
|
+
_trav(ssnode)
|
|
1356
|
+
clist.append(d)
|
|
1357
|
+
clist = []
|
|
1358
|
+
def _trav(rnode):
|
|
1359
|
+
collect(rnode)
|
|
1360
|
+
if rnode.get('children'):
|
|
1361
|
+
for snode in rnode['children']:
|
|
1362
|
+
_trav(snode)
|
|
1363
|
+
_trav(rnode)
|
|
1364
|
+
return clist
|
|
1365
|
+
def _get_flattened_node(self, nodeId=None):
|
|
1366
|
+
if nodeId == None:
|
|
1367
|
+
rnode = self.f.cdp('DOM.getDocument', {'depth':-1, 'pierce':True})['root']
|
|
1368
|
+
else:
|
|
1369
|
+
rnode = self.f.cdp('DOM.describeNode', {'nodeId':nodeId, 'depth':-1, 'pierce':True})['node']
|
|
1370
|
+
mnodes = {"nodes": self._trav_node_tree(rnode)}
|
|
1371
|
+
return mnodes
|
|
1372
|
+
def _filter_shadow_root(self, mnodes):
|
|
1373
|
+
r = []
|
|
1374
|
+
if not mnodes.get('nodes'):
|
|
1375
|
+
return []
|
|
1376
|
+
for n in mnodes['nodes']:
|
|
1377
|
+
srlst = n.get('shadowRoots')
|
|
1378
|
+
if srlst:
|
|
1379
|
+
for sr in srlst:
|
|
1380
|
+
if (sr.get('nodeName') == '#document-fragment'
|
|
1381
|
+
and (
|
|
1382
|
+
sr.get('shadowRootType') == 'closed' or
|
|
1383
|
+
sr.get('shadowRootType') == 'open')
|
|
1384
|
+
):
|
|
1385
|
+
m = self.f.cdp('DOM.resolveNode', {"backendNodeId": sr['backendNodeId']})
|
|
1386
|
+
if m.get('object'):
|
|
1387
|
+
e = Element(self.f, m['object'])
|
|
1388
|
+
r.append(e)
|
|
1389
|
+
return r
|
|
1390
|
+
def _sr_root(self, nodeId=None):
|
|
1391
|
+
mnodes = self._get_flattened_node(nodeId)
|
|
1392
|
+
return self._filter_shadow_root(mnodes)
|
|
1393
|
+
def _make_shadow_root_xpath(self, s):
|
|
1394
|
+
v = re.findall('^([^/]*)(/+)([^/]*)$', s)
|
|
1395
|
+
if v and len(v[0]) == 3:
|
|
1396
|
+
v = v[0]
|
|
1397
|
+
p = v[-1].split('[', 1)
|
|
1398
|
+
tag = p[0]
|
|
1399
|
+
pak = p[1] if len(p) == 2 else ''
|
|
1400
|
+
if pak:
|
|
1401
|
+
pak = "[self::"+tag+' and ' + pak
|
|
1402
|
+
else:
|
|
1403
|
+
pak = "[self::"+tag+']'
|
|
1404
|
+
return '(.|'+v[0]+v[1]+tag+')'+pak
|
|
1405
|
+
else:
|
|
1406
|
+
return s
|
|
1407
|
+
def xpath(self, s, objectId=None):
|
|
1408
|
+
s = s.lstrip()
|
|
1409
|
+
if s[0] != '.' and s[0] != '/': s = './' + s
|
|
1410
|
+
if s[0] == '/': s = '.' + s
|
|
1411
|
+
fscpt = '''
|
|
1412
|
+
function() {
|
|
1413
|
+
var contextNode = window === this ? document : this
|
|
1414
|
+
var node, nodes = [];
|
|
1415
|
+
if (contextNode instanceof ShadowRoot){
|
|
1416
|
+
for (var i = 0; i < contextNode.children.length; i++) {
|
|
1417
|
+
var snode = contextNode.children[i]
|
|
1418
|
+
var iterator = document.evaluate('''+json.dumps(self._make_shadow_root_xpath(s))+''', snode, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
|
|
1419
|
+
while ((node = iterator.iterateNext())) { nodes.push(node); }
|
|
1420
|
+
}
|
|
1421
|
+
}else{
|
|
1422
|
+
var iterator = document.evaluate('''+json.dumps(s)+''', contextNode, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
|
|
1423
|
+
while ((node = iterator.iterateNext())) { nodes.push(node); }
|
|
1424
|
+
}
|
|
1425
|
+
return nodes;
|
|
1426
|
+
}
|
|
1427
|
+
'''
|
|
1428
|
+
r = self._run_iso(fscpt, objectId)
|
|
1429
|
+
return self._parse_array(r)
|
|
1430
|
+
def visibility(self, objectId=None):
|
|
1431
|
+
fscpt = '''
|
|
1432
|
+
function() {
|
|
1433
|
+
const rect = this.getBoundingClientRect();
|
|
1434
|
+
const s = window.getComputedStyle(this);
|
|
1435
|
+
return {
|
|
1436
|
+
inViewport: (
|
|
1437
|
+
rect.top < window.innerHeight &&
|
|
1438
|
+
rect.bottom > 0 &&
|
|
1439
|
+
rect.left < window.innerWidth &&
|
|
1440
|
+
rect.right > 0
|
|
1441
|
+
),
|
|
1442
|
+
style: {
|
|
1443
|
+
display: s.display,
|
|
1444
|
+
visibility: s.visibility,
|
|
1445
|
+
opacity: s.opacity,
|
|
1446
|
+
position: s.position,
|
|
1447
|
+
zIndex: s.zIndex
|
|
1448
|
+
},
|
|
1449
|
+
isDisplayed: !(
|
|
1450
|
+
s.visibility == 'hidden' ||
|
|
1451
|
+
s.display == 'none' ||
|
|
1452
|
+
this.hidden ||
|
|
1453
|
+
rect.width == 0 ||
|
|
1454
|
+
rect.height == 0
|
|
1455
|
+
),
|
|
1456
|
+
rect: {
|
|
1457
|
+
x: rect.x,
|
|
1458
|
+
y: rect.y,
|
|
1459
|
+
width: rect.width,
|
|
1460
|
+
height: rect.height,
|
|
1461
|
+
top: rect.top,
|
|
1462
|
+
right: rect.right,
|
|
1463
|
+
bottom: rect.bottom,
|
|
1464
|
+
left: rect.left
|
|
1465
|
+
},
|
|
1466
|
+
clickPoint: {
|
|
1467
|
+
x: (rect.x + rect.width/2)|0,
|
|
1468
|
+
y: (rect.y + rect.height/2)|0,
|
|
1469
|
+
},
|
|
1470
|
+
inDocument: document.contains(this),
|
|
1471
|
+
isClickable: (
|
|
1472
|
+
s.pointerEvents !== 'none' &&
|
|
1473
|
+
s.cursor !== 'not-allowed'
|
|
1474
|
+
)
|
|
1475
|
+
};
|
|
1476
|
+
}
|
|
1477
|
+
'''
|
|
1478
|
+
r = self._run_iso(fscpt, objectId, returnByValue=True)
|
|
1479
|
+
return r
|
|
1480
|
+
def css(self, s, objectId=None):
|
|
1481
|
+
fscpt = '''
|
|
1482
|
+
function(){
|
|
1483
|
+
var contextNode = window === this ? document : this
|
|
1484
|
+
return Array.from(contextNode.querySelectorAll(''' + json.dumps(s) + '''));
|
|
1485
|
+
}
|
|
1486
|
+
'''
|
|
1487
|
+
r = self._run_iso(fscpt, objectId)
|
|
1488
|
+
return self._parse_array(r)
|
|
1489
|
+
@staticmethod
|
|
1490
|
+
def _frame_element(self):
|
|
1491
|
+
nodeId = self.parent.cdp('DOM.getDocument', {"depth": -1})['root']['nodeId']
|
|
1492
|
+
frs = self.parent.cdp('DOM.querySelectorAll', { "nodeId": nodeId, "selector": 'iframe' })
|
|
1493
|
+
node = None
|
|
1494
|
+
for fr in frs['nodeIds']:
|
|
1495
|
+
nd = self.parent.cdp('DOM.describeNode', { "nodeId": fr })
|
|
1496
|
+
if nd['node']['frameId'] == self.frameId:
|
|
1497
|
+
node = nd['node']
|
|
1498
|
+
break
|
|
1499
|
+
return Element(self.parent, self.parent.cdp('DOM.resolveNode', {"backendNodeId": node['backendNodeId']})['object'])
|
|
1500
|
+
class CDPTools:
|
|
1501
|
+
def __init__(self, f, rootf):
|
|
1502
|
+
self.f = f
|
|
1503
|
+
self.rootf = rootf
|
|
1504
|
+
self.attach(f)
|
|
1505
|
+
def attach(self, f):
|
|
1506
|
+
f.run_js = self.run_js
|
|
1507
|
+
f.run_js_obj = self.run_js_obj
|
|
1508
|
+
f.run_iso_js = self.run_iso_js
|
|
1509
|
+
f.run_iso_js_obj = self.run_iso_js_obj
|
|
1510
|
+
f._js2py = self._js2py
|
|
1511
|
+
f._parse_js2py = self._parse_js2py
|
|
1512
|
+
f._parse_2arg = self._parse_2arg
|
|
1513
|
+
def _init_iso(self, d):
|
|
1514
|
+
if not self.f.iso_contextId:
|
|
1515
|
+
x = self.rootf.cdp('Page.createIsolatedWorld', { "frameId": self.f.frameId }, sessionId=self.f.sessionId)
|
|
1516
|
+
if x.get('message') == 'No frame for given id found':
|
|
1517
|
+
return
|
|
1518
|
+
if 'executionContextId' not in x:
|
|
1519
|
+
print('error create iso:'+str(x))
|
|
1520
|
+
return
|
|
1521
|
+
self.f.iso_contextId = x['executionContextId']
|
|
1522
|
+
if self.f.iso_contextId != None: d["contextId"] = self.f.iso_contextId
|
|
1523
|
+
return d
|
|
1524
|
+
def _parse_2arg(self, v):
|
|
1525
|
+
if isinstance(v, (JSObject, Element)):
|
|
1526
|
+
return {"objectId": v.objectId}
|
|
1527
|
+
else:
|
|
1528
|
+
return {"value": v}
|
|
1529
|
+
def _parse_js2py(self, e, _this=None, iso=False):
|
|
1530
|
+
if type(e) in (int, float, str, bool): return e
|
|
1531
|
+
if e == None: return None
|
|
1532
|
+
if iso and type(e) == dict and e.get('subtype') == 'node':
|
|
1533
|
+
# The ISO here is only used to isolate the impact of code on the main context.
|
|
1534
|
+
return Element(self.f, e, iso=iso)
|
|
1535
|
+
return JSObject(self.f, e, _this, iso)
|
|
1536
|
+
def _js2py(self, code, iso=False):
|
|
1537
|
+
return self._parse_js2py((self.run_iso_js if iso else self.run_js)(code, returnByValue=False), iso=iso)
|
|
1538
|
+
def run_js(self, script, awaitPromise=True, returnByValue=True):
|
|
1539
|
+
d = { "expression": script, "awaitPromise": awaitPromise, "returnByValue": returnByValue }
|
|
1540
|
+
if self.f.contextId != None: d["contextId"] = self.f.contextId
|
|
1541
|
+
return self.rootf.cdp('Runtime.evaluate', d, sessionId=self.f.sessionId)
|
|
1542
|
+
def run_js_obj(self, script, awaitPromise=True, returnByValue=True, objectId=None, arguments=[None], includeCommandLineAPI=False):
|
|
1543
|
+
d = { "functionDeclaration": script, "awaitPromise": awaitPromise, "returnByValue": returnByValue, "objectId": objectId, "includeCommandLineAPI": includeCommandLineAPI }
|
|
1544
|
+
if arguments[0]:
|
|
1545
|
+
arguments[0] = { "objectId": arguments[0].objectId }
|
|
1546
|
+
d['arguments'] = arguments
|
|
1547
|
+
else:
|
|
1548
|
+
d['arguments'] = arguments[1:]
|
|
1549
|
+
if self.f.contextId != None: d["contextId"] = self.f.contextId
|
|
1550
|
+
return self.rootf.cdp('Runtime.callFunctionOn', d, sessionId=self.f.sessionId)
|
|
1551
|
+
def run_iso_js(self, script, awaitPromise=True, returnByValue=True, includeCommandLineAPI=False):
|
|
1552
|
+
d = self._init_iso({ "expression": script, "awaitPromise": awaitPromise,"returnByValue": returnByValue, "includeCommandLineAPI": includeCommandLineAPI })
|
|
1553
|
+
return self.rootf.cdp('Runtime.evaluate', d, sessionId=self.f.sessionId) if d else {}
|
|
1554
|
+
def run_iso_js_obj(self, script, awaitPromise=True, returnByValue=True, objectId=None, arguments=[None], includeCommandLineAPI=False):
|
|
1555
|
+
d = self._init_iso({ "functionDeclaration": script, "awaitPromise": awaitPromise,"returnByValue": returnByValue, "objectId": objectId, "includeCommandLineAPI": includeCommandLineAPI })
|
|
1556
|
+
if not d: return {}
|
|
1557
|
+
if arguments[0]:
|
|
1558
|
+
arguments[0] = { "objectId": arguments[0].objectId }
|
|
1559
|
+
d['arguments'] = arguments
|
|
1560
|
+
else:
|
|
1561
|
+
d['arguments'] = arguments[1:]
|
|
1562
|
+
return self.rootf.cdp('Runtime.callFunctionOn', d, sessionId=self.f.sessionId) if d else {}
|
|
1563
|
+
class AbsFrame:
|
|
1564
|
+
def __init__(self, f, finfo):
|
|
1565
|
+
self.rootf = f
|
|
1566
|
+
self.frameId = finfo['frameId']
|
|
1567
|
+
self.frames = []
|
|
1568
|
+
self.parent = finfo.get('parent')
|
|
1569
|
+
self.url = finfo.get('url')
|
|
1570
|
+
# LocalFrame
|
|
1571
|
+
self.contextId = finfo.get('contextId')
|
|
1572
|
+
self.uniqueId = finfo.get('uniqueId')
|
|
1573
|
+
# RemoteFrame
|
|
1574
|
+
self.sessionId = finfo.get('sessionId')
|
|
1575
|
+
self.iso_contextId = None
|
|
1576
|
+
self.tools = CDPTools(self, self.rootf)
|
|
1577
|
+
self.element = ElementTools(self)
|
|
1578
|
+
@property
|
|
1579
|
+
def type(self):
|
|
1580
|
+
if self.sessionId:
|
|
1581
|
+
return 'RemoteFrame'
|
|
1582
|
+
if self.contextId:
|
|
1583
|
+
return 'LocalFrame'
|
|
1584
|
+
return 'UnknownFrame'
|
|
1585
|
+
def __repr__(self):
|
|
1586
|
+
if self.type == 'RemoteFrame':
|
|
1587
|
+
return '<[{}]|{}|F:{}|S:{}>'.format(self.type, self.url, self.frameId, self.sessionId)
|
|
1588
|
+
return '<[{}]|{}|F:{}>'.format(self.type, self.url, self.frameId)
|
|
1589
|
+
def cdp(self, *a, **kw):
|
|
1590
|
+
return self.rootf.cdp(*a, **kw, sessionId=self.sessionId)
|
|
1591
|
+
def _get_remote_list(self):
|
|
1592
|
+
_self = self
|
|
1593
|
+
_rlst = []
|
|
1594
|
+
while _self:
|
|
1595
|
+
if _self.type == 'RemoteFrame':
|
|
1596
|
+
_rlst.append(_self)
|
|
1597
|
+
_self = _self.parent
|
|
1598
|
+
return _rlst
|
|
1599
|
+
def pic(self,):
|
|
1600
|
+
return self._js2py('document.documentElement', iso=True).pic()
|
|
1601
|
+
class Runtime:
|
|
1602
|
+
def __init__(self, f):
|
|
1603
|
+
self.f = f
|
|
1604
|
+
self.f.set_method_callback('Runtime.executionContextCreated', self.Runtime_executionContextCreated)
|
|
1605
|
+
self.f.set_method_callback('Runtime.executionContextDestroyed', self.Runtime_executionContextDestroyed)
|
|
1606
|
+
self.init()
|
|
1607
|
+
def init(self, sessionId=None):
|
|
1608
|
+
self.f.cdp('Runtime.enable', sessionId=sessionId)
|
|
1609
|
+
def Runtime_executionContextCreated(self, rdata):
|
|
1610
|
+
self.f.root._add_init_check()
|
|
1611
|
+
# {"context":
|
|
1612
|
+
# {"id": 10,
|
|
1613
|
+
# "origin": "http://lc1.test:18000",
|
|
1614
|
+
# "name": "",
|
|
1615
|
+
# "uniqueId": "-2899874725492390568.194300135741888500",
|
|
1616
|
+
# "auxData":
|
|
1617
|
+
# {"isDefault": true,
|
|
1618
|
+
# "type": "default",
|
|
1619
|
+
# "frameId": "6D49B34D12C4719D669CE5DC9BED71A4"}}}
|
|
1620
|
+
params = rdata['params']
|
|
1621
|
+
if 'auxData' not in params['context']:
|
|
1622
|
+
# extension context
|
|
1623
|
+
# {"context": {"id": 1, "origin": "chrome-extension://ognihjbdmbjhecdlpjjonacagooanpfa/background.js", "name": "", "uniqueId": "-5795409506706594697.-3458865582039429610"}}
|
|
1624
|
+
# maybe some blob:url
|
|
1625
|
+
# {"context": {"id": 1, "origin": "blob:https://steamdb.info/fd99df19-bda7-4360-8cff-80c782e38697", "name": "", "uniqueId": "-5913474193475720670.7988369434689538446"}}
|
|
1626
|
+
self.f.root._del_init_check()
|
|
1627
|
+
return
|
|
1628
|
+
if params['context']['auxData']['type'] == 'isolated':
|
|
1629
|
+
# DevTools tools isolate context
|
|
1630
|
+
# {"context": {"id": 100, "origin": "http://lc1.test:18000", "name": "DevTools Performance Metrics", "uniqueId": "-4717759096737431074.6555991471704808792", "auxData": {"isDefault": false, "type": "isolated", "frameId": "65764D3FB5953A54725C764602802733"}}}
|
|
1631
|
+
self.f.root._del_init_check()
|
|
1632
|
+
return
|
|
1633
|
+
frameId = params['context']['auxData']['frameId']
|
|
1634
|
+
uniqueId = params['context']['uniqueId']
|
|
1635
|
+
contextId = params['context']['id']
|
|
1636
|
+
if frameId != self.f.frameId:
|
|
1637
|
+
self.f.root.add_common_frame(self.f, {
|
|
1638
|
+
"contextId": contextId,
|
|
1639
|
+
"uniqueId": uniqueId,
|
|
1640
|
+
"frameId": frameId,
|
|
1641
|
+
}, sessionId=rdata.get('sessionId'))
|
|
1642
|
+
self.f.root._del_init_check()
|
|
1643
|
+
def Runtime_executionContextDestroyed(self, rdata):
|
|
1644
|
+
if 'executionContextUniqueId' in rdata['params']:
|
|
1645
|
+
f = self.f.root.trav_frame(rdata['params'].get('executionContextUniqueId'), 'uniqueId')
|
|
1646
|
+
if f:
|
|
1647
|
+
f.parent.frames.remove(f)
|
|
1648
|
+
class Emulation:
|
|
1649
|
+
def __init__(self, f):
|
|
1650
|
+
self.f = f
|
|
1651
|
+
self.init()
|
|
1652
|
+
def init(self, sessionId=None):
|
|
1653
|
+
self.f.cdp('Emulation.setFocusEmulationEnabled', {'enabled': True}, sessionId=sessionId)
|
|
1654
|
+
# chrome 112+: Emulation.setDeviceMetricsOverride connot work in screenX/screenY.
|
|
1655
|
+
# self.f.cdp('Emulation.setDeviceMetricsOverride', {
|
|
1656
|
+
# "positionX": 100,
|
|
1657
|
+
# "positionY": 200,
|
|
1658
|
+
# }, sessionId=sessionId)
|
|
1659
|
+
class CoreCDP:
|
|
1660
|
+
def __init__(self, ws, root, logger=None):
|
|
1661
|
+
self.ws = ws
|
|
1662
|
+
self.root = root
|
|
1663
|
+
self.logger = logger or (lambda *a,**kw:None)
|
|
1664
|
+
self.id = 0
|
|
1665
|
+
self.xid = 0
|
|
1666
|
+
self.qret = {}
|
|
1667
|
+
self.irun = {}
|
|
1668
|
+
self._start()
|
|
1669
|
+
def _start(self):
|
|
1670
|
+
self.is_running = True
|
|
1671
|
+
self.loop_recv = Thread(target=self.start_loop)
|
|
1672
|
+
self.loop_recv.daemon = True
|
|
1673
|
+
self.loop_recv.start()
|
|
1674
|
+
def _handle_method(self, rdata):
|
|
1675
|
+
method = rdata.get('method')
|
|
1676
|
+
if method in self.irun:
|
|
1677
|
+
for xid in self.irun[method]:
|
|
1678
|
+
m = self.irun[method].get(xid, None)
|
|
1679
|
+
if m:
|
|
1680
|
+
if is_function(m):
|
|
1681
|
+
m(rdata)
|
|
1682
|
+
if isinstance(m, queue.Queue):
|
|
1683
|
+
m.put(rdata['params'])
|
|
1684
|
+
def _handle_return(self, rdata):
|
|
1685
|
+
if rdata.get('id') in self.qret:
|
|
1686
|
+
if rdata.get('result', Err) != Err:
|
|
1687
|
+
self.qret[rdata.get('id')].put(rdata['result'])
|
|
1688
|
+
elif rdata.get('error', Err) != Err:
|
|
1689
|
+
self.qret[rdata.get('id')].put(rdata['error'])
|
|
1690
|
+
else:
|
|
1691
|
+
self.logger(rdata, repr(rdata.get('result')))
|
|
1692
|
+
raise Exception('un expect err.' + repr(rdata.get('result')))
|
|
1693
|
+
def attach(self, f):
|
|
1694
|
+
f.cdp = self.cdp
|
|
1695
|
+
f.wait_once_method = self.wait_once_method
|
|
1696
|
+
f.set_method_callback = self.set_method_callback
|
|
1697
|
+
def start_loop(self):
|
|
1698
|
+
while self.is_running:
|
|
1699
|
+
try:
|
|
1700
|
+
recvd = self.ws.recv()
|
|
1701
|
+
if not recvd:
|
|
1702
|
+
continue
|
|
1703
|
+
rdata = json.loads(recvd)
|
|
1704
|
+
self.logger('recv', rdata)
|
|
1705
|
+
except WebSocketTimeoutException:
|
|
1706
|
+
continue
|
|
1707
|
+
except (
|
|
1708
|
+
OSError,
|
|
1709
|
+
JSONDecodeError,
|
|
1710
|
+
WebSocketException,
|
|
1711
|
+
WebSocketConnectionClosedException,
|
|
1712
|
+
ConnectionResetError,
|
|
1713
|
+
) as e:
|
|
1714
|
+
if not self.is_running:
|
|
1715
|
+
return
|
|
1716
|
+
if (
|
|
1717
|
+
'Connection to remote host was lost.' in str(e)
|
|
1718
|
+
or 'socket is already closed.' in str(e)
|
|
1719
|
+
or isinstance(e, OSError)
|
|
1720
|
+
):
|
|
1721
|
+
# Closing the tab will trigger some return message exceptions and possible exception disconnections
|
|
1722
|
+
return
|
|
1723
|
+
raise Exception('[*] maybe exist ws connect from another python exe! pls close it.')
|
|
1724
|
+
except Exception as e:
|
|
1725
|
+
print(e, 'rasie cdp loop 2')
|
|
1726
|
+
raise 2
|
|
1727
|
+
self._handle_method(rdata)
|
|
1728
|
+
self._handle_return(rdata)
|
|
1729
|
+
def get_id(self):
|
|
1730
|
+
self.id += 1
|
|
1731
|
+
return self.id
|
|
1732
|
+
def get_xid(self):
|
|
1733
|
+
self.xid += 1
|
|
1734
|
+
return self.xid
|
|
1735
|
+
def cdp(self, protocal, data={}, sessionId=None, no_wait=False, limit_time=None):
|
|
1736
|
+
rid = self.get_id()
|
|
1737
|
+
cmd = { "id": rid, "method": protocal, "params": data }
|
|
1738
|
+
if sessionId:
|
|
1739
|
+
cmd['sessionId'] = sessionId
|
|
1740
|
+
self.logger('req', cmd)
|
|
1741
|
+
if not no_wait:
|
|
1742
|
+
self.qret[rid] = queue.Queue()
|
|
1743
|
+
try:
|
|
1744
|
+
self.ws.send(json.dumps(cmd))
|
|
1745
|
+
except (
|
|
1746
|
+
OSError,
|
|
1747
|
+
WebSocketConnectionClosedException
|
|
1748
|
+
) as e:
|
|
1749
|
+
self.logger('[ERROR]', str(e))
|
|
1750
|
+
self.qret.pop(rid, None)
|
|
1751
|
+
return
|
|
1752
|
+
except Exception as e:
|
|
1753
|
+
print(e, 'rasie cdp 1')
|
|
1754
|
+
raise 1
|
|
1755
|
+
if limit_time != None:
|
|
1756
|
+
start = perf_counter()
|
|
1757
|
+
while not no_wait:
|
|
1758
|
+
try:
|
|
1759
|
+
if limit_time != None:
|
|
1760
|
+
if perf_counter() - start > limit_time:
|
|
1761
|
+
self.qret.pop(rid, None)
|
|
1762
|
+
return {'vvv': 'over time'}
|
|
1763
|
+
ret = self.qret[rid].get(timeout=.15)
|
|
1764
|
+
self.qret.pop(rid, None)
|
|
1765
|
+
return try_run_result(ret)
|
|
1766
|
+
except queue.Empty:
|
|
1767
|
+
continue
|
|
1768
|
+
def wait_once_method(self, method, timeout=10, check_ret_func=None, rasie_error=True):
|
|
1769
|
+
self.irun[method] = self.irun.get(method, {})
|
|
1770
|
+
xid = self.get_xid()
|
|
1771
|
+
self.irun[method][xid] = queue.Queue()
|
|
1772
|
+
start = perf_counter()
|
|
1773
|
+
while True:
|
|
1774
|
+
ctime = perf_counter()
|
|
1775
|
+
try:
|
|
1776
|
+
if ctime - start > timeout:
|
|
1777
|
+
self.irun[method].pop(xid, None)
|
|
1778
|
+
return
|
|
1779
|
+
ret = self.irun[method][xid].get(timeout=0.15)
|
|
1780
|
+
if check_ret_func:
|
|
1781
|
+
if not check_ret_func(ret):
|
|
1782
|
+
continue
|
|
1783
|
+
self.irun[method].pop(xid, None)
|
|
1784
|
+
return ret
|
|
1785
|
+
except:
|
|
1786
|
+
if ctime - start > timeout:
|
|
1787
|
+
if rasie_error:
|
|
1788
|
+
raise Exception('wait_once_method {} timeout: {}'.format(method, timeout))
|
|
1789
|
+
continue
|
|
1790
|
+
def set_method_callback(self, method, func):
|
|
1791
|
+
xid = self.get_xid()
|
|
1792
|
+
self.irun[method] = self.irun.get(method, {})
|
|
1793
|
+
self.irun[method][xid] = self.root.pool(func)
|
|
1794
|
+
return xid
|
|
1795
|
+
class Closer:
|
|
1796
|
+
def __init__(self, f):
|
|
1797
|
+
self.f = f
|
|
1798
|
+
self.f.set_method_callback('Inspector.detached', self.close)
|
|
1799
|
+
self.toggle = True
|
|
1800
|
+
def close(self, rdata):
|
|
1801
|
+
self.f.tools.is_running = False
|
|
1802
|
+
self.f.ws.close()
|
|
1803
|
+
if self.f in self.f.root.tabs:
|
|
1804
|
+
self.f.root.tabs.remove(self.f)
|
|
1805
|
+
self.toggle = False
|
|
1806
|
+
class DOM:
|
|
1807
|
+
def __init__(self, f):
|
|
1808
|
+
self.f = f
|
|
1809
|
+
self.f.cdp('DOM.enable')
|
|
1810
|
+
class Cache:
|
|
1811
|
+
def __init__(self, f):
|
|
1812
|
+
self.f = f
|
|
1813
|
+
self.cache_sesslist = [None]
|
|
1814
|
+
self.is_enable = False
|
|
1815
|
+
self.attach(f)
|
|
1816
|
+
def _enable(self):
|
|
1817
|
+
if not self.is_enable:
|
|
1818
|
+
for sessionId in self.cache_sesslist:
|
|
1819
|
+
self._cache_cdp(sessionId)
|
|
1820
|
+
self.is_enable = True
|
|
1821
|
+
def _cache_cdp(self, sessionId):
|
|
1822
|
+
self.f.cdp('Network.enable', sessionId=sessionId)
|
|
1823
|
+
self.f.cdp('Storage.enable', sessionId=sessionId)
|
|
1824
|
+
def attach(self, f):
|
|
1825
|
+
f.clear_cache = self.clear_cache
|
|
1826
|
+
def add_cache_session(self, sessionId):
|
|
1827
|
+
self.cache_sesslist.append(sessionId)
|
|
1828
|
+
if self.is_enable:
|
|
1829
|
+
self._cache_cdp(sessionId)
|
|
1830
|
+
def clear_cache(self):
|
|
1831
|
+
self._enable()
|
|
1832
|
+
for sessionId in self.cache_sesslist:
|
|
1833
|
+
self.f.cdp('Network.clearBrowserCookies',sessionId=sessionId)
|
|
1834
|
+
self.f.cdp('Storage.clearDataForOrigin', {'origin':'*','storageTypes':'local_storage,session_storage,indexeddb'},sessionId=sessionId);
|
|
1835
|
+
class Browser:
|
|
1836
|
+
def __init__(self, f):
|
|
1837
|
+
self.f = f
|
|
1838
|
+
self.attach(f)
|
|
1839
|
+
def attach(self, c):
|
|
1840
|
+
c.get = self.f._go_url
|
|
1841
|
+
c.cdp = self.f.cdp
|
|
1842
|
+
c.init_js = self.f.init_js
|
|
1843
|
+
c.ele = self.f.ele
|
|
1844
|
+
c.eles = self.f.eles
|
|
1845
|
+
c.listen = self.f.listen
|
|
1846
|
+
c.intercept = self.f.intercept
|
|
1847
|
+
c.clear_cache = self.f.clear_cache
|
|
1848
|
+
c.js = self.f.run_js
|
|
1849
|
+
c._js2py = self.f._js2py
|
|
1850
|
+
c.tabs = self.f.root.tabs
|
|
1851
|
+
c.quit = self.f.root.quit
|
|
1852
|
+
c.press = self.f.press
|
|
1853
|
+
c.release = self.f.release
|
|
1854
|
+
c.id = self.f.id
|
|
1855
|
+
c.set_rect = self.f.root.set_rect
|
|
1856
|
+
c.set_fullscreen = self.f.root.set_fullscreen
|
|
1857
|
+
c.set_maxscreen = self.f.root.set_maxscreen
|
|
1858
|
+
c.pic = self.f.pic
|
|
1859
|
+
c.get_dialog = lambda:self.f.page.dialog
|
|
1860
|
+
c.set_dialog = lambda v:setattr(self.f.page, 'dialog', v)
|
|
1861
|
+
c.get_cookies = lambda:self.f._get_cookies()
|
|
1862
|
+
c.set_cookies = lambda v:self.f._set_cookies(v)
|
|
1863
|
+
c.close = self.f.close
|
|
1864
|
+
def __gi(self,a):return self._js2py('window')[a]
|
|
1865
|
+
def __si(self,a,b):self._js2py('window')[a]=b
|
|
1866
|
+
c.__class__.__getitem__ = __gi
|
|
1867
|
+
c.__class__.__setitem__ = __si
|
|
1868
|
+
self.f.root.extension.attach(c)
|
|
1869
|
+
class CookieManager:
|
|
1870
|
+
def __init__(self, f): self.f = f
|
|
1871
|
+
@property
|
|
1872
|
+
def c(self): return self.f.cdp('Network.getCookies', {})['cookies']
|
|
1873
|
+
@property
|
|
1874
|
+
def d(self):
|
|
1875
|
+
dc = {}
|
|
1876
|
+
for d in self.c:
|
|
1877
|
+
dc[d['name']] = d
|
|
1878
|
+
return dc
|
|
1879
|
+
@property
|
|
1880
|
+
def string(self):
|
|
1881
|
+
_r = []
|
|
1882
|
+
for d in self.c:
|
|
1883
|
+
_r.append(d['name'] + '=' + d['value'])
|
|
1884
|
+
return '; '.join(_r)
|
|
1885
|
+
data = property(lambda s:s.c)
|
|
1886
|
+
def __getitem__(self, key): return self.d.get(key, {}).get('value')
|
|
1887
|
+
def __setitem__(self, k, v):
|
|
1888
|
+
s = self.d.get(k)
|
|
1889
|
+
if s:
|
|
1890
|
+
s['value'] = v
|
|
1891
|
+
self.f._set_cookies([s])
|
|
1892
|
+
else:
|
|
1893
|
+
self.f._set_cookies(str(k)+'='+str(v))
|
|
1894
|
+
def __repr__(self): return '<Cookie ['+';'.join(list(self.d.keys()))+']>'
|
|
1895
|
+
class RootFrame:
|
|
1896
|
+
def __init__(self, wsinfo, root, is_auto_create=False):
|
|
1897
|
+
self.root = root
|
|
1898
|
+
self.rootf = self
|
|
1899
|
+
self.wsinfo = wsinfo
|
|
1900
|
+
if not self._check_page(wsinfo): return
|
|
1901
|
+
self.ws = create_connection_saf(wsinfo['webSocketDebuggerUrl'], enable_multithread=True, suppress_origin=True)
|
|
1902
|
+
self.type = wsinfo['type']
|
|
1903
|
+
self.frames = []
|
|
1904
|
+
self._doc_script = []
|
|
1905
|
+
self.url = wsinfo.get('url')
|
|
1906
|
+
self.parent = None
|
|
1907
|
+
self.init_once = False
|
|
1908
|
+
self.logger = Logger(wsinfo['id'], debug)
|
|
1909
|
+
self.frameId = wsinfo['id']
|
|
1910
|
+
self.sessionId = None # Used to maintain consistency with CDPTools objects
|
|
1911
|
+
self.contextId = None # Used to maintain consistency with CDPTools objects
|
|
1912
|
+
self.iso_contextId = None # Used to maintain consistency with CDPTools objects
|
|
1913
|
+
self.cdper = CoreCDP(self.ws, self.root, self.logger)
|
|
1914
|
+
self.cdper.attach(self)
|
|
1915
|
+
if is_auto_create: self._run_init_once()
|
|
1916
|
+
self.root.add_root_frame(self)
|
|
1917
|
+
self.target = Target(self)
|
|
1918
|
+
self.sniff_networt = SniffNetwork(self)
|
|
1919
|
+
self.sniff_fetch = SniffFetch(self)
|
|
1920
|
+
self.cache = Cache(self)
|
|
1921
|
+
self.page = Page(self)
|
|
1922
|
+
self.dom = DOM(self)
|
|
1923
|
+
self.runtime = Runtime(self)
|
|
1924
|
+
self.emulation = Emulation(self)
|
|
1925
|
+
self.tools = CDPTools(self, self)
|
|
1926
|
+
self.closer = Closer(self)
|
|
1927
|
+
self.element = ElementTools(self)
|
|
1928
|
+
self.browser = Browser(self)
|
|
1929
|
+
self.keyboard = Keyboard(self)
|
|
1930
|
+
def id(self, i):
|
|
1931
|
+
return self.ele('#'+ i)
|
|
1932
|
+
def press(self, key): self.cdp('Input.dispatchKeyEvent', self.keyboard._make_down(key))
|
|
1933
|
+
def release(self, key): self.cdp('Input.dispatchKeyEvent', self.keyboard._make_up(key))
|
|
1934
|
+
def close(self):
|
|
1935
|
+
self.cdp('Target.closeTarget', {"targetId": self.frameId})
|
|
1936
|
+
while self.closer.toggle:
|
|
1937
|
+
time.sleep(0.07)
|
|
1938
|
+
def _get_cookies(self, line=False):
|
|
1939
|
+
return CookieManager(self)
|
|
1940
|
+
def _set_cookies(self, v):
|
|
1941
|
+
if type(v) == str:
|
|
1942
|
+
self.tools.run_iso_js('document.cookie='+json.dumps(v))
|
|
1943
|
+
elif type(v) == list:
|
|
1944
|
+
self.cdp('Network.setCookies', {"cookies": v})
|
|
1945
|
+
elif instanceof(v, CookieManager):
|
|
1946
|
+
self.cdp('Network.setCookies', {"cookies": v.data})
|
|
1947
|
+
else:
|
|
1948
|
+
# cookies must be (str/list)
|
|
1949
|
+
# str: document.cookie = 'vvv=123'
|
|
1950
|
+
# list:
|
|
1951
|
+
# [{'domain': '.baidu.com',
|
|
1952
|
+
# 'expires': 1790665876.734538,
|
|
1953
|
+
# 'httpOnly': False,
|
|
1954
|
+
# 'name': 'BIDUPSID',
|
|
1955
|
+
# 'path': '/',
|
|
1956
|
+
# 'priority': 'Medium',
|
|
1957
|
+
# 'sameParty': False,
|
|
1958
|
+
# 'secure': False,
|
|
1959
|
+
# 'session': False,
|
|
1960
|
+
# 'size': 40,
|
|
1961
|
+
# 'sourcePort': 443,
|
|
1962
|
+
# 'sourceScheme': 'Secure',
|
|
1963
|
+
# 'value': '2826F2D60D5E56C841B9E33F82051E8A'},...]
|
|
1964
|
+
# str 作用于当前 url 页面,只能在访问页面后执行。
|
|
1965
|
+
# list 是用于完美还原 cookies 状态,可以在访问页面前执行。
|
|
1966
|
+
raise Exception('cookies must be (str/list/CookieManager)')
|
|
1967
|
+
def _get_remote_list(self):
|
|
1968
|
+
_self = self
|
|
1969
|
+
_rlst = []
|
|
1970
|
+
while _self:
|
|
1971
|
+
if _self.type == 'RemoteFrame':
|
|
1972
|
+
_rlst.append(_self)
|
|
1973
|
+
_self = _self.parent
|
|
1974
|
+
return _rlst
|
|
1975
|
+
def pic(self,):
|
|
1976
|
+
return Screenshot(self.cdp('Page.captureScreenshot', {"format":'png' })['data'])
|
|
1977
|
+
def __repr__(self):
|
|
1978
|
+
return '<[RootFrame]|{}|F:{}>'.format(self.url, self.frameId)
|
|
1979
|
+
def _run_init_once(self):
|
|
1980
|
+
if not self.init_once:
|
|
1981
|
+
self._page_init_js()
|
|
1982
|
+
self.init_once = True
|
|
1983
|
+
def _check_page(self, wsinfo):
|
|
1984
|
+
if (wsinfo['type'] == "service_worker" and wsinfo['url'].startswith('chrome')) \
|
|
1985
|
+
or wsinfo['url'].startswith('devtools') \
|
|
1986
|
+
or wsinfo['type'] != 'page':
|
|
1987
|
+
return False
|
|
1988
|
+
return True
|
|
1989
|
+
def add_sniff_session(self, sessionId):
|
|
1990
|
+
self.sniff_networt.add_listen_session(sessionId)
|
|
1991
|
+
self.sniff_fetch.add_change_session(sessionId)
|
|
1992
|
+
def listen(self, *a,**kw):
|
|
1993
|
+
return self.sniff_networt.listen(*a,**kw)
|
|
1994
|
+
def intercept(self, *a,**kw):
|
|
1995
|
+
return self.sniff_fetch.intercept(*a,**kw)
|
|
1996
|
+
def _go_url(self, url, timeout=None):
|
|
1997
|
+
self.url = url
|
|
1998
|
+
self.iso_contextId = None
|
|
1999
|
+
self._run_init_once()
|
|
2000
|
+
self.cdp("Page.navigate", {"url": url})
|
|
2001
|
+
self.frames = []
|
|
2002
|
+
d_timeout = timeout if timeout != None else 5
|
|
2003
|
+
def check(ret):
|
|
2004
|
+
if self.frameId == ret['frameId']:
|
|
2005
|
+
return True
|
|
2006
|
+
self.wait_once_method("Page.frameStoppedLoading", check_ret_func=check, timeout=d_timeout, rasie_error=False)
|
|
2007
|
+
start = perf_counter()
|
|
2008
|
+
while not timeout:
|
|
2009
|
+
if self.sniff_networt.qlist.empty() and self.sniff_fetch.qlist.empty():
|
|
2010
|
+
break
|
|
2011
|
+
time.sleep(0.1)
|
|
2012
|
+
if perf_counter() - start > 3:
|
|
2013
|
+
break
|
|
2014
|
+
f = self.cdp("Page.getFrameTree")
|
|
2015
|
+
self.url = f['frameTree']['frame']['url']
|
|
2016
|
+
def init_js(self, script):
|
|
2017
|
+
self._doc_script.append(script)
|
|
2018
|
+
def _page_init_js(self, sessionId=None):
|
|
2019
|
+
if self._doc_script:
|
|
2020
|
+
for s in self._doc_script:
|
|
2021
|
+
self.cdp('Page.addScriptToEvaluateOnNewDocument', {"source": s}, sessionId=sessionId)
|
|
2022
|
+
def tree_view(self):
|
|
2023
|
+
r = ''
|
|
2024
|
+
def print_tree(f, level=0, prefix=''):
|
|
2025
|
+
nonlocal r
|
|
2026
|
+
info = str(f)
|
|
2027
|
+
if level == 0:
|
|
2028
|
+
r += info + '\n'
|
|
2029
|
+
else:
|
|
2030
|
+
r += ' ' * (level * 4 - 1) + prefix + info + '\n'
|
|
2031
|
+
for i, item in enumerate(f.frames):
|
|
2032
|
+
if isinstance(item.frames, list):
|
|
2033
|
+
new_prefix = '└──' if i == len(f.frames) - 1 else '├──'
|
|
2034
|
+
print_tree(item, level + 1, new_prefix)
|
|
2035
|
+
print_tree(self)
|
|
2036
|
+
return r.rstrip('\n')
|
|
2037
|
+
class ExtensionConn:
|
|
2038
|
+
def __init__(self, wsinfo, root):
|
|
2039
|
+
self.root = root
|
|
2040
|
+
self.sessionId = None # Used to maintain consistency with CDPTools objects
|
|
2041
|
+
self.contextId = None # Used to maintain consistency with CDPTools objects
|
|
2042
|
+
self.iso_contextId = None # Used to maintain consistency with CDPTools objects
|
|
2043
|
+
self.ws = create_connection_saf(wsinfo['webSocketDebuggerUrl'], enable_multithread=True, suppress_origin=True)
|
|
2044
|
+
self.logger = Logger(wsinfo['id'], debug)
|
|
2045
|
+
self.cdper = CoreCDP(self.ws, self.root, self.logger)
|
|
2046
|
+
self.cdper.attach(self)
|
|
2047
|
+
self.cdp('Runtime.enable')
|
|
2048
|
+
self.tools = CDPTools(self, self)
|
|
2049
|
+
def set_proxy(self, p=None, method='PROXY', script=None):
|
|
2050
|
+
if p:
|
|
2051
|
+
func_script = script or '''
|
|
2052
|
+
function FindProxyForURL(url, host) {
|
|
2053
|
+
return "'''+method+''' '''+p+'''";
|
|
2054
|
+
}'''
|
|
2055
|
+
spt = json.dumps('data:text/plain;base64,' + base64.b64encode((func_script+'''
|
|
2056
|
+
// '''+str(int(time.time()*1000))+'''
|
|
2057
|
+
''').encode()).decode())
|
|
2058
|
+
cfgscript = '''{
|
|
2059
|
+
mode: 'pac_script',
|
|
2060
|
+
pacScript: { url: '''+spt+''' }
|
|
2061
|
+
}'''
|
|
2062
|
+
else:
|
|
2063
|
+
cfgscript = '{mode: "system"}'
|
|
2064
|
+
finscript = '''
|
|
2065
|
+
new Promise(function(r1,r2){
|
|
2066
|
+
chrome.proxy.settings.set({
|
|
2067
|
+
value: '''+cfgscript+''',
|
|
2068
|
+
scope: 'regular', }, function(e){ r1(e) })
|
|
2069
|
+
})
|
|
2070
|
+
'''
|
|
2071
|
+
err = self.run_js(finscript)
|
|
2072
|
+
if err:
|
|
2073
|
+
raise Exception('set proxy:' + json.dumps(err))
|
|
2074
|
+
class ExtensionManager:
|
|
2075
|
+
def __init__(self, root):
|
|
2076
|
+
self.root = root
|
|
2077
|
+
self.e = None
|
|
2078
|
+
def attach(self, c):
|
|
2079
|
+
c.set_proxy = self.set_proxy
|
|
2080
|
+
def set_proxy(self, p=None, method='PROXY', script=None):
|
|
2081
|
+
# Usually there is no need to configure script, script is actually an emergency backup parameter
|
|
2082
|
+
# Perhaps in some cases, someone may need to customize and configure some bypass lists or handle some special proxy modes.
|
|
2083
|
+
t = self.root.rootchrome.new_tab(False)
|
|
2084
|
+
t.get(wvurl)
|
|
2085
|
+
t.close()
|
|
2086
|
+
while not self.e:
|
|
2087
|
+
time.sleep(0.1)
|
|
2088
|
+
self.e.set_proxy(p=p, method=method, script=script)
|
|
2089
|
+
def set_extension(self, e):
|
|
2090
|
+
if self.e:
|
|
2091
|
+
self.e.tools.is_running = False
|
|
2092
|
+
self.e.ws.close()
|
|
2093
|
+
self.e = e
|
|
2094
|
+
class Root:
|
|
2095
|
+
def __init__(self, verison, iframes):
|
|
2096
|
+
self.pool = Pool(10)
|
|
2097
|
+
self.id = 0
|
|
2098
|
+
self.verison = verison
|
|
2099
|
+
self.iframes = iframes
|
|
2100
|
+
self.active = None
|
|
2101
|
+
self.new_tab_frame = {}
|
|
2102
|
+
self.create_tab_must_be_single = queue.Queue(1)
|
|
2103
|
+
self.tabs = []
|
|
2104
|
+
self.is_init = True
|
|
2105
|
+
self.init_sign = 0
|
|
2106
|
+
self.ws = create_connection_saf(verison['webSocketDebuggerUrl'], enable_multithread=True, suppress_origin=True)
|
|
2107
|
+
self.logger = Logger('[[ ---------- ROOT ---------- ]]', debug)
|
|
2108
|
+
self.cdper = CoreCDP(self.ws, self, self.logger)
|
|
2109
|
+
self.cdper.attach(self)
|
|
2110
|
+
self.extension = ExtensionManager(self)
|
|
2111
|
+
self.extension.set_extension(self._get_extension())
|
|
2112
|
+
self.cdp("Target.setAutoAttach", {
|
|
2113
|
+
"autoAttach": True,
|
|
2114
|
+
"waitForDebuggerOnStart": False,
|
|
2115
|
+
"flatten": True,
|
|
2116
|
+
})
|
|
2117
|
+
self.set_method_callback('Target.attachedToTarget', self.Target_attachedToTarget)
|
|
2118
|
+
# self.cdp("Target.setDiscoverTargets", { "discover": True })
|
|
2119
|
+
def filter_extension(self, tinfo):
|
|
2120
|
+
return tinfo['url'].startswith('chrome-extension://') and tinfo['url'].endswith('/vvv.js')
|
|
2121
|
+
def _wid(self):
|
|
2122
|
+
return self.cdp('Browser.getWindowForTarget', {"targetId":self.tabs[0].frameId})['windowId']
|
|
2123
|
+
def set_rect(self, rect):
|
|
2124
|
+
self.cdp('Browser.setWindowBounds', {"windowId":self._wid(),"bounds":{"left":rect[0],"top":rect[1],"width":rect[2],"height":rect[3]}})
|
|
2125
|
+
def set_fullscreen(self, tg=True):
|
|
2126
|
+
self.cdp('Browser.setWindowBounds', {'windowId':self._wid(),'bounds':{'windowState':'fullscreen' if tg else 'normal'}})
|
|
2127
|
+
def set_maxscreen(self, tg=True):
|
|
2128
|
+
self.cdp('Browser.setWindowBounds', {'windowId':self._wid(),'bounds':{'windowState':'maximized' if tg else 'normal'}})
|
|
2129
|
+
def _new_driver(self, background):
|
|
2130
|
+
tid = self.cdp('Target.createTarget', {"url": "about:blank", "background": background})['targetId']
|
|
2131
|
+
tgap = 0.03
|
|
2132
|
+
while True:
|
|
2133
|
+
if self.new_tab_frame.get(tid):
|
|
2134
|
+
break
|
|
2135
|
+
time.sleep(tgap)
|
|
2136
|
+
tgap = tgap * 1.3
|
|
2137
|
+
return self.new_tab_frame.pop(tid)
|
|
2138
|
+
def quit(self):
|
|
2139
|
+
self.cdp('Browser.close')
|
|
2140
|
+
def _get_extension(self):
|
|
2141
|
+
for tinfo in self.cdp('Target.getTargets')['targetInfos']:
|
|
2142
|
+
if self.filter_extension(tinfo):
|
|
2143
|
+
return ExtensionConn({
|
|
2144
|
+
'webSocketDebuggerUrl': make_dev_page_url(tinfo['targetId']),
|
|
2145
|
+
'id': tinfo['targetId'],
|
|
2146
|
+
'type': tinfo['type'],
|
|
2147
|
+
'url': tinfo['url'],
|
|
2148
|
+
}, self)
|
|
2149
|
+
def Target_attachedToTarget(self, rdata):
|
|
2150
|
+
params = rdata['params']
|
|
2151
|
+
sessionId = params.get('sessionId')
|
|
2152
|
+
targetInfo = params.get('targetInfo')
|
|
2153
|
+
url = targetInfo.get('url')
|
|
2154
|
+
d = {
|
|
2155
|
+
'webSocketDebuggerUrl': make_dev_page_url(targetInfo['targetId']),
|
|
2156
|
+
'id': targetInfo['targetId'],
|
|
2157
|
+
'type': targetInfo['type'],
|
|
2158
|
+
'url': url,
|
|
2159
|
+
}
|
|
2160
|
+
if url.startswith('chrome-extension://') and url.endswith('/vvv.js'):
|
|
2161
|
+
self.extension.set_extension(ExtensionConn(d, self))
|
|
2162
|
+
return
|
|
2163
|
+
# TODO
|
|
2164
|
+
# now, Page.addScriptToEvaluateOnNewDocument not fast. cannot work on click open url.
|
|
2165
|
+
f = RootFrame(d, self, is_auto_create=True)
|
|
2166
|
+
if targetInfo.get('openerFrameId'):
|
|
2167
|
+
# TODO
|
|
2168
|
+
# think about how to manager this. bind root? or rootframe?
|
|
2169
|
+
pass
|
|
2170
|
+
else:
|
|
2171
|
+
self.new_tab_frame[targetInfo['targetId']] = f
|
|
2172
|
+
self.cdp('Runtime.runIfWaitingForDebugger', sessionId=sessionId)
|
|
2173
|
+
self.cdp('Target.detachFromTarget', sessionId=sessionId)
|
|
2174
|
+
def _add_init_check(self):
|
|
2175
|
+
if self.is_init:
|
|
2176
|
+
self.init_sign += 1
|
|
2177
|
+
def _del_init_check(self):
|
|
2178
|
+
if self.is_init:
|
|
2179
|
+
self.init_sign -= 1
|
|
2180
|
+
def get_new_id(self):
|
|
2181
|
+
self.id += 1
|
|
2182
|
+
return self.id
|
|
2183
|
+
def add_root_frame(self, f):
|
|
2184
|
+
if not self.active and f.type == 'page':
|
|
2185
|
+
self.active = f
|
|
2186
|
+
self.tabs.append(f)
|
|
2187
|
+
self.trav_init_tree(f)
|
|
2188
|
+
self.trav_remote_tree(f)
|
|
2189
|
+
def add_common_frame(self, rootf, finfo, sessionId=None):
|
|
2190
|
+
f = self.trav_frame(finfo['frameId'])
|
|
2191
|
+
if f:
|
|
2192
|
+
if finfo.get("contextId"): f.contextId = finfo.get("contextId")
|
|
2193
|
+
if finfo.get("uniqueId"): f.uniqueId = finfo.get("uniqueId")
|
|
2194
|
+
if finfo.get('sessionId'): f.sessionId = finfo.get('sessionId')
|
|
2195
|
+
f.frameId = finfo['frameId']
|
|
2196
|
+
else:
|
|
2197
|
+
pf = finfo.get('parent')
|
|
2198
|
+
if not pf:
|
|
2199
|
+
# 初始化阶段,frame 并没有进去对象管理时才可能走的分支
|
|
2200
|
+
return
|
|
2201
|
+
lf = AbsFrame(rootf, {
|
|
2202
|
+
"contextId": finfo.get("contextId"),
|
|
2203
|
+
"uniqueId": finfo.get("uniqueId"),
|
|
2204
|
+
"frameId": finfo['frameId'],
|
|
2205
|
+
"sessionId": getattr(pf, 'sessionId', None),
|
|
2206
|
+
"parent": pf,
|
|
2207
|
+
})
|
|
2208
|
+
pf.frames.append(lf)
|
|
2209
|
+
def trav_remote_tree(self, pf):
|
|
2210
|
+
for f in self.iframes:
|
|
2211
|
+
if f['parentId'] == pf.frameId:
|
|
2212
|
+
lf = AbsFrame(pf, {
|
|
2213
|
+
"frameId": f['id'],
|
|
2214
|
+
"parent": pf,
|
|
2215
|
+
"url": f['url'],
|
|
2216
|
+
})
|
|
2217
|
+
pf.frames.append(lf)
|
|
2218
|
+
def trav_init_tree(self, f, rootf=None, sessionId=None):
|
|
2219
|
+
t = f.cdp('Page.getFrameTree')
|
|
2220
|
+
def _trav(t, pf, rootf):
|
|
2221
|
+
if 'frame' in t:
|
|
2222
|
+
url = t['frame']['url']
|
|
2223
|
+
frameId = t['frame']['id']
|
|
2224
|
+
if frameId != f.frameId:
|
|
2225
|
+
lf = AbsFrame(rootf, {
|
|
2226
|
+
"url": url,
|
|
2227
|
+
"contextId": None,
|
|
2228
|
+
"uniqueId": None,
|
|
2229
|
+
"sessionId": sessionId,
|
|
2230
|
+
"frameId": frameId,
|
|
2231
|
+
"parent": pf,
|
|
2232
|
+
})
|
|
2233
|
+
pf.frames.append(lf)
|
|
2234
|
+
pf = lf
|
|
2235
|
+
if 'childFrames' in t:
|
|
2236
|
+
for i in t['childFrames']:
|
|
2237
|
+
_trav(i, pf, rootf)
|
|
2238
|
+
_trav(t['frameTree'], f, rootf or f)
|
|
2239
|
+
if not rootf and not sessionId:
|
|
2240
|
+
f.url = t['frameTree']['frame']['url']
|
|
2241
|
+
def trav_frame(self, kdata, key='frameId'):
|
|
2242
|
+
def _trav(flist):
|
|
2243
|
+
for f in flist:
|
|
2244
|
+
if getattr(f, key, None) == kdata:
|
|
2245
|
+
return f
|
|
2246
|
+
else:
|
|
2247
|
+
r = _trav(f.frames)
|
|
2248
|
+
if r:
|
|
2249
|
+
return r
|
|
2250
|
+
return _trav(self.tabs)
|
|
2251
|
+
def flat_child_frames(self, flist=None):
|
|
2252
|
+
flst = []
|
|
2253
|
+
def _trav(flist):
|
|
2254
|
+
for f in flist:
|
|
2255
|
+
flst.append(f)
|
|
2256
|
+
_trav(f.frames)
|
|
2257
|
+
_trav(flist)
|
|
2258
|
+
return flst
|
|
2259
|
+
wurl = 'http://{}:{}/json'.format(hostname, port)
|
|
2260
|
+
wvurl = 'http://{}:{}/json/version'.format(hostname, port)
|
|
2261
|
+
wslist = myget(wurl)
|
|
2262
|
+
for idx in range(len(wslist)):
|
|
2263
|
+
wslist[idx]['webSocketDebuggerUrl'] = adj_wsurl(wslist[idx]['webSocketDebuggerUrl'])
|
|
2264
|
+
verison = myget(wvurl)
|
|
2265
|
+
verison['webSocketDebuggerUrl'] = adj_wsurl(verison['webSocketDebuggerUrl'])
|
|
2266
|
+
iframes = [i for i in wslist if i['type'] == 'iframe']
|
|
2267
|
+
root = Root(verison, iframes)
|
|
2268
|
+
for wsinfo in wslist: RootFrame(wsinfo, root)
|
|
2269
|
+
start = perf_counter()
|
|
2270
|
+
while root.init_sign:
|
|
2271
|
+
if perf_counter() - start > 2.5:
|
|
2272
|
+
break
|
|
2273
|
+
time.sleep(0.1)
|
|
2274
|
+
root.is_init = False
|
|
2275
|
+
return root
|
|
2276
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2277
|
+
import sys
|
|
2278
|
+
import re
|
|
2279
|
+
import time
|
|
2280
|
+
import json
|
|
2281
|
+
import shutil
|
|
2282
|
+
import socket
|
|
2283
|
+
import hashlib
|
|
2284
|
+
import platform
|
|
2285
|
+
from os import environ, path, cpu_count, getenv
|
|
2286
|
+
from platform import system
|
|
2287
|
+
from tempfile import gettempdir
|
|
2288
|
+
from pathlib import Path
|
|
2289
|
+
from shutil import rmtree
|
|
2290
|
+
from subprocess import Popen, DEVNULL
|
|
2291
|
+
def find_chrome(path):
|
|
2292
|
+
def get_win_chrome_path():
|
|
2293
|
+
d = Path('C:\\Program Files\\Google\\Chrome\\Application')
|
|
2294
|
+
if d.exists() and (d / 'chrome.exe').exists():
|
|
2295
|
+
return d
|
|
2296
|
+
import os, winreg
|
|
2297
|
+
sub_key = [
|
|
2298
|
+
'SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall',
|
|
2299
|
+
'SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall'
|
|
2300
|
+
]
|
|
2301
|
+
def get_install_list(key, root):
|
|
2302
|
+
try:
|
|
2303
|
+
_key = winreg.OpenKey(root, key, 0, winreg.KEY_ALL_ACCESS)
|
|
2304
|
+
for j in range(0, winreg.QueryInfoKey(_key)[0]-1):
|
|
2305
|
+
try:
|
|
2306
|
+
each_key = winreg.OpenKey(root, key + '\\' + winreg.EnumKey(_key, j), 0, winreg.KEY_ALL_ACCESS)
|
|
2307
|
+
displayname, REG_SZ = winreg.QueryValueEx(each_key, 'DisplayName')
|
|
2308
|
+
install_loc, REG_SZ = winreg.QueryValueEx(each_key, 'InstallLocation')
|
|
2309
|
+
display_var, REG_SZ = winreg.QueryValueEx(each_key, 'DisplayVersion')
|
|
2310
|
+
yield displayname, install_loc, display_var
|
|
2311
|
+
except WindowsError:
|
|
2312
|
+
pass
|
|
2313
|
+
except:
|
|
2314
|
+
pass
|
|
2315
|
+
for key in sub_key:
|
|
2316
|
+
for root in [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]:
|
|
2317
|
+
for name, local, var in get_install_list(key, root):
|
|
2318
|
+
if name == 'Google Chrome':
|
|
2319
|
+
return Path(local)
|
|
2320
|
+
if path:
|
|
2321
|
+
if Path(path).exists():
|
|
2322
|
+
return Path(path)
|
|
2323
|
+
return None
|
|
2324
|
+
sys = system().lower()
|
|
2325
|
+
if sys in ('macos', 'darwin', 'linux'):
|
|
2326
|
+
for p in (
|
|
2327
|
+
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
|
|
2328
|
+
'/usr/bin/google-chrome',
|
|
2329
|
+
'/opt/google/chrome/google-chrome',
|
|
2330
|
+
'/user/lib/chromium-browser/chromium-browser'
|
|
2331
|
+
):
|
|
2332
|
+
if Path(p).exists():
|
|
2333
|
+
return p
|
|
2334
|
+
return None
|
|
2335
|
+
elif sys != 'windows':
|
|
2336
|
+
return None
|
|
2337
|
+
return get_win_chrome_path()
|
|
2338
|
+
def try_some(func, times=7, timegap=0.15):
|
|
2339
|
+
for i in range(times):
|
|
2340
|
+
try:
|
|
2341
|
+
func()
|
|
2342
|
+
return False
|
|
2343
|
+
except:
|
|
2344
|
+
time.sleep(timegap)
|
|
2345
|
+
continue
|
|
2346
|
+
return True
|
|
2347
|
+
def find_free_port(start_port=9233, end_port=60000):
|
|
2348
|
+
class EnvPRNG:
|
|
2349
|
+
def __init__(self):
|
|
2350
|
+
self.seed = int(self.get_env_fingerprint(), 16)
|
|
2351
|
+
self.state = self.seed
|
|
2352
|
+
def get_env_fingerprint(self):
|
|
2353
|
+
try:
|
|
2354
|
+
system = platform.system()
|
|
2355
|
+
release = platform.release()
|
|
2356
|
+
version = platform.version()
|
|
2357
|
+
arch = platform.machine()
|
|
2358
|
+
python_version = platform.python_version()
|
|
2359
|
+
cpu_n = cpu_count()
|
|
2360
|
+
raw = f"{system}-{release}-{version}-{arch}-{python_version}-{cpu_n}-{getenv('USERNAME','')}-{getenv('USER','')}"
|
|
2361
|
+
except:
|
|
2362
|
+
raw = "vvv"
|
|
2363
|
+
return hashlib.sha256(raw.encode('utf-8')).hexdigest()
|
|
2364
|
+
def rand(self):
|
|
2365
|
+
self.state = (1664525 * self.state + 1013904223) % (2**32)
|
|
2366
|
+
return self.state
|
|
2367
|
+
def randint(self, a, b):
|
|
2368
|
+
return a + self.rand() % (b - a + 1)
|
|
2369
|
+
erdn = EnvPRNG()
|
|
2370
|
+
for _ in range(200):
|
|
2371
|
+
try:
|
|
2372
|
+
port = erdn.randint(start_port, end_port)
|
|
2373
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
2374
|
+
s.bind(('0.0.0.0', port))
|
|
2375
|
+
return port
|
|
2376
|
+
except OSError:
|
|
2377
|
+
continue
|
|
2378
|
+
raise ValueError("no free port ({}-{})".format(start_port, end_port))
|
|
2379
|
+
def is_port_open(port, host="127.0.0.1", timeout=0.01):
|
|
2380
|
+
try:
|
|
2381
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
2382
|
+
s.settimeout(timeout)
|
|
2383
|
+
return s.connect_ex((host, port)) == 0
|
|
2384
|
+
except socket.timeout:
|
|
2385
|
+
return False
|
|
2386
|
+
def get_default_user_data_dir():
|
|
2387
|
+
home = path.expanduser("~")
|
|
2388
|
+
if sys.platform.startswith("win"):
|
|
2389
|
+
return path.join(home, "AppData", "Local", "Google", "Chrome", "User Data")
|
|
2390
|
+
elif sys.platform == "darwin":
|
|
2391
|
+
return path.join(home, "Library", "Application Support", "Google", "Chrome")
|
|
2392
|
+
elif sys.platform.startswith("linux"):
|
|
2393
|
+
return path.join(home, ".config", "google-chrome")
|
|
2394
|
+
else:
|
|
2395
|
+
raise Exception("Unsupported platform")
|
|
2396
|
+
class Debug:
|
|
2397
|
+
def __init__(self, debug):
|
|
2398
|
+
self.all = False
|
|
2399
|
+
self.plist = [
|
|
2400
|
+
'env','rest','mouse','proxy',
|
|
2401
|
+
'Network','Fetch','Page','DOM','Target','Runtime','Input','Debugger','DOMDebugger',
|
|
2402
|
+
]
|
|
2403
|
+
if type(debug) == str: self.parse_str(debug)
|
|
2404
|
+
if type(debug) == bool: self.parse_bool(debug)
|
|
2405
|
+
self._bool = self.has_Debug()
|
|
2406
|
+
def __call__(self, tp):
|
|
2407
|
+
if self.all:
|
|
2408
|
+
return True
|
|
2409
|
+
if type(tp) == str:
|
|
2410
|
+
return getattr(self, tp.split('.')[0], None)
|
|
2411
|
+
def __bool__(self):
|
|
2412
|
+
return self._bool
|
|
2413
|
+
def parse_str(self, debug):
|
|
2414
|
+
self.parse_bool(False)
|
|
2415
|
+
dbgls = re.split(r'[,;|/]', debug)
|
|
2416
|
+
for p in dbgls:
|
|
2417
|
+
if p in self.plist:
|
|
2418
|
+
setattr(self, p, True)
|
|
2419
|
+
def parse_bool(self, debug):
|
|
2420
|
+
for p in self.plist: setattr(self, p, debug)
|
|
2421
|
+
self.all = debug
|
|
2422
|
+
def has_Debug(self):
|
|
2423
|
+
tg = False
|
|
2424
|
+
for p in self.plist:
|
|
2425
|
+
if getattr(self, p):
|
|
2426
|
+
tg = True
|
|
2427
|
+
return tg
|
|
2428
|
+
class Chrome:
|
|
2429
|
+
def __init__(self,
|
|
2430
|
+
debug = False,
|
|
2431
|
+
path = None,
|
|
2432
|
+
port = None,
|
|
2433
|
+
hostname = '127.0.0.1',
|
|
2434
|
+
user_path = None,
|
|
2435
|
+
use_system_user_path = False,
|
|
2436
|
+
verison_check = True,
|
|
2437
|
+
headless = False,
|
|
2438
|
+
incognito = False,
|
|
2439
|
+
user_agent = None,
|
|
2440
|
+
arguments = [],
|
|
2441
|
+
proxy = None,
|
|
2442
|
+
extension = None,
|
|
2443
|
+
):
|
|
2444
|
+
self.debug = Debug(debug)
|
|
2445
|
+
self.path = find_chrome(path)
|
|
2446
|
+
self.verison_check = verison_check
|
|
2447
|
+
self.headless = headless
|
|
2448
|
+
self.user_agent = user_agent
|
|
2449
|
+
self.incognito = incognito
|
|
2450
|
+
self.arguments = arguments
|
|
2451
|
+
self.proxy = proxy
|
|
2452
|
+
self.extension = extension
|
|
2453
|
+
self._user_cmd = self._make_user_cmd()
|
|
2454
|
+
if not self.path:
|
|
2455
|
+
raise Exception('chrome path not find.')
|
|
2456
|
+
self.port = port or find_free_port()
|
|
2457
|
+
self.hostname = hostname
|
|
2458
|
+
self.user_path_mode = None
|
|
2459
|
+
# chrome 136+
|
|
2460
|
+
# disable default user path to remote contral
|
|
2461
|
+
if use_system_user_path:
|
|
2462
|
+
self.user_path_mode = 'sys'
|
|
2463
|
+
user_path = Path(get_default_user_data_dir())
|
|
2464
|
+
if user_path:
|
|
2465
|
+
self.user_path_mode = 'user'
|
|
2466
|
+
user_path = Path(user_path)
|
|
2467
|
+
if not user_path.exists() or not user_path.is_dir():
|
|
2468
|
+
raise Exception('user path not exist.')
|
|
2469
|
+
else:
|
|
2470
|
+
self.user_path_mode = 'default'
|
|
2471
|
+
if self.debug.env:
|
|
2472
|
+
print('[*]', 'user_path_mode', self.user_path_mode)
|
|
2473
|
+
self.user_path = user_path or Path(gettempdir()) / 'vchrome' / 'cache_user_temp' / 'userData' / str(self.port)
|
|
2474
|
+
if self.debug.env:
|
|
2475
|
+
print('[*]', self.user_path)
|
|
2476
|
+
self._connect()
|
|
2477
|
+
def _cmd_check_in(self, ls, s):
|
|
2478
|
+
for i in ls:
|
|
2479
|
+
if i.startswith(s.split('=')[0]):
|
|
2480
|
+
return i, s
|
|
2481
|
+
def _make_user_cmd(self):
|
|
2482
|
+
_user_cmd = self.arguments
|
|
2483
|
+
if self.headless and not self._cmd_check_in(_user_cmd, '--headless'):
|
|
2484
|
+
# TODO
|
|
2485
|
+
# think about CDP:Input Event compatible.
|
|
2486
|
+
_user_cmd.append('--headless')
|
|
2487
|
+
if self.incognito and not self._cmd_check_in(_user_cmd, '--incognito'):
|
|
2488
|
+
_user_cmd.append('--incognito')
|
|
2489
|
+
if self.user_agent and not self._cmd_check_in(_user_cmd, '--user-agent'):
|
|
2490
|
+
_user_cmd.append('--user-agent=' + json.dumps(self.user_agent))
|
|
2491
|
+
if self.proxy and not self._cmd_check_in(_user_cmd, '--proxy-server'):
|
|
2492
|
+
_user_cmd.append('--proxy-server=' + self.proxy)
|
|
2493
|
+
if self.extension:
|
|
2494
|
+
if type(self.extension) == str:
|
|
2495
|
+
ep = self.extension
|
|
2496
|
+
_user_cmd.append('--disable-extensions-except=' + ep)
|
|
2497
|
+
_user_cmd.append('--load-extension=' + ep)
|
|
2498
|
+
if type(self.extension) == list:
|
|
2499
|
+
for ep in self.extension:
|
|
2500
|
+
_user_cmd.append('--disable-extensions-except=' + ep)
|
|
2501
|
+
_user_cmd.append('--load-extension=' + ep)
|
|
2502
|
+
return _user_cmd
|
|
2503
|
+
def new_tab(self, foreground=True):
|
|
2504
|
+
class Chrome:
|
|
2505
|
+
def __init__(self, root, dr):
|
|
2506
|
+
self.root = root
|
|
2507
|
+
self.dr = dr
|
|
2508
|
+
self.dr.browser.attach(self)
|
|
2509
|
+
def __str__(self):
|
|
2510
|
+
return str(self.dr.tree_view())
|
|
2511
|
+
def new_tab(self, foreground=True):
|
|
2512
|
+
return Chrome(self.root, self.dr.root._new_driver(not foreground))
|
|
2513
|
+
frames = property(lambda s:s.dr.frames)
|
|
2514
|
+
cookies = property(lambda s:s.get_cookies(), lambda s,v:s.set_cookies(v))
|
|
2515
|
+
dialog = property(lambda s:s.get_dialog(), lambda s,v:s.set_dialog(v))
|
|
2516
|
+
return Chrome(self.root, self.dr.root._new_driver(not foreground))
|
|
2517
|
+
frames = property(lambda s:s.dr.frames)
|
|
2518
|
+
cookies = property(lambda s:s.get_cookies(), lambda s,v:s.set_cookies(v))
|
|
2519
|
+
dialog = property(lambda s:s.get_dialog(), lambda s,v:s.set_dialog(v))
|
|
2520
|
+
def _check_lower_version(self):
|
|
2521
|
+
try:
|
|
2522
|
+
verison = self.dr.root.verison
|
|
2523
|
+
if verison and verison.get('Browser'):
|
|
2524
|
+
ver = int(verison.get('Browser').split('/')[-1].split('.')[0])
|
|
2525
|
+
# Restricting Chrome versions to prevent abnormal states
|
|
2526
|
+
if ver < 100:
|
|
2527
|
+
return True
|
|
2528
|
+
except: pass
|
|
2529
|
+
return False
|
|
2530
|
+
def _connect(self):
|
|
2531
|
+
if not is_port_open(self.port):
|
|
2532
|
+
self._init()
|
|
2533
|
+
self.root = cdp_client(self.hostname, port=self.port, debug=self.debug)
|
|
2534
|
+
self.root.rootchrome = self
|
|
2535
|
+
self.dr = self.root.active
|
|
2536
|
+
if not self.dr:
|
|
2537
|
+
raise Exception('maybe single devtools not close.')
|
|
2538
|
+
self.dr.browser.attach(self)
|
|
2539
|
+
if self.verison_check and self._check_lower_version():
|
|
2540
|
+
raise Exception('chrome verison is less then 100. not reliable. you can set (verison_check=False) for ignore this alert.')
|
|
2541
|
+
if self.debug.mouse:
|
|
2542
|
+
jscode = r'''
|
|
2543
|
+
function f(e){
|
|
2544
|
+
var nDiv = document.createElement('div')
|
|
2545
|
+
var e = e || window.event
|
|
2546
|
+
Object.assign(nDiv.style, {
|
|
2547
|
+
position: 'fixed',
|
|
2548
|
+
left: `${e.clientX+1}px`,
|
|
2549
|
+
top: `${e.clientY+1}px`,
|
|
2550
|
+
width: '5px',
|
|
2551
|
+
height: '5px',
|
|
2552
|
+
backgroundColor: 'red',
|
|
2553
|
+
borderRadius: '50%',
|
|
2554
|
+
pointerEvents: 'none', // 禁止鼠标交互
|
|
2555
|
+
userSelect: 'none', // 禁止选中
|
|
2556
|
+
webkitUserSelect: 'none', // Safari 兼容
|
|
2557
|
+
mozUserSelect: 'none', // Firefox 兼容
|
|
2558
|
+
msUserSelect: 'none', // IE/Edge 兼容
|
|
2559
|
+
zIndex: '2147483647', // 最大 z-index 确保置顶
|
|
2560
|
+
willChange: 'transform', // 优化渲染性能
|
|
2561
|
+
});
|
|
2562
|
+
document.body.appendChild(nDiv)
|
|
2563
|
+
setTimeout(function(){ nDiv.remove(); },1000)
|
|
2564
|
+
}
|
|
2565
|
+
document.addEventListener('mousemove', f, true)
|
|
2566
|
+
document.addEventListener('mousedown', f, true)
|
|
2567
|
+
document.addEventListener('mouseup', f, true)
|
|
2568
|
+
'''
|
|
2569
|
+
self.init_js(jscode)
|
|
2570
|
+
def _merge_config(self, cmd):
|
|
2571
|
+
# The Chrome command line is sequence sensitive and comes with conflict resolution. I just need to add my new command at the end. (Tail instruction priority)
|
|
2572
|
+
return cmd[:-1] + self._user_cmd + cmd[-1:]
|
|
2573
|
+
def _make_files_extension(self, path):
|
|
2574
|
+
def writeF(p, s):
|
|
2575
|
+
with open(p, 'w', encoding='utf8') as f:
|
|
2576
|
+
f.write(s)
|
|
2577
|
+
e = path / 'v_extension'
|
|
2578
|
+
e.mkdir(parents=True, exist_ok=True)
|
|
2579
|
+
d = {
|
|
2580
|
+
"name": "vvv", "version": "0.0.0", "description": "vvv",
|
|
2581
|
+
"permissions": [ "proxy" ],
|
|
2582
|
+
"background": { "service_worker": "vvv.js" },
|
|
2583
|
+
"content_scripts": [{
|
|
2584
|
+
"matches": ["<all_urls>"],
|
|
2585
|
+
"js": ["content_script.js"],
|
|
2586
|
+
"run_at": "document_idle"
|
|
2587
|
+
}],
|
|
2588
|
+
"host_permissions": [ "<all_urls>" ],
|
|
2589
|
+
"manifest_version": 3
|
|
2590
|
+
}
|
|
2591
|
+
writeF(e / 'manifest.json', json.dumps(d))
|
|
2592
|
+
writeF(e / 'content_script.js', '''
|
|
2593
|
+
var port = chrome.runtime.connect({ name: 'keepAlive' });
|
|
2594
|
+
port.postMessage({ ping: Date.now() })
|
|
2595
|
+
''')
|
|
2596
|
+
writeF(e / 'vvv.js', '''
|
|
2597
|
+
var ctime;
|
|
2598
|
+
chrome.runtime.onConnect.addListener((port) => {
|
|
2599
|
+
if (port.name === 'keepAlive') { port.onMessage.addListener((msg) => { ctime = msg }); }
|
|
2600
|
+
});
|
|
2601
|
+
''')
|
|
2602
|
+
return e
|
|
2603
|
+
def _prepare_cmd(self):
|
|
2604
|
+
# TODO
|
|
2605
|
+
# maybe need check writable compatible.
|
|
2606
|
+
path = self.path
|
|
2607
|
+
user_path = self.user_path
|
|
2608
|
+
port = self.port
|
|
2609
|
+
user_path_mode = self.user_path_mode
|
|
2610
|
+
p = Path(path)
|
|
2611
|
+
p = str(p / 'chrome') if p.is_dir() else str(path)
|
|
2612
|
+
u = '--user-data-dir='
|
|
2613
|
+
args = []
|
|
2614
|
+
if user_path_mode == 'sys':
|
|
2615
|
+
args.append(u + str(user_path))
|
|
2616
|
+
if user_path_mode == 'user':
|
|
2617
|
+
args.append(u + str(user_path))
|
|
2618
|
+
if user_path_mode == 'default':
|
|
2619
|
+
if user_path.exists():
|
|
2620
|
+
if try_some(lambda:shutil.rmtree(user_path)):
|
|
2621
|
+
print('[*] error remove cache', user_path)
|
|
2622
|
+
user_path.mkdir(parents=True, exist_ok=True)
|
|
2623
|
+
args.append(u + str(user_path))
|
|
2624
|
+
ep = self._make_files_extension(user_path)
|
|
2625
|
+
# args.append('--disable-extensions-except=' + str(ep))
|
|
2626
|
+
args.append('--load-extension=' + str(ep))
|
|
2627
|
+
return [
|
|
2628
|
+
p,
|
|
2629
|
+
'--remote-debugging-port=' + str(port),
|
|
2630
|
+
'--no-default-browser-check',
|
|
2631
|
+
'--disable-suggestions-ui',
|
|
2632
|
+
'--no-first-run',
|
|
2633
|
+
'--disable-infobars',
|
|
2634
|
+
'--disable-popup-blocking',
|
|
2635
|
+
'--hide-crash-restore-bubble',
|
|
2636
|
+
'--remote-allow-origins=*',
|
|
2637
|
+
'--enable-features=NetworkService',
|
|
2638
|
+
'--enable-features=NetworkServiceInProcess',
|
|
2639
|
+
'--disable-features=PaymentRequest',
|
|
2640
|
+
'--disable-features=DigitalGoodsApi',
|
|
2641
|
+
'--disable-features=PrivacySandboxSettings4',
|
|
2642
|
+
'--disable-features=DisableLoadExtensionCommandLineSwitch', # Starting from a higher version, the command line to load plugins has been disabled. This line needs to be added
|
|
2643
|
+
'--disable-component-extensions-with-background-pages',
|
|
2644
|
+
# # Disable automatic password saving prompt, the new version is invalid
|
|
2645
|
+
# '--password-store=basic',
|
|
2646
|
+
# # Disable automatic password saving prompt, the new version is invalid
|
|
2647
|
+
# '--disable-features=AutofillEnableSavePasswordBubble',
|
|
2648
|
+
# '--disable-features=PasswordManagerOnboarding',
|
|
2649
|
+
# '--disable-features=PasswordImport',
|
|
2650
|
+
# '--disable-features=PasswordManagerRedesign',
|
|
2651
|
+
# # Disable automatic password saving prompt, the new version is invalid
|
|
2652
|
+
# '--disable-save-password-bubble',
|
|
2653
|
+
# '--disable-autofill',
|
|
2654
|
+
# '--disable-password-manager-reauthentication',
|
|
2655
|
+
# '--disable-autofill-keyboard-accessory-view',
|
|
2656
|
+
# from pyppeteer
|
|
2657
|
+
'--disable-background-networking',
|
|
2658
|
+
'--disable-background-timer-throttling',
|
|
2659
|
+
'--disable-breakpad',
|
|
2660
|
+
'--disable-browser-side-navigation',
|
|
2661
|
+
'--disable-client-side-phishing-detection',
|
|
2662
|
+
'--disable-default-apps',
|
|
2663
|
+
'--disable-dev-shm-usage',
|
|
2664
|
+
'--disable-hang-monitor',
|
|
2665
|
+
'--disable-prompt-on-repost',
|
|
2666
|
+
'--disable-sync',
|
|
2667
|
+
'--disable-translate',
|
|
2668
|
+
'--metrics-recording-only',
|
|
2669
|
+
'--no-first-run',
|
|
2670
|
+
'--safebrowsing-disable-auto-update',
|
|
2671
|
+
# '--disable-component-update',
|
|
2672
|
+
# '--site-per-process',
|
|
2673
|
+
# '--disable-extensions',
|
|
2674
|
+
# '--disable-web-security',
|
|
2675
|
+
*args,
|
|
2676
|
+
'about:blank',
|
|
2677
|
+
# # This part may be related to performance
|
|
2678
|
+
# '--disable-extensions',
|
|
2679
|
+
# '--disable-gpu',
|
|
2680
|
+
# '--no-sandbox',
|
|
2681
|
+
# '--disable-dev-shm-usage',
|
|
2682
|
+
]
|
|
2683
|
+
def _init(self):
|
|
2684
|
+
def _start_browser(cmd):
|
|
2685
|
+
try:
|
|
2686
|
+
return Popen(cmd, shell=False, stdout=DEVNULL, stderr=DEVNULL)
|
|
2687
|
+
except FileNotFoundError:
|
|
2688
|
+
raise FileNotFoundError('browser not find.')
|
|
2689
|
+
cmd = self._prepare_cmd()
|
|
2690
|
+
cmd = self._merge_config(cmd)
|
|
2691
|
+
if self.debug.env:
|
|
2692
|
+
print('[*]', json.dumps(cmd, indent=4))
|
|
2693
|
+
_start_browser(cmd)
|
|
2694
|
+
def __str__(self):
|
|
2695
|
+
return str(self.dr.tree_view())
|
|
2696
|
+
# ----------------------------------------------------------------------------------------------------
|
|
2697
|
+
"""
|
|
2698
|
+
websocket - WebSocket client library for Python
|
|
2699
|
+
|
|
2700
|
+
Copyright 2024 engn33r
|
|
2701
|
+
|
|
2702
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
2703
|
+
you may not use this file except in compliance with the License.
|
|
2704
|
+
You may obtain a copy of the License at
|
|
2705
|
+
|
|
2706
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
2707
|
+
|
|
2708
|
+
Unless required by applicable law or agreed to in writing, software
|
|
2709
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
2710
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
2711
|
+
See the License for the specific language governing permissions and
|
|
2712
|
+
limitations under the License.
|
|
2713
|
+
"""
|
|
2714
|
+
# websocket-client:1.8.0
|
|
2715
|
+
import array
|
|
2716
|
+
import os
|
|
2717
|
+
import struct
|
|
2718
|
+
import sys
|
|
2719
|
+
from threading import Lock
|
|
2720
|
+
from typing import Callable, Optional, Union
|
|
2721
|
+
class WebSocketException(Exception):pass
|
|
2722
|
+
class WebSocketProtocolException(WebSocketException):pass
|
|
2723
|
+
class WebSocketPayloadException(WebSocketException):pass
|
|
2724
|
+
class WebSocketConnectionClosedException(WebSocketException):pass
|
|
2725
|
+
class WebSocketTimeoutException(WebSocketException):pass
|
|
2726
|
+
class WebSocketProxyException(WebSocketException):pass
|
|
2727
|
+
class WebSocketBadStatusException(WebSocketException):
|
|
2728
|
+
def __init__(self, message: str, status_code: int, status_message=None, resp_headers=None, resp_body=None):
|
|
2729
|
+
super().__init__(message)
|
|
2730
|
+
self.status_code = status_code
|
|
2731
|
+
self.resp_headers = resp_headers
|
|
2732
|
+
self.resp_body = resp_body
|
|
2733
|
+
class WebSocketAddressException(WebSocketException):pass
|
|
2734
|
+
from typing import Union
|
|
2735
|
+
class NoLock:
|
|
2736
|
+
def __enter__(self) -> None:pass
|
|
2737
|
+
def __exit__(self, exc_type, exc_value, traceback) -> None:pass
|
|
2738
|
+
try:
|
|
2739
|
+
from wsaccel.utf8validator import Utf8Validator
|
|
2740
|
+
def _validate_utf8(utfbytes: Union[str, bytes]) -> bool:
|
|
2741
|
+
result: bool = Utf8Validator().validate(utfbytes)[0]
|
|
2742
|
+
return result
|
|
2743
|
+
except ImportError:
|
|
2744
|
+
_UTF8_ACCEPT = 0
|
|
2745
|
+
_UTF8_REJECT = 12
|
|
2746
|
+
_UTF8D = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]
|
|
2747
|
+
def _decode(state: int, codep: int, ch: int) -> tuple:
|
|
2748
|
+
tp = _UTF8D[ch]
|
|
2749
|
+
codep = ch & 63 | codep << 6 if state != _UTF8_ACCEPT else 255 >> tp & ch
|
|
2750
|
+
state = _UTF8D[256 + state + tp]
|
|
2751
|
+
return (state, codep)
|
|
2752
|
+
def _validate_utf8(utfbytes: Union[str, bytes]) -> bool:
|
|
2753
|
+
state = _UTF8_ACCEPT
|
|
2754
|
+
codep = 0
|
|
2755
|
+
for i in utfbytes:
|
|
2756
|
+
(state, codep) = _decode(state, codep, int(i))
|
|
2757
|
+
if state == _UTF8_REJECT:
|
|
2758
|
+
return False
|
|
2759
|
+
return True
|
|
2760
|
+
def validate_utf8(utfbytes: Union[str, bytes]) -> bool:
|
|
2761
|
+
return _validate_utf8(utfbytes)
|
|
2762
|
+
def extract_err_message(exception: Exception) -> Union[str, None]:
|
|
2763
|
+
if exception.args:
|
|
2764
|
+
exception_message: str = exception.args[0]
|
|
2765
|
+
return exception_message
|
|
2766
|
+
else:
|
|
2767
|
+
return None
|
|
2768
|
+
def extract_error_code(exception: Exception) -> Union[int, None]:
|
|
2769
|
+
if exception.args and len(exception.args) > 1:
|
|
2770
|
+
return exception.args[0] if isinstance(exception.args[0], int) else None
|
|
2771
|
+
try:
|
|
2772
|
+
from wsaccel.xormask import XorMaskerSimple
|
|
2773
|
+
def _mask(mask_value: array.array, data_value: array.array) -> bytes:
|
|
2774
|
+
mask_result: bytes = XorMaskerSimple(mask_value).process(data_value)
|
|
2775
|
+
return mask_result
|
|
2776
|
+
except ImportError:
|
|
2777
|
+
native_byteorder = sys.byteorder
|
|
2778
|
+
def _mask(mask_value: array.array, data_value: array.array) -> bytes:
|
|
2779
|
+
datalen = len(data_value)
|
|
2780
|
+
int_data_value = int.from_bytes(data_value, native_byteorder)
|
|
2781
|
+
int_mask_value = int.from_bytes(mask_value * (datalen // 4) + mask_value[:datalen % 4], native_byteorder)
|
|
2782
|
+
return (int_data_value ^ int_mask_value).to_bytes(datalen, native_byteorder)
|
|
2783
|
+
STATUS_NORMAL = 1000
|
|
2784
|
+
STATUS_GOING_AWAY = 1001
|
|
2785
|
+
STATUS_PROTOCOL_ERROR = 1002
|
|
2786
|
+
STATUS_UNSUPPORTED_DATA_TYPE = 1003
|
|
2787
|
+
STATUS_STATUS_NOT_AVAILABLE = 1005
|
|
2788
|
+
STATUS_ABNORMAL_CLOSED = 1006
|
|
2789
|
+
STATUS_INVALID_PAYLOAD = 1007
|
|
2790
|
+
STATUS_POLICY_VIOLATION = 1008
|
|
2791
|
+
STATUS_MESSAGE_TOO_BIG = 1009
|
|
2792
|
+
STATUS_INVALID_EXTENSION = 1010
|
|
2793
|
+
STATUS_UNEXPECTED_CONDITION = 1011
|
|
2794
|
+
STATUS_SERVICE_RESTART = 1012
|
|
2795
|
+
STATUS_TRY_AGAIN_LATER = 1013
|
|
2796
|
+
STATUS_BAD_GATEWAY = 1014
|
|
2797
|
+
STATUS_TLS_HANDSHAKE_ERROR = 1015
|
|
2798
|
+
VALID_CLOSE_STATUS = (STATUS_NORMAL, STATUS_GOING_AWAY, STATUS_PROTOCOL_ERROR, STATUS_UNSUPPORTED_DATA_TYPE, STATUS_INVALID_PAYLOAD, STATUS_POLICY_VIOLATION, STATUS_MESSAGE_TOO_BIG, STATUS_INVALID_EXTENSION, STATUS_UNEXPECTED_CONDITION, STATUS_SERVICE_RESTART, STATUS_TRY_AGAIN_LATER, STATUS_BAD_GATEWAY)
|
|
2799
|
+
class ABNF:
|
|
2800
|
+
OPCODE_CONT = 0
|
|
2801
|
+
OPCODE_TEXT = 1
|
|
2802
|
+
OPCODE_BINARY = 2
|
|
2803
|
+
OPCODE_CLOSE = 8
|
|
2804
|
+
OPCODE_PING = 9
|
|
2805
|
+
OPCODE_PONG = 10
|
|
2806
|
+
OPCODES = (OPCODE_CONT, OPCODE_TEXT, OPCODE_BINARY, OPCODE_CLOSE, OPCODE_PING, OPCODE_PONG)
|
|
2807
|
+
OPCODE_MAP = {OPCODE_CONT: 'cont', OPCODE_TEXT: 'text', OPCODE_BINARY: 'binary', OPCODE_CLOSE: 'close', OPCODE_PING: 'ping', OPCODE_PONG: 'pong'}
|
|
2808
|
+
LENGTH_7 = 126
|
|
2809
|
+
LENGTH_16 = 1 << 16
|
|
2810
|
+
LENGTH_63 = 1 << 63
|
|
2811
|
+
def __init__(self, fin: int=0, rsv1: int=0, rsv2: int=0, rsv3: int=0, opcode: int=OPCODE_TEXT, mask_value: int=1, data: Union[str, bytes, None]='') -> None:
|
|
2812
|
+
self.fin = fin
|
|
2813
|
+
self.rsv1 = rsv1
|
|
2814
|
+
self.rsv2 = rsv2
|
|
2815
|
+
self.rsv3 = rsv3
|
|
2816
|
+
self.opcode = opcode
|
|
2817
|
+
self.mask_value = mask_value
|
|
2818
|
+
if data is None:
|
|
2819
|
+
data = ''
|
|
2820
|
+
self.data = data
|
|
2821
|
+
self.get_mask_key = os.urandom
|
|
2822
|
+
def validate(self, skip_utf8_validation: bool=False) -> None:
|
|
2823
|
+
if self.rsv1 or self.rsv2 or self.rsv3:
|
|
2824
|
+
raise WebSocketProtocolException('rsv is not implemented, yet')
|
|
2825
|
+
if self.opcode not in ABNF.OPCODES:
|
|
2826
|
+
raise WebSocketProtocolException('Invalid opcode %r', self.opcode)
|
|
2827
|
+
if self.opcode == ABNF.OPCODE_PING and (not self.fin):
|
|
2828
|
+
raise WebSocketProtocolException('Invalid ping frame.')
|
|
2829
|
+
if self.opcode == ABNF.OPCODE_CLOSE:
|
|
2830
|
+
l = len(self.data)
|
|
2831
|
+
if not l:
|
|
2832
|
+
return
|
|
2833
|
+
if l == 1 or l >= 126:
|
|
2834
|
+
raise WebSocketProtocolException('Invalid close frame.')
|
|
2835
|
+
if l > 2 and (not skip_utf8_validation) and (not validate_utf8(self.data[2:])):
|
|
2836
|
+
raise WebSocketProtocolException('Invalid close frame.')
|
|
2837
|
+
code = 256 * int(self.data[0]) + int(self.data[1])
|
|
2838
|
+
if not self._is_valid_close_status(code):
|
|
2839
|
+
raise WebSocketProtocolException('Invalid close opcode %r', code)
|
|
2840
|
+
@staticmethod
|
|
2841
|
+
def _is_valid_close_status(code: int) -> bool:
|
|
2842
|
+
return code in VALID_CLOSE_STATUS or 3000 <= code < 5000
|
|
2843
|
+
def __str__(self) -> str:
|
|
2844
|
+
return f'fin={self.fin} opcode={self.opcode} data={self.data}'
|
|
2845
|
+
@staticmethod
|
|
2846
|
+
def create_frame(data: Union[bytes, str], opcode: int, fin: int=1) -> 'ABNF':
|
|
2847
|
+
if opcode == ABNF.OPCODE_TEXT and isinstance(data, str):
|
|
2848
|
+
data = data.encode('utf-8')
|
|
2849
|
+
return ABNF(fin, 0, 0, 0, opcode, 1, data)
|
|
2850
|
+
def format(self) -> bytes:
|
|
2851
|
+
if any((x not in (0, 1) for x in [self.fin, self.rsv1, self.rsv2, self.rsv3])):
|
|
2852
|
+
raise ValueError('not 0 or 1')
|
|
2853
|
+
if self.opcode not in ABNF.OPCODES:
|
|
2854
|
+
raise ValueError('Invalid OPCODE')
|
|
2855
|
+
length = len(self.data)
|
|
2856
|
+
if length >= ABNF.LENGTH_63:
|
|
2857
|
+
raise ValueError('data is too long')
|
|
2858
|
+
frame_header = chr(self.fin << 7 | self.rsv1 << 6 | self.rsv2 << 5 | self.rsv3 << 4 | self.opcode).encode('latin-1')
|
|
2859
|
+
if length < ABNF.LENGTH_7:
|
|
2860
|
+
frame_header += chr(self.mask_value << 7 | length).encode('latin-1')
|
|
2861
|
+
elif length < ABNF.LENGTH_16:
|
|
2862
|
+
frame_header += chr(self.mask_value << 7 | 126).encode('latin-1')
|
|
2863
|
+
frame_header += struct.pack('!H', length)
|
|
2864
|
+
else:
|
|
2865
|
+
frame_header += chr(self.mask_value << 7 | 127).encode('latin-1')
|
|
2866
|
+
frame_header += struct.pack('!Q', length)
|
|
2867
|
+
if not self.mask_value:
|
|
2868
|
+
if isinstance(self.data, str):
|
|
2869
|
+
self.data = self.data.encode('utf-8')
|
|
2870
|
+
return frame_header + self.data
|
|
2871
|
+
mask_key = self.get_mask_key(4)
|
|
2872
|
+
return frame_header + self._get_masked(mask_key)
|
|
2873
|
+
def _get_masked(self, mask_key: Union[str, bytes]) -> bytes:
|
|
2874
|
+
s = ABNF.mask(mask_key, self.data)
|
|
2875
|
+
if isinstance(mask_key, str):
|
|
2876
|
+
mask_key = mask_key.encode('utf-8')
|
|
2877
|
+
return mask_key + s
|
|
2878
|
+
@staticmethod
|
|
2879
|
+
def mask(mask_key: Union[str, bytes], data: Union[str, bytes]) -> bytes:
|
|
2880
|
+
if data is None:
|
|
2881
|
+
data = ''
|
|
2882
|
+
if isinstance(mask_key, str):
|
|
2883
|
+
mask_key = mask_key.encode('latin-1')
|
|
2884
|
+
if isinstance(data, str):
|
|
2885
|
+
data = data.encode('latin-1')
|
|
2886
|
+
return _mask(array.array('B', mask_key), array.array('B', data))
|
|
2887
|
+
class frame_buffer:
|
|
2888
|
+
_HEADER_MASK_INDEX = 5
|
|
2889
|
+
_HEADER_LENGTH_INDEX = 6
|
|
2890
|
+
def __init__(self, recv_fn: Callable[[int], int], skip_utf8_validation: bool) -> None:
|
|
2891
|
+
self.recv = recv_fn
|
|
2892
|
+
self.skip_utf8_validation = skip_utf8_validation
|
|
2893
|
+
self.recv_buffer: list = []
|
|
2894
|
+
self.clear()
|
|
2895
|
+
self.lock = Lock()
|
|
2896
|
+
def clear(self) -> None:
|
|
2897
|
+
self.header: Optional[tuple] = None
|
|
2898
|
+
self.length: Optional[int] = None
|
|
2899
|
+
self.mask_value: Union[bytes, str, None] = None
|
|
2900
|
+
def has_received_header(self) -> bool:
|
|
2901
|
+
return self.header is None
|
|
2902
|
+
def recv_header(self) -> None:
|
|
2903
|
+
header = self.recv_strict(2)
|
|
2904
|
+
b1 = header[0]
|
|
2905
|
+
fin = b1 >> 7 & 1
|
|
2906
|
+
rsv1 = b1 >> 6 & 1
|
|
2907
|
+
rsv2 = b1 >> 5 & 1
|
|
2908
|
+
rsv3 = b1 >> 4 & 1
|
|
2909
|
+
opcode = b1 & 15
|
|
2910
|
+
b2 = header[1]
|
|
2911
|
+
has_mask = b2 >> 7 & 1
|
|
2912
|
+
length_bits = b2 & 127
|
|
2913
|
+
self.header = (fin, rsv1, rsv2, rsv3, opcode, has_mask, length_bits)
|
|
2914
|
+
def has_mask(self) -> Union[bool, int]:
|
|
2915
|
+
if not self.header:
|
|
2916
|
+
return False
|
|
2917
|
+
header_val: int = self.header[frame_buffer._HEADER_MASK_INDEX]
|
|
2918
|
+
return header_val
|
|
2919
|
+
def has_received_length(self) -> bool:
|
|
2920
|
+
return self.length is None
|
|
2921
|
+
def recv_length(self) -> None:
|
|
2922
|
+
bits = self.header[frame_buffer._HEADER_LENGTH_INDEX]
|
|
2923
|
+
length_bits = bits & 127
|
|
2924
|
+
if length_bits == 126:
|
|
2925
|
+
v = self.recv_strict(2)
|
|
2926
|
+
self.length = struct.unpack('!H', v)[0]
|
|
2927
|
+
elif length_bits == 127:
|
|
2928
|
+
v = self.recv_strict(8)
|
|
2929
|
+
self.length = struct.unpack('!Q', v)[0]
|
|
2930
|
+
else:
|
|
2931
|
+
self.length = length_bits
|
|
2932
|
+
def has_received_mask(self) -> bool:
|
|
2933
|
+
return self.mask_value is None
|
|
2934
|
+
def recv_mask(self) -> None:
|
|
2935
|
+
self.mask_value = self.recv_strict(4) if self.has_mask() else ''
|
|
2936
|
+
def recv_frame(self) -> ABNF:
|
|
2937
|
+
with self.lock:
|
|
2938
|
+
if self.has_received_header():
|
|
2939
|
+
self.recv_header()
|
|
2940
|
+
(fin, rsv1, rsv2, rsv3, opcode, has_mask, _) = self.header
|
|
2941
|
+
if self.has_received_length():
|
|
2942
|
+
self.recv_length()
|
|
2943
|
+
length = self.length
|
|
2944
|
+
if self.has_received_mask():
|
|
2945
|
+
self.recv_mask()
|
|
2946
|
+
mask_value = self.mask_value
|
|
2947
|
+
payload = self.recv_strict(length)
|
|
2948
|
+
if has_mask:
|
|
2949
|
+
payload = ABNF.mask(mask_value, payload)
|
|
2950
|
+
self.clear()
|
|
2951
|
+
frame = ABNF(fin, rsv1, rsv2, rsv3, opcode, has_mask, payload)
|
|
2952
|
+
frame.validate(self.skip_utf8_validation)
|
|
2953
|
+
return frame
|
|
2954
|
+
def recv_strict(self, bufsize: int) -> bytes:
|
|
2955
|
+
shortage = bufsize - sum(map(len, self.recv_buffer))
|
|
2956
|
+
while shortage > 0:
|
|
2957
|
+
bytes_ = self.recv(min(16384, shortage))
|
|
2958
|
+
self.recv_buffer.append(bytes_)
|
|
2959
|
+
shortage -= len(bytes_)
|
|
2960
|
+
unified = b''.join(self.recv_buffer)
|
|
2961
|
+
if shortage == 0:
|
|
2962
|
+
self.recv_buffer = []
|
|
2963
|
+
return unified
|
|
2964
|
+
else:
|
|
2965
|
+
self.recv_buffer = [unified[bufsize:]]
|
|
2966
|
+
return unified[:bufsize]
|
|
2967
|
+
class continuous_frame:
|
|
2968
|
+
def __init__(self, fire_cont_frame: bool, skip_utf8_validation: bool) -> None:
|
|
2969
|
+
self.fire_cont_frame = fire_cont_frame
|
|
2970
|
+
self.skip_utf8_validation = skip_utf8_validation
|
|
2971
|
+
self.cont_data: Optional[list] = None
|
|
2972
|
+
self.recving_frames: Optional[int] = None
|
|
2973
|
+
def validate(self, frame: ABNF) -> None:
|
|
2974
|
+
if not self.recving_frames and frame.opcode == ABNF.OPCODE_CONT:
|
|
2975
|
+
raise WebSocketProtocolException('Illegal frame')
|
|
2976
|
+
if self.recving_frames and frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
|
|
2977
|
+
raise WebSocketProtocolException('Illegal frame')
|
|
2978
|
+
def add(self, frame: ABNF) -> None:
|
|
2979
|
+
if self.cont_data:
|
|
2980
|
+
self.cont_data[1] += frame.data
|
|
2981
|
+
else:
|
|
2982
|
+
if frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
|
|
2983
|
+
self.recving_frames = frame.opcode
|
|
2984
|
+
self.cont_data = [frame.opcode, frame.data]
|
|
2985
|
+
if frame.fin:
|
|
2986
|
+
self.recving_frames = None
|
|
2987
|
+
def is_fire(self, frame: ABNF) -> Union[bool, int]:
|
|
2988
|
+
return frame.fin or self.fire_cont_frame
|
|
2989
|
+
def extract(self, frame: ABNF) -> tuple:
|
|
2990
|
+
data = self.cont_data
|
|
2991
|
+
self.cont_data = None
|
|
2992
|
+
frame.data = data[1]
|
|
2993
|
+
if not self.fire_cont_frame and data[0] == ABNF.OPCODE_TEXT and (not self.skip_utf8_validation) and (not validate_utf8(frame.data)):
|
|
2994
|
+
raise WebSocketPayloadException(f'cannot decode: {repr(frame.data)}')
|
|
2995
|
+
return (data[0], frame)
|
|
2996
|
+
import inspect
|
|
2997
|
+
import selectors
|
|
2998
|
+
import socket
|
|
2999
|
+
import threading
|
|
3000
|
+
import time
|
|
3001
|
+
from typing import Any, Callable, Optional, Union
|
|
3002
|
+
import logging
|
|
3003
|
+
_logger = logging.getLogger('websocket')
|
|
3004
|
+
try:
|
|
3005
|
+
from logging import NullHandler
|
|
3006
|
+
except ImportError:
|
|
3007
|
+
class NullHandler(logging.Handler):
|
|
3008
|
+
def emit(self, record) -> None:pass
|
|
3009
|
+
_logger.addHandler(NullHandler())
|
|
3010
|
+
_traceEnabled = False
|
|
3011
|
+
def enableTrace(traceable: bool, handler: logging.StreamHandler=logging.StreamHandler(), level: str='DEBUG') -> None:
|
|
3012
|
+
global _traceEnabled
|
|
3013
|
+
_traceEnabled = traceable
|
|
3014
|
+
if traceable:
|
|
3015
|
+
_logger.addHandler(handler)
|
|
3016
|
+
_logger.setLevel(getattr(logging, level))
|
|
3017
|
+
def dump(title: str, message: str) -> None:
|
|
3018
|
+
if _traceEnabled:
|
|
3019
|
+
_logger.debug(f'--- {title} ---')
|
|
3020
|
+
_logger.debug(message)
|
|
3021
|
+
_logger.debug('-----------------------')
|
|
3022
|
+
def error(msg: str) -> None:
|
|
3023
|
+
_logger.error(msg)
|
|
3024
|
+
def warning(msg: str) -> None:
|
|
3025
|
+
_logger.warning(msg)
|
|
3026
|
+
def debug(msg: str) -> None:
|
|
3027
|
+
_logger.debug(msg)
|
|
3028
|
+
def info(msg: str) -> None:
|
|
3029
|
+
_logger.info(msg)
|
|
3030
|
+
def trace(msg: str) -> None:
|
|
3031
|
+
if _traceEnabled:
|
|
3032
|
+
_logger.debug(msg)
|
|
3033
|
+
def isEnabledForError() -> bool:
|
|
3034
|
+
return _logger.isEnabledFor(logging.ERROR)
|
|
3035
|
+
def isEnabledForDebug() -> bool:
|
|
3036
|
+
return _logger.isEnabledFor(logging.DEBUG)
|
|
3037
|
+
def isEnabledForTrace() -> bool:
|
|
3038
|
+
return _traceEnabled
|
|
3039
|
+
import socket
|
|
3040
|
+
import struct
|
|
3041
|
+
import threading
|
|
3042
|
+
import time
|
|
3043
|
+
from typing import Optional, Union
|
|
3044
|
+
import hashlib
|
|
3045
|
+
import hmac
|
|
3046
|
+
import os
|
|
3047
|
+
from base64 import encodebytes as base64encode
|
|
3048
|
+
from http import HTTPStatus
|
|
3049
|
+
import http.cookies
|
|
3050
|
+
from typing import Optional
|
|
3051
|
+
class SimpleCookieJar:
|
|
3052
|
+
def __init__(self) -> None:
|
|
3053
|
+
self.jar: dict = {}
|
|
3054
|
+
def add(self, set_cookie: Optional[str]) -> None:
|
|
3055
|
+
if set_cookie:
|
|
3056
|
+
simple_cookie = http.cookies.SimpleCookie(set_cookie)
|
|
3057
|
+
for v in simple_cookie.values():
|
|
3058
|
+
if (domain := v.get('domain')):
|
|
3059
|
+
if not domain.startswith('.'):
|
|
3060
|
+
domain = f'.{domain}'
|
|
3061
|
+
cookie = self.jar.get(domain) if self.jar.get(domain) else http.cookies.SimpleCookie()
|
|
3062
|
+
cookie.update(simple_cookie)
|
|
3063
|
+
self.jar[domain.lower()] = cookie
|
|
3064
|
+
def set(self, set_cookie: str) -> None:
|
|
3065
|
+
if set_cookie:
|
|
3066
|
+
simple_cookie = http.cookies.SimpleCookie(set_cookie)
|
|
3067
|
+
for v in simple_cookie.values():
|
|
3068
|
+
if (domain := v.get('domain')):
|
|
3069
|
+
if not domain.startswith('.'):
|
|
3070
|
+
domain = f'.{domain}'
|
|
3071
|
+
self.jar[domain.lower()] = simple_cookie
|
|
3072
|
+
def get(self, host: str) -> str:
|
|
3073
|
+
if not host:
|
|
3074
|
+
return ''
|
|
3075
|
+
cookies = []
|
|
3076
|
+
for (domain, _) in self.jar.items():
|
|
3077
|
+
host = host.lower()
|
|
3078
|
+
if host.endswith(domain) or host == domain[1:]:
|
|
3079
|
+
cookies.append(self.jar.get(domain))
|
|
3080
|
+
return '; '.join(filter(None, sorted([f'{k}={v.value}' for cookie in filter(None, cookies) for (k, v) in cookie.items()])))
|
|
3081
|
+
import errno
|
|
3082
|
+
import os
|
|
3083
|
+
import socket
|
|
3084
|
+
from base64 import encodebytes as base64encode
|
|
3085
|
+
import errno
|
|
3086
|
+
import selectors
|
|
3087
|
+
import socket
|
|
3088
|
+
from typing import Union
|
|
3089
|
+
try:
|
|
3090
|
+
import ssl
|
|
3091
|
+
from ssl import SSLError, SSLEOFError, SSLWantReadError, SSLWantWriteError
|
|
3092
|
+
HAVE_SSL = True
|
|
3093
|
+
except ImportError:
|
|
3094
|
+
class SSLError(Exception):pass
|
|
3095
|
+
class SSLEOFError(Exception):pass
|
|
3096
|
+
class SSLWantReadError(Exception):pass
|
|
3097
|
+
class SSLWantWriteError(Exception):pass
|
|
3098
|
+
ssl = None
|
|
3099
|
+
HAVE_SSL = False
|
|
3100
|
+
DEFAULT_SOCKET_OPTION = [(socket.SOL_TCP, socket.TCP_NODELAY, 1)]
|
|
3101
|
+
if hasattr(socket, 'SO_KEEPALIVE'):
|
|
3102
|
+
DEFAULT_SOCKET_OPTION.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
|
|
3103
|
+
if hasattr(socket, 'TCP_KEEPIDLE'):
|
|
3104
|
+
DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPIDLE, 30))
|
|
3105
|
+
if hasattr(socket, 'TCP_KEEPINTVL'):
|
|
3106
|
+
DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPINTVL, 10))
|
|
3107
|
+
if hasattr(socket, 'TCP_KEEPCNT'):
|
|
3108
|
+
DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPCNT, 3))
|
|
3109
|
+
_default_timeout = None
|
|
3110
|
+
class sock_opt:
|
|
3111
|
+
def __init__(self, sockopt: list, sslopt: dict) -> None:
|
|
3112
|
+
if sockopt is None:
|
|
3113
|
+
sockopt = []
|
|
3114
|
+
if sslopt is None:
|
|
3115
|
+
sslopt = {}
|
|
3116
|
+
self.sockopt = sockopt
|
|
3117
|
+
self.sslopt = sslopt
|
|
3118
|
+
self.timeout = None
|
|
3119
|
+
def setdefaulttimeout(timeout: Union[int, float, None]) -> None:
|
|
3120
|
+
global _default_timeout
|
|
3121
|
+
_default_timeout = timeout
|
|
3122
|
+
def getdefaulttimeout() -> Union[int, float, None]:
|
|
3123
|
+
return _default_timeout
|
|
3124
|
+
def recv(sock: socket.socket, bufsize: int) -> bytes:
|
|
3125
|
+
if not sock:
|
|
3126
|
+
raise WebSocketConnectionClosedException('socket is already closed.')
|
|
3127
|
+
def _recv():
|
|
3128
|
+
try:
|
|
3129
|
+
return sock.recv(bufsize)
|
|
3130
|
+
except SSLWantReadError:pass
|
|
3131
|
+
except socket.error as exc:
|
|
3132
|
+
error_code = extract_error_code(exc)
|
|
3133
|
+
if error_code not in [errno.EAGAIN, errno.EWOULDBLOCK]:
|
|
3134
|
+
raise
|
|
3135
|
+
sel = selectors.DefaultSelector()
|
|
3136
|
+
sel.register(sock, selectors.EVENT_READ)
|
|
3137
|
+
r = sel.select(sock.gettimeout())
|
|
3138
|
+
sel.close()
|
|
3139
|
+
if r:
|
|
3140
|
+
return sock.recv(bufsize)
|
|
3141
|
+
try:
|
|
3142
|
+
if sock.gettimeout() == 0:
|
|
3143
|
+
bytes_ = sock.recv(bufsize)
|
|
3144
|
+
else:
|
|
3145
|
+
bytes_ = _recv()
|
|
3146
|
+
except TimeoutError:
|
|
3147
|
+
raise WebSocketTimeoutException('Connection timed out')
|
|
3148
|
+
except socket.timeout as e:
|
|
3149
|
+
message = extract_err_message(e)
|
|
3150
|
+
raise WebSocketTimeoutException(message)
|
|
3151
|
+
except SSLError as e:
|
|
3152
|
+
message = extract_err_message(e)
|
|
3153
|
+
if isinstance(message, str) and 'timed out' in message:
|
|
3154
|
+
raise WebSocketTimeoutException(message)
|
|
3155
|
+
else:
|
|
3156
|
+
raise
|
|
3157
|
+
if not bytes_:
|
|
3158
|
+
raise WebSocketConnectionClosedException('Connection to remote host was lost.')
|
|
3159
|
+
return bytes_
|
|
3160
|
+
def recv_line(sock: socket.socket) -> bytes:
|
|
3161
|
+
line = []
|
|
3162
|
+
while True:
|
|
3163
|
+
c = recv(sock, 1)
|
|
3164
|
+
line.append(c)
|
|
3165
|
+
if c == b'\n':
|
|
3166
|
+
break
|
|
3167
|
+
return b''.join(line)
|
|
3168
|
+
def send(sock: socket.socket, data: Union[bytes, str]) -> int:
|
|
3169
|
+
if isinstance(data, str):
|
|
3170
|
+
data = data.encode('utf-8')
|
|
3171
|
+
if not sock:
|
|
3172
|
+
raise WebSocketConnectionClosedException('socket is already closed.')
|
|
3173
|
+
def _send():
|
|
3174
|
+
try:
|
|
3175
|
+
return sock.send(data)
|
|
3176
|
+
except SSLWantWriteError:pass
|
|
3177
|
+
except socket.error as exc:
|
|
3178
|
+
error_code = extract_error_code(exc)
|
|
3179
|
+
if error_code is None:
|
|
3180
|
+
raise
|
|
3181
|
+
if error_code not in [errno.EAGAIN, errno.EWOULDBLOCK]:
|
|
3182
|
+
raise
|
|
3183
|
+
sel = selectors.DefaultSelector()
|
|
3184
|
+
sel.register(sock, selectors.EVENT_WRITE)
|
|
3185
|
+
w = sel.select(sock.gettimeout())
|
|
3186
|
+
sel.close()
|
|
3187
|
+
if w:
|
|
3188
|
+
return sock.send(data)
|
|
3189
|
+
try:
|
|
3190
|
+
if sock.gettimeout() == 0:
|
|
3191
|
+
return sock.send(data)
|
|
3192
|
+
else:
|
|
3193
|
+
return _send()
|
|
3194
|
+
except socket.timeout as e:
|
|
3195
|
+
message = extract_err_message(e)
|
|
3196
|
+
raise WebSocketTimeoutException(message)
|
|
3197
|
+
except Exception as e:
|
|
3198
|
+
message = extract_err_message(e)
|
|
3199
|
+
if isinstance(message, str) and 'timed out' in message:
|
|
3200
|
+
raise WebSocketTimeoutException(message)
|
|
3201
|
+
else:
|
|
3202
|
+
raise
|
|
3203
|
+
import os
|
|
3204
|
+
import socket
|
|
3205
|
+
import struct
|
|
3206
|
+
from typing import Optional
|
|
3207
|
+
from urllib.parse import unquote, urlparse
|
|
3208
|
+
def parse_url(url: str) -> tuple:
|
|
3209
|
+
if ':' not in url:
|
|
3210
|
+
raise ValueError('url is invalid')
|
|
3211
|
+
(scheme, url) = url.split(':', 1)
|
|
3212
|
+
parsed = urlparse(url, scheme='http')
|
|
3213
|
+
if parsed.hostname:
|
|
3214
|
+
hostname = parsed.hostname
|
|
3215
|
+
else:
|
|
3216
|
+
raise ValueError('hostname is invalid')
|
|
3217
|
+
port = 0
|
|
3218
|
+
if parsed.port:
|
|
3219
|
+
port = parsed.port
|
|
3220
|
+
is_secure = False
|
|
3221
|
+
if scheme == 'ws':
|
|
3222
|
+
if not port:
|
|
3223
|
+
port = 80
|
|
3224
|
+
elif scheme == 'wss':
|
|
3225
|
+
is_secure = True
|
|
3226
|
+
if not port:
|
|
3227
|
+
port = 443
|
|
3228
|
+
else:
|
|
3229
|
+
raise ValueError('scheme %s is invalid' % scheme)
|
|
3230
|
+
if parsed.path:
|
|
3231
|
+
resource = parsed.path
|
|
3232
|
+
else:
|
|
3233
|
+
resource = '/'
|
|
3234
|
+
if parsed.query:
|
|
3235
|
+
resource += f'?{parsed.query}'
|
|
3236
|
+
return (hostname, port, resource, is_secure)
|
|
3237
|
+
DEFAULT_NO_PROXY_HOST = ['localhost', '127.0.0.1']
|
|
3238
|
+
def _is_ip_address(addr: str) -> bool:
|
|
3239
|
+
try:
|
|
3240
|
+
socket.inet_aton(addr)
|
|
3241
|
+
except socket.error:
|
|
3242
|
+
return False
|
|
3243
|
+
else:
|
|
3244
|
+
return True
|
|
3245
|
+
def _is_subnet_address(hostname: str) -> bool:
|
|
3246
|
+
try:
|
|
3247
|
+
(addr, netmask) = hostname.split('/')
|
|
3248
|
+
return _is_ip_address(addr) and 0 <= int(netmask) < 32
|
|
3249
|
+
except ValueError:
|
|
3250
|
+
return False
|
|
3251
|
+
def _is_address_in_network(ip: str, net: str) -> bool:
|
|
3252
|
+
ipaddr: int = struct.unpack('!I', socket.inet_aton(ip))[0]
|
|
3253
|
+
(netaddr, netmask) = net.split('/')
|
|
3254
|
+
netaddr: int = struct.unpack('!I', socket.inet_aton(netaddr))[0]
|
|
3255
|
+
netmask = 4294967295 << 32 - int(netmask) & 4294967295
|
|
3256
|
+
return ipaddr & netmask == netaddr
|
|
3257
|
+
def _is_no_proxy_host(hostname: str, no_proxy: Optional[list]) -> bool:
|
|
3258
|
+
if not no_proxy:
|
|
3259
|
+
if (v := os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')).replace(' ', '')):
|
|
3260
|
+
no_proxy = v.split(',')
|
|
3261
|
+
if not no_proxy:
|
|
3262
|
+
no_proxy = DEFAULT_NO_PROXY_HOST
|
|
3263
|
+
if '*' in no_proxy:
|
|
3264
|
+
return True
|
|
3265
|
+
if hostname in no_proxy:
|
|
3266
|
+
return True
|
|
3267
|
+
if _is_ip_address(hostname):
|
|
3268
|
+
return any([_is_address_in_network(hostname, subnet) for subnet in no_proxy if _is_subnet_address(subnet)])
|
|
3269
|
+
for domain in [domain for domain in no_proxy if domain.startswith('.')]:
|
|
3270
|
+
if hostname.endswith(domain):
|
|
3271
|
+
return True
|
|
3272
|
+
return False
|
|
3273
|
+
def get_proxy_info(hostname: str, is_secure: bool, proxy_host: Optional[str]=None, proxy_port: int=0, proxy_auth: Optional[tuple]=None, no_proxy: Optional[list]=None, proxy_type: str='http') -> tuple:
|
|
3274
|
+
if _is_no_proxy_host(hostname, no_proxy):
|
|
3275
|
+
return (None, 0, None)
|
|
3276
|
+
if proxy_host:
|
|
3277
|
+
if not proxy_port:
|
|
3278
|
+
raise WebSocketProxyException('Cannot use port 0 when proxy_host specified')
|
|
3279
|
+
port = proxy_port
|
|
3280
|
+
auth = proxy_auth
|
|
3281
|
+
return (proxy_host, port, auth)
|
|
3282
|
+
env_key = 'https_proxy' if is_secure else 'http_proxy'
|
|
3283
|
+
value = os.environ.get(env_key, os.environ.get(env_key.upper(), '')).replace(' ', '')
|
|
3284
|
+
if value:
|
|
3285
|
+
proxy = urlparse(value)
|
|
3286
|
+
auth = (unquote(proxy.username), unquote(proxy.password)) if proxy.username else None
|
|
3287
|
+
return (proxy.hostname, proxy.port, auth)
|
|
3288
|
+
return (None, 0, None)
|
|
3289
|
+
try:
|
|
3290
|
+
from python_socks._errors import *
|
|
3291
|
+
from python_socks._types import ProxyType
|
|
3292
|
+
from python_socks.sync import Proxy
|
|
3293
|
+
HAVE_PYTHON_SOCKS = True
|
|
3294
|
+
except:
|
|
3295
|
+
HAVE_PYTHON_SOCKS = False
|
|
3296
|
+
class ProxyError(Exception):pass
|
|
3297
|
+
class ProxyTimeoutError(Exception):pass
|
|
3298
|
+
class ProxyConnectionError(Exception):pass
|
|
3299
|
+
class proxy_info:
|
|
3300
|
+
def __init__(self, **options):
|
|
3301
|
+
self.proxy_host = options.get('http_proxy_host', None)
|
|
3302
|
+
if self.proxy_host:
|
|
3303
|
+
self.proxy_port = options.get('http_proxy_port', 0)
|
|
3304
|
+
self.auth = options.get('http_proxy_auth', None)
|
|
3305
|
+
self.no_proxy = options.get('http_no_proxy', None)
|
|
3306
|
+
self.proxy_protocol = options.get('proxy_type', 'http')
|
|
3307
|
+
self.proxy_timeout = options.get('http_proxy_timeout', None)
|
|
3308
|
+
if self.proxy_protocol not in ['http', 'socks4', 'socks4a', 'socks5', 'socks5h']:
|
|
3309
|
+
raise ProxyError('Only http, socks4, socks5 proxy protocols are supported')
|
|
3310
|
+
else:
|
|
3311
|
+
self.proxy_port = 0
|
|
3312
|
+
self.auth = None
|
|
3313
|
+
self.no_proxy = None
|
|
3314
|
+
self.proxy_protocol = 'http'
|
|
3315
|
+
def _start_proxied_socket(url: str, options, proxy) -> tuple:
|
|
3316
|
+
if not HAVE_PYTHON_SOCKS:
|
|
3317
|
+
raise WebSocketException('Python Socks is needed for SOCKS proxying but is not available')
|
|
3318
|
+
(hostname, port, resource, is_secure) = parse_url(url)
|
|
3319
|
+
if proxy.proxy_protocol == 'socks4':
|
|
3320
|
+
rdns = False
|
|
3321
|
+
proxy_type = ProxyType.SOCKS4
|
|
3322
|
+
elif proxy.proxy_protocol == 'socks4a':
|
|
3323
|
+
rdns = True
|
|
3324
|
+
proxy_type = ProxyType.SOCKS4
|
|
3325
|
+
elif proxy.proxy_protocol == 'socks5':
|
|
3326
|
+
rdns = False
|
|
3327
|
+
proxy_type = ProxyType.SOCKS5
|
|
3328
|
+
elif proxy.proxy_protocol == 'socks5h':
|
|
3329
|
+
rdns = True
|
|
3330
|
+
proxy_type = ProxyType.SOCKS5
|
|
3331
|
+
ws_proxy = Proxy.create(proxy_type=proxy_type, host=proxy.proxy_host, port=int(proxy.proxy_port), username=proxy.auth[0] if proxy.auth else None, password=proxy.auth[1] if proxy.auth else None, rdns=rdns)
|
|
3332
|
+
sock = ws_proxy.connect(hostname, port, timeout=proxy.proxy_timeout)
|
|
3333
|
+
if is_secure:
|
|
3334
|
+
if HAVE_SSL:
|
|
3335
|
+
sock = _ssl_socket(sock, options.sslopt, hostname)
|
|
3336
|
+
else:
|
|
3337
|
+
raise WebSocketException('SSL not available.')
|
|
3338
|
+
return (sock, (hostname, port, resource))
|
|
3339
|
+
def connect(url: str, options, proxy, socket):
|
|
3340
|
+
if proxy.proxy_host and (not socket) and (proxy.proxy_protocol != 'http'):
|
|
3341
|
+
return _start_proxied_socket(url, options, proxy)
|
|
3342
|
+
(hostname, port_from_url, resource, is_secure) = parse_url(url)
|
|
3343
|
+
if socket:
|
|
3344
|
+
return (socket, (hostname, port_from_url, resource))
|
|
3345
|
+
(addrinfo_list, need_tunnel, auth) = _get_addrinfo_list(hostname, port_from_url, is_secure, proxy)
|
|
3346
|
+
if not addrinfo_list:
|
|
3347
|
+
raise WebSocketException(f'Host not found.: {hostname}:{port_from_url}')
|
|
3348
|
+
sock = None
|
|
3349
|
+
try:
|
|
3350
|
+
sock = _open_socket(addrinfo_list, options.sockopt, options.timeout)
|
|
3351
|
+
if need_tunnel:
|
|
3352
|
+
sock = _tunnel(sock, hostname, port_from_url, auth)
|
|
3353
|
+
if is_secure:
|
|
3354
|
+
if HAVE_SSL:
|
|
3355
|
+
sock = _ssl_socket(sock, options.sslopt, hostname)
|
|
3356
|
+
else:
|
|
3357
|
+
raise WebSocketException('SSL not available.')
|
|
3358
|
+
return (sock, (hostname, port_from_url, resource))
|
|
3359
|
+
except:
|
|
3360
|
+
if sock:
|
|
3361
|
+
sock.close()
|
|
3362
|
+
raise
|
|
3363
|
+
def _get_addrinfo_list(hostname, port: int, is_secure: bool, proxy) -> tuple:
|
|
3364
|
+
(phost, pport, pauth) = get_proxy_info(hostname, is_secure, proxy.proxy_host, proxy.proxy_port, proxy.auth, proxy.no_proxy)
|
|
3365
|
+
try:
|
|
3366
|
+
if not phost:
|
|
3367
|
+
addrinfo_list = socket.getaddrinfo(hostname, port, 0, socket.SOCK_STREAM, socket.SOL_TCP)
|
|
3368
|
+
return (addrinfo_list, False, None)
|
|
3369
|
+
else:
|
|
3370
|
+
pport = pport and pport or 80
|
|
3371
|
+
addrinfo_list = socket.getaddrinfo(phost, pport, 0, socket.SOCK_STREAM, socket.SOL_TCP)
|
|
3372
|
+
return (addrinfo_list, True, pauth)
|
|
3373
|
+
except socket.gaierror as e:
|
|
3374
|
+
raise WebSocketAddressException(e)
|
|
3375
|
+
def _open_socket(addrinfo_list, sockopt, timeout):
|
|
3376
|
+
err = None
|
|
3377
|
+
for addrinfo in addrinfo_list:
|
|
3378
|
+
(family, socktype, proto) = addrinfo[:3]
|
|
3379
|
+
sock = socket.socket(family, socktype, proto)
|
|
3380
|
+
sock.settimeout(timeout)
|
|
3381
|
+
for opts in DEFAULT_SOCKET_OPTION:
|
|
3382
|
+
sock.setsockopt(*opts)
|
|
3383
|
+
for opts in sockopt:
|
|
3384
|
+
sock.setsockopt(*opts)
|
|
3385
|
+
address = addrinfo[4]
|
|
3386
|
+
err = None
|
|
3387
|
+
while not err:
|
|
3388
|
+
try:
|
|
3389
|
+
sock.connect(address)
|
|
3390
|
+
except socket.error as error:
|
|
3391
|
+
sock.close()
|
|
3392
|
+
error.remote_ip = str(address[0])
|
|
3393
|
+
try:
|
|
3394
|
+
eConnRefused = (errno.ECONNREFUSED, errno.WSAECONNREFUSED, errno.ENETUNREACH)
|
|
3395
|
+
except AttributeError:
|
|
3396
|
+
eConnRefused = (errno.ECONNREFUSED, errno.ENETUNREACH)
|
|
3397
|
+
if error.errno not in eConnRefused:
|
|
3398
|
+
raise error
|
|
3399
|
+
err = error
|
|
3400
|
+
continue
|
|
3401
|
+
else:
|
|
3402
|
+
break
|
|
3403
|
+
else:
|
|
3404
|
+
continue
|
|
3405
|
+
break
|
|
3406
|
+
else:
|
|
3407
|
+
if err:
|
|
3408
|
+
raise err
|
|
3409
|
+
return sock
|
|
3410
|
+
def _wrap_sni_socket(sock: socket.socket, sslopt: dict, hostname, check_hostname):
|
|
3411
|
+
context = sslopt.get('context', None)
|
|
3412
|
+
if not context:
|
|
3413
|
+
context = ssl.SSLContext(sslopt.get('ssl_version', ssl.PROTOCOL_TLS_CLIENT))
|
|
3414
|
+
context.keylog_filename = os.environ.get('SSLKEYLOGFILE', None)
|
|
3415
|
+
if sslopt.get('cert_reqs', ssl.CERT_NONE) != ssl.CERT_NONE:
|
|
3416
|
+
cafile = sslopt.get('ca_certs', None)
|
|
3417
|
+
capath = sslopt.get('ca_cert_path', None)
|
|
3418
|
+
if cafile or capath:
|
|
3419
|
+
context.load_verify_locations(cafile=cafile, capath=capath)
|
|
3420
|
+
elif hasattr(context, 'load_default_certs'):
|
|
3421
|
+
context.load_default_certs(ssl.Purpose.SERVER_AUTH)
|
|
3422
|
+
if sslopt.get('certfile', None):
|
|
3423
|
+
context.load_cert_chain(sslopt['certfile'], sslopt.get('keyfile', None), sslopt.get('password', None))
|
|
3424
|
+
if sslopt.get('cert_reqs', ssl.CERT_NONE) == ssl.CERT_NONE and (not sslopt.get('check_hostname', False)):
|
|
3425
|
+
context.check_hostname = False
|
|
3426
|
+
context.verify_mode = ssl.CERT_NONE
|
|
3427
|
+
else:
|
|
3428
|
+
context.check_hostname = sslopt.get('check_hostname', True)
|
|
3429
|
+
context.verify_mode = sslopt.get('cert_reqs', ssl.CERT_REQUIRED)
|
|
3430
|
+
if 'ciphers' in sslopt:
|
|
3431
|
+
context.set_ciphers(sslopt['ciphers'])
|
|
3432
|
+
if 'cert_chain' in sslopt:
|
|
3433
|
+
(certfile, keyfile, password) = sslopt['cert_chain']
|
|
3434
|
+
context.load_cert_chain(certfile, keyfile, password)
|
|
3435
|
+
if 'ecdh_curve' in sslopt:
|
|
3436
|
+
context.set_ecdh_curve(sslopt['ecdh_curve'])
|
|
3437
|
+
return context.wrap_socket(sock, do_handshake_on_connect=sslopt.get('do_handshake_on_connect', True), suppress_ragged_eofs=sslopt.get('suppress_ragged_eofs', True), server_hostname=hostname)
|
|
3438
|
+
def _ssl_socket(sock: socket.socket, user_sslopt: dict, hostname):
|
|
3439
|
+
sslopt: dict = {'cert_reqs': ssl.CERT_REQUIRED}
|
|
3440
|
+
sslopt.update(user_sslopt)
|
|
3441
|
+
cert_path = os.environ.get('WEBSOCKET_CLIENT_CA_BUNDLE')
|
|
3442
|
+
if cert_path and os.path.isfile(cert_path) and (user_sslopt.get('ca_certs', None) is None):
|
|
3443
|
+
sslopt['ca_certs'] = cert_path
|
|
3444
|
+
elif cert_path and os.path.isdir(cert_path) and (user_sslopt.get('ca_cert_path', None) is None):
|
|
3445
|
+
sslopt['ca_cert_path'] = cert_path
|
|
3446
|
+
if sslopt.get('server_hostname', None):
|
|
3447
|
+
hostname = sslopt['server_hostname']
|
|
3448
|
+
check_hostname = sslopt.get('check_hostname', True)
|
|
3449
|
+
sock = _wrap_sni_socket(sock, sslopt, hostname, check_hostname)
|
|
3450
|
+
return sock
|
|
3451
|
+
def _tunnel(sock: socket.socket, host, port: int, auth) -> socket.socket:
|
|
3452
|
+
debug('Connecting proxy...')
|
|
3453
|
+
connect_header = f'CONNECT {host}:{port} HTTP/1.1\r\n'
|
|
3454
|
+
connect_header += f'Host: {host}:{port}\r\n'
|
|
3455
|
+
if auth and auth[0]:
|
|
3456
|
+
auth_str = auth[0]
|
|
3457
|
+
if auth[1]:
|
|
3458
|
+
auth_str += f':{auth[1]}'
|
|
3459
|
+
encoded_str = base64encode(auth_str.encode()).strip().decode().replace('\n', '')
|
|
3460
|
+
connect_header += f'Proxy-Authorization: Basic {encoded_str}\r\n'
|
|
3461
|
+
connect_header += '\r\n'
|
|
3462
|
+
dump('request header', connect_header)
|
|
3463
|
+
send(sock, connect_header)
|
|
3464
|
+
try:
|
|
3465
|
+
(status, _, _) = read_headers(sock)
|
|
3466
|
+
except Exception as e:
|
|
3467
|
+
raise WebSocketProxyException(str(e))
|
|
3468
|
+
if status != 200:
|
|
3469
|
+
raise WebSocketProxyException(f'failed CONNECT via proxy status: {status}')
|
|
3470
|
+
return sock
|
|
3471
|
+
def read_headers(sock: socket.socket) -> tuple:
|
|
3472
|
+
status = None
|
|
3473
|
+
status_message = None
|
|
3474
|
+
headers: dict = {}
|
|
3475
|
+
trace('--- response header ---')
|
|
3476
|
+
while True:
|
|
3477
|
+
line = recv_line(sock)
|
|
3478
|
+
line = line.decode('utf-8').strip()
|
|
3479
|
+
if not line:
|
|
3480
|
+
break
|
|
3481
|
+
trace(line)
|
|
3482
|
+
if not status:
|
|
3483
|
+
status_info = line.split(' ', 2)
|
|
3484
|
+
status = int(status_info[1])
|
|
3485
|
+
if len(status_info) > 2:
|
|
3486
|
+
status_message = status_info[2]
|
|
3487
|
+
else:
|
|
3488
|
+
kv = line.split(':', 1)
|
|
3489
|
+
if len(kv) != 2:
|
|
3490
|
+
raise WebSocketException('Invalid header')
|
|
3491
|
+
(key, value) = kv
|
|
3492
|
+
if key.lower() == 'set-cookie' and headers.get('set-cookie'):
|
|
3493
|
+
headers['set-cookie'] = headers.get('set-cookie') + '; ' + value.strip()
|
|
3494
|
+
else:
|
|
3495
|
+
headers[key.lower()] = value.strip()
|
|
3496
|
+
trace('-----------------------')
|
|
3497
|
+
return (status, headers, status_message)
|
|
3498
|
+
VERSION = 13
|
|
3499
|
+
SUPPORTED_REDIRECT_STATUSES = (HTTPStatus.MOVED_PERMANENTLY, HTTPStatus.FOUND, HTTPStatus.SEE_OTHER, HTTPStatus.TEMPORARY_REDIRECT, HTTPStatus.PERMANENT_REDIRECT)
|
|
3500
|
+
SUCCESS_STATUSES = SUPPORTED_REDIRECT_STATUSES + (HTTPStatus.SWITCHING_PROTOCOLS,)
|
|
3501
|
+
CookieJar = SimpleCookieJar()
|
|
3502
|
+
class handshake_response:
|
|
3503
|
+
def __init__(self, status: int, headers: dict, subprotocol):
|
|
3504
|
+
self.status = status
|
|
3505
|
+
self.headers = headers
|
|
3506
|
+
self.subprotocol = subprotocol
|
|
3507
|
+
CookieJar.add(headers.get('set-cookie'))
|
|
3508
|
+
def handshake(sock, url: str, hostname: str, port: int, resource: str, **options) -> handshake_response:
|
|
3509
|
+
(headers, key) = _get_handshake_headers(resource, url, hostname, port, options)
|
|
3510
|
+
header_str = '\r\n'.join(headers)
|
|
3511
|
+
send(sock, header_str)
|
|
3512
|
+
dump('request header', header_str)
|
|
3513
|
+
(status, resp) = _get_resp_headers(sock)
|
|
3514
|
+
if status in SUPPORTED_REDIRECT_STATUSES:
|
|
3515
|
+
return handshake_response(status, resp, None)
|
|
3516
|
+
(success, subproto) = _validate(resp, key, options.get('subprotocols'))
|
|
3517
|
+
if not success:
|
|
3518
|
+
raise WebSocketException('Invalid WebSocket Header')
|
|
3519
|
+
return handshake_response(status, resp, subproto)
|
|
3520
|
+
def _pack_hostname(hostname: str) -> str:
|
|
3521
|
+
if ':' in hostname:
|
|
3522
|
+
return f'[{hostname}]'
|
|
3523
|
+
return hostname
|
|
3524
|
+
def _get_handshake_headers(resource: str, url: str, host: str, port: int, options: dict) -> tuple:
|
|
3525
|
+
headers = [f'GET {resource} HTTP/1.1', 'Upgrade: websocket']
|
|
3526
|
+
if port in [80, 443]:
|
|
3527
|
+
hostport = _pack_hostname(host)
|
|
3528
|
+
else:
|
|
3529
|
+
hostport = f'{_pack_hostname(host)}:{port}'
|
|
3530
|
+
if options.get('host'):
|
|
3531
|
+
headers.append(f"Host: {options['host']}")
|
|
3532
|
+
else:
|
|
3533
|
+
headers.append(f'Host: {hostport}')
|
|
3534
|
+
(scheme, url) = url.split(':', 1)
|
|
3535
|
+
if not options.get('suppress_origin'):
|
|
3536
|
+
if 'origin' in options and options['origin'] is not None:
|
|
3537
|
+
headers.append(f"Origin: {options['origin']}")
|
|
3538
|
+
elif scheme == 'wss':
|
|
3539
|
+
headers.append(f'Origin: https://{hostport}')
|
|
3540
|
+
else:
|
|
3541
|
+
headers.append(f'Origin: http://{hostport}')
|
|
3542
|
+
key = _create_sec_websocket_key()
|
|
3543
|
+
if not options.get('header') or 'Sec-WebSocket-Key' not in options['header']:
|
|
3544
|
+
headers.append(f'Sec-WebSocket-Key: {key}')
|
|
3545
|
+
else:
|
|
3546
|
+
key = options['header']['Sec-WebSocket-Key']
|
|
3547
|
+
if not options.get('header') or 'Sec-WebSocket-Version' not in options['header']:
|
|
3548
|
+
headers.append(f'Sec-WebSocket-Version: {VERSION}')
|
|
3549
|
+
if not options.get('connection'):
|
|
3550
|
+
headers.append('Connection: Upgrade')
|
|
3551
|
+
else:
|
|
3552
|
+
headers.append(options['connection'])
|
|
3553
|
+
if (subprotocols := options.get('subprotocols')):
|
|
3554
|
+
headers.append(f"Sec-WebSocket-Protocol: {','.join(subprotocols)}")
|
|
3555
|
+
if (header := options.get('header')):
|
|
3556
|
+
if isinstance(header, dict):
|
|
3557
|
+
header = [': '.join([k, v]) for (k, v) in header.items() if v is not None]
|
|
3558
|
+
headers.extend(header)
|
|
3559
|
+
server_cookie = CookieJar.get(host)
|
|
3560
|
+
client_cookie = options.get('cookie', None)
|
|
3561
|
+
if (cookie := '; '.join(filter(None, [server_cookie, client_cookie]))):
|
|
3562
|
+
headers.append(f'Cookie: {cookie}')
|
|
3563
|
+
headers.extend(('', ''))
|
|
3564
|
+
return (headers, key)
|
|
3565
|
+
def _get_resp_headers(sock, success_statuses: tuple=SUCCESS_STATUSES) -> tuple:
|
|
3566
|
+
(status, resp_headers, status_message) = read_headers(sock)
|
|
3567
|
+
if status not in success_statuses:
|
|
3568
|
+
content_len = resp_headers.get('content-length')
|
|
3569
|
+
if content_len:
|
|
3570
|
+
response_body = sock.recv(int(content_len))
|
|
3571
|
+
else:
|
|
3572
|
+
response_body = None
|
|
3573
|
+
raise WebSocketBadStatusException(f'Handshake status {status} {status_message} -+-+- {resp_headers} -+-+- {response_body}', status, status_message, resp_headers, response_body)
|
|
3574
|
+
return (status, resp_headers)
|
|
3575
|
+
_HEADERS_TO_CHECK = {'upgrade': 'websocket', 'connection': 'upgrade'}
|
|
3576
|
+
def _validate(headers, key: str, subprotocols) -> tuple:
|
|
3577
|
+
subproto = None
|
|
3578
|
+
for (k, v) in _HEADERS_TO_CHECK.items():
|
|
3579
|
+
r = headers.get(k, None)
|
|
3580
|
+
if not r:
|
|
3581
|
+
return (False, None)
|
|
3582
|
+
r = [x.strip().lower() for x in r.split(',')]
|
|
3583
|
+
if v not in r:
|
|
3584
|
+
return (False, None)
|
|
3585
|
+
if subprotocols:
|
|
3586
|
+
subproto = headers.get('sec-websocket-protocol', None)
|
|
3587
|
+
if not subproto or subproto.lower() not in [s.lower() for s in subprotocols]:
|
|
3588
|
+
error(f'Invalid subprotocol: {subprotocols}')
|
|
3589
|
+
return (False, None)
|
|
3590
|
+
subproto = subproto.lower()
|
|
3591
|
+
result = headers.get('sec-websocket-accept', None)
|
|
3592
|
+
if not result:
|
|
3593
|
+
return (False, None)
|
|
3594
|
+
result = result.lower()
|
|
3595
|
+
if isinstance(result, str):
|
|
3596
|
+
result = result.encode('utf-8')
|
|
3597
|
+
value = f'{key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11'.encode('utf-8')
|
|
3598
|
+
hashed = base64encode(hashlib.sha1(value).digest()).strip().lower()
|
|
3599
|
+
if hmac.compare_digest(hashed, result):
|
|
3600
|
+
return (True, subproto)
|
|
3601
|
+
else:
|
|
3602
|
+
return (False, None)
|
|
3603
|
+
def _create_sec_websocket_key() -> str:
|
|
3604
|
+
randomness = os.urandom(16)
|
|
3605
|
+
return base64encode(randomness).decode('utf-8').strip()
|
|
3606
|
+
class WebSocket:
|
|
3607
|
+
def __init__(self, get_mask_key=None, sockopt=None, sslopt=None, fire_cont_frame: bool=False, enable_multithread: bool=True, skip_utf8_validation: bool=False, **_):
|
|
3608
|
+
self.sock_opt = sock_opt(sockopt, sslopt)
|
|
3609
|
+
self.handshake_response = None
|
|
3610
|
+
self.sock: Optional[socket.socket] = None
|
|
3611
|
+
self.connected = False
|
|
3612
|
+
self.get_mask_key = get_mask_key
|
|
3613
|
+
self.frame_buffer = frame_buffer(self._recv, skip_utf8_validation)
|
|
3614
|
+
self.cont_frame = continuous_frame(fire_cont_frame, skip_utf8_validation)
|
|
3615
|
+
if enable_multithread:
|
|
3616
|
+
self.lock = threading.Lock()
|
|
3617
|
+
self.readlock = threading.Lock()
|
|
3618
|
+
else:
|
|
3619
|
+
self.lock = NoLock()
|
|
3620
|
+
self.readlock = NoLock()
|
|
3621
|
+
def __iter__(self):
|
|
3622
|
+
while True:
|
|
3623
|
+
yield self.recv()
|
|
3624
|
+
def __next__(self):
|
|
3625
|
+
return self.recv()
|
|
3626
|
+
def next(self):
|
|
3627
|
+
return self.__next__()
|
|
3628
|
+
def fileno(self):
|
|
3629
|
+
return self.sock.fileno()
|
|
3630
|
+
def set_mask_key(self, func):
|
|
3631
|
+
self.get_mask_key = func
|
|
3632
|
+
def gettimeout(self) -> Union[float, int, None]:
|
|
3633
|
+
return self.sock_opt.timeout
|
|
3634
|
+
def settimeout(self, timeout: Union[float, int, None]):
|
|
3635
|
+
self.sock_opt.timeout = timeout
|
|
3636
|
+
if self.sock:
|
|
3637
|
+
self.sock.settimeout(timeout)
|
|
3638
|
+
timeout = property(gettimeout, settimeout)
|
|
3639
|
+
def getsubprotocol(self):
|
|
3640
|
+
if self.handshake_response:
|
|
3641
|
+
return self.handshake_response.subprotocol
|
|
3642
|
+
else:
|
|
3643
|
+
return None
|
|
3644
|
+
subprotocol = property(getsubprotocol)
|
|
3645
|
+
def getstatus(self):
|
|
3646
|
+
if self.handshake_response:
|
|
3647
|
+
return self.handshake_response.status
|
|
3648
|
+
else:
|
|
3649
|
+
return None
|
|
3650
|
+
status = property(getstatus)
|
|
3651
|
+
def getheaders(self):
|
|
3652
|
+
if self.handshake_response:
|
|
3653
|
+
return self.handshake_response.headers
|
|
3654
|
+
else:
|
|
3655
|
+
return None
|
|
3656
|
+
def is_ssl(self):
|
|
3657
|
+
try:
|
|
3658
|
+
return isinstance(self.sock, ssl.SSLSocket)
|
|
3659
|
+
except:
|
|
3660
|
+
return False
|
|
3661
|
+
headers = property(getheaders)
|
|
3662
|
+
def connect(self, url, **options):
|
|
3663
|
+
self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
|
|
3664
|
+
(self.sock, addrs) = connect(url, self.sock_opt, proxy_info(**options), options.pop('socket', None))
|
|
3665
|
+
try:
|
|
3666
|
+
self.handshake_response = handshake(self.sock, url, *addrs, **options)
|
|
3667
|
+
for _ in range(options.pop('redirect_limit', 3)):
|
|
3668
|
+
if self.handshake_response.status in SUPPORTED_REDIRECT_STATUSES:
|
|
3669
|
+
url = self.handshake_response.headers['location']
|
|
3670
|
+
self.sock.close()
|
|
3671
|
+
(self.sock, addrs) = connect(url, self.sock_opt, proxy_info(**options), options.pop('socket', None))
|
|
3672
|
+
self.handshake_response = handshake(self.sock, url, *addrs, **options)
|
|
3673
|
+
self.connected = True
|
|
3674
|
+
except:
|
|
3675
|
+
if self.sock:
|
|
3676
|
+
self.sock.close()
|
|
3677
|
+
self.sock = None
|
|
3678
|
+
raise
|
|
3679
|
+
def send(self, payload: Union[bytes, str], opcode: int=ABNF.OPCODE_TEXT) -> int:
|
|
3680
|
+
frame = ABNF.create_frame(payload, opcode)
|
|
3681
|
+
return self.send_frame(frame)
|
|
3682
|
+
def send_text(self, text_data: str) -> int:
|
|
3683
|
+
return self.send(text_data, ABNF.OPCODE_TEXT)
|
|
3684
|
+
def send_bytes(self, data: Union[bytes, bytearray]) -> int:
|
|
3685
|
+
return self.send(data, ABNF.OPCODE_BINARY)
|
|
3686
|
+
def send_frame(self, frame) -> int:
|
|
3687
|
+
if self.get_mask_key:
|
|
3688
|
+
frame.get_mask_key = self.get_mask_key
|
|
3689
|
+
data = frame.format()
|
|
3690
|
+
length = len(data)
|
|
3691
|
+
if isEnabledForTrace():
|
|
3692
|
+
trace(f'++Sent raw: {repr(data)}')
|
|
3693
|
+
trace(f'++Sent decoded: {frame.__str__()}')
|
|
3694
|
+
with self.lock:
|
|
3695
|
+
while data:
|
|
3696
|
+
l = self._send(data)
|
|
3697
|
+
data = data[l:]
|
|
3698
|
+
return length
|
|
3699
|
+
def send_binary(self, payload: bytes) -> int:
|
|
3700
|
+
return self.send(payload, ABNF.OPCODE_BINARY)
|
|
3701
|
+
def ping(self, payload: Union[str, bytes]=''):
|
|
3702
|
+
if isinstance(payload, str):
|
|
3703
|
+
payload = payload.encode('utf-8')
|
|
3704
|
+
self.send(payload, ABNF.OPCODE_PING)
|
|
3705
|
+
def pong(self, payload: Union[str, bytes]=''):
|
|
3706
|
+
if isinstance(payload, str):
|
|
3707
|
+
payload = payload.encode('utf-8')
|
|
3708
|
+
self.send(payload, ABNF.OPCODE_PONG)
|
|
3709
|
+
def recv(self) -> Union[str, bytes]:
|
|
3710
|
+
with self.readlock:
|
|
3711
|
+
(opcode, data) = self.recv_data()
|
|
3712
|
+
if opcode == ABNF.OPCODE_TEXT:
|
|
3713
|
+
data_received: Union[bytes, str] = data
|
|
3714
|
+
if isinstance(data_received, bytes):
|
|
3715
|
+
return data_received.decode('utf-8')
|
|
3716
|
+
elif isinstance(data_received, str):
|
|
3717
|
+
return data_received
|
|
3718
|
+
elif opcode == ABNF.OPCODE_BINARY:
|
|
3719
|
+
data_binary: bytes = data
|
|
3720
|
+
return data_binary
|
|
3721
|
+
else:
|
|
3722
|
+
return ''
|
|
3723
|
+
def recv_data(self, control_frame: bool=False) -> tuple:
|
|
3724
|
+
(opcode, frame) = self.recv_data_frame(control_frame)
|
|
3725
|
+
return (opcode, frame.data)
|
|
3726
|
+
def recv_data_frame(self, control_frame: bool=False) -> tuple:
|
|
3727
|
+
while True:
|
|
3728
|
+
frame = self.recv_frame()
|
|
3729
|
+
if isEnabledForTrace():
|
|
3730
|
+
trace(f'++Rcv raw: {repr(frame.format())}')
|
|
3731
|
+
trace(f'++Rcv decoded: {frame.__str__()}')
|
|
3732
|
+
if not frame:
|
|
3733
|
+
raise WebSocketProtocolException(f'Not a valid frame {frame}')
|
|
3734
|
+
elif frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY, ABNF.OPCODE_CONT):
|
|
3735
|
+
self.cont_frame.validate(frame)
|
|
3736
|
+
self.cont_frame.add(frame)
|
|
3737
|
+
if self.cont_frame.is_fire(frame):
|
|
3738
|
+
return self.cont_frame.extract(frame)
|
|
3739
|
+
elif frame.opcode == ABNF.OPCODE_CLOSE:
|
|
3740
|
+
self.send_close()
|
|
3741
|
+
return (frame.opcode, frame)
|
|
3742
|
+
elif frame.opcode == ABNF.OPCODE_PING:
|
|
3743
|
+
if len(frame.data) < 126:
|
|
3744
|
+
self.pong(frame.data)
|
|
3745
|
+
else:
|
|
3746
|
+
raise WebSocketProtocolException('Ping message is too long')
|
|
3747
|
+
if control_frame:
|
|
3748
|
+
return (frame.opcode, frame)
|
|
3749
|
+
elif frame.opcode == ABNF.OPCODE_PONG:
|
|
3750
|
+
if control_frame:
|
|
3751
|
+
return (frame.opcode, frame)
|
|
3752
|
+
def recv_frame(self):
|
|
3753
|
+
return self.frame_buffer.recv_frame()
|
|
3754
|
+
def send_close(self, status: int=STATUS_NORMAL, reason: bytes=b''):
|
|
3755
|
+
if status < 0 or status >= ABNF.LENGTH_16:
|
|
3756
|
+
raise ValueError('code is invalid range')
|
|
3757
|
+
self.connected = False
|
|
3758
|
+
self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE)
|
|
3759
|
+
def close(self, status: int=STATUS_NORMAL, reason: bytes=b'', timeout: int=3):
|
|
3760
|
+
if not self.connected:
|
|
3761
|
+
return
|
|
3762
|
+
if status < 0 or status >= ABNF.LENGTH_16:
|
|
3763
|
+
raise ValueError('code is invalid range')
|
|
3764
|
+
try:
|
|
3765
|
+
self.connected = False
|
|
3766
|
+
self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE)
|
|
3767
|
+
sock_timeout = self.sock.gettimeout()
|
|
3768
|
+
self.sock.settimeout(timeout)
|
|
3769
|
+
start_time = time.time()
|
|
3770
|
+
while timeout is None or time.time() - start_time < timeout:
|
|
3771
|
+
try:
|
|
3772
|
+
frame = self.recv_frame()
|
|
3773
|
+
if frame.opcode != ABNF.OPCODE_CLOSE:
|
|
3774
|
+
continue
|
|
3775
|
+
if isEnabledForError():
|
|
3776
|
+
recv_status = struct.unpack('!H', frame.data[0:2])[0]
|
|
3777
|
+
if recv_status >= 3000 and recv_status <= 4999:
|
|
3778
|
+
debug(f'close status: {repr(recv_status)}')
|
|
3779
|
+
elif recv_status != STATUS_NORMAL:
|
|
3780
|
+
error(f'close status: {repr(recv_status)}')
|
|
3781
|
+
break
|
|
3782
|
+
except:
|
|
3783
|
+
break
|
|
3784
|
+
self.sock.settimeout(sock_timeout)
|
|
3785
|
+
self.sock.shutdown(socket.SHUT_RDWR)
|
|
3786
|
+
except:pass
|
|
3787
|
+
self.shutdown()
|
|
3788
|
+
def abort(self):
|
|
3789
|
+
if self.connected:
|
|
3790
|
+
self.sock.shutdown(socket.SHUT_RDWR)
|
|
3791
|
+
def shutdown(self):
|
|
3792
|
+
if self.sock:
|
|
3793
|
+
self.sock.close()
|
|
3794
|
+
self.sock = None
|
|
3795
|
+
self.connected = False
|
|
3796
|
+
def _send(self, data: Union[str, bytes]):
|
|
3797
|
+
return send(self.sock, data)
|
|
3798
|
+
def _recv(self, bufsize):
|
|
3799
|
+
try:
|
|
3800
|
+
return recv(self.sock, bufsize)
|
|
3801
|
+
except WebSocketConnectionClosedException:
|
|
3802
|
+
if self.sock:
|
|
3803
|
+
self.sock.close()
|
|
3804
|
+
self.sock = None
|
|
3805
|
+
self.connected = False
|
|
3806
|
+
raise
|
|
3807
|
+
def create_connection(url: str, timeout=None, class_=WebSocket, **options):
|
|
3808
|
+
sockopt = options.pop('sockopt', [])
|
|
3809
|
+
sslopt = options.pop('sslopt', {})
|
|
3810
|
+
fire_cont_frame = options.pop('fire_cont_frame', False)
|
|
3811
|
+
enable_multithread = options.pop('enable_multithread', True)
|
|
3812
|
+
skip_utf8_validation = options.pop('skip_utf8_validation', False)
|
|
3813
|
+
websock = class_(sockopt=sockopt, sslopt=sslopt, fire_cont_frame=fire_cont_frame, enable_multithread=enable_multithread, skip_utf8_validation=skip_utf8_validation, **options)
|
|
3814
|
+
websock.settimeout(timeout if timeout is not None else getdefaulttimeout())
|
|
3815
|
+
websock.connect(url, **options)
|
|
3816
|
+
return websock
|
|
3817
|
+
RECONNECT = 0
|
|
3818
|
+
def setReconnect(reconnectInterval: int) -> None:
|
|
3819
|
+
global RECONNECT
|
|
3820
|
+
RECONNECT = reconnectInterval
|
|
3821
|
+
class DispatcherBase:
|
|
3822
|
+
def __init__(self, app: Any, ping_timeout: Union[float, int, None]) -> None:
|
|
3823
|
+
self.app = app
|
|
3824
|
+
self.ping_timeout = ping_timeout
|
|
3825
|
+
def timeout(self, seconds: Union[float, int, None], callback: Callable) -> None:
|
|
3826
|
+
time.sleep(seconds)
|
|
3827
|
+
callback()
|
|
3828
|
+
def reconnect(self, seconds: int, reconnector: Callable) -> None:
|
|
3829
|
+
try:
|
|
3830
|
+
_logging.info(f'reconnect() - retrying in {seconds} seconds [{len(inspect.stack())} frames in stack]')
|
|
3831
|
+
time.sleep(seconds)
|
|
3832
|
+
reconnector(reconnecting=True)
|
|
3833
|
+
except KeyboardInterrupt as e:
|
|
3834
|
+
_logging.info(f'User exited {e}')
|
|
3835
|
+
raise e
|
|
3836
|
+
class Dispatcher(DispatcherBase):
|
|
3837
|
+
def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
|
|
3838
|
+
sel = selectors.DefaultSelector()
|
|
3839
|
+
sel.register(self.app.sock.sock, selectors.EVENT_READ)
|
|
3840
|
+
try:
|
|
3841
|
+
while self.app.keep_running:
|
|
3842
|
+
if sel.select(self.ping_timeout):
|
|
3843
|
+
if not read_callback():
|
|
3844
|
+
break
|
|
3845
|
+
check_callback()
|
|
3846
|
+
finally:
|
|
3847
|
+
sel.close()
|
|
3848
|
+
class SSLDispatcher(DispatcherBase):
|
|
3849
|
+
def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
|
|
3850
|
+
sock = self.app.sock.sock
|
|
3851
|
+
sel = selectors.DefaultSelector()
|
|
3852
|
+
sel.register(sock, selectors.EVENT_READ)
|
|
3853
|
+
try:
|
|
3854
|
+
while self.app.keep_running:
|
|
3855
|
+
if self.select(sock, sel):
|
|
3856
|
+
if not read_callback():
|
|
3857
|
+
break
|
|
3858
|
+
check_callback()
|
|
3859
|
+
finally:
|
|
3860
|
+
sel.close()
|
|
3861
|
+
def select(self, sock, sel: selectors.DefaultSelector):
|
|
3862
|
+
sock = self.app.sock.sock
|
|
3863
|
+
if sock.pending():
|
|
3864
|
+
return [sock]
|
|
3865
|
+
r = sel.select(self.ping_timeout)
|
|
3866
|
+
if len(r) > 0:
|
|
3867
|
+
return r[0][0]
|
|
3868
|
+
class WrappedDispatcher:
|
|
3869
|
+
def __init__(self, app, ping_timeout: Union[float, int, None], dispatcher) -> None:
|
|
3870
|
+
self.app = app
|
|
3871
|
+
self.ping_timeout = ping_timeout
|
|
3872
|
+
self.dispatcher = dispatcher
|
|
3873
|
+
dispatcher.signal(2, dispatcher.abort)
|
|
3874
|
+
def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
|
|
3875
|
+
self.dispatcher.read(sock, read_callback)
|
|
3876
|
+
self.ping_timeout and self.timeout(self.ping_timeout, check_callback)
|
|
3877
|
+
def timeout(self, seconds: float, callback: Callable) -> None:
|
|
3878
|
+
self.dispatcher.timeout(seconds, callback)
|
|
3879
|
+
def reconnect(self, seconds: int, reconnector: Callable) -> None:
|
|
3880
|
+
self.timeout(seconds, reconnector)
|
|
3881
|
+
class WebSocketApp:
|
|
3882
|
+
def __init__(self, url: str, header: Union[list, dict, Callable, None]=None, on_open: Optional[Callable[[WebSocket], None]]=None, on_reconnect: Optional[Callable[[WebSocket], None]]=None, on_message: Optional[Callable[[WebSocket, Any], None]]=None, on_error: Optional[Callable[[WebSocket, Any], None]]=None, on_close: Optional[Callable[[WebSocket, Any, Any], None]]=None, on_ping: Optional[Callable]=None, on_pong: Optional[Callable]=None, on_cont_message: Optional[Callable]=None, keep_running: bool=True, get_mask_key: Optional[Callable]=None, cookie: Optional[str]=None, subprotocols: Optional[list]=None, on_data: Optional[Callable]=None, socket: Optional[socket.socket]=None) -> None:
|
|
3883
|
+
self.url = url
|
|
3884
|
+
self.header = header if header is not None else []
|
|
3885
|
+
self.cookie = cookie
|
|
3886
|
+
self.on_open = on_open
|
|
3887
|
+
self.on_reconnect = on_reconnect
|
|
3888
|
+
self.on_message = on_message
|
|
3889
|
+
self.on_data = on_data
|
|
3890
|
+
self.on_error = on_error
|
|
3891
|
+
self.on_close = on_close
|
|
3892
|
+
self.on_ping = on_ping
|
|
3893
|
+
self.on_pong = on_pong
|
|
3894
|
+
self.on_cont_message = on_cont_message
|
|
3895
|
+
self.keep_running = False
|
|
3896
|
+
self.get_mask_key = get_mask_key
|
|
3897
|
+
self.sock: Optional[WebSocket] = None
|
|
3898
|
+
self.last_ping_tm = float(0)
|
|
3899
|
+
self.last_pong_tm = float(0)
|
|
3900
|
+
self.ping_thread: Optional[threading.Thread] = None
|
|
3901
|
+
self.stop_ping: Optional[threading.Event] = None
|
|
3902
|
+
self.ping_interval = float(0)
|
|
3903
|
+
self.ping_timeout: Union[float, int, None] = None
|
|
3904
|
+
self.ping_payload = ''
|
|
3905
|
+
self.subprotocols = subprotocols
|
|
3906
|
+
self.prepared_socket = socket
|
|
3907
|
+
self.has_errored = False
|
|
3908
|
+
self.has_done_teardown = False
|
|
3909
|
+
self.has_done_teardown_lock = threading.Lock()
|
|
3910
|
+
def send(self, data: Union[bytes, str], opcode: int=ABNF.OPCODE_TEXT) -> None:
|
|
3911
|
+
if not self.sock or self.sock.send(data, opcode) == 0:
|
|
3912
|
+
raise WebSocketConnectionClosedException('Connection is already closed.')
|
|
3913
|
+
def send_text(self, text_data: str) -> None:
|
|
3914
|
+
if not self.sock or self.sock.send(text_data, ABNF.OPCODE_TEXT) == 0:
|
|
3915
|
+
raise WebSocketConnectionClosedException('Connection is already closed.')
|
|
3916
|
+
def send_bytes(self, data: Union[bytes, bytearray]) -> None:
|
|
3917
|
+
if not self.sock or self.sock.send(data, ABNF.OPCODE_BINARY) == 0:
|
|
3918
|
+
raise WebSocketConnectionClosedException('Connection is already closed.')
|
|
3919
|
+
def close(self, **kwargs) -> None:
|
|
3920
|
+
self.keep_running = False
|
|
3921
|
+
if self.sock:
|
|
3922
|
+
self.sock.close(**kwargs)
|
|
3923
|
+
self.sock = None
|
|
3924
|
+
def _start_ping_thread(self) -> None:
|
|
3925
|
+
self.last_ping_tm = self.last_pong_tm = float(0)
|
|
3926
|
+
self.stop_ping = threading.Event()
|
|
3927
|
+
self.ping_thread = threading.Thread(target=self._send_ping)
|
|
3928
|
+
self.ping_thread.daemon = True
|
|
3929
|
+
self.ping_thread.start()
|
|
3930
|
+
def _stop_ping_thread(self) -> None:
|
|
3931
|
+
if self.stop_ping:
|
|
3932
|
+
self.stop_ping.set()
|
|
3933
|
+
if self.ping_thread and self.ping_thread.is_alive():
|
|
3934
|
+
self.ping_thread.join(3)
|
|
3935
|
+
self.last_ping_tm = self.last_pong_tm = float(0)
|
|
3936
|
+
def _send_ping(self) -> None:
|
|
3937
|
+
if self.stop_ping.wait(self.ping_interval) or self.keep_running is False:
|
|
3938
|
+
return
|
|
3939
|
+
while not self.stop_ping.wait(self.ping_interval) and self.keep_running is True:
|
|
3940
|
+
if self.sock:
|
|
3941
|
+
self.last_ping_tm = time.time()
|
|
3942
|
+
try:
|
|
3943
|
+
_logging.debug('Sending ping')
|
|
3944
|
+
self.sock.ping(self.ping_payload)
|
|
3945
|
+
except Exception as e:
|
|
3946
|
+
_logging.debug(f'Failed to send ping: {e}')
|
|
3947
|
+
def run_forever(self, sockopt: tuple=None, sslopt: dict=None, ping_interval: Union[float, int]=0, ping_timeout: Union[float, int, None]=None, ping_payload: str='', http_proxy_host: str=None, http_proxy_port: Union[int, str]=None, http_no_proxy: list=None, http_proxy_auth: tuple=None, http_proxy_timeout: Optional[float]=None, skip_utf8_validation: bool=False, host: str=None, origin: str=None, dispatcher=None, suppress_origin: bool=False, proxy_type: str=None, reconnect: int=None) -> bool:
|
|
3948
|
+
if reconnect is None:
|
|
3949
|
+
reconnect = RECONNECT
|
|
3950
|
+
if ping_timeout is not None and ping_timeout <= 0:
|
|
3951
|
+
raise WebSocketException('Ensure ping_timeout > 0')
|
|
3952
|
+
if ping_interval is not None and ping_interval < 0:
|
|
3953
|
+
raise WebSocketException('Ensure ping_interval >= 0')
|
|
3954
|
+
if ping_timeout and ping_interval and (ping_interval <= ping_timeout):
|
|
3955
|
+
raise WebSocketException('Ensure ping_interval > ping_timeout')
|
|
3956
|
+
if not sockopt:
|
|
3957
|
+
sockopt = ()
|
|
3958
|
+
if not sslopt:
|
|
3959
|
+
sslopt = {}
|
|
3960
|
+
if self.sock:
|
|
3961
|
+
raise WebSocketException('socket is already opened')
|
|
3962
|
+
self.ping_interval = ping_interval
|
|
3963
|
+
self.ping_timeout = ping_timeout
|
|
3964
|
+
self.ping_payload = ping_payload
|
|
3965
|
+
self.has_done_teardown = False
|
|
3966
|
+
self.keep_running = True
|
|
3967
|
+
def teardown(close_frame: ABNF=None):
|
|
3968
|
+
with self.has_done_teardown_lock:
|
|
3969
|
+
if self.has_done_teardown:
|
|
3970
|
+
return
|
|
3971
|
+
self.has_done_teardown = True
|
|
3972
|
+
self._stop_ping_thread()
|
|
3973
|
+
self.keep_running = False
|
|
3974
|
+
if self.sock:
|
|
3975
|
+
self.sock.close()
|
|
3976
|
+
(close_status_code, close_reason) = self._get_close_args(close_frame if close_frame else None)
|
|
3977
|
+
self.sock = None
|
|
3978
|
+
self._callback(self.on_close, close_status_code, close_reason)
|
|
3979
|
+
def setSock(reconnecting: bool=False) -> None:
|
|
3980
|
+
if reconnecting and self.sock:
|
|
3981
|
+
self.sock.shutdown()
|
|
3982
|
+
self.sock = WebSocket(self.get_mask_key, sockopt=sockopt, sslopt=sslopt, fire_cont_frame=self.on_cont_message is not None, skip_utf8_validation=skip_utf8_validation, enable_multithread=True)
|
|
3983
|
+
self.sock.settimeout(getdefaulttimeout())
|
|
3984
|
+
try:
|
|
3985
|
+
header = self.header() if callable(self.header) else self.header
|
|
3986
|
+
self.sock.connect(self.url, header=header, cookie=self.cookie, http_proxy_host=http_proxy_host, http_proxy_port=http_proxy_port, http_no_proxy=http_no_proxy, http_proxy_auth=http_proxy_auth, http_proxy_timeout=http_proxy_timeout, subprotocols=self.subprotocols, host=host, origin=origin, suppress_origin=suppress_origin, proxy_type=proxy_type, socket=self.prepared_socket)
|
|
3987
|
+
_logging.info('Websocket connected')
|
|
3988
|
+
if self.ping_interval:
|
|
3989
|
+
self._start_ping_thread()
|
|
3990
|
+
if reconnecting and self.on_reconnect:
|
|
3991
|
+
self._callback(self.on_reconnect)
|
|
3992
|
+
else:
|
|
3993
|
+
self._callback(self.on_open)
|
|
3994
|
+
dispatcher.read(self.sock.sock, read, check)
|
|
3995
|
+
except (WebSocketConnectionClosedException, ConnectionRefusedError, KeyboardInterrupt, SystemExit, Exception) as e:
|
|
3996
|
+
handleDisconnect(e, reconnecting)
|
|
3997
|
+
def read() -> bool:
|
|
3998
|
+
if not self.keep_running:
|
|
3999
|
+
return teardown()
|
|
4000
|
+
try:
|
|
4001
|
+
(op_code, frame) = self.sock.recv_data_frame(True)
|
|
4002
|
+
except (WebSocketConnectionClosedException, KeyboardInterrupt, SSLEOFError) as e:
|
|
4003
|
+
if custom_dispatcher:
|
|
4004
|
+
return handleDisconnect(e, bool(reconnect))
|
|
4005
|
+
else:
|
|
4006
|
+
raise e
|
|
4007
|
+
if op_code == ABNF.OPCODE_CLOSE:
|
|
4008
|
+
return teardown(frame)
|
|
4009
|
+
elif op_code == ABNF.OPCODE_PING:
|
|
4010
|
+
self._callback(self.on_ping, frame.data)
|
|
4011
|
+
elif op_code == ABNF.OPCODE_PONG:
|
|
4012
|
+
self.last_pong_tm = time.time()
|
|
4013
|
+
self._callback(self.on_pong, frame.data)
|
|
4014
|
+
elif op_code == ABNF.OPCODE_CONT and self.on_cont_message:
|
|
4015
|
+
self._callback(self.on_data, frame.data, frame.opcode, frame.fin)
|
|
4016
|
+
self._callback(self.on_cont_message, frame.data, frame.fin)
|
|
4017
|
+
else:
|
|
4018
|
+
data = frame.data
|
|
4019
|
+
if op_code == ABNF.OPCODE_TEXT and (not skip_utf8_validation):
|
|
4020
|
+
data = data.decode('utf-8')
|
|
4021
|
+
self._callback(self.on_data, data, frame.opcode, True)
|
|
4022
|
+
self._callback(self.on_message, data)
|
|
4023
|
+
return True
|
|
4024
|
+
def check() -> bool:
|
|
4025
|
+
if self.ping_timeout:
|
|
4026
|
+
has_timeout_expired = time.time() - self.last_ping_tm > self.ping_timeout
|
|
4027
|
+
has_pong_not_arrived_after_last_ping = self.last_pong_tm - self.last_ping_tm < 0
|
|
4028
|
+
has_pong_arrived_too_late = self.last_pong_tm - self.last_ping_tm > self.ping_timeout
|
|
4029
|
+
if self.last_ping_tm and has_timeout_expired and (has_pong_not_arrived_after_last_ping or has_pong_arrived_too_late):
|
|
4030
|
+
raise WebSocketTimeoutException('ping/pong timed out')
|
|
4031
|
+
return True
|
|
4032
|
+
def handleDisconnect(e: Union[WebSocketConnectionClosedException, ConnectionRefusedError, KeyboardInterrupt, SystemExit, Exception], reconnecting: bool=False) -> bool:
|
|
4033
|
+
self.has_errored = True
|
|
4034
|
+
self._stop_ping_thread()
|
|
4035
|
+
if not reconnecting:
|
|
4036
|
+
self._callback(self.on_error, e)
|
|
4037
|
+
if isinstance(e, (KeyboardInterrupt, SystemExit)):
|
|
4038
|
+
teardown()
|
|
4039
|
+
raise
|
|
4040
|
+
if reconnect:
|
|
4041
|
+
_logging.info(f'{e} - reconnect')
|
|
4042
|
+
if custom_dispatcher:
|
|
4043
|
+
_logging.debug(f'Calling custom dispatcher reconnect [{len(inspect.stack())} frames in stack]')
|
|
4044
|
+
dispatcher.reconnect(reconnect, setSock)
|
|
4045
|
+
else:
|
|
4046
|
+
_logging.error(f'{e} - goodbye')
|
|
4047
|
+
teardown()
|
|
4048
|
+
custom_dispatcher = bool(dispatcher)
|
|
4049
|
+
dispatcher = self.create_dispatcher(ping_timeout, dispatcher, parse_url(self.url)[3])
|
|
4050
|
+
try:
|
|
4051
|
+
setSock()
|
|
4052
|
+
if not custom_dispatcher and reconnect:
|
|
4053
|
+
while self.keep_running:
|
|
4054
|
+
_logging.debug(f'Calling dispatcher reconnect [{len(inspect.stack())} frames in stack]')
|
|
4055
|
+
dispatcher.reconnect(reconnect, setSock)
|
|
4056
|
+
except (KeyboardInterrupt, Exception) as e:
|
|
4057
|
+
_logging.info(f'tearing down on exception {e}')
|
|
4058
|
+
teardown()
|
|
4059
|
+
finally:
|
|
4060
|
+
if not custom_dispatcher:
|
|
4061
|
+
teardown()
|
|
4062
|
+
return self.has_errored
|
|
4063
|
+
def create_dispatcher(self, ping_timeout: Union[float, int, None], dispatcher: Optional[DispatcherBase]=None, is_ssl: bool=False) -> Union[Dispatcher, SSLDispatcher, WrappedDispatcher]:
|
|
4064
|
+
if dispatcher:
|
|
4065
|
+
return WrappedDispatcher(self, ping_timeout, dispatcher)
|
|
4066
|
+
timeout = ping_timeout or 10
|
|
4067
|
+
if is_ssl:
|
|
4068
|
+
return SSLDispatcher(self, timeout)
|
|
4069
|
+
return Dispatcher(self, timeout)
|
|
4070
|
+
def _get_close_args(self, close_frame: ABNF) -> list:
|
|
4071
|
+
if not self.on_close or not close_frame:
|
|
4072
|
+
return [None, None]
|
|
4073
|
+
if close_frame.data and len(close_frame.data) >= 2:
|
|
4074
|
+
close_status_code = 256 * int(close_frame.data[0]) + int(close_frame.data[1])
|
|
4075
|
+
reason = close_frame.data[2:]
|
|
4076
|
+
if isinstance(reason, bytes):
|
|
4077
|
+
reason = reason.decode('utf-8')
|
|
4078
|
+
return [close_status_code, reason]
|
|
4079
|
+
else:
|
|
4080
|
+
return [None, None]
|
|
4081
|
+
def _callback(self, callback, *args) -> None:
|
|
4082
|
+
if callback:
|
|
4083
|
+
try:
|
|
4084
|
+
callback(self, *args)
|
|
4085
|
+
except Exception as e:
|
|
4086
|
+
_logging.error(f'error from callback {callback}: {e}')
|
|
4087
|
+
if self.on_error:
|
|
4088
|
+
self.on_error(self, e)
|