kkutils 1.8.2__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kkutils-1.8.2 → kkutils-2.0.0}/PKG-INFO +1 -1
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/base_utils.py +24 -13
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/config_utils.py +4 -2
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/crypto.py +11 -1
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/db_utils.py +57 -29
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/email_utils.py +35 -25
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/http_utils.py +15 -7
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/rabbitmq.py +64 -36
- {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/PKG-INFO +1 -1
- kkutils-2.0.0/kkutils.egg-info/SOURCES.txt +29 -0
- {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/top_level.txt +1 -1
- {kkutils-1.8.2 → kkutils-2.0.0}/processor.py +1 -1
- {kkutils-1.8.2 → kkutils-2.0.0}/setup.py +2 -2
- {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/application.py +1 -1
- {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/basehandler.py +6 -3
- {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/userhandler.py +8 -1
- {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/utils.py +1 -1
- kkutils-1.8.2/kkutils.egg-info/SOURCES.txt +0 -29
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/__init__.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/curl_utils.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/decorator.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/fire.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/generate_machine_id.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/log_utils.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/stopwatch.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/xdb_searcher.py +0 -0
- {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/xor_file.py +0 -0
- {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/dependency_links.txt +0 -0
- {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/requires.txt +0 -0
- {kkutils-1.8.2 → kkutils-2.0.0}/requirements.txt +0 -0
- {kkutils-1.8.2 → kkutils-2.0.0}/setup.cfg +0 -0
- {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/__init__.py +0 -0
|
@@ -38,9 +38,14 @@ from tqdm import tqdm as _tqdm
|
|
|
38
38
|
def _include_yaml(loader, node):
|
|
39
39
|
root = Path(loader.name).parent
|
|
40
40
|
if isinstance(node, yaml.SequenceNode):
|
|
41
|
-
|
|
41
|
+
result = []
|
|
42
|
+
for x in node.value:
|
|
43
|
+
with open(root / x.value) as f:
|
|
44
|
+
result.append(yaml.load(f, Loader=yaml.FullLoader))
|
|
45
|
+
return result
|
|
42
46
|
else:
|
|
43
|
-
|
|
47
|
+
with open(root / node.value) as f:
|
|
48
|
+
return yaml.load(f, Loader=yaml.FullLoader)
|
|
44
49
|
|
|
45
50
|
|
|
46
51
|
def yaml_load(stream, **kwargs):
|
|
@@ -117,13 +122,14 @@ class Cache:
|
|
|
117
122
|
self.data = {}
|
|
118
123
|
|
|
119
124
|
def get(self, key):
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
with self.lock:
|
|
126
|
+
item = self.data.get(key)
|
|
127
|
+
if item is not None:
|
|
128
|
+
value, ttl = item
|
|
129
|
+
if ttl is None:
|
|
130
|
+
return value
|
|
131
|
+
if ttl >= int(time.time()):
|
|
132
|
+
return value
|
|
127
133
|
|
|
128
134
|
def set(self, key, value, ttl=None):
|
|
129
135
|
with self.lock:
|
|
@@ -132,9 +138,14 @@ class Cache:
|
|
|
132
138
|
|
|
133
139
|
def incr(self, key, value=1):
|
|
134
140
|
with self.lock:
|
|
135
|
-
self.data.
|
|
136
|
-
|
|
137
|
-
|
|
141
|
+
item = self.data.get(key)
|
|
142
|
+
if item is None:
|
|
143
|
+
current = 0
|
|
144
|
+
else:
|
|
145
|
+
current = item[0]
|
|
146
|
+
new_value = current + value
|
|
147
|
+
self.data[key] = new_value, None
|
|
148
|
+
return new_value
|
|
138
149
|
|
|
139
150
|
def delete(self, key):
|
|
140
151
|
with self.lock:
|
|
@@ -148,7 +159,7 @@ class Cache:
|
|
|
148
159
|
|
|
149
160
|
def ttl(self, key, ttl):
|
|
150
161
|
with self.lock:
|
|
151
|
-
item = self.data
|
|
162
|
+
item = self.data.get(key)
|
|
152
163
|
if item is not None:
|
|
153
164
|
self.data[key] = item[0], int(time.time()) + ttl
|
|
154
165
|
|
|
@@ -60,9 +60,11 @@ class Config(Dict):
|
|
|
60
60
|
if config.endswith('.ini'):
|
|
61
61
|
cfg.update(Dict(Config._load_ini(config)))
|
|
62
62
|
elif config.endswith('.json'):
|
|
63
|
-
|
|
63
|
+
with open(config) as f:
|
|
64
|
+
cfg.update(Dict(json.load(f)))
|
|
64
65
|
elif config.endswith('.yaml') or config.endswith('.yml'):
|
|
65
|
-
|
|
66
|
+
with open(config) as f:
|
|
67
|
+
cfg.update(Dict(yaml.load(f, Loader=yaml.FullLoader)))
|
|
66
68
|
|
|
67
69
|
if cfg.extends:
|
|
68
70
|
base = Dict()
|
|
@@ -49,8 +49,18 @@ def decode(text, lib='base62'):
|
|
|
49
49
|
return binascii.a2b_hex(text)
|
|
50
50
|
|
|
51
51
|
|
|
52
|
+
_XOR_TABLES = {}
|
|
53
|
+
|
|
54
|
+
|
|
52
55
|
def xor_encrypt(text: str, key: int, lib='base64'):
|
|
53
|
-
|
|
56
|
+
text = to_str(text)
|
|
57
|
+
if text.isascii() and 0 <= key < 128:
|
|
58
|
+
# Fast path: ASCII xor a 7-bit key stays single-byte, so a byte table gives identical output
|
|
59
|
+
table = _XOR_TABLES.get(key)
|
|
60
|
+
if table is None:
|
|
61
|
+
table = _XOR_TABLES[key] = bytes(i ^ key for i in range(256))
|
|
62
|
+
return encode(text.encode().translate(table), lib)
|
|
63
|
+
encrypted = ''.join([chr(ord(x) ^ ord(chr(key))) for x in text])
|
|
54
64
|
return encode(encrypted, lib)
|
|
55
65
|
|
|
56
66
|
|
|
@@ -5,6 +5,7 @@ Last modified: 2018-01-05 11:24:17
|
|
|
5
5
|
'''
|
|
6
6
|
|
|
7
7
|
import asyncio
|
|
8
|
+
import logging
|
|
8
9
|
import os
|
|
9
10
|
import re
|
|
10
11
|
from urllib.parse import parse_qs, quote_plus, unquote_plus
|
|
@@ -13,6 +14,7 @@ import aiomysql
|
|
|
13
14
|
import pymongo
|
|
14
15
|
import pymysql
|
|
15
16
|
import redis
|
|
17
|
+
import redis.exceptions
|
|
16
18
|
from dbutils.pooled_db import PooledDB
|
|
17
19
|
from motor import core
|
|
18
20
|
from motor.docstrings import get_database_doc
|
|
@@ -124,12 +126,12 @@ class MongoClient(pymongo.MongoClient):
|
|
|
124
126
|
kwargs.setdefault('document_class', Dict)
|
|
125
127
|
kwargs.setdefault('maxPoolSize', 50)
|
|
126
128
|
kwargs.setdefault('minPoolSize', 5)
|
|
127
|
-
kwargs.setdefault('maxIdleTimeMS', 30000)
|
|
128
|
-
kwargs.setdefault('waitQueueTimeoutMS', 10000)
|
|
129
|
-
kwargs.setdefault('serverSelectionTimeoutMS', 10000)
|
|
130
|
-
kwargs.setdefault('connectTimeoutMS', 10000)
|
|
131
|
-
kwargs.setdefault('socketTimeoutMS', 30000)
|
|
132
|
-
kwargs.setdefault('heartbeatFrequencyMS',
|
|
129
|
+
# kwargs.setdefault('maxIdleTimeMS', 30000)
|
|
130
|
+
# kwargs.setdefault('waitQueueTimeoutMS', 10000)
|
|
131
|
+
# kwargs.setdefault('serverSelectionTimeoutMS', 10000)
|
|
132
|
+
# kwargs.setdefault('connectTimeoutMS', 10000)
|
|
133
|
+
# kwargs.setdefault('socketTimeoutMS', 30000)
|
|
134
|
+
# kwargs.setdefault('heartbeatFrequencyMS', 10000)
|
|
133
135
|
kwargs.setdefault('retryWrites', True)
|
|
134
136
|
kwargs.setdefault('retryReads', True)
|
|
135
137
|
super(MongoClient, self).__init__(uri, **kwargs)
|
|
@@ -290,9 +292,26 @@ class AioRedis(aioredis.StrictRedis):
|
|
|
290
292
|
uri = f"redis://:{password}@{host}:{port}/{db}" if password else f"redis://{host}:{port}/{db}"
|
|
291
293
|
|
|
292
294
|
kwargs.setdefault('decode_responses', True)
|
|
295
|
+
kwargs.setdefault('health_check_interval', 30)
|
|
296
|
+
kwargs.setdefault('max_connections', 5000)
|
|
297
|
+
kwargs.setdefault('retry_on_timeout', True)
|
|
298
|
+
kwargs.setdefault('socket_keepalive', True)
|
|
293
299
|
pool = aioredis.ConnectionPool.from_url(uri, **kwargs)
|
|
294
300
|
super().__init__(connection_pool=pool)
|
|
295
301
|
|
|
302
|
+
async def execute_command(self, *args, **kwargs):
|
|
303
|
+
try:
|
|
304
|
+
return await super().execute_command(*args, **kwargs)
|
|
305
|
+
except redis.exceptions.ConnectionError as e:
|
|
306
|
+
pool = self.connection_pool
|
|
307
|
+
logging.getLogger().error(
|
|
308
|
+
f'redis connection error: {e}, cmd: {args[:1]}, '
|
|
309
|
+
f'in_use: {len(pool._in_use_connections)}, '
|
|
310
|
+
f'available: {len(pool._available_connections)}, '
|
|
311
|
+
f'max: {pool.max_connections}, pid: {os.getpid()}'
|
|
312
|
+
)
|
|
313
|
+
raise
|
|
314
|
+
|
|
296
315
|
async def clear(self, pattern='*'):
|
|
297
316
|
if pattern == '*':
|
|
298
317
|
await self.flushdb()
|
|
@@ -385,17 +404,15 @@ class AioMysql(Mysql):
|
|
|
385
404
|
|
|
386
405
|
def __init__(self, *args, **kwargs):
|
|
387
406
|
super().__init__(*args, **kwargs)
|
|
388
|
-
self._lock = asyncio.Lock()
|
|
389
|
-
|
|
390
|
-
async def _execute(self, cursor, sql, args):
|
|
391
|
-
async with self._lock:
|
|
392
|
-
await cursor.execute(sql, args)
|
|
393
407
|
|
|
394
|
-
async def
|
|
408
|
+
async def _get_conn(self):
|
|
395
409
|
if self._pool is None:
|
|
396
410
|
self._pool = await aiomysql.create_pool(minsize=0, maxsize=10, **self._kwargs)
|
|
397
|
-
|
|
398
|
-
|
|
411
|
+
return await self._pool.acquire()
|
|
412
|
+
|
|
413
|
+
async def _release_conn(self, conn):
|
|
414
|
+
if self._pool is not None:
|
|
415
|
+
await self._pool.release(conn)
|
|
399
416
|
|
|
400
417
|
async def close(self):
|
|
401
418
|
if self._pool is not None:
|
|
@@ -404,53 +421,64 @@ class AioMysql(Mysql):
|
|
|
404
421
|
self._pool = None
|
|
405
422
|
|
|
406
423
|
async def iter(self, sql, *args):
|
|
407
|
-
|
|
424
|
+
conn = await self._get_conn()
|
|
425
|
+
cursor = await conn.cursor(aiomysql.cursors.SSDictCursor)
|
|
408
426
|
try:
|
|
409
|
-
await
|
|
427
|
+
await cursor.execute(sql, args)
|
|
410
428
|
async for row in cursor:
|
|
411
429
|
yield Dict(row)
|
|
412
430
|
finally:
|
|
413
431
|
await cursor.close()
|
|
432
|
+
await self._release_conn(conn)
|
|
414
433
|
|
|
415
434
|
async def query(self, sql, *args):
|
|
416
|
-
|
|
435
|
+
conn = await self._get_conn()
|
|
436
|
+
cursor = await conn.cursor(aiomysql.DictCursor)
|
|
417
437
|
try:
|
|
418
|
-
await
|
|
438
|
+
await cursor.execute(sql, args)
|
|
419
439
|
rows = await cursor.fetchall()
|
|
420
440
|
return [Dict(row) for row in rows]
|
|
421
441
|
finally:
|
|
422
442
|
await cursor.close()
|
|
443
|
+
await self._release_conn(conn)
|
|
423
444
|
|
|
424
445
|
async def query_td(self, sql, *args):
|
|
425
|
-
|
|
446
|
+
conn = await self._get_conn()
|
|
447
|
+
cursor = await conn.cursor()
|
|
426
448
|
try:
|
|
427
|
-
await
|
|
449
|
+
await cursor.execute(sql, args)
|
|
428
450
|
rows = await cursor.fetchall()
|
|
429
|
-
return {'th': [col[0] for col in cursor.
|
|
451
|
+
return {'th': [col[0] for col in cursor.description], 'td': rows}
|
|
430
452
|
finally:
|
|
431
453
|
await cursor.close()
|
|
454
|
+
await self._release_conn(conn)
|
|
432
455
|
|
|
433
456
|
async def get(self, sql, *args):
|
|
434
|
-
|
|
457
|
+
conn = await self._get_conn()
|
|
458
|
+
cursor = await conn.cursor(aiomysql.DictCursor)
|
|
435
459
|
try:
|
|
436
|
-
await
|
|
460
|
+
await cursor.execute(sql, args)
|
|
437
461
|
row = await cursor.fetchone()
|
|
438
462
|
return Dict(row) if row else None
|
|
439
463
|
finally:
|
|
440
464
|
await cursor.close()
|
|
465
|
+
await self._release_conn(conn)
|
|
441
466
|
|
|
442
467
|
async def execute(self, sql, *args):
|
|
443
|
-
|
|
468
|
+
conn = await self._get_conn()
|
|
469
|
+
cursor = await conn.cursor()
|
|
444
470
|
try:
|
|
445
|
-
await
|
|
446
|
-
return cursor.
|
|
471
|
+
await cursor.execute(sql, args)
|
|
472
|
+
return cursor.lastrowid
|
|
447
473
|
finally:
|
|
448
474
|
await cursor.close()
|
|
475
|
+
await self._release_conn(conn)
|
|
449
476
|
|
|
450
477
|
async def executemany(self, sql, args):
|
|
451
|
-
|
|
478
|
+
conn = await self._get_conn()
|
|
479
|
+
cursor = await conn.cursor()
|
|
452
480
|
try:
|
|
453
|
-
|
|
454
|
-
return await cursor.executemany(sql, args)
|
|
481
|
+
await cursor.executemany(sql, args)
|
|
455
482
|
finally:
|
|
456
483
|
await cursor.close()
|
|
484
|
+
await self._release_conn(conn)
|
|
@@ -17,15 +17,30 @@ import aiosmtplib
|
|
|
17
17
|
|
|
18
18
|
__all__ = ['Email', 'AioEmail']
|
|
19
19
|
|
|
20
|
+
# Hard cap on the connect/login/send/quit cycle. Without this, a stalled or
|
|
21
|
+
# unresponsive MTA hangs the caller indefinitely — smtplib's default is no
|
|
22
|
+
# timeout at all.
|
|
23
|
+
DEFAULT_TIMEOUT = 30
|
|
24
|
+
|
|
20
25
|
|
|
21
26
|
class EmailBase:
|
|
27
|
+
"""Shared message-building logic for ``Email``/``AioEmail``.
|
|
28
|
+
|
|
29
|
+
Subclasses build a fresh SMTP client on every ``send()`` call rather
|
|
30
|
+
than reusing one across the instance's lifetime — a stale/half-closed
|
|
31
|
+
socket or half-completed auth left over from a prior failure can then
|
|
32
|
+
never leak into the next send. Each call gets a clean
|
|
33
|
+
connect/login/send/quit cycle, with ``with``/``async with`` guaranteeing
|
|
34
|
+
cleanup even on error.
|
|
35
|
+
"""
|
|
22
36
|
|
|
23
|
-
def __init__(self, sender=None, smtp=None, user=None, pwd=None, use_tls=False):
|
|
37
|
+
def __init__(self, sender=None, smtp=None, user=None, pwd=None, use_tls=False, timeout=DEFAULT_TIMEOUT):
|
|
24
38
|
self.sender = sender or os.environ.get('EMAIL_SENDER')
|
|
25
39
|
self.smtp = smtp or os.environ.get('EMAIL_SMTP')
|
|
26
40
|
self.user = user or os.environ.get('EMAIL_USER')
|
|
27
41
|
self.pwd = pwd or os.environ.get('EMAIL_PWD')
|
|
28
42
|
self.use_tls = use_tls or os.environ.get('EMAIL_TLS') == 'true'
|
|
43
|
+
self.timeout = timeout
|
|
29
44
|
|
|
30
45
|
@staticmethod
|
|
31
46
|
def _format_addr(s):
|
|
@@ -33,7 +48,7 @@ class EmailBase:
|
|
|
33
48
|
name, addr = parseaddr(s)
|
|
34
49
|
return formataddr((Header(name, 'utf-8').encode(), addr)) if name else addr
|
|
35
50
|
|
|
36
|
-
def pack(self, receivers, title=None, content=None, files=None, cc=None):
|
|
51
|
+
def pack(self, receivers, title=None, content=None, files=None, cc=None, sender=None):
|
|
37
52
|
msg = MIMEMultipart()
|
|
38
53
|
msg.set_charset('utf8')
|
|
39
54
|
|
|
@@ -60,8 +75,12 @@ class EmailBase:
|
|
|
60
75
|
|
|
61
76
|
msg['subject'] = title
|
|
62
77
|
msg['date'] = formatdate(localtime=True)
|
|
63
|
-
|
|
64
|
-
|
|
78
|
+
# Per-call sender overrides the instance default (self.sender, set at
|
|
79
|
+
# construction time) so one client can send under different display
|
|
80
|
+
# names without needing a separate Email/AioEmail instance per name.
|
|
81
|
+
display_name = sender if sender is not None else self.sender
|
|
82
|
+
from_addr = f'{display_name}<{self.user}>' if display_name else self.user
|
|
83
|
+
msg['from'] = self._format_addr(from_addr)
|
|
65
84
|
if not isinstance(receivers, (list, tuple)):
|
|
66
85
|
receivers = [receivers]
|
|
67
86
|
msg['to'] = COMMASPACE.join([self._format_addr(r) for r in receivers])
|
|
@@ -70,32 +89,23 @@ class EmailBase:
|
|
|
70
89
|
|
|
71
90
|
class Email(EmailBase):
|
|
72
91
|
|
|
73
|
-
def __init__(self, *args, **kwargs):
|
|
74
|
-
super().__init__(*args, **kwargs)
|
|
75
|
-
if self.use_tls:
|
|
76
|
-
self.client = smtplib.SMTP_SSL(self.smtp, 465)
|
|
77
|
-
else:
|
|
78
|
-
self.client = smtplib.SMTP(self.smtp)
|
|
79
|
-
|
|
80
92
|
def send(self, *args, **kwargs):
|
|
81
93
|
msg = self.pack(*args, **kwargs)
|
|
82
|
-
|
|
83
|
-
self.
|
|
84
|
-
self.
|
|
85
|
-
|
|
86
|
-
|
|
94
|
+
# See EmailBase docstring for why this connects fresh each call.
|
|
95
|
+
cls = smtplib.SMTP_SSL if self.use_tls else smtplib.SMTP
|
|
96
|
+
with cls(self.smtp, 465 if self.use_tls else 0, timeout=self.timeout) as client:
|
|
97
|
+
client.ehlo()
|
|
98
|
+
client.login(self.user, self.pwd)
|
|
99
|
+
client.send_message(msg)
|
|
87
100
|
|
|
88
101
|
|
|
89
102
|
class AioEmail(EmailBase):
|
|
90
103
|
|
|
91
|
-
def __init__(self, *args, **kwargs):
|
|
92
|
-
super().__init__(*args, **kwargs)
|
|
93
|
-
kwargs = {'port': 465, 'use_tls': True} if self.use_tls else {}
|
|
94
|
-
self.client = aiosmtplib.SMTP(hostname=self.smtp, **kwargs)
|
|
95
|
-
|
|
96
104
|
async def send(self, *args, **kwargs):
|
|
97
105
|
msg = self.pack(*args, **kwargs)
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
106
|
+
# See EmailBase docstring for why this connects fresh each call.
|
|
107
|
+
extra = {'port': 465, 'use_tls': True} if self.use_tls else {}
|
|
108
|
+
client = aiosmtplib.SMTP(hostname=self.smtp, timeout=self.timeout, **extra)
|
|
109
|
+
async with client:
|
|
110
|
+
await client.login(self.user, self.pwd)
|
|
111
|
+
await client.send_message(msg)
|
|
@@ -140,6 +140,9 @@ class Response:
|
|
|
140
140
|
encoding = params['charset'].strip("'\"")
|
|
141
141
|
if encoding is None:
|
|
142
142
|
encoding = chardet.detect(self.body)['encoding']
|
|
143
|
+
# chardet misdetects ASCII text containing '+' (e.g. base64) as UTF-7
|
|
144
|
+
if encoding and encoding.lower() == 'utf-7':
|
|
145
|
+
encoding = 'utf-8'
|
|
143
146
|
return 'gbk' if encoding and encoding.lower() == 'gb2312' else encoding
|
|
144
147
|
|
|
145
148
|
@cached_property
|
|
@@ -150,7 +153,9 @@ class Response:
|
|
|
150
153
|
return self.body.decode()
|
|
151
154
|
|
|
152
155
|
def json(self, **kwargs):
|
|
153
|
-
|
|
156
|
+
# json.loads accepts bytes and detects UTF-8/16/32 per the JSON spec,
|
|
157
|
+
# bypassing charset guessing on responses like text/plain JSON
|
|
158
|
+
return DictWrapper(json.loads(self.body, **kwargs))
|
|
154
159
|
|
|
155
160
|
def soup(self, features='html5lib', **kwargs):
|
|
156
161
|
return BeautifulSoup(self.body, features=features, **kwargs)
|
|
@@ -203,7 +208,7 @@ class BaseRequest(metaclass=RequestMeta):
|
|
|
203
208
|
files: dict, { key: filename or file descriptor }
|
|
204
209
|
proxy: str, 'http://user:password@114.112.93.35:8080'
|
|
205
210
|
'''
|
|
206
|
-
self.proxy = kwargs.get('proxy', os.environ.get('
|
|
211
|
+
self.proxy = kwargs.get('proxy', os.environ.get('HTTP_PROXY'))
|
|
207
212
|
self.cookie = kwargs.get('cookie')
|
|
208
213
|
self.cookies = kwargs.get('cookies', {})
|
|
209
214
|
self.timeout = kwargs.get('timeout', 30)
|
|
@@ -218,7 +223,8 @@ class BaseRequest(metaclass=RequestMeta):
|
|
|
218
223
|
'connection': 'keep-alive',
|
|
219
224
|
'accept-encoding': 'gzip,deflate,sdch',
|
|
220
225
|
'accept-language': 'zh-CN,zh;q=0.8,en;q=0.6',
|
|
221
|
-
'
|
|
226
|
+
'sec-ch-ua-platform': '"macOS"',
|
|
227
|
+
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
|
222
228
|
}
|
|
223
229
|
self.headers.update(kwargs.get('headers', {}))
|
|
224
230
|
|
|
@@ -598,11 +604,13 @@ class Aiohttp(AsyncRequest):
|
|
|
598
604
|
self.load_cookie()
|
|
599
605
|
|
|
600
606
|
def __del__(self):
|
|
601
|
-
|
|
602
|
-
|
|
607
|
+
if self.session.closed:
|
|
608
|
+
return
|
|
609
|
+
try:
|
|
610
|
+
loop = asyncio.get_running_loop()
|
|
603
611
|
loop.create_task(self.session.close())
|
|
604
|
-
|
|
605
|
-
|
|
612
|
+
except RuntimeError:
|
|
613
|
+
pass
|
|
606
614
|
|
|
607
615
|
def save_cookie(self):
|
|
608
616
|
if self.cookie:
|
|
@@ -13,8 +13,6 @@ from functools import partial
|
|
|
13
13
|
import aio_pika
|
|
14
14
|
import pika
|
|
15
15
|
|
|
16
|
-
from .decorator import aioretry, retry
|
|
17
|
-
|
|
18
16
|
logging.getLogger('pika').setLevel(logging.ERROR)
|
|
19
17
|
|
|
20
18
|
__all__ = ['Pika', 'AioPika']
|
|
@@ -53,7 +51,7 @@ class Pika:
|
|
|
53
51
|
if not (queue in self._channels and not self._channels[queue].is_closed):
|
|
54
52
|
channel = self._connection.channel()
|
|
55
53
|
channel.basic_qos(prefetch_count=1)
|
|
56
|
-
q = channel.queue_declare(queue=queue, auto_delete=False, durable=
|
|
54
|
+
q = channel.queue_declare(queue=queue, auto_delete=False, durable=True)
|
|
57
55
|
self._channels[queue] = channel
|
|
58
56
|
self._queues[queue] = q
|
|
59
57
|
|
|
@@ -83,14 +81,14 @@ class Pika:
|
|
|
83
81
|
channel.stop_consuming()
|
|
84
82
|
self.close()
|
|
85
83
|
|
|
86
|
-
@retry
|
|
87
84
|
def get(self, queue=None, auto_ack=True):
|
|
88
85
|
with self._lock:
|
|
89
86
|
queue = self.init(queue, auto_ack)
|
|
90
87
|
msg = self._channels[queue].basic_get(queue, auto_ack=auto_ack)
|
|
88
|
+
if msg[0] is None:
|
|
89
|
+
return None
|
|
91
90
|
return pickle.loads(msg[-1])
|
|
92
91
|
|
|
93
|
-
@retry
|
|
94
92
|
def publish(self, msg, queue=None):
|
|
95
93
|
with self._lock:
|
|
96
94
|
queue = self.init(queue)
|
|
@@ -111,14 +109,15 @@ class AioPika(Pika):
|
|
|
111
109
|
def __init__(self, queue, workers=1, **kwargs):
|
|
112
110
|
super().__init__(queue, **kwargs)
|
|
113
111
|
self.workers = workers
|
|
114
|
-
self.loop = asyncio.get_event_loop()
|
|
115
112
|
self.lock = asyncio.Lock()
|
|
113
|
+
self._consumer_tasks = {}
|
|
114
|
+
self._stop_events = {}
|
|
116
115
|
|
|
117
116
|
async def connect(self):
|
|
118
117
|
async with self.lock:
|
|
119
118
|
if self._connection is None or self._connection.is_closed:
|
|
120
|
-
self._connection = await aio_pika.connect_robust(self.uri
|
|
121
|
-
self.logger.info(f'
|
|
119
|
+
self._connection = await aio_pika.connect_robust(self.uri)
|
|
120
|
+
self.logger.info(f'initialize {self._connection}')
|
|
122
121
|
|
|
123
122
|
async def size(self, queue=None):
|
|
124
123
|
queue = await self.init(queue)
|
|
@@ -130,49 +129,78 @@ class AioPika(Pika):
|
|
|
130
129
|
|
|
131
130
|
async with self.lock:
|
|
132
131
|
if not (queue in self._channels and not self._channels[queue].is_closed):
|
|
133
|
-
|
|
132
|
+
try:
|
|
133
|
+
channel = await self._connection.channel()
|
|
134
|
+
except RuntimeError:
|
|
135
|
+
self._connection = await aio_pika.connect_robust(self.uri)
|
|
136
|
+
self.logger.info(f're-initialize {self._connection}')
|
|
137
|
+
channel = await self._connection.channel()
|
|
134
138
|
await channel.set_qos(prefetch_count=1)
|
|
135
|
-
q = await channel.declare_queue(queue, auto_delete=False, durable=
|
|
139
|
+
q = await channel.declare_queue(queue, auto_delete=False, durable=True)
|
|
136
140
|
self._channels[queue] = channel
|
|
137
141
|
self._queues[queue] = q
|
|
138
142
|
|
|
139
143
|
return queue
|
|
140
144
|
|
|
141
|
-
async def
|
|
142
|
-
async
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
145
|
+
async def _consume_with_channel(self, process, queue_name, stop_event, auto_ack):
|
|
146
|
+
async with self.lock:
|
|
147
|
+
channel = await self._connection.channel()
|
|
148
|
+
await channel.set_qos(prefetch_count=1)
|
|
149
|
+
q = await channel.declare_queue(queue_name, auto_delete=False, durable=True)
|
|
150
|
+
|
|
151
|
+
async with q.iterator() as queue_iter:
|
|
152
|
+
async for msg in queue_iter:
|
|
153
|
+
if stop_event.is_set():
|
|
154
|
+
break
|
|
155
|
+
try:
|
|
156
|
+
if auto_ack:
|
|
157
|
+
await msg.ack()
|
|
158
|
+
body = pickle.loads(msg.body)
|
|
159
|
+
await process(body)
|
|
160
|
+
if not auto_ack:
|
|
161
|
+
await msg.ack()
|
|
162
|
+
except Exception as e:
|
|
163
|
+
self.logger.exception(e)
|
|
164
|
+
if not auto_ack:
|
|
165
|
+
await msg.nack(requeue=True)
|
|
151
166
|
|
|
152
167
|
async def consume(self, process, queue=None, auto_ack=False):
|
|
153
168
|
queue = await self.init(queue, auto_ack)
|
|
169
|
+
|
|
170
|
+
if queue not in self._stop_events:
|
|
171
|
+
self._stop_events[queue] = asyncio.Event()
|
|
172
|
+
self._consumer_tasks[queue] = []
|
|
173
|
+
|
|
174
|
+
stop_event = self._stop_events[queue]
|
|
175
|
+
|
|
154
176
|
for _ in range(self.workers):
|
|
155
|
-
|
|
177
|
+
task = asyncio.create_task(self._consume_with_channel(process, queue, stop_event, auto_ack))
|
|
178
|
+
self._consumer_tasks[queue].append(task)
|
|
156
179
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
async def publish(self, msg, queue=None):
|
|
180
|
+
return self._consumer_tasks[queue]
|
|
181
|
+
|
|
182
|
+
async def stop_consume(self, queue=None):
|
|
183
|
+
queue = queue or self.queue
|
|
184
|
+
if queue in self._stop_events:
|
|
185
|
+
self._stop_events[queue].set()
|
|
186
|
+
if queue in self._consumer_tasks:
|
|
187
|
+
await asyncio.gather(*self._consumer_tasks[queue], return_exceptions=True)
|
|
188
|
+
del self._consumer_tasks[queue]
|
|
189
|
+
del self._stop_events[queue]
|
|
190
|
+
|
|
191
|
+
async def publish(self, msg, queue=None, timeout=5):
|
|
169
192
|
queue = await self.init(queue)
|
|
170
|
-
|
|
171
|
-
|
|
193
|
+
message = aio_pika.Message(pickle.dumps(msg))
|
|
194
|
+
coro = self._channels[queue].default_exchange.publish(message, routing_key=queue)
|
|
195
|
+
await asyncio.wait_for(coro, timeout=timeout)
|
|
172
196
|
|
|
173
197
|
async def close(self):
|
|
174
|
-
for
|
|
198
|
+
for queue in list(self._stop_events.keys()):
|
|
199
|
+
await self.stop_consume(queue)
|
|
200
|
+
|
|
201
|
+
for channel in list(self._channels.values()):
|
|
175
202
|
if not channel.is_closed:
|
|
176
203
|
await channel.close()
|
|
204
|
+
|
|
177
205
|
if self._connection is not None and not self._connection.is_closed:
|
|
178
206
|
await self._connection.close()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
processor.py
|
|
2
|
+
requirements.txt
|
|
3
|
+
setup.py
|
|
4
|
+
kkutils/__init__.py
|
|
5
|
+
kkutils/base_utils.py
|
|
6
|
+
kkutils/config_utils.py
|
|
7
|
+
kkutils/crypto.py
|
|
8
|
+
kkutils/curl_utils.py
|
|
9
|
+
kkutils/db_utils.py
|
|
10
|
+
kkutils/decorator.py
|
|
11
|
+
kkutils/email_utils.py
|
|
12
|
+
kkutils/fire.py
|
|
13
|
+
kkutils/generate_machine_id.py
|
|
14
|
+
kkutils/http_utils.py
|
|
15
|
+
kkutils/log_utils.py
|
|
16
|
+
kkutils/rabbitmq.py
|
|
17
|
+
kkutils/stopwatch.py
|
|
18
|
+
kkutils/xdb_searcher.py
|
|
19
|
+
kkutils/xor_file.py
|
|
20
|
+
kkutils.egg-info/PKG-INFO
|
|
21
|
+
kkutils.egg-info/SOURCES.txt
|
|
22
|
+
kkutils.egg-info/dependency_links.txt
|
|
23
|
+
kkutils.egg-info/requires.txt
|
|
24
|
+
kkutils.egg-info/top_level.txt
|
|
25
|
+
tornado_utils/__init__.py
|
|
26
|
+
tornado_utils/application.py
|
|
27
|
+
tornado_utils/basehandler.py
|
|
28
|
+
tornado_utils/userhandler.py
|
|
29
|
+
tornado_utils/utils.py
|
|
@@ -5,7 +5,7 @@ from setuptools import setup
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name="kkutils",
|
|
8
|
-
version="
|
|
8
|
+
version="2.0.0",
|
|
9
9
|
description="python utils",
|
|
10
10
|
author="digua",
|
|
11
11
|
author_email="digua@gmail.com",
|
|
@@ -17,7 +17,7 @@ setup(
|
|
|
17
17
|
include_package_data=True,
|
|
18
18
|
extras_require={'curl': ['pycurl']},
|
|
19
19
|
py_modules=['processor'],
|
|
20
|
-
packages=['tornado_utils', '
|
|
20
|
+
packages=['tornado_utils', 'kkutils'],
|
|
21
21
|
classifiers=[
|
|
22
22
|
'Development Status :: 4 - Beta',
|
|
23
23
|
'Intended Audience :: Developers',
|
|
@@ -13,7 +13,7 @@ import tornado.netutil
|
|
|
13
13
|
import tornado.process
|
|
14
14
|
import tornado.web
|
|
15
15
|
from tornado.options import define, options
|
|
16
|
-
from
|
|
16
|
+
from kkutils import Cache, Dict, Logger, get_ip
|
|
17
17
|
|
|
18
18
|
__all__ = ['Blueprint', 'Application']
|
|
19
19
|
|
|
@@ -20,7 +20,7 @@ from functools import cached_property
|
|
|
20
20
|
|
|
21
21
|
import tornado.web
|
|
22
22
|
from bson import ObjectId
|
|
23
|
-
from
|
|
23
|
+
from kkutils import Dict, JSONEncoder
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class BaseHandler(tornado.web.RequestHandler):
|
|
@@ -161,7 +161,9 @@ class BaseHandler(tornado.web.RequestHandler):
|
|
|
161
161
|
return user
|
|
162
162
|
|
|
163
163
|
async def get_current_user(self):
|
|
164
|
-
|
|
164
|
+
auth_header = self.request.headers.get('Authorization', '')
|
|
165
|
+
header_token = auth_header.removeprefix('Bearer ').strip() if auth_header else None
|
|
166
|
+
token = self.args.get('token') or self.get_cookie('token') or header_token
|
|
165
167
|
if token and hasattr(self.app, 'db'):
|
|
166
168
|
user = await self.db.users.find_one({'token': token})
|
|
167
169
|
if user:
|
|
@@ -217,7 +219,8 @@ class BaseHandler(tornado.web.RequestHandler):
|
|
|
217
219
|
super().write_error(status_code, **kwargs)
|
|
218
220
|
|
|
219
221
|
def render(self, template_name, **kwargs):
|
|
220
|
-
|
|
222
|
+
accept = self.request.headers.get('Accept', '')
|
|
223
|
+
if self.get_argument('f', None) == 'json' or 'application/json' in accept:
|
|
221
224
|
self.finish(kwargs)
|
|
222
225
|
else:
|
|
223
226
|
super().render(template_name, **kwargs)
|
|
@@ -13,7 +13,7 @@ from datetime import datetime
|
|
|
13
13
|
import pymongo
|
|
14
14
|
import tornado.web
|
|
15
15
|
from bson import ObjectId
|
|
16
|
-
from
|
|
16
|
+
from kkutils import Dict
|
|
17
17
|
|
|
18
18
|
from .application import Blueprint
|
|
19
19
|
from .basehandler import BaseHandler
|
|
@@ -151,6 +151,13 @@ class LogoutHandler(BaseHandler):
|
|
|
151
151
|
class SignupHandler(BaseHandler):
|
|
152
152
|
|
|
153
153
|
def get(self):
|
|
154
|
+
link = self.args.get('link') or self.args.get('from_id')
|
|
155
|
+
if link:
|
|
156
|
+
self.set_cookie('invite_link', str(link), expires_days=1)
|
|
157
|
+
elif not self.args.get('link') and not self.args.get('from_id'):
|
|
158
|
+
cookie_link = self.get_cookie('invite_link')
|
|
159
|
+
if cookie_link:
|
|
160
|
+
self.args['link'] = cookie_link
|
|
154
161
|
self.render('signup.html')
|
|
155
162
|
|
|
156
163
|
async def post(self):
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
processor.py
|
|
2
|
-
requirements.txt
|
|
3
|
-
setup.py
|
|
4
|
-
kkutils.egg-info/PKG-INFO
|
|
5
|
-
kkutils.egg-info/SOURCES.txt
|
|
6
|
-
kkutils.egg-info/dependency_links.txt
|
|
7
|
-
kkutils.egg-info/requires.txt
|
|
8
|
-
kkutils.egg-info/top_level.txt
|
|
9
|
-
tornado_utils/__init__.py
|
|
10
|
-
tornado_utils/application.py
|
|
11
|
-
tornado_utils/basehandler.py
|
|
12
|
-
tornado_utils/userhandler.py
|
|
13
|
-
tornado_utils/utils.py
|
|
14
|
-
utils/__init__.py
|
|
15
|
-
utils/base_utils.py
|
|
16
|
-
utils/config_utils.py
|
|
17
|
-
utils/crypto.py
|
|
18
|
-
utils/curl_utils.py
|
|
19
|
-
utils/db_utils.py
|
|
20
|
-
utils/decorator.py
|
|
21
|
-
utils/email_utils.py
|
|
22
|
-
utils/fire.py
|
|
23
|
-
utils/generate_machine_id.py
|
|
24
|
-
utils/http_utils.py
|
|
25
|
-
utils/log_utils.py
|
|
26
|
-
utils/rabbitmq.py
|
|
27
|
-
utils/stopwatch.py
|
|
28
|
-
utils/xdb_searcher.py
|
|
29
|
-
utils/xor_file.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|