kkutils 1.8.2__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {kkutils-1.8.2 → kkutils-2.0.0}/PKG-INFO +1 -1
  2. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/base_utils.py +24 -13
  3. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/config_utils.py +4 -2
  4. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/crypto.py +11 -1
  5. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/db_utils.py +57 -29
  6. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/email_utils.py +35 -25
  7. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/http_utils.py +15 -7
  8. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/rabbitmq.py +64 -36
  9. {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/PKG-INFO +1 -1
  10. kkutils-2.0.0/kkutils.egg-info/SOURCES.txt +29 -0
  11. {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/top_level.txt +1 -1
  12. {kkutils-1.8.2 → kkutils-2.0.0}/processor.py +1 -1
  13. {kkutils-1.8.2 → kkutils-2.0.0}/setup.py +2 -2
  14. {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/application.py +1 -1
  15. {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/basehandler.py +6 -3
  16. {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/userhandler.py +8 -1
  17. {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/utils.py +1 -1
  18. kkutils-1.8.2/kkutils.egg-info/SOURCES.txt +0 -29
  19. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/__init__.py +0 -0
  20. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/curl_utils.py +0 -0
  21. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/decorator.py +0 -0
  22. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/fire.py +0 -0
  23. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/generate_machine_id.py +0 -0
  24. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/log_utils.py +0 -0
  25. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/stopwatch.py +0 -0
  26. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/xdb_searcher.py +0 -0
  27. {kkutils-1.8.2/utils → kkutils-2.0.0/kkutils}/xor_file.py +0 -0
  28. {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/dependency_links.txt +0 -0
  29. {kkutils-1.8.2 → kkutils-2.0.0}/kkutils.egg-info/requires.txt +0 -0
  30. {kkutils-1.8.2 → kkutils-2.0.0}/requirements.txt +0 -0
  31. {kkutils-1.8.2 → kkutils-2.0.0}/setup.cfg +0 -0
  32. {kkutils-1.8.2 → kkutils-2.0.0}/tornado_utils/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kkutils
3
- Version: 1.8.2
3
+ Version: 2.0.0
4
4
  Summary: python utils
5
5
  Home-page: https://www.github.com
6
6
  Author: digua
@@ -38,9 +38,14 @@ from tqdm import tqdm as _tqdm
38
38
  def _include_yaml(loader, node):
39
39
  root = Path(loader.name).parent
40
40
  if isinstance(node, yaml.SequenceNode):
41
- return [yaml.load(open(root / x.value), Loader=yaml.FullLoader) for x in node.value]
41
+ result = []
42
+ for x in node.value:
43
+ with open(root / x.value) as f:
44
+ result.append(yaml.load(f, Loader=yaml.FullLoader))
45
+ return result
42
46
  else:
43
- return yaml.load(open(root / node.value), Loader=yaml.FullLoader)
47
+ with open(root / node.value) as f:
48
+ return yaml.load(f, Loader=yaml.FullLoader)
44
49
 
45
50
 
46
51
  def yaml_load(stream, **kwargs):
@@ -117,13 +122,14 @@ class Cache:
117
122
  self.data = {}
118
123
 
119
124
  def get(self, key):
120
- item = self.data.get(key)
121
- if item is not None:
122
- value, ttl = item
123
- if ttl is None:
124
- return value
125
- if ttl >= int(time.time()):
126
- return value
125
+ with self.lock:
126
+ item = self.data.get(key)
127
+ if item is not None:
128
+ value, ttl = item
129
+ if ttl is None:
130
+ return value
131
+ if ttl >= int(time.time()):
132
+ return value
127
133
 
128
134
  def set(self, key, value, ttl=None):
129
135
  with self.lock:
@@ -132,9 +138,14 @@ class Cache:
132
138
 
133
139
  def incr(self, key, value=1):
134
140
  with self.lock:
135
- self.data.setdefault(key, 0)
136
- self.data[key] += value
137
- return self.data[key]
141
+ item = self.data.get(key)
142
+ if item is None:
143
+ current = 0
144
+ else:
145
+ current = item[0]
146
+ new_value = current + value
147
+ self.data[key] = new_value, None
148
+ return new_value
138
149
 
139
150
  def delete(self, key):
140
151
  with self.lock:
@@ -148,7 +159,7 @@ class Cache:
148
159
 
149
160
  def ttl(self, key, ttl):
150
161
  with self.lock:
151
- item = self.data[key]
162
+ item = self.data.get(key)
152
163
  if item is not None:
153
164
  self.data[key] = item[0], int(time.time()) + ttl
154
165
 
@@ -60,9 +60,11 @@ class Config(Dict):
60
60
  if config.endswith('.ini'):
61
61
  cfg.update(Dict(Config._load_ini(config)))
62
62
  elif config.endswith('.json'):
63
- cfg.update(Dict(json.load(open(config))))
63
+ with open(config) as f:
64
+ cfg.update(Dict(json.load(f)))
64
65
  elif config.endswith('.yaml') or config.endswith('.yml'):
65
- cfg.update(Dict(yaml.load(open(config), Loader=yaml.FullLoader)))
66
+ with open(config) as f:
67
+ cfg.update(Dict(yaml.load(f, Loader=yaml.FullLoader)))
66
68
 
67
69
  if cfg.extends:
68
70
  base = Dict()
@@ -49,8 +49,18 @@ def decode(text, lib='base62'):
49
49
  return binascii.a2b_hex(text)
50
50
 
51
51
 
52
+ _XOR_TABLES = {}
53
+
54
+
52
55
  def xor_encrypt(text: str, key: int, lib='base64'):
53
- encrypted = ''.join([chr(ord(x) ^ ord(chr(key))) for x in to_str(text)])
56
+ text = to_str(text)
57
+ if text.isascii() and 0 <= key < 128:
58
+ # Fast path: ASCII xor a 7-bit key stays single-byte, so a byte table gives identical output
59
+ table = _XOR_TABLES.get(key)
60
+ if table is None:
61
+ table = _XOR_TABLES[key] = bytes(i ^ key for i in range(256))
62
+ return encode(text.encode().translate(table), lib)
63
+ encrypted = ''.join([chr(ord(x) ^ ord(chr(key))) for x in text])
54
64
  return encode(encrypted, lib)
55
65
 
56
66
 
@@ -5,6 +5,7 @@ Last modified: 2018-01-05 11:24:17
5
5
  '''
6
6
 
7
7
  import asyncio
8
+ import logging
8
9
  import os
9
10
  import re
10
11
  from urllib.parse import parse_qs, quote_plus, unquote_plus
@@ -13,6 +14,7 @@ import aiomysql
13
14
  import pymongo
14
15
  import pymysql
15
16
  import redis
17
+ import redis.exceptions
16
18
  from dbutils.pooled_db import PooledDB
17
19
  from motor import core
18
20
  from motor.docstrings import get_database_doc
@@ -124,12 +126,12 @@ class MongoClient(pymongo.MongoClient):
124
126
  kwargs.setdefault('document_class', Dict)
125
127
  kwargs.setdefault('maxPoolSize', 50)
126
128
  kwargs.setdefault('minPoolSize', 5)
127
- kwargs.setdefault('maxIdleTimeMS', 30000)
128
- kwargs.setdefault('waitQueueTimeoutMS', 10000)
129
- kwargs.setdefault('serverSelectionTimeoutMS', 10000)
130
- kwargs.setdefault('connectTimeoutMS', 10000)
131
- kwargs.setdefault('socketTimeoutMS', 30000)
132
- kwargs.setdefault('heartbeatFrequencyMS', 30000)
129
+ # kwargs.setdefault('maxIdleTimeMS', 30000)
130
+ # kwargs.setdefault('waitQueueTimeoutMS', 10000)
131
+ # kwargs.setdefault('serverSelectionTimeoutMS', 10000)
132
+ # kwargs.setdefault('connectTimeoutMS', 10000)
133
+ # kwargs.setdefault('socketTimeoutMS', 30000)
134
+ # kwargs.setdefault('heartbeatFrequencyMS', 10000)
133
135
  kwargs.setdefault('retryWrites', True)
134
136
  kwargs.setdefault('retryReads', True)
135
137
  super(MongoClient, self).__init__(uri, **kwargs)
@@ -290,9 +292,26 @@ class AioRedis(aioredis.StrictRedis):
290
292
  uri = f"redis://:{password}@{host}:{port}/{db}" if password else f"redis://{host}:{port}/{db}"
291
293
 
292
294
  kwargs.setdefault('decode_responses', True)
295
+ kwargs.setdefault('health_check_interval', 30)
296
+ kwargs.setdefault('max_connections', 5000)
297
+ kwargs.setdefault('retry_on_timeout', True)
298
+ kwargs.setdefault('socket_keepalive', True)
293
299
  pool = aioredis.ConnectionPool.from_url(uri, **kwargs)
294
300
  super().__init__(connection_pool=pool)
295
301
 
302
+ async def execute_command(self, *args, **kwargs):
303
+ try:
304
+ return await super().execute_command(*args, **kwargs)
305
+ except redis.exceptions.ConnectionError as e:
306
+ pool = self.connection_pool
307
+ logging.getLogger().error(
308
+ f'redis connection error: {e}, cmd: {args[:1]}, '
309
+ f'in_use: {len(pool._in_use_connections)}, '
310
+ f'available: {len(pool._available_connections)}, '
311
+ f'max: {pool.max_connections}, pid: {os.getpid()}'
312
+ )
313
+ raise
314
+
296
315
  async def clear(self, pattern='*'):
297
316
  if pattern == '*':
298
317
  await self.flushdb()
@@ -385,17 +404,15 @@ class AioMysql(Mysql):
385
404
 
386
405
  def __init__(self, *args, **kwargs):
387
406
  super().__init__(*args, **kwargs)
388
- self._lock = asyncio.Lock()
389
-
390
- async def _execute(self, cursor, sql, args):
391
- async with self._lock:
392
- await cursor.execute(sql, args)
393
407
 
394
- async def cursor(self, cursorclass=aiomysql.cursors.DictCursor):
408
+ async def _get_conn(self):
395
409
  if self._pool is None:
396
410
  self._pool = await aiomysql.create_pool(minsize=0, maxsize=10, **self._kwargs)
397
- self._conn = await self._pool.acquire()
398
- return await self._conn.cursor(cursorclass)
411
+ return await self._pool.acquire()
412
+
413
+ async def _release_conn(self, conn):
414
+ if self._pool is not None:
415
+ await self._pool.release(conn)
399
416
 
400
417
  async def close(self):
401
418
  if self._pool is not None:
@@ -404,53 +421,64 @@ class AioMysql(Mysql):
404
421
  self._pool = None
405
422
 
406
423
  async def iter(self, sql, *args):
407
- cursor = await self.cursor(aiomysql.cursors.SSDictCursor)
424
+ conn = await self._get_conn()
425
+ cursor = await conn.cursor(aiomysql.cursors.SSDictCursor)
408
426
  try:
409
- await self._execute(cursor, sql, args)
427
+ await cursor.execute(sql, args)
410
428
  async for row in cursor:
411
429
  yield Dict(row)
412
430
  finally:
413
431
  await cursor.close()
432
+ await self._release_conn(conn)
414
433
 
415
434
  async def query(self, sql, *args):
416
- cursor = await self.cursor()
435
+ conn = await self._get_conn()
436
+ cursor = await conn.cursor(aiomysql.DictCursor)
417
437
  try:
418
- await self._execute(cursor, sql, args)
438
+ await cursor.execute(sql, args)
419
439
  rows = await cursor.fetchall()
420
440
  return [Dict(row) for row in rows]
421
441
  finally:
422
442
  await cursor.close()
443
+ await self._release_conn(conn)
423
444
 
424
445
  async def query_td(self, sql, *args):
425
- cursor = await self.cursor()
446
+ conn = await self._get_conn()
447
+ cursor = await conn.cursor()
426
448
  try:
427
- await self._execute(cursor, sql, args)
449
+ await cursor.execute(sql, args)
428
450
  rows = await cursor.fetchall()
429
- return {'th': [col[0] for col in cursor._description], 'td': rows}
451
+ return {'th': [col[0] for col in cursor.description], 'td': rows}
430
452
  finally:
431
453
  await cursor.close()
454
+ await self._release_conn(conn)
432
455
 
433
456
  async def get(self, sql, *args):
434
- cursor = await self.cursor()
457
+ conn = await self._get_conn()
458
+ cursor = await conn.cursor(aiomysql.DictCursor)
435
459
  try:
436
- await self._execute(cursor, sql, args)
460
+ await cursor.execute(sql, args)
437
461
  row = await cursor.fetchone()
438
462
  return Dict(row) if row else None
439
463
  finally:
440
464
  await cursor.close()
465
+ await self._release_conn(conn)
441
466
 
442
467
  async def execute(self, sql, *args):
443
- cursor = await self.cursor()
468
+ conn = await self._get_conn()
469
+ cursor = await conn.cursor()
444
470
  try:
445
- await self._execute(cursor, sql, args)
446
- return cursor._lastrowid
471
+ await cursor.execute(sql, args)
472
+ return cursor.lastrowid
447
473
  finally:
448
474
  await cursor.close()
475
+ await self._release_conn(conn)
449
476
 
450
477
  async def executemany(self, sql, args):
451
- cursor = await self.cursor()
478
+ conn = await self._get_conn()
479
+ cursor = await conn.cursor()
452
480
  try:
453
- async with self._lock:
454
- return await cursor.executemany(sql, args)
481
+ await cursor.executemany(sql, args)
455
482
  finally:
456
483
  await cursor.close()
484
+ await self._release_conn(conn)
@@ -17,15 +17,30 @@ import aiosmtplib
17
17
 
18
18
  __all__ = ['Email', 'AioEmail']
19
19
 
20
+ # Hard cap on the connect/login/send/quit cycle. Without this, a stalled or
21
+ # unresponsive MTA hangs the caller indefinitely — smtplib's default is no
22
+ # timeout at all.
23
+ DEFAULT_TIMEOUT = 30
24
+
20
25
 
21
26
  class EmailBase:
27
+ """Shared message-building logic for ``Email``/``AioEmail``.
28
+
29
+ Subclasses build a fresh SMTP client on every ``send()`` call rather
30
+ than reusing one across the instance's lifetime — a stale/half-closed
31
+ socket or half-completed auth left over from a prior failure can then
32
+ never leak into the next send. Each call gets a clean
33
+ connect/login/send/quit cycle, with ``with``/``async with`` guaranteeing
34
+ cleanup even on error.
35
+ """
22
36
 
23
- def __init__(self, sender=None, smtp=None, user=None, pwd=None, use_tls=False):
37
+ def __init__(self, sender=None, smtp=None, user=None, pwd=None, use_tls=False, timeout=DEFAULT_TIMEOUT):
24
38
  self.sender = sender or os.environ.get('EMAIL_SENDER')
25
39
  self.smtp = smtp or os.environ.get('EMAIL_SMTP')
26
40
  self.user = user or os.environ.get('EMAIL_USER')
27
41
  self.pwd = pwd or os.environ.get('EMAIL_PWD')
28
42
  self.use_tls = use_tls or os.environ.get('EMAIL_TLS') == 'true'
43
+ self.timeout = timeout
29
44
 
30
45
  @staticmethod
31
46
  def _format_addr(s):
@@ -33,7 +48,7 @@ class EmailBase:
33
48
  name, addr = parseaddr(s)
34
49
  return formataddr((Header(name, 'utf-8').encode(), addr)) if name else addr
35
50
 
36
- def pack(self, receivers, title=None, content=None, files=None, cc=None):
51
+ def pack(self, receivers, title=None, content=None, files=None, cc=None, sender=None):
37
52
  msg = MIMEMultipart()
38
53
  msg.set_charset('utf8')
39
54
 
@@ -60,8 +75,12 @@ class EmailBase:
60
75
 
61
76
  msg['subject'] = title
62
77
  msg['date'] = formatdate(localtime=True)
63
- sender = f'{self.sender}<{self.user}>' if self.sender else self.user
64
- msg['from'] = self._format_addr(sender)
78
+ # Per-call sender overrides the instance default (self.sender, set at
79
+ # construction time) so one client can send under different display
80
+ # names without needing a separate Email/AioEmail instance per name.
81
+ display_name = sender if sender is not None else self.sender
82
+ from_addr = f'{display_name}<{self.user}>' if display_name else self.user
83
+ msg['from'] = self._format_addr(from_addr)
65
84
  if not isinstance(receivers, (list, tuple)):
66
85
  receivers = [receivers]
67
86
  msg['to'] = COMMASPACE.join([self._format_addr(r) for r in receivers])
@@ -70,32 +89,23 @@ class EmailBase:
70
89
 
71
90
  class Email(EmailBase):
72
91
 
73
- def __init__(self, *args, **kwargs):
74
- super().__init__(*args, **kwargs)
75
- if self.use_tls:
76
- self.client = smtplib.SMTP_SSL(self.smtp, 465)
77
- else:
78
- self.client = smtplib.SMTP(self.smtp)
79
-
80
92
  def send(self, *args, **kwargs):
81
93
  msg = self.pack(*args, **kwargs)
82
- self.client.connect(self.smtp)
83
- self.client.ehlo()
84
- self.client.login(self.user, self.pwd)
85
- self.client.send_message(msg)
86
- self.client.quit()
94
+ # See EmailBase docstring for why this connects fresh each call.
95
+ cls = smtplib.SMTP_SSL if self.use_tls else smtplib.SMTP
96
+ with cls(self.smtp, 465 if self.use_tls else 0, timeout=self.timeout) as client:
97
+ client.ehlo()
98
+ client.login(self.user, self.pwd)
99
+ client.send_message(msg)
87
100
 
88
101
 
89
102
  class AioEmail(EmailBase):
90
103
 
91
- def __init__(self, *args, **kwargs):
92
- super().__init__(*args, **kwargs)
93
- kwargs = {'port': 465, 'use_tls': True} if self.use_tls else {}
94
- self.client = aiosmtplib.SMTP(hostname=self.smtp, **kwargs)
95
-
96
104
  async def send(self, *args, **kwargs):
97
105
  msg = self.pack(*args, **kwargs)
98
- await self.client.connect()
99
- await self.client.login(self.user, self.pwd)
100
- await self.client.send_message(msg)
101
- await self.client.quit()
106
+ # See EmailBase docstring for why this connects fresh each call.
107
+ extra = {'port': 465, 'use_tls': True} if self.use_tls else {}
108
+ client = aiosmtplib.SMTP(hostname=self.smtp, timeout=self.timeout, **extra)
109
+ async with client:
110
+ await client.login(self.user, self.pwd)
111
+ await client.send_message(msg)
@@ -140,6 +140,9 @@ class Response:
140
140
  encoding = params['charset'].strip("'\"")
141
141
  if encoding is None:
142
142
  encoding = chardet.detect(self.body)['encoding']
143
+ # chardet misdetects ASCII text containing '+' (e.g. base64) as UTF-7
144
+ if encoding and encoding.lower() == 'utf-7':
145
+ encoding = 'utf-8'
143
146
  return 'gbk' if encoding and encoding.lower() == 'gb2312' else encoding
144
147
 
145
148
  @cached_property
@@ -150,7 +153,9 @@ class Response:
150
153
  return self.body.decode()
151
154
 
152
155
  def json(self, **kwargs):
153
- return DictWrapper(json.loads(self.text, **kwargs))
156
+ # json.loads accepts bytes and detects UTF-8/16/32 per the JSON spec,
157
+ # bypassing charset guessing on responses like text/plain JSON
158
+ return DictWrapper(json.loads(self.body, **kwargs))
154
159
 
155
160
  def soup(self, features='html5lib', **kwargs):
156
161
  return BeautifulSoup(self.body, features=features, **kwargs)
@@ -203,7 +208,7 @@ class BaseRequest(metaclass=RequestMeta):
203
208
  files: dict, { key: filename or file descriptor }
204
209
  proxy: str, 'http://user:password@114.112.93.35:8080'
205
210
  '''
206
- self.proxy = kwargs.get('proxy', os.environ.get('http_proxy'))
211
+ self.proxy = kwargs.get('proxy', os.environ.get('HTTP_PROXY'))
207
212
  self.cookie = kwargs.get('cookie')
208
213
  self.cookies = kwargs.get('cookies', {})
209
214
  self.timeout = kwargs.get('timeout', 30)
@@ -218,7 +223,8 @@ class BaseRequest(metaclass=RequestMeta):
218
223
  'connection': 'keep-alive',
219
224
  'accept-encoding': 'gzip,deflate,sdch',
220
225
  'accept-language': 'zh-CN,zh;q=0.8,en;q=0.6',
221
- 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36'
226
+ 'sec-ch-ua-platform': '"macOS"',
227
+ 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
222
228
  }
223
229
  self.headers.update(kwargs.get('headers', {}))
224
230
 
@@ -598,11 +604,13 @@ class Aiohttp(AsyncRequest):
598
604
  self.load_cookie()
599
605
 
600
606
  def __del__(self):
601
- loop = asyncio.get_event_loop()
602
- if loop.is_running():
607
+ if self.session.closed:
608
+ return
609
+ try:
610
+ loop = asyncio.get_running_loop()
603
611
  loop.create_task(self.session.close())
604
- else:
605
- loop.run_until_complete(self.session.close())
612
+ except RuntimeError:
613
+ pass
606
614
 
607
615
  def save_cookie(self):
608
616
  if self.cookie:
@@ -13,8 +13,6 @@ from functools import partial
13
13
  import aio_pika
14
14
  import pika
15
15
 
16
- from .decorator import aioretry, retry
17
-
18
16
  logging.getLogger('pika').setLevel(logging.ERROR)
19
17
 
20
18
  __all__ = ['Pika', 'AioPika']
@@ -53,7 +51,7 @@ class Pika:
53
51
  if not (queue in self._channels and not self._channels[queue].is_closed):
54
52
  channel = self._connection.channel()
55
53
  channel.basic_qos(prefetch_count=1)
56
- q = channel.queue_declare(queue=queue, auto_delete=False, durable=False)
54
+ q = channel.queue_declare(queue=queue, auto_delete=False, durable=True)
57
55
  self._channels[queue] = channel
58
56
  self._queues[queue] = q
59
57
 
@@ -83,14 +81,14 @@ class Pika:
83
81
  channel.stop_consuming()
84
82
  self.close()
85
83
 
86
- @retry
87
84
  def get(self, queue=None, auto_ack=True):
88
85
  with self._lock:
89
86
  queue = self.init(queue, auto_ack)
90
87
  msg = self._channels[queue].basic_get(queue, auto_ack=auto_ack)
88
+ if msg[0] is None:
89
+ return None
91
90
  return pickle.loads(msg[-1])
92
91
 
93
- @retry
94
92
  def publish(self, msg, queue=None):
95
93
  with self._lock:
96
94
  queue = self.init(queue)
@@ -111,14 +109,15 @@ class AioPika(Pika):
111
109
  def __init__(self, queue, workers=1, **kwargs):
112
110
  super().__init__(queue, **kwargs)
113
111
  self.workers = workers
114
- self.loop = asyncio.get_event_loop()
115
112
  self.lock = asyncio.Lock()
113
+ self._consumer_tasks = {}
114
+ self._stop_events = {}
116
115
 
117
116
  async def connect(self):
118
117
  async with self.lock:
119
118
  if self._connection is None or self._connection.is_closed:
120
- self._connection = await aio_pika.connect_robust(self.uri, loop=self.loop)
121
- self.logger.info(f'initintize {self._connection}')
119
+ self._connection = await aio_pika.connect_robust(self.uri)
120
+ self.logger.info(f'initialize {self._connection}')
122
121
 
123
122
  async def size(self, queue=None):
124
123
  queue = await self.init(queue)
@@ -130,49 +129,78 @@ class AioPika(Pika):
130
129
 
131
130
  async with self.lock:
132
131
  if not (queue in self._channels and not self._channels[queue].is_closed):
133
- channel = await self._connection.channel()
132
+ try:
133
+ channel = await self._connection.channel()
134
+ except RuntimeError:
135
+ self._connection = await aio_pika.connect_robust(self.uri)
136
+ self.logger.info(f're-initialize {self._connection}')
137
+ channel = await self._connection.channel()
134
138
  await channel.set_qos(prefetch_count=1)
135
- q = await channel.declare_queue(queue, auto_delete=False, durable=False)
139
+ q = await channel.declare_queue(queue, auto_delete=False, durable=True)
136
140
  self._channels[queue] = channel
137
141
  self._queues[queue] = q
138
142
 
139
143
  return queue
140
144
 
141
- async def _consume(self, process, queue, auto_ack):
142
- async for msg in queue:
143
- try:
144
- if auto_ack:
145
- await msg.ack()
146
- await process(pickle.loads(msg.body))
147
- if not auto_ack:
148
- await msg.ack()
149
- except Exception as e:
150
- self.logger.exception(e)
145
+ async def _consume_with_channel(self, process, queue_name, stop_event, auto_ack):
146
+ async with self.lock:
147
+ channel = await self._connection.channel()
148
+ await channel.set_qos(prefetch_count=1)
149
+ q = await channel.declare_queue(queue_name, auto_delete=False, durable=True)
150
+
151
+ async with q.iterator() as queue_iter:
152
+ async for msg in queue_iter:
153
+ if stop_event.is_set():
154
+ break
155
+ try:
156
+ if auto_ack:
157
+ await msg.ack()
158
+ body = pickle.loads(msg.body)
159
+ await process(body)
160
+ if not auto_ack:
161
+ await msg.ack()
162
+ except Exception as e:
163
+ self.logger.exception(e)
164
+ if not auto_ack:
165
+ await msg.nack(requeue=True)
151
166
 
152
167
  async def consume(self, process, queue=None, auto_ack=False):
153
168
  queue = await self.init(queue, auto_ack)
169
+
170
+ if queue not in self._stop_events:
171
+ self._stop_events[queue] = asyncio.Event()
172
+ self._consumer_tasks[queue] = []
173
+
174
+ stop_event = self._stop_events[queue]
175
+
154
176
  for _ in range(self.workers):
155
- self.loop.create_task(self._consume(process, self._queues[queue], auto_ack))
177
+ task = asyncio.create_task(self._consume_with_channel(process, queue, stop_event, auto_ack))
178
+ self._consumer_tasks[queue].append(task)
156
179
 
157
- @aioretry
158
- async def get(self, queue=None, auto_ack=True):
159
- queue = await self.init(queue, auto_ack)
160
- while True:
161
- msg = await self._queues[queue].get(no_ack=auto_ack, fail=False, timeout=None)
162
- if msg:
163
- return pickle.loads(msg.body)
164
- else:
165
- await asyncio.sleep(1)
166
-
167
- @aioretry
168
- async def publish(self, msg, queue=None):
180
+ return self._consumer_tasks[queue]
181
+
182
+ async def stop_consume(self, queue=None):
183
+ queue = queue or self.queue
184
+ if queue in self._stop_events:
185
+ self._stop_events[queue].set()
186
+ if queue in self._consumer_tasks:
187
+ await asyncio.gather(*self._consumer_tasks[queue], return_exceptions=True)
188
+ del self._consumer_tasks[queue]
189
+ del self._stop_events[queue]
190
+
191
+ async def publish(self, msg, queue=None, timeout=5):
169
192
  queue = await self.init(queue)
170
- return await self._channels[queue].default_exchange.publish(aio_pika.Message(pickle.dumps(msg)),
171
- routing_key=queue)
193
+ message = aio_pika.Message(pickle.dumps(msg))
194
+ coro = self._channels[queue].default_exchange.publish(message, routing_key=queue)
195
+ await asyncio.wait_for(coro, timeout=timeout)
172
196
 
173
197
  async def close(self):
174
- for channel in self._channels.values():
198
+ for queue in list(self._stop_events.keys()):
199
+ await self.stop_consume(queue)
200
+
201
+ for channel in list(self._channels.values()):
175
202
  if not channel.is_closed:
176
203
  await channel.close()
204
+
177
205
  if self._connection is not None and not self._connection.is_closed:
178
206
  await self._connection.close()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kkutils
3
- Version: 1.8.2
3
+ Version: 2.0.0
4
4
  Summary: python utils
5
5
  Home-page: https://www.github.com
6
6
  Author: digua
@@ -0,0 +1,29 @@
1
+ processor.py
2
+ requirements.txt
3
+ setup.py
4
+ kkutils/__init__.py
5
+ kkutils/base_utils.py
6
+ kkutils/config_utils.py
7
+ kkutils/crypto.py
8
+ kkutils/curl_utils.py
9
+ kkutils/db_utils.py
10
+ kkutils/decorator.py
11
+ kkutils/email_utils.py
12
+ kkutils/fire.py
13
+ kkutils/generate_machine_id.py
14
+ kkutils/http_utils.py
15
+ kkutils/log_utils.py
16
+ kkutils/rabbitmq.py
17
+ kkutils/stopwatch.py
18
+ kkutils/xdb_searcher.py
19
+ kkutils/xor_file.py
20
+ kkutils.egg-info/PKG-INFO
21
+ kkutils.egg-info/SOURCES.txt
22
+ kkutils.egg-info/dependency_links.txt
23
+ kkutils.egg-info/requires.txt
24
+ kkutils.egg-info/top_level.txt
25
+ tornado_utils/__init__.py
26
+ tornado_utils/application.py
27
+ tornado_utils/basehandler.py
28
+ tornado_utils/userhandler.py
29
+ tornado_utils/utils.py
@@ -1,3 +1,3 @@
1
+ kkutils
1
2
  processor
2
3
  tornado_utils
3
- utils
@@ -16,7 +16,7 @@ from inspect import isclass
16
16
  from signal import SIGINT, SIGTERM
17
17
 
18
18
  import uvloop
19
- from utils import Config, Dict, Logger
19
+ from kkutils import Config, Dict, Logger
20
20
 
21
21
  asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
22
22
 
@@ -5,7 +5,7 @@ from setuptools import setup
5
5
 
6
6
  setup(
7
7
  name="kkutils",
8
- version="1.8.2",
8
+ version="2.0.0",
9
9
  description="python utils",
10
10
  author="digua",
11
11
  author_email="digua@gmail.com",
@@ -17,7 +17,7 @@ setup(
17
17
  include_package_data=True,
18
18
  extras_require={'curl': ['pycurl']},
19
19
  py_modules=['processor'],
20
- packages=['tornado_utils', 'utils'],
20
+ packages=['tornado_utils', 'kkutils'],
21
21
  classifiers=[
22
22
  'Development Status :: 4 - Beta',
23
23
  'Intended Audience :: Developers',
@@ -13,7 +13,7 @@ import tornado.netutil
13
13
  import tornado.process
14
14
  import tornado.web
15
15
  from tornado.options import define, options
16
- from utils import Cache, Dict, Logger, get_ip
16
+ from kkutils import Cache, Dict, Logger, get_ip
17
17
 
18
18
  __all__ = ['Blueprint', 'Application']
19
19
 
@@ -20,7 +20,7 @@ from functools import cached_property
20
20
 
21
21
  import tornado.web
22
22
  from bson import ObjectId
23
- from utils import Dict, JSONEncoder
23
+ from kkutils import Dict, JSONEncoder
24
24
 
25
25
 
26
26
  class BaseHandler(tornado.web.RequestHandler):
@@ -161,7 +161,9 @@ class BaseHandler(tornado.web.RequestHandler):
161
161
  return user
162
162
 
163
163
  async def get_current_user(self):
164
- token = self.args.get('token', self.get_cookie('token'))
164
+ auth_header = self.request.headers.get('Authorization', '')
165
+ header_token = auth_header.removeprefix('Bearer ').strip() if auth_header else None
166
+ token = self.args.get('token') or self.get_cookie('token') or header_token
165
167
  if token and hasattr(self.app, 'db'):
166
168
  user = await self.db.users.find_one({'token': token})
167
169
  if user:
@@ -217,7 +219,8 @@ class BaseHandler(tornado.web.RequestHandler):
217
219
  super().write_error(status_code, **kwargs)
218
220
 
219
221
  def render(self, template_name, **kwargs):
220
- if self.get_argument('f', None) == 'json':
222
+ accept = self.request.headers.get('Accept', '')
223
+ if self.get_argument('f', None) == 'json' or 'application/json' in accept:
221
224
  self.finish(kwargs)
222
225
  else:
223
226
  super().render(template_name, **kwargs)
@@ -13,7 +13,7 @@ from datetime import datetime
13
13
  import pymongo
14
14
  import tornado.web
15
15
  from bson import ObjectId
16
- from utils import Dict
16
+ from kkutils import Dict
17
17
 
18
18
  from .application import Blueprint
19
19
  from .basehandler import BaseHandler
@@ -151,6 +151,13 @@ class LogoutHandler(BaseHandler):
151
151
  class SignupHandler(BaseHandler):
152
152
 
153
153
  def get(self):
154
+ link = self.args.get('link') or self.args.get('from_id')
155
+ if link:
156
+ self.set_cookie('invite_link', str(link), expires_days=1)
157
+ elif not self.args.get('link') and not self.args.get('from_id'):
158
+ cookie_link = self.get_cookie('invite_link')
159
+ if cookie_link:
160
+ self.args['link'] = cookie_link
154
161
  self.render('signup.html')
155
162
 
156
163
  async def post(self):
@@ -9,7 +9,7 @@ import urllib.parse
9
9
 
10
10
  from tornado.template import Template
11
11
  from tornado.web import HTTPError, UIModule
12
- from utils import awaitable
12
+ from kkutils import awaitable
13
13
 
14
14
 
15
15
  def authorized(method):
@@ -1,29 +0,0 @@
1
- processor.py
2
- requirements.txt
3
- setup.py
4
- kkutils.egg-info/PKG-INFO
5
- kkutils.egg-info/SOURCES.txt
6
- kkutils.egg-info/dependency_links.txt
7
- kkutils.egg-info/requires.txt
8
- kkutils.egg-info/top_level.txt
9
- tornado_utils/__init__.py
10
- tornado_utils/application.py
11
- tornado_utils/basehandler.py
12
- tornado_utils/userhandler.py
13
- tornado_utils/utils.py
14
- utils/__init__.py
15
- utils/base_utils.py
16
- utils/config_utils.py
17
- utils/crypto.py
18
- utils/curl_utils.py
19
- utils/db_utils.py
20
- utils/decorator.py
21
- utils/email_utils.py
22
- utils/fire.py
23
- utils/generate_machine_id.py
24
- utils/http_utils.py
25
- utils/log_utils.py
26
- utils/rabbitmq.py
27
- utils/stopwatch.py
28
- utils/xdb_searcher.py
29
- utils/xor_file.py
File without changes
File without changes
File without changes