kkutils 1.8.3__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {kkutils-1.8.3 → kkutils-2.0.0}/PKG-INFO +1 -1
  2. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/crypto.py +11 -1
  3. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/db_utils.py +16 -0
  4. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/email_utils.py +35 -25
  5. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/http_utils.py +7 -2
  6. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/rabbitmq.py +12 -6
  7. {kkutils-1.8.3 → kkutils-2.0.0}/kkutils.egg-info/PKG-INFO +1 -1
  8. kkutils-2.0.0/kkutils.egg-info/SOURCES.txt +29 -0
  9. {kkutils-1.8.3 → kkutils-2.0.0}/kkutils.egg-info/top_level.txt +1 -1
  10. {kkutils-1.8.3 → kkutils-2.0.0}/processor.py +1 -1
  11. {kkutils-1.8.3 → kkutils-2.0.0}/setup.py +2 -2
  12. {kkutils-1.8.3 → kkutils-2.0.0}/tornado_utils/application.py +1 -1
  13. {kkutils-1.8.3 → kkutils-2.0.0}/tornado_utils/basehandler.py +6 -3
  14. {kkutils-1.8.3 → kkutils-2.0.0}/tornado_utils/userhandler.py +8 -1
  15. {kkutils-1.8.3 → kkutils-2.0.0}/tornado_utils/utils.py +1 -1
  16. kkutils-1.8.3/kkutils.egg-info/SOURCES.txt +0 -29
  17. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/__init__.py +0 -0
  18. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/base_utils.py +0 -0
  19. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/config_utils.py +0 -0
  20. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/curl_utils.py +0 -0
  21. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/decorator.py +0 -0
  22. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/fire.py +0 -0
  23. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/generate_machine_id.py +0 -0
  24. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/log_utils.py +0 -0
  25. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/stopwatch.py +0 -0
  26. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/xdb_searcher.py +0 -0
  27. {kkutils-1.8.3/utils → kkutils-2.0.0/kkutils}/xor_file.py +0 -0
  28. {kkutils-1.8.3 → kkutils-2.0.0}/kkutils.egg-info/dependency_links.txt +0 -0
  29. {kkutils-1.8.3 → kkutils-2.0.0}/kkutils.egg-info/requires.txt +0 -0
  30. {kkutils-1.8.3 → kkutils-2.0.0}/requirements.txt +0 -0
  31. {kkutils-1.8.3 → kkutils-2.0.0}/setup.cfg +0 -0
  32. {kkutils-1.8.3 → kkutils-2.0.0}/tornado_utils/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kkutils
3
- Version: 1.8.3
3
+ Version: 2.0.0
4
4
  Summary: python utils
5
5
  Home-page: https://www.github.com
6
6
  Author: digua
@@ -49,8 +49,18 @@ def decode(text, lib='base62'):
49
49
  return binascii.a2b_hex(text)
50
50
 
51
51
 
52
+ _XOR_TABLES = {}
53
+
54
+
52
55
  def xor_encrypt(text: str, key: int, lib='base64'):
53
- encrypted = ''.join([chr(ord(x) ^ ord(chr(key))) for x in to_str(text)])
56
+ text = to_str(text)
57
+ if text.isascii() and 0 <= key < 128:
58
+ # Fast path: ASCII xor a 7-bit key stays single-byte, so a byte table gives identical output
59
+ table = _XOR_TABLES.get(key)
60
+ if table is None:
61
+ table = _XOR_TABLES[key] = bytes(i ^ key for i in range(256))
62
+ return encode(text.encode().translate(table), lib)
63
+ encrypted = ''.join([chr(ord(x) ^ ord(chr(key))) for x in text])
54
64
  return encode(encrypted, lib)
55
65
 
56
66
 
@@ -5,6 +5,7 @@ Last modified: 2018-01-05 11:24:17
5
5
  '''
6
6
 
7
7
  import asyncio
8
+ import logging
8
9
  import os
9
10
  import re
10
11
  from urllib.parse import parse_qs, quote_plus, unquote_plus
@@ -13,6 +14,7 @@ import aiomysql
13
14
  import pymongo
14
15
  import pymysql
15
16
  import redis
17
+ import redis.exceptions
16
18
  from dbutils.pooled_db import PooledDB
17
19
  from motor import core
18
20
  from motor.docstrings import get_database_doc
@@ -291,11 +293,25 @@ class AioRedis(aioredis.StrictRedis):
291
293
 
292
294
  kwargs.setdefault('decode_responses', True)
293
295
  kwargs.setdefault('health_check_interval', 30)
296
+ kwargs.setdefault('max_connections', 5000)
294
297
  kwargs.setdefault('retry_on_timeout', True)
295
298
  kwargs.setdefault('socket_keepalive', True)
296
299
  pool = aioredis.ConnectionPool.from_url(uri, **kwargs)
297
300
  super().__init__(connection_pool=pool)
298
301
 
302
+ async def execute_command(self, *args, **kwargs):
303
+ try:
304
+ return await super().execute_command(*args, **kwargs)
305
+ except redis.exceptions.ConnectionError as e:
306
+ pool = self.connection_pool
307
+ logging.getLogger().error(
308
+ f'redis connection error: {e}, cmd: {args[:1]}, '
309
+ f'in_use: {len(pool._in_use_connections)}, '
310
+ f'available: {len(pool._available_connections)}, '
311
+ f'max: {pool.max_connections}, pid: {os.getpid()}'
312
+ )
313
+ raise
314
+
299
315
  async def clear(self, pattern='*'):
300
316
  if pattern == '*':
301
317
  await self.flushdb()
@@ -17,15 +17,30 @@ import aiosmtplib
17
17
 
18
18
  __all__ = ['Email', 'AioEmail']
19
19
 
20
+ # Hard cap on the connect/login/send/quit cycle. Without this, a stalled or
21
+ # unresponsive MTA hangs the caller indefinitely — smtplib's default is no
22
+ # timeout at all.
23
+ DEFAULT_TIMEOUT = 30
24
+
20
25
 
21
26
  class EmailBase:
27
+ """Shared message-building logic for ``Email``/``AioEmail``.
28
+
29
+ Subclasses build a fresh SMTP client on every ``send()`` call rather
30
+ than reusing one across the instance's lifetime — a stale/half-closed
31
+ socket or half-completed auth left over from a prior failure can then
32
+ never leak into the next send. Each call gets a clean
33
+ connect/login/send/quit cycle, with ``with``/``async with`` guaranteeing
34
+ cleanup even on error.
35
+ """
22
36
 
23
- def __init__(self, sender=None, smtp=None, user=None, pwd=None, use_tls=False):
37
+ def __init__(self, sender=None, smtp=None, user=None, pwd=None, use_tls=False, timeout=DEFAULT_TIMEOUT):
24
38
  self.sender = sender or os.environ.get('EMAIL_SENDER')
25
39
  self.smtp = smtp or os.environ.get('EMAIL_SMTP')
26
40
  self.user = user or os.environ.get('EMAIL_USER')
27
41
  self.pwd = pwd or os.environ.get('EMAIL_PWD')
28
42
  self.use_tls = use_tls or os.environ.get('EMAIL_TLS') == 'true'
43
+ self.timeout = timeout
29
44
 
30
45
  @staticmethod
31
46
  def _format_addr(s):
@@ -33,7 +48,7 @@ class EmailBase:
33
48
  name, addr = parseaddr(s)
34
49
  return formataddr((Header(name, 'utf-8').encode(), addr)) if name else addr
35
50
 
36
- def pack(self, receivers, title=None, content=None, files=None, cc=None):
51
+ def pack(self, receivers, title=None, content=None, files=None, cc=None, sender=None):
37
52
  msg = MIMEMultipart()
38
53
  msg.set_charset('utf8')
39
54
 
@@ -60,8 +75,12 @@ class EmailBase:
60
75
 
61
76
  msg['subject'] = title
62
77
  msg['date'] = formatdate(localtime=True)
63
- sender = f'{self.sender}<{self.user}>' if self.sender else self.user
64
- msg['from'] = self._format_addr(sender)
78
+ # Per-call sender overrides the instance default (self.sender, set at
79
+ # construction time) so one client can send under different display
80
+ # names without needing a separate Email/AioEmail instance per name.
81
+ display_name = sender if sender is not None else self.sender
82
+ from_addr = f'{display_name}<{self.user}>' if display_name else self.user
83
+ msg['from'] = self._format_addr(from_addr)
65
84
  if not isinstance(receivers, (list, tuple)):
66
85
  receivers = [receivers]
67
86
  msg['to'] = COMMASPACE.join([self._format_addr(r) for r in receivers])
@@ -70,32 +89,23 @@ class EmailBase:
70
89
 
71
90
  class Email(EmailBase):
72
91
 
73
- def __init__(self, *args, **kwargs):
74
- super().__init__(*args, **kwargs)
75
- if self.use_tls:
76
- self.client = smtplib.SMTP_SSL(self.smtp, 465)
77
- else:
78
- self.client = smtplib.SMTP(self.smtp)
79
-
80
92
  def send(self, *args, **kwargs):
81
93
  msg = self.pack(*args, **kwargs)
82
- self.client.connect(self.smtp)
83
- self.client.ehlo()
84
- self.client.login(self.user, self.pwd)
85
- self.client.send_message(msg)
86
- self.client.quit()
94
+ # See EmailBase docstring for why this connects fresh each call.
95
+ cls = smtplib.SMTP_SSL if self.use_tls else smtplib.SMTP
96
+ with cls(self.smtp, 465 if self.use_tls else 0, timeout=self.timeout) as client:
97
+ client.ehlo()
98
+ client.login(self.user, self.pwd)
99
+ client.send_message(msg)
87
100
 
88
101
 
89
102
  class AioEmail(EmailBase):
90
103
 
91
- def __init__(self, *args, **kwargs):
92
- super().__init__(*args, **kwargs)
93
- kwargs = {'port': 465, 'use_tls': True} if self.use_tls else {}
94
- self.client = aiosmtplib.SMTP(hostname=self.smtp, **kwargs)
95
-
96
104
  async def send(self, *args, **kwargs):
97
105
  msg = self.pack(*args, **kwargs)
98
- await self.client.connect()
99
- await self.client.login(self.user, self.pwd)
100
- await self.client.send_message(msg)
101
- await self.client.quit()
106
+ # See EmailBase docstring for why this connects fresh each call.
107
+ extra = {'port': 465, 'use_tls': True} if self.use_tls else {}
108
+ client = aiosmtplib.SMTP(hostname=self.smtp, timeout=self.timeout, **extra)
109
+ async with client:
110
+ await client.login(self.user, self.pwd)
111
+ await client.send_message(msg)
@@ -140,6 +140,9 @@ class Response:
140
140
  encoding = params['charset'].strip("'\"")
141
141
  if encoding is None:
142
142
  encoding = chardet.detect(self.body)['encoding']
143
+ # chardet misdetects ASCII text containing '+' (e.g. base64) as UTF-7
144
+ if encoding and encoding.lower() == 'utf-7':
145
+ encoding = 'utf-8'
143
146
  return 'gbk' if encoding and encoding.lower() == 'gb2312' else encoding
144
147
 
145
148
  @cached_property
@@ -150,7 +153,9 @@ class Response:
150
153
  return self.body.decode()
151
154
 
152
155
  def json(self, **kwargs):
153
- return DictWrapper(json.loads(self.text, **kwargs))
156
+ # json.loads accepts bytes and detects UTF-8/16/32 per the JSON spec,
157
+ # bypassing charset guessing on responses like text/plain JSON
158
+ return DictWrapper(json.loads(self.body, **kwargs))
154
159
 
155
160
  def soup(self, features='html5lib', **kwargs):
156
161
  return BeautifulSoup(self.body, features=features, **kwargs)
@@ -203,7 +208,7 @@ class BaseRequest(metaclass=RequestMeta):
203
208
  files: dict, { key: filename or file descriptor }
204
209
  proxy: str, 'http://user:password@114.112.93.35:8080'
205
210
  '''
206
- self.proxy = kwargs.get('proxy', os.environ.get('http_proxy'))
211
+ self.proxy = kwargs.get('proxy', os.environ.get('HTTP_PROXY'))
207
212
  self.cookie = kwargs.get('cookie')
208
213
  self.cookies = kwargs.get('cookies', {})
209
214
  self.timeout = kwargs.get('timeout', 30)
@@ -51,7 +51,7 @@ class Pika:
51
51
  if not (queue in self._channels and not self._channels[queue].is_closed):
52
52
  channel = self._connection.channel()
53
53
  channel.basic_qos(prefetch_count=1)
54
- q = channel.queue_declare(queue=queue, auto_delete=False, durable=False)
54
+ q = channel.queue_declare(queue=queue, auto_delete=False, durable=True)
55
55
  self._channels[queue] = channel
56
56
  self._queues[queue] = q
57
57
 
@@ -129,9 +129,14 @@ class AioPika(Pika):
129
129
 
130
130
  async with self.lock:
131
131
  if not (queue in self._channels and not self._channels[queue].is_closed):
132
- channel = await self._connection.channel()
132
+ try:
133
+ channel = await self._connection.channel()
134
+ except RuntimeError:
135
+ self._connection = await aio_pika.connect_robust(self.uri)
136
+ self.logger.info(f're-initialize {self._connection}')
137
+ channel = await self._connection.channel()
133
138
  await channel.set_qos(prefetch_count=1)
134
- q = await channel.declare_queue(queue, auto_delete=False, durable=False)
139
+ q = await channel.declare_queue(queue, auto_delete=False, durable=True)
135
140
  self._channels[queue] = channel
136
141
  self._queues[queue] = q
137
142
 
@@ -141,7 +146,7 @@ class AioPika(Pika):
141
146
  async with self.lock:
142
147
  channel = await self._connection.channel()
143
148
  await channel.set_qos(prefetch_count=1)
144
- q = await channel.declare_queue(queue_name, auto_delete=False, durable=False)
149
+ q = await channel.declare_queue(queue_name, auto_delete=False, durable=True)
145
150
 
146
151
  async with q.iterator() as queue_iter:
147
152
  async for msg in queue_iter:
@@ -183,10 +188,11 @@ class AioPika(Pika):
183
188
  del self._consumer_tasks[queue]
184
189
  del self._stop_events[queue]
185
190
 
186
- async def publish(self, msg, queue=None):
191
+ async def publish(self, msg, queue=None, timeout=5):
187
192
  queue = await self.init(queue)
188
193
  message = aio_pika.Message(pickle.dumps(msg))
189
- await self._channels[queue].default_exchange.publish(message, routing_key=queue)
194
+ coro = self._channels[queue].default_exchange.publish(message, routing_key=queue)
195
+ await asyncio.wait_for(coro, timeout=timeout)
190
196
 
191
197
  async def close(self):
192
198
  for queue in list(self._stop_events.keys()):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kkutils
3
- Version: 1.8.3
3
+ Version: 2.0.0
4
4
  Summary: python utils
5
5
  Home-page: https://www.github.com
6
6
  Author: digua
@@ -0,0 +1,29 @@
1
+ processor.py
2
+ requirements.txt
3
+ setup.py
4
+ kkutils/__init__.py
5
+ kkutils/base_utils.py
6
+ kkutils/config_utils.py
7
+ kkutils/crypto.py
8
+ kkutils/curl_utils.py
9
+ kkutils/db_utils.py
10
+ kkutils/decorator.py
11
+ kkutils/email_utils.py
12
+ kkutils/fire.py
13
+ kkutils/generate_machine_id.py
14
+ kkutils/http_utils.py
15
+ kkutils/log_utils.py
16
+ kkutils/rabbitmq.py
17
+ kkutils/stopwatch.py
18
+ kkutils/xdb_searcher.py
19
+ kkutils/xor_file.py
20
+ kkutils.egg-info/PKG-INFO
21
+ kkutils.egg-info/SOURCES.txt
22
+ kkutils.egg-info/dependency_links.txt
23
+ kkutils.egg-info/requires.txt
24
+ kkutils.egg-info/top_level.txt
25
+ tornado_utils/__init__.py
26
+ tornado_utils/application.py
27
+ tornado_utils/basehandler.py
28
+ tornado_utils/userhandler.py
29
+ tornado_utils/utils.py
@@ -1,3 +1,3 @@
1
+ kkutils
1
2
  processor
2
3
  tornado_utils
3
- utils
@@ -16,7 +16,7 @@ from inspect import isclass
16
16
  from signal import SIGINT, SIGTERM
17
17
 
18
18
  import uvloop
19
- from utils import Config, Dict, Logger
19
+ from kkutils import Config, Dict, Logger
20
20
 
21
21
  asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
22
22
 
@@ -5,7 +5,7 @@ from setuptools import setup
5
5
 
6
6
  setup(
7
7
  name="kkutils",
8
- version="1.8.3",
8
+ version="2.0.0",
9
9
  description="python utils",
10
10
  author="digua",
11
11
  author_email="digua@gmail.com",
@@ -17,7 +17,7 @@ setup(
17
17
  include_package_data=True,
18
18
  extras_require={'curl': ['pycurl']},
19
19
  py_modules=['processor'],
20
- packages=['tornado_utils', 'utils'],
20
+ packages=['tornado_utils', 'kkutils'],
21
21
  classifiers=[
22
22
  'Development Status :: 4 - Beta',
23
23
  'Intended Audience :: Developers',
@@ -13,7 +13,7 @@ import tornado.netutil
13
13
  import tornado.process
14
14
  import tornado.web
15
15
  from tornado.options import define, options
16
- from utils import Cache, Dict, Logger, get_ip
16
+ from kkutils import Cache, Dict, Logger, get_ip
17
17
 
18
18
  __all__ = ['Blueprint', 'Application']
19
19
 
@@ -20,7 +20,7 @@ from functools import cached_property
20
20
 
21
21
  import tornado.web
22
22
  from bson import ObjectId
23
- from utils import Dict, JSONEncoder
23
+ from kkutils import Dict, JSONEncoder
24
24
 
25
25
 
26
26
  class BaseHandler(tornado.web.RequestHandler):
@@ -161,7 +161,9 @@ class BaseHandler(tornado.web.RequestHandler):
161
161
  return user
162
162
 
163
163
  async def get_current_user(self):
164
- token = self.args.get('token', self.get_cookie('token'))
164
+ auth_header = self.request.headers.get('Authorization', '')
165
+ header_token = auth_header.removeprefix('Bearer ').strip() if auth_header else None
166
+ token = self.args.get('token') or self.get_cookie('token') or header_token
165
167
  if token and hasattr(self.app, 'db'):
166
168
  user = await self.db.users.find_one({'token': token})
167
169
  if user:
@@ -217,7 +219,8 @@ class BaseHandler(tornado.web.RequestHandler):
217
219
  super().write_error(status_code, **kwargs)
218
220
 
219
221
  def render(self, template_name, **kwargs):
220
- if self.get_argument('f', None) == 'json':
222
+ accept = self.request.headers.get('Accept', '')
223
+ if self.get_argument('f', None) == 'json' or 'application/json' in accept:
221
224
  self.finish(kwargs)
222
225
  else:
223
226
  super().render(template_name, **kwargs)
@@ -13,7 +13,7 @@ from datetime import datetime
13
13
  import pymongo
14
14
  import tornado.web
15
15
  from bson import ObjectId
16
- from utils import Dict
16
+ from kkutils import Dict
17
17
 
18
18
  from .application import Blueprint
19
19
  from .basehandler import BaseHandler
@@ -151,6 +151,13 @@ class LogoutHandler(BaseHandler):
151
151
  class SignupHandler(BaseHandler):
152
152
 
153
153
  def get(self):
154
+ link = self.args.get('link') or self.args.get('from_id')
155
+ if link:
156
+ self.set_cookie('invite_link', str(link), expires_days=1)
157
+ elif not self.args.get('link') and not self.args.get('from_id'):
158
+ cookie_link = self.get_cookie('invite_link')
159
+ if cookie_link:
160
+ self.args['link'] = cookie_link
154
161
  self.render('signup.html')
155
162
 
156
163
  async def post(self):
@@ -9,7 +9,7 @@ import urllib.parse
9
9
 
10
10
  from tornado.template import Template
11
11
  from tornado.web import HTTPError, UIModule
12
- from utils import awaitable
12
+ from kkutils import awaitable
13
13
 
14
14
 
15
15
  def authorized(method):
@@ -1,29 +0,0 @@
1
- processor.py
2
- requirements.txt
3
- setup.py
4
- kkutils.egg-info/PKG-INFO
5
- kkutils.egg-info/SOURCES.txt
6
- kkutils.egg-info/dependency_links.txt
7
- kkutils.egg-info/requires.txt
8
- kkutils.egg-info/top_level.txt
9
- tornado_utils/__init__.py
10
- tornado_utils/application.py
11
- tornado_utils/basehandler.py
12
- tornado_utils/userhandler.py
13
- tornado_utils/utils.py
14
- utils/__init__.py
15
- utils/base_utils.py
16
- utils/config_utils.py
17
- utils/crypto.py
18
- utils/curl_utils.py
19
- utils/db_utils.py
20
- utils/decorator.py
21
- utils/email_utils.py
22
- utils/fire.py
23
- utils/generate_machine_id.py
24
- utils/http_utils.py
25
- utils/log_utils.py
26
- utils/rabbitmq.py
27
- utils/stopwatch.py
28
- utils/xdb_searcher.py
29
- utils/xor_file.py
File without changes
File without changes
File without changes