gomyck-tools 1.0.0__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/PKG-INFO +2 -2
- gomyck-tools-1.0.2/ctools/api_result.py +51 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/bottle_server.py +34 -1
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/call.py +3 -3
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/cjson.py +3 -9
- gomyck-tools-1.0.2/ctools/ckafka.py +152 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/cron_lite.py +111 -99
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/database.py +13 -5
- gomyck-tools-1.0.2/ctools/douglas_rarefy.py +130 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/ex.py +1 -1
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/http_utils.py +2 -2
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/metrics.py +0 -19
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/pacth.py +1 -1
- gomyck-tools-1.0.2/ctools/process_pool.py +35 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sys_log.py +26 -14
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/thread_pool.py +4 -0
- gomyck-tools-1.0.2/ctools/web_base.py +143 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/PKG-INFO +2 -2
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/SOURCES.txt +5 -3
- gomyck-tools-1.0.2/gomyck_tools.egg-info/requires.txt +8 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/setup.py +17 -8
- gomyck-tools-1.0.0/ctools/log.py +0 -28
- gomyck-tools-1.0.0/ctools/mvc.py +0 -56
- gomyck-tools-1.0.0/ctools/token_tools.py +0 -13
- gomyck-tools-1.0.0/gomyck_tools.egg-info/requires.txt +0 -6
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/README.md +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/__init__.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/aes_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/application.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/b64.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/bashPath.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/browser_element_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/compile_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/console.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/date_utils.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/download_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/enums.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/excelOpt.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/html_soup.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/id_worker_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/images_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/imgDialog.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/license.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/mqtt_utils.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/obj.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/plan_area_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/pty_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/resource_bundle_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/screenshot_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sign.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sm_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/ssh.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/strDiff.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/string_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sys_info.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/upload_tools.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/win_canvas.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/win_control.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/wordFill.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/word_fill.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/word_fill_entity.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/work_path.py +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/dependency_links.txt +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/top_level.txt +0 -0
- {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/setup.cfg +0 -0
@@ -1,7 +1,7 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: gomyck-tools
|
3
|
-
Version: 1.0.
|
4
|
-
Summary: A ctools for python development
|
3
|
+
Version: 1.0.2
|
4
|
+
Summary: A ctools for python development by hao474798383
|
5
5
|
Home-page: https://blog.gomyck.com
|
6
6
|
Author: gomyck
|
7
7
|
Author-email: hao474798383@163.com
|
@@ -0,0 +1,51 @@
|
|
1
|
+
from ctools import cjson
|
2
|
+
|
3
|
+
cjson.str_value_keys = [
|
4
|
+
"obj_id",
|
5
|
+
"script_id",
|
6
|
+
"template_id"
|
7
|
+
]
|
8
|
+
|
9
|
+
|
10
|
+
class _ResEnum(object):
|
11
|
+
def __init__(self, code: int, message: str):
|
12
|
+
self.code = code
|
13
|
+
self.message = message
|
14
|
+
|
15
|
+
def __eq__(self, o: object) -> bool:
|
16
|
+
return self.code == o
|
17
|
+
|
18
|
+
class R(object):
|
19
|
+
class Code:
|
20
|
+
|
21
|
+
@staticmethod
|
22
|
+
def cus_code(code, msg):
|
23
|
+
return _ResEnum(code, msg)
|
24
|
+
|
25
|
+
SUCCESS = _ResEnum(200, "成功")
|
26
|
+
FAIL = _ResEnum(400, "失败")
|
27
|
+
ERROR = _ResEnum(500, "异常")
|
28
|
+
|
29
|
+
def __init__(self, code: int, message: str, data=""):
|
30
|
+
self.code = code
|
31
|
+
self.message = message
|
32
|
+
self.data = data
|
33
|
+
|
34
|
+
def _to_json(self):
|
35
|
+
return cjson.unify_to_str(cjson.dumps(self))
|
36
|
+
|
37
|
+
@staticmethod
|
38
|
+
def parser(r_json: str):
|
39
|
+
return R(**cjson.loads(r_json))
|
40
|
+
|
41
|
+
@staticmethod
|
42
|
+
def ok(data=None, resp=Code.SUCCESS, msg=None):
|
43
|
+
return R(resp.code, msg if msg is not None else resp.message, data)._to_json()
|
44
|
+
|
45
|
+
@staticmethod
|
46
|
+
def fail(msg=None, resp=Code.FAIL, data=None):
|
47
|
+
return R(resp.code, msg if msg is not None else resp.message, data)._to_json()
|
48
|
+
|
49
|
+
@staticmethod
|
50
|
+
def error(msg=None, resp=Code.ERROR, data=None):
|
51
|
+
return R(resp.code, msg if msg is not None else resp.message, data)._to_json()
|
@@ -4,6 +4,39 @@ from wsgiref.simple_server import WSGIServer, WSGIRequestHandler, make_server
|
|
4
4
|
from geventwebsocket.handler import WebSocketHandler
|
5
5
|
from ctools import sys_log
|
6
6
|
|
7
|
+
"""
|
8
|
+
app = init_app('/doc_download')
|
9
|
+
|
10
|
+
@app.get('/queryList')
|
11
|
+
@mvc.parameter_handler()
|
12
|
+
@rule('DOC:DOWNLOAD')
|
13
|
+
def query_list(params, pageInfo):
|
14
|
+
"""
|
15
|
+
|
16
|
+
"""
|
17
|
+
module_names = list(globals().keys())
|
18
|
+
def get_modules():
|
19
|
+
mods = []
|
20
|
+
for modname in module_names:
|
21
|
+
if modname == 'base' or modname == 'online' or modname.startswith('__') or modname == 'importlib': continue
|
22
|
+
module = globals()[modname]
|
23
|
+
mods.append(module)
|
24
|
+
return mods
|
25
|
+
|
26
|
+
def get_ws_modules():
|
27
|
+
from . import websocket
|
28
|
+
return [websocket]
|
29
|
+
"""
|
30
|
+
|
31
|
+
"""
|
32
|
+
http_app = Bottle()
|
33
|
+
for module in controller.get_modules():
|
34
|
+
log.debug('正在挂载http路由: {}'.format(module.app.context_path))
|
35
|
+
http_app.mount(module.app.context_path, module.app)
|
36
|
+
http_server = bottle_server.WSGIRefServer(port=application.Server.port)
|
37
|
+
http_app.run(server=http_server, quiet=True)
|
38
|
+
"""
|
39
|
+
|
7
40
|
class ThreadedWSGIServer(ThreadingMixIn, WSGIServer): pass
|
8
41
|
|
9
42
|
class CustomWSGIHandler(WSGIRequestHandler):
|
@@ -36,7 +69,7 @@ class WSGIRefServer(ServerAdapter):
|
|
36
69
|
|
37
70
|
class WebSocketServer(ServerAdapter):
|
38
71
|
|
39
|
-
def __init__(self, host='0.0.0.0', port=
|
72
|
+
def __init__(self, host='0.0.0.0', port=8011):
|
40
73
|
super().__init__(host, port)
|
41
74
|
self.server = None
|
42
75
|
|
@@ -3,7 +3,7 @@ import threading
|
|
3
3
|
import time
|
4
4
|
from functools import wraps
|
5
5
|
|
6
|
-
|
6
|
+
# annotation
|
7
7
|
def once(func):
|
8
8
|
"""
|
9
9
|
decorator to initialize a function once
|
@@ -24,7 +24,7 @@ def once(func):
|
|
24
24
|
|
25
25
|
return wrapper
|
26
26
|
|
27
|
-
|
27
|
+
# annotation
|
28
28
|
def init(func):
|
29
29
|
"""
|
30
30
|
decorator to initialize a function automic
|
@@ -38,7 +38,7 @@ def init(func):
|
|
38
38
|
|
39
39
|
return wrapper
|
40
40
|
|
41
|
-
|
41
|
+
# annotation
|
42
42
|
def schd(interval_seconds, start_by_call: bool = False):
|
43
43
|
start_flag = False
|
44
44
|
run_flag = False
|
@@ -1,10 +1,11 @@
|
|
1
1
|
import jsonpickle
|
2
2
|
|
3
|
+
# 需要转换成str的属性
|
4
|
+
str_value_keys = []
|
3
5
|
jsonpickle.set_preferred_backend('json')
|
4
6
|
jsonpickle.set_encoder_options('json', ensure_ascii=False)
|
5
7
|
jsonpickle.set_decoder_options('json', encoding='utf-8')
|
6
8
|
|
7
|
-
|
8
9
|
def dumps(obj) -> str:
|
9
10
|
"""
|
10
11
|
将对象转换为json字符串
|
@@ -12,9 +13,9 @@ def dumps(obj) -> str:
|
|
12
13
|
:return: json 字符串
|
13
14
|
"""
|
14
15
|
if obj is None: return None
|
16
|
+
if type(obj) == str: return obj
|
15
17
|
return f'{jsonpickle.encode(obj, unpicklable=False)}'
|
16
18
|
|
17
|
-
|
18
19
|
def loads(json_str: str) -> dict:
|
19
20
|
"""
|
20
21
|
将json字符串转换为对象
|
@@ -23,11 +24,6 @@ def loads(json_str: str) -> dict:
|
|
23
24
|
"""
|
24
25
|
return jsonpickle.decode(json_str)
|
25
26
|
|
26
|
-
|
27
|
-
# 需要转换成str的属性
|
28
|
-
str_value_keys = []
|
29
|
-
|
30
|
-
|
31
27
|
def unify_to_str(json_str: str) -> str:
|
32
28
|
if not str_value_keys: return json_str
|
33
29
|
obj = loads(json_str)
|
@@ -37,7 +33,6 @@ def unify_to_str(json_str: str) -> str:
|
|
37
33
|
_handle_dict(obj)
|
38
34
|
return dumps(obj)
|
39
35
|
|
40
|
-
|
41
36
|
def _handle_list(data):
|
42
37
|
for o in data:
|
43
38
|
if isinstance(o, list):
|
@@ -45,7 +40,6 @@ def _handle_list(data):
|
|
45
40
|
elif isinstance(o, dict):
|
46
41
|
_handle_dict(o)
|
47
42
|
|
48
|
-
|
49
43
|
def _handle_dict(data):
|
50
44
|
for k, v in data.items():
|
51
45
|
if isinstance(v, list):
|
@@ -0,0 +1,152 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: UTF-8 -*-
|
3
|
+
__author__ = 'haoyang'
|
4
|
+
__date__ = '2024/9/5 10:39'
|
5
|
+
|
6
|
+
import signal
|
7
|
+
import sys
|
8
|
+
import time
|
9
|
+
from threading import Thread, Lock
|
10
|
+
|
11
|
+
from kafka import KafkaProducer, errors, KafkaConsumer
|
12
|
+
from kafka.producer.future import FutureRecordMetadata
|
13
|
+
|
14
|
+
from ctools import thread_pool
|
15
|
+
from ctools.cjson import dumps
|
16
|
+
|
17
|
+
"""
|
18
|
+
import time
|
19
|
+
|
20
|
+
from ctools import thread_pool
|
21
|
+
from ctools.ckafka import CKafka
|
22
|
+
|
23
|
+
c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
|
24
|
+
|
25
|
+
def send_msg():
|
26
|
+
while True:
|
27
|
+
time.sleep(3)
|
28
|
+
c.send_msg('test', 'test')
|
29
|
+
|
30
|
+
thread_pool.submit(send_msg)
|
31
|
+
c.get_msg('test')
|
32
|
+
"""
|
33
|
+
|
34
|
+
class CKafka:
|
35
|
+
|
36
|
+
def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password', consumer_group: str = 'ck-py-kafka-consumer'):
|
37
|
+
self.consumer: KafkaConsumer = None
|
38
|
+
self.producer: KafkaProducer = None
|
39
|
+
self.start_consumer = False
|
40
|
+
self.consumer_callback = {"topic_key": []}
|
41
|
+
self.consumer_group = consumer_group
|
42
|
+
self.kafka_url = kafka_url
|
43
|
+
self.init_producer = False
|
44
|
+
self.init_consumer = False
|
45
|
+
self.secure = secure
|
46
|
+
self.username = username
|
47
|
+
self.password = password
|
48
|
+
self.locker = Lock()
|
49
|
+
self.quited = False
|
50
|
+
|
51
|
+
def _create_producer(self) -> KafkaProducer:
|
52
|
+
print("[ Producer ] Connecting to Kafka brokers")
|
53
|
+
for i in range(0, 6):
|
54
|
+
try:
|
55
|
+
if self.secure:
|
56
|
+
self.producer = KafkaProducer(
|
57
|
+
bootstrap_servers=self.kafka_url,
|
58
|
+
sasl_plain_username=self.username,
|
59
|
+
sasl_plain_password=self.password,
|
60
|
+
security_protocol='SASL_PLAINTEXT',
|
61
|
+
sasl_mechanism='PLAIN',
|
62
|
+
value_serializer=lambda x: dumps(x).encode('utf-8'))
|
63
|
+
else:
|
64
|
+
self.producer = KafkaProducer(
|
65
|
+
bootstrap_servers=self.kafka_url,
|
66
|
+
value_serializer=lambda x: dumps(x).encode('utf-8'))
|
67
|
+
print("[ Producer ] Connected to Kafka...")
|
68
|
+
self.init_producer = True
|
69
|
+
return self.producer
|
70
|
+
except errors.NoBrokersAvailable:
|
71
|
+
print("[ Producer ] Waiting for brokers to become available...")
|
72
|
+
time.sleep(3)
|
73
|
+
raise RuntimeError("[ Producer ] Failed to connect to brokers within 60 seconds")
|
74
|
+
|
75
|
+
def _create_consumer(self) -> KafkaProducer:
|
76
|
+
print("[ Consumer ] Connecting to Kafka brokers")
|
77
|
+
for i in range(0, 6):
|
78
|
+
try:
|
79
|
+
if self.secure:
|
80
|
+
self.consumer = KafkaConsumer(
|
81
|
+
group_id=self.consumer_group,
|
82
|
+
bootstrap_servers=self.kafka_url,
|
83
|
+
sasl_plain_username=self.username,
|
84
|
+
sasl_plain_password=self.password,
|
85
|
+
security_protocol='SASL_PLAINTEXT',
|
86
|
+
sasl_mechanism='PLAIN',
|
87
|
+
value_deserializer=lambda x: x.decode('utf-8'))
|
88
|
+
else:
|
89
|
+
self.consumer = KafkaProducer(
|
90
|
+
bootstrap_servers=self.kafka_url,
|
91
|
+
value_deserializer=lambda x: x.decode('utf-8'))
|
92
|
+
print("[ Consumer ] Connected to Kafka...")
|
93
|
+
self.init_consumer = True
|
94
|
+
return self.consumer
|
95
|
+
except errors.NoBrokersAvailable:
|
96
|
+
print("[ Consumer ] Waiting for brokers to become available...")
|
97
|
+
time.sleep(3)
|
98
|
+
raise RuntimeError("[ Consumer ] Failed to connect to brokers within 60 seconds")
|
99
|
+
|
100
|
+
# FutureRecordMetadata 可以添加回调, 来监听是否发送成功
|
101
|
+
# r.add_callback(lambda x: print(x))
|
102
|
+
# r.get() 可以同步获取结果
|
103
|
+
def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
|
104
|
+
if self.quited: return None
|
105
|
+
if not self.init_producer:
|
106
|
+
with self.locker:
|
107
|
+
if not self.init_producer:
|
108
|
+
self._create_producer()
|
109
|
+
return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
|
110
|
+
|
111
|
+
def get_msg(self, topics: str, callBack=print):
|
112
|
+
if not self.init_consumer:
|
113
|
+
with self.locker:
|
114
|
+
if not self.init_consumer:
|
115
|
+
self._create_consumer()
|
116
|
+
for topic in topics.split(','):
|
117
|
+
if topic not in self.consumer_callback.keys():
|
118
|
+
self.consumer_callback[topic] = []
|
119
|
+
self.consumer.subscribe(self.consumer_callback.keys())
|
120
|
+
self.consumer_callback[topic].append(callBack)
|
121
|
+
if not self.start_consumer:
|
122
|
+
t = Thread(target=self._start_consumer_poll)
|
123
|
+
t.start()
|
124
|
+
|
125
|
+
def _start_consumer_poll(self):
|
126
|
+
self.start_consumer = True
|
127
|
+
for msg in self.consumer:
|
128
|
+
if self.quited: break
|
129
|
+
taskList = []
|
130
|
+
funcList = []
|
131
|
+
begin_time = time.time()
|
132
|
+
for func in self.consumer_callback[msg.topic]:
|
133
|
+
if self.quited: break
|
134
|
+
f = thread_pool.submit(func, (msg, ))
|
135
|
+
taskList.append(f)
|
136
|
+
funcList.append(func.__name__)
|
137
|
+
for f in taskList:
|
138
|
+
if self.quited: break
|
139
|
+
f.result()
|
140
|
+
end_time = time.time()
|
141
|
+
if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
|
142
|
+
taskList.clear()
|
143
|
+
funcList.clear()
|
144
|
+
|
145
|
+
def shutdown(self):
|
146
|
+
self.quited = True
|
147
|
+
try: self.consumer.close()
|
148
|
+
except Exception: pass
|
149
|
+
try: self.producer.close()
|
150
|
+
except Exception: pass
|
151
|
+
thread_pool.shutdown(wait=True)
|
152
|
+
|
@@ -7,10 +7,26 @@ import traceback
|
|
7
7
|
from datetime import datetime
|
8
8
|
from functools import wraps
|
9
9
|
from typing import Optional, Dict
|
10
|
-
|
11
10
|
import pytz
|
12
11
|
from croniter import croniter
|
13
12
|
|
13
|
+
"""
|
14
|
+
@cron_lite.cron_task('0/1 * * * * ? *')
|
15
|
+
def demo():
|
16
|
+
print('hello world')
|
17
|
+
|
18
|
+
@cron_lite.cron_task('0/1 * * * * ? *')
|
19
|
+
def demo1():
|
20
|
+
print('hello world111')
|
21
|
+
|
22
|
+
def demo2(xx, fff):
|
23
|
+
print('hello world222', xx, fff)
|
24
|
+
|
25
|
+
cron_lite.apply_cron_task('0/1 * * * * ? *', demo2, (123123123, 34534534))
|
26
|
+
print(123123)
|
27
|
+
|
28
|
+
cron_lite.start_all()
|
29
|
+
"""
|
14
30
|
|
15
31
|
class SchedulerMeta:
|
16
32
|
timer_task_name: str = None
|
@@ -22,8 +38,8 @@ class SchedulerMeta:
|
|
22
38
|
|
23
39
|
scheduler_map: Dict[str, SchedulerMeta] = {} # {timer_task_name: SchedulerMeta}
|
24
40
|
_switch = False
|
41
|
+
_info_handler = print
|
25
42
|
_error_handler = print
|
26
|
-
_info_handler = print
|
27
43
|
_time_zone: Optional[pytz.BaseTzInfo] = None
|
28
44
|
|
29
45
|
|
@@ -31,75 +47,8 @@ def set_time_zone(time_zone_name: str):
|
|
31
47
|
global _time_zone
|
32
48
|
_time_zone = pytz.timezone(time_zone_name)
|
33
49
|
|
34
|
-
|
35
|
-
def
|
36
|
-
cron_obj = croniter(cron_expr)
|
37
|
-
if _time_zone:
|
38
|
-
cron_obj.set_current(datetime.now(tz=_time_zone))
|
39
|
-
next_time = int(cron_obj.get_next())
|
40
|
-
if scheduler_map.get(timer_task_name) is None:
|
41
|
-
scheduler_meta = SchedulerMeta()
|
42
|
-
scheduler_meta.timer_task_name = timer_task_name
|
43
|
-
scheduler_meta.switch = True
|
44
|
-
scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
|
45
|
-
scheduler_map[timer_task_name] = scheduler_meta
|
46
|
-
if till_time_stamp is None or next_time <= till_time_stamp:
|
47
|
-
scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
|
48
|
-
|
49
|
-
|
50
|
-
def _run_sched(scheduler_meta: SchedulerMeta):
|
51
|
-
active(scheduler_meta.timer_task_name)
|
52
|
-
while True:
|
53
|
-
scheduler = scheduler_meta.scheduler
|
54
|
-
if not _switch or not scheduler_meta.switch:
|
55
|
-
scheduler.empty()
|
56
|
-
inactive(scheduler_meta.timer_task_name)
|
57
|
-
return
|
58
|
-
t = scheduler.run(False)
|
59
|
-
if t is None:
|
60
|
-
inactive(scheduler_meta.timer_task_name)
|
61
|
-
return
|
62
|
-
st = time.time()
|
63
|
-
while time.time() - st < t:
|
64
|
-
if not _switch or not scheduler_meta.switch:
|
65
|
-
scheduler.empty()
|
66
|
-
inactive(scheduler_meta.timer_task_name)
|
67
|
-
return
|
68
|
-
time.sleep(0.5)
|
69
|
-
|
70
|
-
|
71
|
-
def _start():
|
72
|
-
global _switch
|
73
|
-
_info_handler("cron started")
|
74
|
-
tl = []
|
75
|
-
for timer_task_name, scheduler_meta in scheduler_map.items():
|
76
|
-
print("Registering Job:", timer_task_name)
|
77
|
-
t = threading.Thread(target=_run_sched, args=(scheduler_meta,), daemon=True)
|
78
|
-
# 有些task非常耗时,会影响退出。目前设计改为退出时不保证task完成
|
79
|
-
t.start()
|
80
|
-
tl.append(t)
|
81
|
-
|
82
|
-
for t in tl:
|
83
|
-
t.join()
|
84
|
-
_info_handler("cron finished")
|
85
|
-
_switch = False # ensure close when there are no more tasks with switch open
|
86
|
-
scheduler_map.clear()
|
87
|
-
|
88
|
-
|
89
|
-
def convert_cron(cron_expr):
|
90
|
-
res_cron = ""
|
91
|
-
cron_list = cron_expr.split(" ")
|
92
|
-
if len(cron_list) > 6:
|
93
|
-
for cron in cron_list[1:]:
|
94
|
-
if cron != "?":
|
95
|
-
res_cron += "%s " % cron
|
96
|
-
res_cron += "%s" % cron_list[0]
|
97
|
-
else:
|
98
|
-
res_cron = cron_expr
|
99
|
-
return res_cron
|
100
|
-
|
101
|
-
|
102
|
-
def cron_task(cron_expr: str, till_time_stamp: int = None):
|
50
|
+
# @annotation
|
51
|
+
def cron_task(cron_expr: str, task_name: str = None, till_time_stamp: int = None):
|
103
52
|
"""
|
104
53
|
cron_task decorator to register a function as crontab task
|
105
54
|
:param cron_expr: the croniter accepted cron_expression. NOTICE: the default timezone is UTC and can be changed by
|
@@ -107,7 +56,7 @@ def cron_task(cron_expr: str, till_time_stamp: int = None):
|
|
107
56
|
:param till_time_stamp: run this jog till when. None means forever
|
108
57
|
:return: the real decorator
|
109
58
|
"""
|
110
|
-
cron_expr =
|
59
|
+
cron_expr = _convert_cron(cron_expr)
|
111
60
|
assert len(cron_expr.split(" ")) in (5, 6), \
|
112
61
|
"only supported <min hour day month weekday> and <min hour day month weekday sec>"
|
113
62
|
|
@@ -121,15 +70,15 @@ def cron_task(cron_expr: str, till_time_stamp: int = None):
|
|
121
70
|
_error_handler(f"run {func.__name__} failed\n" + traceback.format_exc())
|
122
71
|
except Exception:
|
123
72
|
_error_handler(f"run {func.__name__} failed\n")
|
124
|
-
_register_next(inner, inner, cron_expr, till_time_stamp)
|
73
|
+
_register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp)
|
125
74
|
|
126
|
-
_register_next(inner, inner, cron_expr, till_time_stamp)
|
75
|
+
_register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp, init=True)
|
127
76
|
return inner
|
128
77
|
|
129
78
|
return deco
|
130
79
|
|
131
80
|
|
132
|
-
def apply_cron_task(
|
81
|
+
def apply_cron_task(cron_expr, func, params, timer_task_name=None, till_time_stamp=None):
|
133
82
|
"""
|
134
83
|
cron_task decorator to register a function as crontab task
|
135
84
|
:param func: task callback function
|
@@ -140,37 +89,32 @@ def apply_cron_task(timer_task_name, func, params, cron_expr, till_time_stamp=No
|
|
140
89
|
:param till_time_stamp: run this jog till when. None means forever
|
141
90
|
:return: the real decorator
|
142
91
|
"""
|
143
|
-
cron_expr =
|
144
|
-
assert len(cron_expr.split(" ")) in (5, 6),
|
145
|
-
|
146
|
-
|
92
|
+
cron_expr = _convert_cron(cron_expr)
|
93
|
+
assert len(cron_expr.split(" ")) in (5, 6), "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
|
94
|
+
task_name = func.__name__ if timer_task_name is None else timer_task_name
|
147
95
|
@wraps(func)
|
148
96
|
def wrapper(*args, **kwargs):
|
149
97
|
try:
|
150
|
-
|
98
|
+
nonlocal params
|
99
|
+
func.__taskName__ = task_name
|
100
|
+
func(*params, *args, **kwargs)
|
151
101
|
except Exception as exc:
|
152
102
|
_error_handler(f"Run {func.__name__} failed with error: {str(exc)}")
|
153
103
|
finally:
|
154
|
-
_register_next(
|
104
|
+
_register_next(task_name, wrapper, cron_expr, till_time_stamp)
|
155
105
|
|
156
|
-
_register_next(
|
106
|
+
_register_next(task_name, wrapper, cron_expr, till_time_stamp, init=True)
|
107
|
+
# 不使用 submit, 因为提交的任务, 不是 daemon 线程
|
108
|
+
t = threading.Thread(target=_start, args=(timer_task_name, ))
|
109
|
+
t.setDaemon(True)
|
110
|
+
t.start()
|
111
|
+
return t
|
157
112
|
|
158
|
-
|
159
|
-
_switch = True
|
160
|
-
|
161
|
-
scheduler = scheduler_map.get(timer_task_name)
|
162
|
-
if scheduler:
|
163
|
-
scheduler.switch = True
|
164
|
-
t = threading.Thread(target=_run_sched, name=timer_task_name, args=(scheduler,), daemon=True)
|
165
|
-
# 有些task非常耗时,会影响退出。目前设计改为退出时不保证task完成
|
166
|
-
t.start()
|
167
|
-
return wrapper
|
168
|
-
|
169
|
-
|
170
|
-
def start_all(spawn: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
|
113
|
+
def start_all(spawn: bool = True, daemon: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
|
171
114
|
"""
|
172
115
|
start_all starts all cron tasks registered before.
|
173
116
|
:param spawn: whether to start a new thread for scheduler. If not, the action will block the current thread
|
117
|
+
:param daemon: the new thread is daemon if True
|
174
118
|
:param info_handler: handle info output (scheduler start / stop), default = print, can use logging.info
|
175
119
|
:param error_handler: handle error output (task execute exception), default = print, can use logging.error
|
176
120
|
:raise RuntimeError: if the tasks are already started and still running we cannot start again. The feature is not
|
@@ -179,16 +123,14 @@ def start_all(spawn: bool = True, info_handler=None, error_handler=None) -> Opti
|
|
179
123
|
"""
|
180
124
|
global _switch, _info_handler, _error_handler
|
181
125
|
if _switch:
|
182
|
-
raise RuntimeError("the crontab was already started")
|
126
|
+
raise RuntimeError("the crontab was already started...")
|
183
127
|
if info_handler:
|
184
128
|
_info_handler = info_handler
|
185
129
|
if error_handler:
|
186
130
|
_error_handler = error_handler
|
187
|
-
|
188
|
-
_switch = True
|
189
131
|
if spawn:
|
190
132
|
t = threading.Thread(target=_start)
|
191
|
-
t.setDaemon(
|
133
|
+
t.setDaemon(daemon)
|
192
134
|
t.start()
|
193
135
|
return t
|
194
136
|
else:
|
@@ -237,3 +179,73 @@ def stop_all(wait_thread: Optional[threading.Thread] = None):
|
|
237
179
|
scheduler_map.get(timer_task_name).switch = False
|
238
180
|
if wait_thread:
|
239
181
|
wait_thread.join()
|
182
|
+
|
183
|
+
|
184
|
+
def _register_next(timer_task_name, base_func, cron_expr, till_time_stamp, init: bool = False):
|
185
|
+
cron_obj = croniter(cron_expr)
|
186
|
+
if _time_zone:
|
187
|
+
cron_obj.set_current(datetime.now(tz=_time_zone))
|
188
|
+
next_time = int(cron_obj.get_next())
|
189
|
+
if scheduler_map.get(timer_task_name) is None:
|
190
|
+
scheduler_meta = SchedulerMeta()
|
191
|
+
scheduler_meta.timer_task_name = timer_task_name
|
192
|
+
scheduler_meta.switch = True
|
193
|
+
scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
|
194
|
+
scheduler_map[timer_task_name] = scheduler_meta
|
195
|
+
elif init:
|
196
|
+
raise ValueError(f"task name: {timer_task_name} already exists!!!!!")
|
197
|
+
if till_time_stamp is None or next_time <= till_time_stamp:
|
198
|
+
scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
|
199
|
+
|
200
|
+
|
201
|
+
def _run_sched(scheduler_meta: SchedulerMeta):
|
202
|
+
active(scheduler_meta.timer_task_name)
|
203
|
+
while True:
|
204
|
+
scheduler = scheduler_meta.scheduler
|
205
|
+
if not _switch or not scheduler_meta.switch:
|
206
|
+
scheduler.empty()
|
207
|
+
inactive(scheduler_meta.timer_task_name)
|
208
|
+
return
|
209
|
+
t = scheduler.run(False)
|
210
|
+
if t is None:
|
211
|
+
inactive(scheduler_meta.timer_task_name)
|
212
|
+
return
|
213
|
+
st = time.time()
|
214
|
+
while time.time() - st < t:
|
215
|
+
if not _switch or not scheduler_meta.switch:
|
216
|
+
scheduler.empty()
|
217
|
+
inactive(scheduler_meta.timer_task_name)
|
218
|
+
return
|
219
|
+
time.sleep(0.5)
|
220
|
+
|
221
|
+
|
222
|
+
def _start(taskName: str = None):
|
223
|
+
global _switch
|
224
|
+
_switch = True
|
225
|
+
_info_handler("cron job begin start...")
|
226
|
+
taskList = []
|
227
|
+
for timer_task_name, scheduler_meta in scheduler_map.items():
|
228
|
+
if taskName is not None and timer_task_name != taskName: continue
|
229
|
+
print("register job: ", timer_task_name)
|
230
|
+
thread = threading.Thread(target=_run_sched, args=(scheduler_meta, ))
|
231
|
+
thread.setDaemon(True)
|
232
|
+
thread.start()
|
233
|
+
taskList.append(thread)
|
234
|
+
for task in taskList: task.join()
|
235
|
+
_info_handler("cron job execute finished...")
|
236
|
+
_switch = False
|
237
|
+
scheduler_map.clear()
|
238
|
+
|
239
|
+
|
240
|
+
def _convert_cron(cron_expr):
|
241
|
+
res_cron = ""
|
242
|
+
cron_list = cron_expr.split(" ")
|
243
|
+
if len(cron_list) > 6:
|
244
|
+
for cron in cron_list[1:]:
|
245
|
+
if cron != "?":
|
246
|
+
res_cron += "%s " % cron
|
247
|
+
res_cron += "%s" % cron_list[0]
|
248
|
+
else:
|
249
|
+
res_cron = cron_expr
|
250
|
+
return res_cron
|
251
|
+
|
@@ -8,6 +8,13 @@ from sqlalchemy.sql import text
|
|
8
8
|
from ctools import call, string_tools
|
9
9
|
from ctools.thread_pool import thread_local
|
10
10
|
|
11
|
+
"""
|
12
|
+
database.init_db('postgresql://postgres:123456@192.168.3.107:32566/abc', default_schema='public', db_key='source', pool_size=100)
|
13
|
+
with database.get_session('source') as s:
|
14
|
+
s.execute(text('insert into xxx (name) values (:name)'), {'name': string_tools.get_random_str(5)})
|
15
|
+
s.commit()
|
16
|
+
"""
|
17
|
+
|
11
18
|
Base = None
|
12
19
|
inited_db = {}
|
13
20
|
engines = {}
|
@@ -21,11 +28,8 @@ def _init():
|
|
21
28
|
global Base
|
22
29
|
Base = declarative_base()
|
23
30
|
|
24
|
-
def set_search_path(dbapi_connection, default_schema):
|
25
|
-
with dbapi_connection.cursor() as cursor:
|
26
|
-
cursor.execute(f'SET search_path TO {default_schema}')
|
27
|
-
|
28
31
|
# 密码里的@ 要替换成 %40
|
32
|
+
# sqlite connect_args={"check_same_thread": False} sqlite:///{}.format(db_url)
|
29
33
|
def init_db(db_url: str, db_key: str='default', connect_args: dict={}, default_schema: str=None, pool_size: int=5, max_overflow: int=25, echo: bool=False):
|
30
34
|
if inited_db.get(db_key): raise Exception('db {} already init!!!'.format(db_key))
|
31
35
|
global engines, sessionMakers
|
@@ -33,7 +37,11 @@ def init_db(db_url: str, db_key: str='default', connect_args: dict={}, default_s
|
|
33
37
|
engines[db_key] = engine
|
34
38
|
sessionMakers[db_key] = sessionMaker
|
35
39
|
inited_db[db_key] = True
|
36
|
-
if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record:
|
40
|
+
if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: _set_search_path(dbapi_connection, default_schema))
|
41
|
+
|
42
|
+
def _set_search_path(dbapi_connection, default_schema):
|
43
|
+
with dbapi_connection.cursor() as cursor:
|
44
|
+
cursor.execute(f'SET search_path TO {default_schema}')
|
37
45
|
|
38
46
|
def _create_connection(db_url: str, pool_size: int=5, max_overflow: int=25, connect_args={}, echo: bool=False):
|
39
47
|
engine = create_engine('{}'.format(db_url),
|