gomyck-tools 1.0.0__tar.gz → 1.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/PKG-INFO +2 -2
  2. gomyck-tools-1.0.2/ctools/api_result.py +51 -0
  3. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/bottle_server.py +34 -1
  4. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/call.py +3 -3
  5. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/cjson.py +3 -9
  6. gomyck-tools-1.0.2/ctools/ckafka.py +152 -0
  7. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/cron_lite.py +111 -99
  8. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/database.py +13 -5
  9. gomyck-tools-1.0.2/ctools/douglas_rarefy.py +130 -0
  10. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/ex.py +1 -1
  11. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/http_utils.py +2 -2
  12. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/metrics.py +0 -19
  13. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/pacth.py +1 -1
  14. gomyck-tools-1.0.2/ctools/process_pool.py +35 -0
  15. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sys_log.py +26 -14
  16. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/thread_pool.py +4 -0
  17. gomyck-tools-1.0.2/ctools/web_base.py +143 -0
  18. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/PKG-INFO +2 -2
  19. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/SOURCES.txt +5 -3
  20. gomyck-tools-1.0.2/gomyck_tools.egg-info/requires.txt +8 -0
  21. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/setup.py +17 -8
  22. gomyck-tools-1.0.0/ctools/log.py +0 -28
  23. gomyck-tools-1.0.0/ctools/mvc.py +0 -56
  24. gomyck-tools-1.0.0/ctools/token_tools.py +0 -13
  25. gomyck-tools-1.0.0/gomyck_tools.egg-info/requires.txt +0 -6
  26. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/README.md +0 -0
  27. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/__init__.py +0 -0
  28. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/aes_tools.py +0 -0
  29. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/application.py +0 -0
  30. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/b64.py +0 -0
  31. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/bashPath.py +0 -0
  32. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/browser_element_tools.py +0 -0
  33. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/compile_tools.py +0 -0
  34. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/console.py +0 -0
  35. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/date_utils.py +0 -0
  36. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/download_tools.py +0 -0
  37. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/enums.py +0 -0
  38. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/excelOpt.py +0 -0
  39. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/html_soup.py +0 -0
  40. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/id_worker_tools.py +0 -0
  41. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/images_tools.py +0 -0
  42. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/imgDialog.py +0 -0
  43. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/license.py +0 -0
  44. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/mqtt_utils.py +0 -0
  45. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/obj.py +0 -0
  46. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/plan_area_tools.py +0 -0
  47. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/pty_tools.py +0 -0
  48. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/resource_bundle_tools.py +0 -0
  49. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/screenshot_tools.py +0 -0
  50. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sign.py +0 -0
  51. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sm_tools.py +0 -0
  52. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/ssh.py +0 -0
  53. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/strDiff.py +0 -0
  54. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/string_tools.py +0 -0
  55. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/sys_info.py +0 -0
  56. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/upload_tools.py +0 -0
  57. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/win_canvas.py +0 -0
  58. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/win_control.py +0 -0
  59. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/wordFill.py +0 -0
  60. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/word_fill.py +0 -0
  61. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/word_fill_entity.py +0 -0
  62. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/ctools/work_path.py +0 -0
  63. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/dependency_links.txt +0 -0
  64. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/gomyck_tools.egg-info/top_level.txt +0 -0
  65. {gomyck-tools-1.0.0 → gomyck-tools-1.0.2}/setup.cfg +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: gomyck-tools
3
- Version: 1.0.0
4
- Summary: A ctools for python development
3
+ Version: 1.0.2
4
+ Summary: A ctools for python development by hao474798383
5
5
  Home-page: https://blog.gomyck.com
6
6
  Author: gomyck
7
7
  Author-email: hao474798383@163.com
@@ -0,0 +1,51 @@
1
+ from ctools import cjson
2
+
3
+ cjson.str_value_keys = [
4
+ "obj_id",
5
+ "script_id",
6
+ "template_id"
7
+ ]
8
+
9
+
10
+ class _ResEnum(object):
11
+ def __init__(self, code: int, message: str):
12
+ self.code = code
13
+ self.message = message
14
+
15
+ def __eq__(self, o: object) -> bool:
16
+ return self.code == o
17
+
18
+ class R(object):
19
+ class Code:
20
+
21
+ @staticmethod
22
+ def cus_code(code, msg):
23
+ return _ResEnum(code, msg)
24
+
25
+ SUCCESS = _ResEnum(200, "成功")
26
+ FAIL = _ResEnum(400, "失败")
27
+ ERROR = _ResEnum(500, "异常")
28
+
29
+ def __init__(self, code: int, message: str, data=""):
30
+ self.code = code
31
+ self.message = message
32
+ self.data = data
33
+
34
+ def _to_json(self):
35
+ return cjson.unify_to_str(cjson.dumps(self))
36
+
37
+ @staticmethod
38
+ def parser(r_json: str):
39
+ return R(**cjson.loads(r_json))
40
+
41
+ @staticmethod
42
+ def ok(data=None, resp=Code.SUCCESS, msg=None):
43
+ return R(resp.code, msg if msg is not None else resp.message, data)._to_json()
44
+
45
+ @staticmethod
46
+ def fail(msg=None, resp=Code.FAIL, data=None):
47
+ return R(resp.code, msg if msg is not None else resp.message, data)._to_json()
48
+
49
+ @staticmethod
50
+ def error(msg=None, resp=Code.ERROR, data=None):
51
+ return R(resp.code, msg if msg is not None else resp.message, data)._to_json()
@@ -4,6 +4,39 @@ from wsgiref.simple_server import WSGIServer, WSGIRequestHandler, make_server
4
4
  from geventwebsocket.handler import WebSocketHandler
5
5
  from ctools import sys_log
6
6
 
7
+ """
8
+ app = init_app('/doc_download')
9
+
10
+ @app.get('/queryList')
11
+ @mvc.parameter_handler()
12
+ @rule('DOC:DOWNLOAD')
13
+ def query_list(params, pageInfo):
14
+ """
15
+
16
+ """
17
+ module_names = list(globals().keys())
18
+ def get_modules():
19
+ mods = []
20
+ for modname in module_names:
21
+ if modname == 'base' or modname == 'online' or modname.startswith('__') or modname == 'importlib': continue
22
+ module = globals()[modname]
23
+ mods.append(module)
24
+ return mods
25
+
26
+ def get_ws_modules():
27
+ from . import websocket
28
+ return [websocket]
29
+ """
30
+
31
+ """
32
+ http_app = Bottle()
33
+ for module in controller.get_modules():
34
+ log.debug('正在挂载http路由: {}'.format(module.app.context_path))
35
+ http_app.mount(module.app.context_path, module.app)
36
+ http_server = bottle_server.WSGIRefServer(port=application.Server.port)
37
+ http_app.run(server=http_server, quiet=True)
38
+ """
39
+
7
40
  class ThreadedWSGIServer(ThreadingMixIn, WSGIServer): pass
8
41
 
9
42
  class CustomWSGIHandler(WSGIRequestHandler):
@@ -36,7 +69,7 @@ class WSGIRefServer(ServerAdapter):
36
69
 
37
70
  class WebSocketServer(ServerAdapter):
38
71
 
39
- def __init__(self, host='0.0.0.0', port=8012):
72
+ def __init__(self, host='0.0.0.0', port=8011):
40
73
  super().__init__(host, port)
41
74
  self.server = None
42
75
 
@@ -3,7 +3,7 @@ import threading
3
3
  import time
4
4
  from functools import wraps
5
5
 
6
-
6
+ # annotation
7
7
  def once(func):
8
8
  """
9
9
  decorator to initialize a function once
@@ -24,7 +24,7 @@ def once(func):
24
24
 
25
25
  return wrapper
26
26
 
27
-
27
+ # annotation
28
28
  def init(func):
29
29
  """
30
30
  decorator to initialize a function automic
@@ -38,7 +38,7 @@ def init(func):
38
38
 
39
39
  return wrapper
40
40
 
41
-
41
+ # annotation
42
42
  def schd(interval_seconds, start_by_call: bool = False):
43
43
  start_flag = False
44
44
  run_flag = False
@@ -1,10 +1,11 @@
1
1
  import jsonpickle
2
2
 
3
+ # 需要转换成str的属性
4
+ str_value_keys = []
3
5
  jsonpickle.set_preferred_backend('json')
4
6
  jsonpickle.set_encoder_options('json', ensure_ascii=False)
5
7
  jsonpickle.set_decoder_options('json', encoding='utf-8')
6
8
 
7
-
8
9
  def dumps(obj) -> str:
9
10
  """
10
11
  将对象转换为json字符串
@@ -12,9 +13,9 @@ def dumps(obj) -> str:
12
13
  :return: json 字符串
13
14
  """
14
15
  if obj is None: return None
16
+ if type(obj) == str: return obj
15
17
  return f'{jsonpickle.encode(obj, unpicklable=False)}'
16
18
 
17
-
18
19
  def loads(json_str: str) -> dict:
19
20
  """
20
21
  将json字符串转换为对象
@@ -23,11 +24,6 @@ def loads(json_str: str) -> dict:
23
24
  """
24
25
  return jsonpickle.decode(json_str)
25
26
 
26
-
27
- # 需要转换成str的属性
28
- str_value_keys = []
29
-
30
-
31
27
  def unify_to_str(json_str: str) -> str:
32
28
  if not str_value_keys: return json_str
33
29
  obj = loads(json_str)
@@ -37,7 +33,6 @@ def unify_to_str(json_str: str) -> str:
37
33
  _handle_dict(obj)
38
34
  return dumps(obj)
39
35
 
40
-
41
36
  def _handle_list(data):
42
37
  for o in data:
43
38
  if isinstance(o, list):
@@ -45,7 +40,6 @@ def _handle_list(data):
45
40
  elif isinstance(o, dict):
46
41
  _handle_dict(o)
47
42
 
48
-
49
43
  def _handle_dict(data):
50
44
  for k, v in data.items():
51
45
  if isinstance(v, list):
@@ -0,0 +1,152 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: UTF-8 -*-
3
+ __author__ = 'haoyang'
4
+ __date__ = '2024/9/5 10:39'
5
+
6
+ import signal
7
+ import sys
8
+ import time
9
+ from threading import Thread, Lock
10
+
11
+ from kafka import KafkaProducer, errors, KafkaConsumer
12
+ from kafka.producer.future import FutureRecordMetadata
13
+
14
+ from ctools import thread_pool
15
+ from ctools.cjson import dumps
16
+
17
+ """
18
+ import time
19
+
20
+ from ctools import thread_pool
21
+ from ctools.ckafka import CKafka
22
+
23
+ c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
24
+
25
+ def send_msg():
26
+ while True:
27
+ time.sleep(3)
28
+ c.send_msg('test', 'test')
29
+
30
+ thread_pool.submit(send_msg)
31
+ c.get_msg('test')
32
+ """
33
+
34
+ class CKafka:
35
+
36
+ def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password', consumer_group: str = 'ck-py-kafka-consumer'):
37
+ self.consumer: KafkaConsumer = None
38
+ self.producer: KafkaProducer = None
39
+ self.start_consumer = False
40
+ self.consumer_callback = {"topic_key": []}
41
+ self.consumer_group = consumer_group
42
+ self.kafka_url = kafka_url
43
+ self.init_producer = False
44
+ self.init_consumer = False
45
+ self.secure = secure
46
+ self.username = username
47
+ self.password = password
48
+ self.locker = Lock()
49
+ self.quited = False
50
+
51
+ def _create_producer(self) -> KafkaProducer:
52
+ print("[ Producer ] Connecting to Kafka brokers")
53
+ for i in range(0, 6):
54
+ try:
55
+ if self.secure:
56
+ self.producer = KafkaProducer(
57
+ bootstrap_servers=self.kafka_url,
58
+ sasl_plain_username=self.username,
59
+ sasl_plain_password=self.password,
60
+ security_protocol='SASL_PLAINTEXT',
61
+ sasl_mechanism='PLAIN',
62
+ value_serializer=lambda x: dumps(x).encode('utf-8'))
63
+ else:
64
+ self.producer = KafkaProducer(
65
+ bootstrap_servers=self.kafka_url,
66
+ value_serializer=lambda x: dumps(x).encode('utf-8'))
67
+ print("[ Producer ] Connected to Kafka...")
68
+ self.init_producer = True
69
+ return self.producer
70
+ except errors.NoBrokersAvailable:
71
+ print("[ Producer ] Waiting for brokers to become available...")
72
+ time.sleep(3)
73
+ raise RuntimeError("[ Producer ] Failed to connect to brokers within 60 seconds")
74
+
75
+ def _create_consumer(self) -> KafkaProducer:
76
+ print("[ Consumer ] Connecting to Kafka brokers")
77
+ for i in range(0, 6):
78
+ try:
79
+ if self.secure:
80
+ self.consumer = KafkaConsumer(
81
+ group_id=self.consumer_group,
82
+ bootstrap_servers=self.kafka_url,
83
+ sasl_plain_username=self.username,
84
+ sasl_plain_password=self.password,
85
+ security_protocol='SASL_PLAINTEXT',
86
+ sasl_mechanism='PLAIN',
87
+ value_deserializer=lambda x: x.decode('utf-8'))
88
+ else:
89
+ self.consumer = KafkaProducer(
90
+ bootstrap_servers=self.kafka_url,
91
+ value_deserializer=lambda x: x.decode('utf-8'))
92
+ print("[ Consumer ] Connected to Kafka...")
93
+ self.init_consumer = True
94
+ return self.consumer
95
+ except errors.NoBrokersAvailable:
96
+ print("[ Consumer ] Waiting for brokers to become available...")
97
+ time.sleep(3)
98
+ raise RuntimeError("[ Consumer ] Failed to connect to brokers within 60 seconds")
99
+
100
+ # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
101
+ # r.add_callback(lambda x: print(x))
102
+ # r.get() 可以同步获取结果
103
+ def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
104
+ if self.quited: return None
105
+ if not self.init_producer:
106
+ with self.locker:
107
+ if not self.init_producer:
108
+ self._create_producer()
109
+ return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
110
+
111
+ def get_msg(self, topics: str, callBack=print):
112
+ if not self.init_consumer:
113
+ with self.locker:
114
+ if not self.init_consumer:
115
+ self._create_consumer()
116
+ for topic in topics.split(','):
117
+ if topic not in self.consumer_callback.keys():
118
+ self.consumer_callback[topic] = []
119
+ self.consumer.subscribe(self.consumer_callback.keys())
120
+ self.consumer_callback[topic].append(callBack)
121
+ if not self.start_consumer:
122
+ t = Thread(target=self._start_consumer_poll)
123
+ t.start()
124
+
125
+ def _start_consumer_poll(self):
126
+ self.start_consumer = True
127
+ for msg in self.consumer:
128
+ if self.quited: break
129
+ taskList = []
130
+ funcList = []
131
+ begin_time = time.time()
132
+ for func in self.consumer_callback[msg.topic]:
133
+ if self.quited: break
134
+ f = thread_pool.submit(func, (msg, ))
135
+ taskList.append(f)
136
+ funcList.append(func.__name__)
137
+ for f in taskList:
138
+ if self.quited: break
139
+ f.result()
140
+ end_time = time.time()
141
+ if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
142
+ taskList.clear()
143
+ funcList.clear()
144
+
145
+ def shutdown(self):
146
+ self.quited = True
147
+ try: self.consumer.close()
148
+ except Exception: pass
149
+ try: self.producer.close()
150
+ except Exception: pass
151
+ thread_pool.shutdown(wait=True)
152
+
@@ -7,10 +7,26 @@ import traceback
7
7
  from datetime import datetime
8
8
  from functools import wraps
9
9
  from typing import Optional, Dict
10
-
11
10
  import pytz
12
11
  from croniter import croniter
13
12
 
13
+ """
14
+ @cron_lite.cron_task('0/1 * * * * ? *')
15
+ def demo():
16
+ print('hello world')
17
+
18
+ @cron_lite.cron_task('0/1 * * * * ? *')
19
+ def demo1():
20
+ print('hello world111')
21
+
22
+ def demo2(xx, fff):
23
+ print('hello world222', xx, fff)
24
+
25
+ cron_lite.apply_cron_task('0/1 * * * * ? *', demo2, (123123123, 34534534))
26
+ print(123123)
27
+
28
+ cron_lite.start_all()
29
+ """
14
30
 
15
31
  class SchedulerMeta:
16
32
  timer_task_name: str = None
@@ -22,8 +38,8 @@ class SchedulerMeta:
22
38
 
23
39
  scheduler_map: Dict[str, SchedulerMeta] = {} # {timer_task_name: SchedulerMeta}
24
40
  _switch = False
41
+ _info_handler = print
25
42
  _error_handler = print
26
- _info_handler = print
27
43
  _time_zone: Optional[pytz.BaseTzInfo] = None
28
44
 
29
45
 
@@ -31,75 +47,8 @@ def set_time_zone(time_zone_name: str):
31
47
  global _time_zone
32
48
  _time_zone = pytz.timezone(time_zone_name)
33
49
 
34
-
35
- def _register_next(timer_task_name, base_func, cron_expr, till_time_stamp):
36
- cron_obj = croniter(cron_expr)
37
- if _time_zone:
38
- cron_obj.set_current(datetime.now(tz=_time_zone))
39
- next_time = int(cron_obj.get_next())
40
- if scheduler_map.get(timer_task_name) is None:
41
- scheduler_meta = SchedulerMeta()
42
- scheduler_meta.timer_task_name = timer_task_name
43
- scheduler_meta.switch = True
44
- scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
45
- scheduler_map[timer_task_name] = scheduler_meta
46
- if till_time_stamp is None or next_time <= till_time_stamp:
47
- scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
48
-
49
-
50
- def _run_sched(scheduler_meta: SchedulerMeta):
51
- active(scheduler_meta.timer_task_name)
52
- while True:
53
- scheduler = scheduler_meta.scheduler
54
- if not _switch or not scheduler_meta.switch:
55
- scheduler.empty()
56
- inactive(scheduler_meta.timer_task_name)
57
- return
58
- t = scheduler.run(False)
59
- if t is None:
60
- inactive(scheduler_meta.timer_task_name)
61
- return
62
- st = time.time()
63
- while time.time() - st < t:
64
- if not _switch or not scheduler_meta.switch:
65
- scheduler.empty()
66
- inactive(scheduler_meta.timer_task_name)
67
- return
68
- time.sleep(0.5)
69
-
70
-
71
- def _start():
72
- global _switch
73
- _info_handler("cron started")
74
- tl = []
75
- for timer_task_name, scheduler_meta in scheduler_map.items():
76
- print("Registering Job:", timer_task_name)
77
- t = threading.Thread(target=_run_sched, args=(scheduler_meta,), daemon=True)
78
- # 有些task非常耗时,会影响退出。目前设计改为退出时不保证task完成
79
- t.start()
80
- tl.append(t)
81
-
82
- for t in tl:
83
- t.join()
84
- _info_handler("cron finished")
85
- _switch = False # ensure close when there are no more tasks with switch open
86
- scheduler_map.clear()
87
-
88
-
89
- def convert_cron(cron_expr):
90
- res_cron = ""
91
- cron_list = cron_expr.split(" ")
92
- if len(cron_list) > 6:
93
- for cron in cron_list[1:]:
94
- if cron != "?":
95
- res_cron += "%s " % cron
96
- res_cron += "%s" % cron_list[0]
97
- else:
98
- res_cron = cron_expr
99
- return res_cron
100
-
101
-
102
- def cron_task(cron_expr: str, till_time_stamp: int = None):
50
+ # @annotation
51
+ def cron_task(cron_expr: str, task_name: str = None, till_time_stamp: int = None):
103
52
  """
104
53
  cron_task decorator to register a function as crontab task
105
54
  :param cron_expr: the croniter accepted cron_expression. NOTICE: the default timezone is UTC and can be changed by
@@ -107,7 +56,7 @@ def cron_task(cron_expr: str, till_time_stamp: int = None):
107
56
  :param till_time_stamp: run this jog till when. None means forever
108
57
  :return: the real decorator
109
58
  """
110
- cron_expr = convert_cron(cron_expr)
59
+ cron_expr = _convert_cron(cron_expr)
111
60
  assert len(cron_expr.split(" ")) in (5, 6), \
112
61
  "only supported <min hour day month weekday> and <min hour day month weekday sec>"
113
62
 
@@ -121,15 +70,15 @@ def cron_task(cron_expr: str, till_time_stamp: int = None):
121
70
  _error_handler(f"run {func.__name__} failed\n" + traceback.format_exc())
122
71
  except Exception:
123
72
  _error_handler(f"run {func.__name__} failed\n")
124
- _register_next(inner, inner, cron_expr, till_time_stamp)
73
+ _register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp)
125
74
 
126
- _register_next(inner, inner, cron_expr, till_time_stamp)
75
+ _register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp, init=True)
127
76
  return inner
128
77
 
129
78
  return deco
130
79
 
131
80
 
132
- def apply_cron_task(timer_task_name, func, params, cron_expr, till_time_stamp=None):
81
+ def apply_cron_task(cron_expr, func, params, timer_task_name=None, till_time_stamp=None):
133
82
  """
134
83
  cron_task decorator to register a function as crontab task
135
84
  :param func: task callback function
@@ -140,37 +89,32 @@ def apply_cron_task(timer_task_name, func, params, cron_expr, till_time_stamp=No
140
89
  :param till_time_stamp: run this jog till when. None means forever
141
90
  :return: the real decorator
142
91
  """
143
- cron_expr = convert_cron(cron_expr)
144
- assert len(cron_expr.split(" ")) in (5, 6), \
145
- "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
146
-
92
+ cron_expr = _convert_cron(cron_expr)
93
+ assert len(cron_expr.split(" ")) in (5, 6), "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
94
+ task_name = func.__name__ if timer_task_name is None else timer_task_name
147
95
  @wraps(func)
148
96
  def wrapper(*args, **kwargs):
149
97
  try:
150
- func(timer_task_name, params, *args, **kwargs)
98
+ nonlocal params
99
+ func.__taskName__ = task_name
100
+ func(*params, *args, **kwargs)
151
101
  except Exception as exc:
152
102
  _error_handler(f"Run {func.__name__} failed with error: {str(exc)}")
153
103
  finally:
154
- _register_next(timer_task_name, wrapper, cron_expr, till_time_stamp)
104
+ _register_next(task_name, wrapper, cron_expr, till_time_stamp)
155
105
 
156
- _register_next(timer_task_name, wrapper, cron_expr, till_time_stamp)
106
+ _register_next(task_name, wrapper, cron_expr, till_time_stamp, init=True)
107
+ # 不使用 submit, 因为提交的任务, 不是 daemon 线程
108
+ t = threading.Thread(target=_start, args=(timer_task_name, ))
109
+ t.setDaemon(True)
110
+ t.start()
111
+ return t
157
112
 
158
- global _switch
159
- _switch = True
160
-
161
- scheduler = scheduler_map.get(timer_task_name)
162
- if scheduler:
163
- scheduler.switch = True
164
- t = threading.Thread(target=_run_sched, name=timer_task_name, args=(scheduler,), daemon=True)
165
- # 有些task非常耗时,会影响退出。目前设计改为退出时不保证task完成
166
- t.start()
167
- return wrapper
168
-
169
-
170
- def start_all(spawn: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
113
+ def start_all(spawn: bool = True, daemon: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
171
114
  """
172
115
  start_all starts all cron tasks registered before.
173
116
  :param spawn: whether to start a new thread for scheduler. If not, the action will block the current thread
117
+ :param daemon: the new thread is daemon if True
174
118
  :param info_handler: handle info output (scheduler start / stop), default = print, can use logging.info
175
119
  :param error_handler: handle error output (task execute exception), default = print, can use logging.error
176
120
  :raise RuntimeError: if the tasks are already started and still running we cannot start again. The feature is not
@@ -179,16 +123,14 @@ def start_all(spawn: bool = True, info_handler=None, error_handler=None) -> Opti
179
123
  """
180
124
  global _switch, _info_handler, _error_handler
181
125
  if _switch:
182
- raise RuntimeError("the crontab was already started")
126
+ raise RuntimeError("the crontab was already started...")
183
127
  if info_handler:
184
128
  _info_handler = info_handler
185
129
  if error_handler:
186
130
  _error_handler = error_handler
187
-
188
- _switch = True
189
131
  if spawn:
190
132
  t = threading.Thread(target=_start)
191
- t.setDaemon(True)
133
+ t.setDaemon(daemon)
192
134
  t.start()
193
135
  return t
194
136
  else:
@@ -237,3 +179,73 @@ def stop_all(wait_thread: Optional[threading.Thread] = None):
237
179
  scheduler_map.get(timer_task_name).switch = False
238
180
  if wait_thread:
239
181
  wait_thread.join()
182
+
183
+
184
+ def _register_next(timer_task_name, base_func, cron_expr, till_time_stamp, init: bool = False):
185
+ cron_obj = croniter(cron_expr)
186
+ if _time_zone:
187
+ cron_obj.set_current(datetime.now(tz=_time_zone))
188
+ next_time = int(cron_obj.get_next())
189
+ if scheduler_map.get(timer_task_name) is None:
190
+ scheduler_meta = SchedulerMeta()
191
+ scheduler_meta.timer_task_name = timer_task_name
192
+ scheduler_meta.switch = True
193
+ scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
194
+ scheduler_map[timer_task_name] = scheduler_meta
195
+ elif init:
196
+ raise ValueError(f"task name: {timer_task_name} already exists!!!!!")
197
+ if till_time_stamp is None or next_time <= till_time_stamp:
198
+ scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
199
+
200
+
201
+ def _run_sched(scheduler_meta: SchedulerMeta):
202
+ active(scheduler_meta.timer_task_name)
203
+ while True:
204
+ scheduler = scheduler_meta.scheduler
205
+ if not _switch or not scheduler_meta.switch:
206
+ scheduler.empty()
207
+ inactive(scheduler_meta.timer_task_name)
208
+ return
209
+ t = scheduler.run(False)
210
+ if t is None:
211
+ inactive(scheduler_meta.timer_task_name)
212
+ return
213
+ st = time.time()
214
+ while time.time() - st < t:
215
+ if not _switch or not scheduler_meta.switch:
216
+ scheduler.empty()
217
+ inactive(scheduler_meta.timer_task_name)
218
+ return
219
+ time.sleep(0.5)
220
+
221
+
222
+ def _start(taskName: str = None):
223
+ global _switch
224
+ _switch = True
225
+ _info_handler("cron job begin start...")
226
+ taskList = []
227
+ for timer_task_name, scheduler_meta in scheduler_map.items():
228
+ if taskName is not None and timer_task_name != taskName: continue
229
+ print("register job: ", timer_task_name)
230
+ thread = threading.Thread(target=_run_sched, args=(scheduler_meta, ))
231
+ thread.setDaemon(True)
232
+ thread.start()
233
+ taskList.append(thread)
234
+ for task in taskList: task.join()
235
+ _info_handler("cron job execute finished...")
236
+ _switch = False
237
+ scheduler_map.clear()
238
+
239
+
240
+ def _convert_cron(cron_expr):
241
+ res_cron = ""
242
+ cron_list = cron_expr.split(" ")
243
+ if len(cron_list) > 6:
244
+ for cron in cron_list[1:]:
245
+ if cron != "?":
246
+ res_cron += "%s " % cron
247
+ res_cron += "%s" % cron_list[0]
248
+ else:
249
+ res_cron = cron_expr
250
+ return res_cron
251
+
@@ -8,6 +8,13 @@ from sqlalchemy.sql import text
8
8
  from ctools import call, string_tools
9
9
  from ctools.thread_pool import thread_local
10
10
 
11
+ """
12
+ database.init_db('postgresql://postgres:123456@192.168.3.107:32566/abc', default_schema='public', db_key='source', pool_size=100)
13
+ with database.get_session('source') as s:
14
+ s.execute(text('insert into xxx (name) values (:name)'), {'name': string_tools.get_random_str(5)})
15
+ s.commit()
16
+ """
17
+
11
18
  Base = None
12
19
  inited_db = {}
13
20
  engines = {}
@@ -21,11 +28,8 @@ def _init():
21
28
  global Base
22
29
  Base = declarative_base()
23
30
 
24
- def set_search_path(dbapi_connection, default_schema):
25
- with dbapi_connection.cursor() as cursor:
26
- cursor.execute(f'SET search_path TO {default_schema}')
27
-
28
31
  # 密码里的@ 要替换成 %40
32
+ # sqlite connect_args={"check_same_thread": False} sqlite:///{}.format(db_url)
29
33
  def init_db(db_url: str, db_key: str='default', connect_args: dict={}, default_schema: str=None, pool_size: int=5, max_overflow: int=25, echo: bool=False):
30
34
  if inited_db.get(db_key): raise Exception('db {} already init!!!'.format(db_key))
31
35
  global engines, sessionMakers
@@ -33,7 +37,11 @@ def init_db(db_url: str, db_key: str='default', connect_args: dict={}, default_s
33
37
  engines[db_key] = engine
34
38
  sessionMakers[db_key] = sessionMaker
35
39
  inited_db[db_key] = True
36
- if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: set_search_path(dbapi_connection, default_schema))
40
+ if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: _set_search_path(dbapi_connection, default_schema))
41
+
42
+ def _set_search_path(dbapi_connection, default_schema):
43
+ with dbapi_connection.cursor() as cursor:
44
+ cursor.execute(f'SET search_path TO {default_schema}')
37
45
 
38
46
  def _create_connection(db_url: str, pool_size: int=5, max_overflow: int=25, connect_args={}, echo: bool=False):
39
47
  engine = create_engine('{}'.format(db_url),