toolkits 0.2.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolkits/3des/3des.py +93 -0
- toolkits/3des/__init__.py +0 -0
- toolkits/__init__.py +2 -0
- toolkits/basic/__init__.py +0 -0
- toolkits/basic/list_helper.py +26 -0
- toolkits/config/__init__.py +0 -0
- toolkits/config/config_demo.py +43 -0
- toolkits/databases/__init__.py +0 -0
- toolkits/databases/database_client_util.py +143 -0
- toolkits/databases/es_client.py +88 -0
- toolkits/databases/hive_client.py +72 -0
- toolkits/databases/hive_cmd.py +113 -0
- toolkits/databases/hive_helper.py +220 -0
- toolkits/databases/redis_mgmt.py +95 -0
- toolkits/databases/sql_helper.py +291 -0
- toolkits/databases/sqlalchemy_helper.py +71 -0
- toolkits/databases/status_check.py +162 -0
- toolkits/db_query_demo.py +72 -0
- toolkits/libs_core/__init__.py +0 -0
- toolkits/libs_core/config_groups_helper.py +60 -0
- toolkits/libs_core/config_helper.py +22 -0
- toolkits/libs_core/env_prepare.py +145 -0
- toolkits/libs_core/load_module.py +46 -0
- toolkits/libs_core/mysql_helper.py +151 -0
- toolkits/network/__init__.py +0 -0
- toolkits/network/ip_helper.py +32 -0
- toolkits/network/pdi_helper.py +206 -0
- toolkits/network/send_mail.py +105 -0
- toolkits/system/__init__.py +0 -0
- toolkits/system/aes_cipher.py +44 -0
- toolkits/system/basic_utils.py +20 -0
- toolkits/system/collections_helper.py +72 -0
- toolkits/system/crpyt_helper.py +39 -0
- toolkits/system/dict2xml.py +416 -0
- toolkits/system/dict_helper.py +29 -0
- toolkits/system/excel_helper.py +101 -0
- toolkits/system/file_helper.py +52 -0
- toolkits/system/load_module.py +47 -0
- toolkits/system/priority_tasks.py +199 -0
- toolkits/system/process_monitor/__init__.py +0 -0
- toolkits/system/process_monitor/process_monitor.py +349 -0
- toolkits/system/shell_helper.py +263 -0
- toolkits/system/str_helper.py +187 -0
- toolkits/system/tasks_deamon/__init__.py +0 -0
- toolkits/system/tasks_deamon/tasks_controller.py +70 -0
- toolkits/system/tasks_deamon/tasks_multiprocessing.py +134 -0
- toolkits/system/tasks_deamon/tasks_process.py +137 -0
- toolkits/system/test_shell_helper.py +2 -0
- toolkits/system/time_helper.py +175 -0
- toolkits/system/win32_env.py +49 -0
- toolkits/tookits_app.py +17 -0
- toolkits/tookits_cli.py +126 -0
- toolkits-0.2.7.dist-info/METADATA +35 -0
- toolkits-0.2.7.dist-info/RECORD +56 -0
- toolkits-0.2.7.dist-info/WHEEL +4 -0
- toolkits-0.2.7.dist-info/entry_points.txt +5 -0
@@ -0,0 +1,137 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
import json
|
3
|
+
import os
|
4
|
+
import pprint
|
5
|
+
import sys
|
6
|
+
import arrow
|
7
|
+
import fire
|
8
|
+
from log4python.Log4python import log
|
9
|
+
import traceback
|
10
|
+
import importlib
|
11
|
+
|
12
|
+
path_cur = os.path.dirname(os.path.realpath(__file__))
|
13
|
+
path_parent = "%s/../../" % path_cur
|
14
|
+
sys.path.append(path_parent)
|
15
|
+
|
16
|
+
from system.priority_tasks import PriorityTasks
|
17
|
+
from system.tasks_deamon.tasks_controller import TasksController
|
18
|
+
from system.tasks_deamon.tasks_multiprocessing import TasksMultiprocessing
|
19
|
+
from system.time_helper import get_time_range_interval
|
20
|
+
|
21
|
+
importlib.reload(sys)
|
22
|
+
logger = log("TasksProcess")
|
23
|
+
worker_func = None
|
24
|
+
|
25
|
+
|
26
|
+
def load_json(task_data):
|
27
|
+
task_obj = None
|
28
|
+
try:
|
29
|
+
if type(task_data) is str:
|
30
|
+
task_obj = json.loads(task_data)
|
31
|
+
else:
|
32
|
+
logger.error("Error: wrong parameters[%s]" % pprint.pformat(task_data))
|
33
|
+
except Exception as ex:
|
34
|
+
logger.error("Task[%s] Process Error" % pprint.pformat(task_data))
|
35
|
+
logger.error("Error: %s" % ex)
|
36
|
+
logger.error(traceback.format_exc())
|
37
|
+
return task_obj
|
38
|
+
|
39
|
+
|
40
|
+
def install_worker(exec_func):
|
41
|
+
def tasks_worker(task_data):
|
42
|
+
global worker_func
|
43
|
+
worker_func = exec_func
|
44
|
+
task_obj = load_json(task_data)
|
45
|
+
if task_obj:
|
46
|
+
time_begin = task_obj['time_begin']
|
47
|
+
time_end = task_obj['time_end']
|
48
|
+
logger.debug("Before ExecTimeRange:[%s]" % json.dumps(task_obj, ensure_ascii=False))
|
49
|
+
# do some work
|
50
|
+
worker_func(time_begin, time_end)
|
51
|
+
logger.debug("After ExecTimeRange:[%s]" % json.dumps(task_obj, ensure_ascii=False))
|
52
|
+
return tasks_worker
|
53
|
+
|
54
|
+
|
55
|
+
class TasksRework(TasksMultiprocessing):
|
56
|
+
def __init__(self, tasks_queue_name, tasks_switch):
|
57
|
+
TasksMultiprocessing.__init__(self, tasks_queue_name, tasks_switch)
|
58
|
+
self.__task_queue_name = tasks_queue_name
|
59
|
+
self.__tasks_switch = tasks_switch
|
60
|
+
self.__date_white_list = None
|
61
|
+
self.__database_config = None
|
62
|
+
self.__task_worker = None
|
63
|
+
self.__init_tasks_flag = None
|
64
|
+
|
65
|
+
def init_tasks(self, database_config, tasks_worker_func, task_worker_number=10):
|
66
|
+
"""do not resort the order of below method call !!!
|
67
|
+
:param task_worker_number:
|
68
|
+
:param database_config:
|
69
|
+
:param tasks_worker_func:
|
70
|
+
"""
|
71
|
+
self.__database_config = database_config
|
72
|
+
self.__task_worker = tasks_worker_func
|
73
|
+
self.init_db(database_config, task_worker_number)
|
74
|
+
self.set_worker(tasks_worker_func)
|
75
|
+
self.__init_tasks_flag = True
|
76
|
+
|
77
|
+
def set_date_white_list(self, date_list):
|
78
|
+
self.__date_white_list = date_list
|
79
|
+
|
80
|
+
def __get_process_white_list(self):
|
81
|
+
date_list = []
|
82
|
+
if not self.__date_white_list:
|
83
|
+
date_list = self.__date_white_list
|
84
|
+
return date_list
|
85
|
+
|
86
|
+
def tasks_send(self, time_begin, time_end):
|
87
|
+
if not self.__init_tasks_flag:
|
88
|
+
msg = "Please first call init_tasks function!!!"
|
89
|
+
logger.debug(msg)
|
90
|
+
raise Exception(msg)
|
91
|
+
|
92
|
+
task = PriorityTasks(self.__database_config['redis'],
|
93
|
+
self.__task_queue_name,
|
94
|
+
work_status_queue_name=self.__tasks_switch)
|
95
|
+
time_range_list = get_time_range_interval(time_begin, time_end, "1d")
|
96
|
+
date_white_process_list = self.__get_process_white_list()
|
97
|
+
|
98
|
+
for time_range in time_range_list:
|
99
|
+
ts_begin = arrow.get(time_range['time_begin']).format('YYYY-MM-DD HH:mm:ss')
|
100
|
+
ts_end = arrow.get(time_range['time_end']).format('YYYY-MM-DD HH:mm:ss')
|
101
|
+
|
102
|
+
if date_white_process_list and arrow.get(time_range['time_begin']).format('YYYY-MM-DD') in date_white_process_list:
|
103
|
+
continue
|
104
|
+
|
105
|
+
task_info = {
|
106
|
+
"time_begin": ts_begin,
|
107
|
+
"time_end": ts_end
|
108
|
+
}
|
109
|
+
|
110
|
+
print((json.dumps(task_info, ensure_ascii=False)))
|
111
|
+
task.send_high_task(json.dumps(task_info, ensure_ascii=False))
|
112
|
+
|
113
|
+
|
114
|
+
class TasksProcess:
|
115
|
+
def __init__(self, db_config, task_queue_name, task_switch_name):
|
116
|
+
self.__database_config = db_config
|
117
|
+
self.__task_controller = TasksController()
|
118
|
+
self.__task_rework = TasksRework(task_queue_name, task_switch_name)
|
119
|
+
|
120
|
+
def worker_schedule(self, worker_exec, time_daily="06:33"):
|
121
|
+
self.__task_controller.run_schedule(worker_exec, time_daily)
|
122
|
+
|
123
|
+
def worker_redo(self, exec_func, time_begin, time_end):
|
124
|
+
self.__task_rework.init_tasks(self.__database_config, exec_func)
|
125
|
+
self.__task_rework.tasks_send(time_begin, time_end)
|
126
|
+
|
127
|
+
def worker_daemon(self, exec_func, task_worker_number=5):
|
128
|
+
self.__task_rework.init_tasks(self.__database_config, exec_func, task_worker_number=task_worker_number)
|
129
|
+
self.__task_rework.daemon_start("start")
|
130
|
+
|
131
|
+
|
132
|
+
if __name__ == '__main__':
|
133
|
+
try:
|
134
|
+
fire.Fire(TasksProcess)
|
135
|
+
except Exception as ex:
|
136
|
+
logger.error("Error: %s" % ex)
|
137
|
+
logger.error(traceback.format_exc())
|
@@ -0,0 +1,175 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
import argparse
|
3
|
+
import re
|
4
|
+
import sys
|
5
|
+
import arrow
|
6
|
+
from log4python.Log4python import log
|
7
|
+
import traceback
|
8
|
+
import importlib
|
9
|
+
importlib.reload(sys)
|
10
|
+
logger = log("timeHelper")
|
11
|
+
|
12
|
+
|
13
|
+
class TimeCheck:
|
14
|
+
def __init__(self):
|
15
|
+
self.time_begin = 0
|
16
|
+
self.time_end = 0
|
17
|
+
|
18
|
+
def time_audit_begin(self):
|
19
|
+
self.time_begin = arrow.now().timestamp
|
20
|
+
|
21
|
+
def time_audit_end(self):
|
22
|
+
self.time_end = arrow.now().timestamp
|
23
|
+
return self.time_end - self.time_begin
|
24
|
+
|
25
|
+
|
26
|
+
def get_arrow_time(time_str):
|
27
|
+
ret_time = None
|
28
|
+
try:
|
29
|
+
ret = re.search("([0-9]{4})([0-9]{2})([0-9]{2})", time_str, re.MULTILINE)
|
30
|
+
if not ret:
|
31
|
+
ret = arrow.get(time_str)
|
32
|
+
if ret:
|
33
|
+
ret_time = ret.timestamp
|
34
|
+
else:
|
35
|
+
ret_time = arrow.get("%s-%s-%s 00:00:00" % (ret.group(1), ret.group(2), ret.group(3))).timestamp
|
36
|
+
except Exception as ex:
|
37
|
+
logger.debug("Error: %s" % ex)
|
38
|
+
logger.debug(traceback.format_exc())
|
39
|
+
finally:
|
40
|
+
return ret_time
|
41
|
+
|
42
|
+
|
43
|
+
def get_month_end(timestamp):
|
44
|
+
arr = arrow.get(timestamp)
|
45
|
+
return arrow.get(arr.shift(months=1).format("YYYY-MM-01 00:00:00")).timestamp
|
46
|
+
|
47
|
+
|
48
|
+
def get_local_timestamp(datetime_str, time_zone="Asia/Shanghai"):
|
49
|
+
"""
|
50
|
+
:param datetime_str: "2018-08-17 00:00:00"
|
51
|
+
:param time_zone: ""Asia/Shanghai"
|
52
|
+
:return: unix timestamp
|
53
|
+
"""
|
54
|
+
return arrow.get(datetime_str).replace(tzinfo=time_zone).timestamp
|
55
|
+
|
56
|
+
|
57
|
+
def get_time_range_interval(time_begin, time_end, interval="1d", month_split=False):
|
58
|
+
""" usage:
|
59
|
+
time_day_begin = arrow.get(ret["time_begin"]).format('YYYY-MM-DD')
|
60
|
+
time_day_end = arrow.get(ret["time_end"]).format('YYYY-MM-DD')
|
61
|
+
:param month_split: split time by month, default is False
|
62
|
+
:param interval:
|
63
|
+
:param time_end:
|
64
|
+
:param time_begin: """
|
65
|
+
ts_begin = get_arrow_time(time_begin)
|
66
|
+
ts_end = get_arrow_time(time_end)
|
67
|
+
ts_query_end = ts_begin
|
68
|
+
|
69
|
+
time_range = []
|
70
|
+
unit = interval[-1:]
|
71
|
+
number = interval[:-1]
|
72
|
+
secs_unit = {
|
73
|
+
"h": 60 * 60,
|
74
|
+
"m": 60,
|
75
|
+
"d": 60 * 60 * 24
|
76
|
+
}
|
77
|
+
time_interval = 1
|
78
|
+
|
79
|
+
if unit in list(secs_unit.keys()):
|
80
|
+
time_interval = secs_unit[unit] * int(number)
|
81
|
+
else:
|
82
|
+
logger.error("Err: interval format error[%s]!!" % interval)
|
83
|
+
exit(-1)
|
84
|
+
if ts_begin is None or ts_end is None:
|
85
|
+
logger.error("time format error..")
|
86
|
+
exit(-1)
|
87
|
+
if ts_begin > ts_end:
|
88
|
+
logger.error("Err: time_begin bigger than time_end!!")
|
89
|
+
exit(-1)
|
90
|
+
if ((ts_end - ts_begin) < time_interval) and (ts_end != ts_begin):
|
91
|
+
logger.error("Err: 'interval' is smaller than the gap of between time_begin and time_end !!")
|
92
|
+
exit(-1)
|
93
|
+
|
94
|
+
while True:
|
95
|
+
ts_query_begin = ts_query_end
|
96
|
+
ts_query_end = ts_query_begin + time_interval
|
97
|
+
ts_month_end = get_month_end(ts_query_begin)
|
98
|
+
|
99
|
+
if month_split:
|
100
|
+
if ts_query_end > ts_month_end:
|
101
|
+
if ts_query_end > ts_end:
|
102
|
+
ts_query_end = ts_end
|
103
|
+
else:
|
104
|
+
ts_query_end = ts_month_end
|
105
|
+
else:
|
106
|
+
if ts_query_end > ts_end:
|
107
|
+
ts_query_end = ts_end
|
108
|
+
else:
|
109
|
+
if ts_query_end > ts_end:
|
110
|
+
ts_query_end = ts_end
|
111
|
+
|
112
|
+
time_range.append({
|
113
|
+
"time_begin": ts_query_begin,
|
114
|
+
"time_end": ts_query_end
|
115
|
+
})
|
116
|
+
|
117
|
+
if ts_query_end >= ts_end:
|
118
|
+
break
|
119
|
+
return time_range
|
120
|
+
|
121
|
+
|
122
|
+
def get_time_range(time_begin, time_end, days_interval):
|
123
|
+
ts_begin = get_arrow_time(time_begin)
|
124
|
+
ts_end = get_arrow_time(time_end)
|
125
|
+
time_range = []
|
126
|
+
|
127
|
+
if ts_begin is None or ts_end is None:
|
128
|
+
print("time format error..")
|
129
|
+
exit(-1)
|
130
|
+
ts_query_begin = ts_begin
|
131
|
+
day_secs = 60 * 60 * 24
|
132
|
+
ts_query_end = ts_query_begin - day_secs * 1
|
133
|
+
|
134
|
+
while True:
|
135
|
+
if ts_query_end >= ts_end:
|
136
|
+
break
|
137
|
+
|
138
|
+
time_interval = day_secs * 1
|
139
|
+
if (ts_end - ts_begin) > (day_secs * days_interval):
|
140
|
+
time_interval = day_secs * days_interval
|
141
|
+
ts_query_begin = ts_query_end + day_secs * 1
|
142
|
+
ts_query_end = ts_query_begin + time_interval
|
143
|
+
ts_month_end = get_month_end(ts_query_begin)
|
144
|
+
|
145
|
+
if ts_query_end > ts_month_end:
|
146
|
+
if ts_query_end > ts_end:
|
147
|
+
ts_query_end = ts_end
|
148
|
+
else:
|
149
|
+
ts_query_end = ts_month_end
|
150
|
+
else:
|
151
|
+
if ts_query_end > ts_end:
|
152
|
+
ts_query_end = ts_end
|
153
|
+
|
154
|
+
time_day_begin = arrow.get(ts_query_begin).format('YYYY-MM-DD')
|
155
|
+
time_day_end = arrow.get(ts_query_end).format('YYYY-MM-DD')
|
156
|
+
|
157
|
+
time_range.append({
|
158
|
+
"time_begin": ts_query_begin,
|
159
|
+
"time_end": ts_query_end
|
160
|
+
})
|
161
|
+
|
162
|
+
return time_range
|
163
|
+
|
164
|
+
|
165
|
+
if __name__ == '__main__':
|
166
|
+
try:
|
167
|
+
# parser = argparse.ArgumentParser()
|
168
|
+
# parser.add_argument("logFile", type=str, help="specify the log file's path")
|
169
|
+
# args = parser.parse_args()
|
170
|
+
time_range = get_time_range_interval("2018-01-02 00:00:00", "2018-01-02 03:00:00", "1h")
|
171
|
+
for time_item in time_range:
|
172
|
+
print(("TimeRange:%s-%s" % (arrow.get(time_item['time_begin']).format('YYYY-MM-DD HH:mm:ss'), arrow.get(time_item['time_end']).format('YYYY-MM-DD HH:mm:ss'))))
|
173
|
+
except Exception as ex:
|
174
|
+
logger.debug("Error: %s" % ex)
|
175
|
+
logger.debug(traceback.format_exc())
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# coding=utf-8
|
2
|
+
import os
|
3
|
+
import sys
|
4
|
+
from subprocess import check_call
|
5
|
+
if sys.hexversion > 0x03000000:
|
6
|
+
import winreg
|
7
|
+
else:
|
8
|
+
import winreg as winreg
|
9
|
+
|
10
|
+
|
11
|
+
class Win32Env:
|
12
|
+
"""Utility class to get/set windows environment variable"""
|
13
|
+
|
14
|
+
def __init__(self, scope):
|
15
|
+
assert scope in ('user', 'system')
|
16
|
+
self.scope = scope
|
17
|
+
if scope == 'user':
|
18
|
+
self.root = winreg.HKEY_CURRENT_USER
|
19
|
+
self.subkey = 'Environment'
|
20
|
+
else:
|
21
|
+
self.root = winreg.HKEY_LOCAL_MACHINE
|
22
|
+
self.subkey = r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment'
|
23
|
+
|
24
|
+
def getenv(self, name):
|
25
|
+
key = winreg.OpenKey(self.root, self.subkey, 0, winreg.KEY_READ)
|
26
|
+
try:
|
27
|
+
value, _ = winreg.QueryValueEx(key, name)
|
28
|
+
except WindowsError:
|
29
|
+
value = ''
|
30
|
+
return value
|
31
|
+
|
32
|
+
def setenv(self, name, value):
|
33
|
+
# Note: for 'system' scope, you must run this as Administrator
|
34
|
+
key = winreg.OpenKey(self.root, self.subkey, 0, winreg.KEY_ALL_ACCESS)
|
35
|
+
winreg.SetValueEx(key, name, 0, winreg.REG_EXPAND_SZ, value)
|
36
|
+
winreg.CloseKey(key)
|
37
|
+
# For some strange reason, calling SendMessage from the current process
|
38
|
+
# doesn't propagate environment changes at all.
|
39
|
+
# TODO: handle CalledProcessError (for assert)
|
40
|
+
check_call('''\
|
41
|
+
"%s" -c "import win32api, win32con; assert win32api.SendMessage(win32con.HWND_BROADCAST, win32con.WM_SETTINGCHANGE, 0, 'Environment')"''' % sys.executable)
|
42
|
+
|
43
|
+
|
44
|
+
if __name__ == "__main__":
|
45
|
+
# 程序将显示PATH变量的值,并设置用户的PATHONPATH变量的值。
|
46
|
+
e1 = Win32Env(scope="system")
|
47
|
+
print((e1.getenv('PATH')))
|
48
|
+
e2 = Win32Env(scope="user")
|
49
|
+
e2.setenv('PYTHONPATH', os.path.expanduser(r'~\mymodules'))
|
toolkits/tookits_app.py
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
from log4python.Log4python import log
|
3
|
+
from toolkits.system.basic_utils import get_script_directory, add_relative_search_path
|
4
|
+
|
5
|
+
logger = log("tookitsApp")
|
6
|
+
|
7
|
+
|
8
|
+
class tookitsApp:
|
9
|
+
app_name = 'tookits'
|
10
|
+
config_file_path = None
|
11
|
+
|
12
|
+
def __init__(self):
|
13
|
+
self.__base_path = get_script_directory()
|
14
|
+
|
15
|
+
|
16
|
+
if __name__ == '__main__':
|
17
|
+
pass
|
toolkits/tookits_cli.py
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
import sys
|
3
|
+
import traceback
|
4
|
+
|
5
|
+
import fire
|
6
|
+
from log4python.Log4python import log
|
7
|
+
from toolkits.system.basic_utils import get_script_directory
|
8
|
+
|
9
|
+
from libs_core.env_prepare import EnvPrepare
|
10
|
+
from tookits_app import tookitsApp
|
11
|
+
from libs_core.process_admin import ProcessAdmin
|
12
|
+
from db_query_demo import DbQueryDemo
|
13
|
+
|
14
|
+
logger = log("tookits")
|
15
|
+
|
16
|
+
|
17
|
+
class tookits:
|
18
|
+
def __init__(self, config_path=None):
|
19
|
+
self.__base_path = get_script_directory()
|
20
|
+
if config_path is None:
|
21
|
+
self.__env = EnvPrepare(tookitsApp.app_name)
|
22
|
+
self.__env.check_env()
|
23
|
+
self.__config_path = self.__env.get_config_path("config.py")
|
24
|
+
self.__env.check_config_ready(self.__config_path, "请先初始化配置文件、在数据库创建相应的数据表")
|
25
|
+
else:
|
26
|
+
tookitsApp.config_file_path = config_path
|
27
|
+
self.db_query = DbQueryDemo
|
28
|
+
self.__db_list = self.__env.get_config("db_list")
|
29
|
+
# self.auto_stat_stock = AutoStatStock(self.__db_list['dev_ops'])
|
30
|
+
self.__configChatGroups = ConfigGroupsHelper(self.__env.get_config('db_list'), 'dict')
|
31
|
+
self.__process_admin = ProcessAdmin()
|
32
|
+
|
33
|
+
def show_db_list(self):
|
34
|
+
"""
|
35
|
+
列表查看数据库信息
|
36
|
+
:return:
|
37
|
+
"""
|
38
|
+
self.__configChatGroups.list_config_groups("RDS-连接")
|
39
|
+
|
40
|
+
def init_config(self, path_target):
|
41
|
+
src_path = "%s/config/" % self.__base_path
|
42
|
+
self.__env.init_default_config("directory", src_path, path_target)
|
43
|
+
|
44
|
+
def init_database(self):
|
45
|
+
ddl_sql_list = self.__get_db_ddl_list("policy_status_monitor")
|
46
|
+
for ddl_sql in ddl_sql_list:
|
47
|
+
database_config = self.__env.get_config(self.__config_path, "database_config")
|
48
|
+
self.__env.init_db(database_config['mysql'], ddl_sql)
|
49
|
+
|
50
|
+
def __schedule_tasks(self):
|
51
|
+
self.__process_admin.loop_check_exit()
|
52
|
+
# self.auto_stat_stock.schedule_start()
|
53
|
+
|
54
|
+
def __recall_tasks(self):
|
55
|
+
self.__process_admin.loop_check_exit()
|
56
|
+
logger.info("DBinfo: %s", json.dumps(self.__db_list['dev_ops'], ensure_ascii=True))
|
57
|
+
# app = StrategyRecall(mysql_conn_info=self.__db_list['dev_ops'])
|
58
|
+
# app.process_buy_sale_data("stock_buy_sale_data", "group_test_01")
|
59
|
+
|
60
|
+
def start_all_tasks(self):
|
61
|
+
tasks_list = [
|
62
|
+
self.__schedule_tasks,
|
63
|
+
self.__recall_tasks
|
64
|
+
]
|
65
|
+
self.__process_admin.processes_start(tasks_list)
|
66
|
+
|
67
|
+
@staticmethod
|
68
|
+
def __get_db_ddl_list(table_name):
|
69
|
+
ddl_sql = ["""CREATE TABLE `%s` (
|
70
|
+
`task_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
|
71
|
+
`task_name` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
|
72
|
+
`task_status` varchar(10) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '0:关闭,1:启用,8:暂停',
|
73
|
+
`desc` varchar(500) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
74
|
+
`task_type` varchar(30) COLLATE utf8mb4_unicode_ci NOT NULL,
|
75
|
+
`task_params` text COLLATE utf8mb4_unicode_ci NOT NULL,
|
76
|
+
`task_crontab` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
|
77
|
+
`action_type` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
|
78
|
+
`action_params` text COLLATE utf8mb4_unicode_ci NOT NULL,
|
79
|
+
`event_id` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
80
|
+
`check_condition` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
81
|
+
`db_config` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
82
|
+
`extra_data` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
83
|
+
PRIMARY KEY (`task_id`)
|
84
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;""" % table_name,
|
85
|
+
"""INSERT INTO """ + table_name + """
|
86
|
+
(task_id, task_name, task_status, `desc`, task_type, task_params, task_crontab, action_type, action_params, event_id, check_condition, db_config, extra_data)
|
87
|
+
VALUES('00065df8eb99442db2808ee19592000x', 'task_50031_dlp_dchat', '0', '策略产出监控-DLP-Dchat安装状态监控', 'sql', 'select mc_rule as event_id, FROM_UNIXTIME(occur_ts/1000,''%%Y-%%m-%%d'') as event_time, count(*) as event_value, "" as event_detail
|
88
|
+
from online_alarm oa
|
89
|
+
where occur_ts >= unix_timestamp(concat(CURDATE(), " 00:00:00")) * 1000
|
90
|
+
and occur_ts < unix_timestamp(concat(ADDDATE(CURDATE(), 1), " 00:00:00")) * 1000
|
91
|
+
and mc_rule = ''20071''
|
92
|
+
group by FROM_UNIXTIME(occur_ts/1000,''%%Y-%%m-%%d''),mc_rule
|
93
|
+
limit 1', '15 20 * * *', 'email', '{
|
94
|
+
"event_id": "50031",
|
95
|
+
"event_action": {
|
96
|
+
"ding_talk": {
|
97
|
+
"group_info": {
|
98
|
+
"name": "long_term_query_talk",
|
99
|
+
"desc": "长期查询监控群",
|
100
|
+
"web_hook": "https://oapi.dingtalk.com/robot/send?access_token=69b8e2bae810c9f392b197072699a8241e0a25552291789128cbe5c805cfd5da"
|
101
|
+
},
|
102
|
+
"group_target": "long_term_query_talk"
|
103
|
+
}
|
104
|
+
}
|
105
|
+
}', '50031', 'event_value < 5000', '{
|
106
|
+
"user": "db_admin",
|
107
|
+
"pwd": "gYrmHFNOpkIb/UNM5zCuiDScMdCPwsneOoV38RbEYOs=",
|
108
|
+
"host": "10.83.16.15",
|
109
|
+
"port": 8012,
|
110
|
+
"db_name": "sec_admin"
|
111
|
+
}', NULL);
|
112
|
+
;"""
|
113
|
+
]
|
114
|
+
return ddl_sql
|
115
|
+
|
116
|
+
|
117
|
+
def main():
|
118
|
+
try:
|
119
|
+
fire.Fire(tookits)
|
120
|
+
except Exception as ex:
|
121
|
+
logger.error("Error: %s" % ex)
|
122
|
+
logger.error(traceback.format_exc())
|
123
|
+
|
124
|
+
|
125
|
+
if __name__ == '__main__':
|
126
|
+
main()
|
@@ -0,0 +1,35 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: toolkits
|
3
|
+
Version: 0.2.7
|
4
|
+
Summary: toolkits for quickly reference
|
5
|
+
Author-Email: li_jia_yue <59727816@qq.com>
|
6
|
+
License: MIT
|
7
|
+
Requires-Python: >=3.9
|
8
|
+
Requires-Dist: fire>=0.6.0
|
9
|
+
Requires-Dist: arrow>=1.3.0
|
10
|
+
Requires-Dist: pytest>=8.2.2
|
11
|
+
Requires-Dist: icecream>=2.1.3
|
12
|
+
Requires-Dist: schedule>=1.2.2
|
13
|
+
Requires-Dist: jmespath>=1.0.1
|
14
|
+
Requires-Dist: unicodecsv>=0.14.1
|
15
|
+
Requires-Dist: log4python>=1.0.11
|
16
|
+
Requires-Dist: unipath>=1.1
|
17
|
+
Description-Content-Type: text/markdown
|
18
|
+
|
19
|
+
# toolkits
|
20
|
+
## 迁移PDM项目
|
21
|
+
1, Create virtual env
|
22
|
+
pdm venv create
|
23
|
+
|
24
|
+
2, Install dependency
|
25
|
+
pdm install
|
26
|
+
|
27
|
+
3, basic python libs
|
28
|
+
pdm add arrow toolkits requests dingtalkchatbot exchangelib redis PyYAML sqlalchemy kafka-python-ng urllib3==1.26.15 wheel jmespath log4python PyMySQL pytest crython icecream
|
29
|
+
|
30
|
+
## 使用包中的Python脚本
|
31
|
+
### 设置Python环境变量
|
32
|
+
export PYTHONPATH="/home/pythonDemo/toolkits/src/:$PYTHONPATH"
|
33
|
+
### 源码目录中不要有[与包名一致的Python脚本文件]
|
34
|
+
### 直接调用Python脚本
|
35
|
+
python toolkits/db_query_demo.py --config_path="/home/pythonDemo/toolkits/src/toolkits/config/config_demo.py" worker test_01.txt
|
@@ -0,0 +1,56 @@
|
|
1
|
+
toolkits-0.2.7.dist-info/METADATA,sha256=UAc6iTuOM6I0yDPPPMP6Y7AwYSLtKhRoytG4jyl3ytw,1157
|
2
|
+
toolkits-0.2.7.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
|
3
|
+
toolkits-0.2.7.dist-info/entry_points.txt,sha256=ioUpExuekI3N4XaBZ8NKV1mx0DZnKWLkg2PoGICq6OY,72
|
4
|
+
toolkits/3des/3des.py,sha256=DWfAUGtU7U74GrpnIgA8ZEw7R-kKWyKGfv3Ycqachlw,3400
|
5
|
+
toolkits/3des/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
+
toolkits/__init__.py,sha256=CBKKalWuRi-mXuf2iwrAI5zzzGcUU5e8xhQNgqBERmg,50
|
7
|
+
toolkits/basic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
toolkits/basic/list_helper.py,sha256=jvD-YFneCC-LSCgsJ4PWKhBd-hxuE8PAFXScmtbg3kc,717
|
9
|
+
toolkits/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
+
toolkits/config/config_demo.py,sha256=F9gfD5o7psX6gBXcYc4PVcavTAGECJ5U8Qv98xysJ3k,1087
|
11
|
+
toolkits/databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
12
|
+
toolkits/databases/database_client_util.py,sha256=Jy1d7xbKmbkxfUWNiz1K-BY1OB1vNgJ0WiCBfmEA8fk,4658
|
13
|
+
toolkits/databases/es_client.py,sha256=08VYRLMIKXz8FK9E_NDDdsKVrFLZdNwWLu4GzCAfQik,2904
|
14
|
+
toolkits/databases/hive_client.py,sha256=3qPpGP2nlMOCULkvwvxbVLfgE1dVKztUmsuJGQaL79w,2154
|
15
|
+
toolkits/databases/hive_cmd.py,sha256=pPr5-6z4y1Yx0J5GoCom2wBFD7x9HOSk8nDIYfZK7Pc,3652
|
16
|
+
toolkits/databases/hive_helper.py,sha256=LCt4rff5meLJ1uCozPzL_S7rpGC8wOWvpe2pk0JXlsQ,8233
|
17
|
+
toolkits/databases/redis_mgmt.py,sha256=f02qbYNGfvNGvmVfau5QMK6R92YusbZ-XcZnaRMpgWg,3386
|
18
|
+
toolkits/databases/sql_helper.py,sha256=JHB9UtvXVhcZNVmuM9teWjsNoLYNxRn0RxQaZwi2N-8,9707
|
19
|
+
toolkits/databases/sqlalchemy_helper.py,sha256=26lgOiJDNmuJpX0YWAzZkNylBFGlmaY4jnbcQJIY4Rg,2027
|
20
|
+
toolkits/databases/status_check.py,sha256=12oeY94n5ZRnPoNSrY_P8uB6JMWMStAnuRO6gmTrm9w,6474
|
21
|
+
toolkits/db_query_demo.py,sha256=0xPiOaAwY4ulZ91x4IhQ1gNRoOIHVk558FKGMAjnHGY,2577
|
22
|
+
toolkits/libs_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
|
+
toolkits/libs_core/config_groups_helper.py,sha256=k3ji_4iZVqTL5jB6xrIUKNbATM7Aoc0gAi7rJ1HyJeA,1987
|
24
|
+
toolkits/libs_core/config_helper.py,sha256=t4j10aGcrOR6Zb6-qFZUgELO9tShPJs1NNIB3uVCuVg,494
|
25
|
+
toolkits/libs_core/env_prepare.py,sha256=1zrxrdKcXR9eouEXuqfH03n1z8LSJ10QZ7HEogtA1_4,6103
|
26
|
+
toolkits/libs_core/load_module.py,sha256=AAFPDx4_N6e3KdaI_LKciSqAD0zuHNfZxyu1FImk4o4,1325
|
27
|
+
toolkits/libs_core/mysql_helper.py,sha256=XI5b5oAkcqDQ8_r3cBDFjjHIv3ufACq22Ls9dq8XdKQ,6153
|
28
|
+
toolkits/network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
|
+
toolkits/network/ip_helper.py,sha256=VgSnQyXOAViAZuTnXeOZXtPiL38kbf3rD3-SUxRqNsM,665
|
30
|
+
toolkits/network/pdi_helper.py,sha256=RR8Jdqyd-g1rnrf98UaZgOOks7Z1jcZFaWXzUbf4fQo,7406
|
31
|
+
toolkits/network/send_mail.py,sha256=bHxS2zOQOQXMLMYOOup_KITWecYzMzFIiD0eNb2lzz0,3985
|
32
|
+
toolkits/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
|
+
toolkits/system/aes_cipher.py,sha256=2O-xpT3B1I-od7lczVz9z7kp8Q1OB6-DMT9dqeRVkPU,1143
|
34
|
+
toolkits/system/basic_utils.py,sha256=yv22VyuHEFVAE9oQsycb3Lzl_cB5YlgzwjThvlmKuHQ,548
|
35
|
+
toolkits/system/collections_helper.py,sha256=esFbjEjRrPL6f2MjahEKUB4uTLLEnxw-felpMXuFsaI,2186
|
36
|
+
toolkits/system/crpyt_helper.py,sha256=fobpWDS0uDzgSPeXYhxJN2yhIe9Vu8OXLJjG0WjXKdw,1173
|
37
|
+
toolkits/system/dict2xml.py,sha256=ofjrkVYlARgXWv4n_lS6_n5oh7cmFsN3JdREsLeB6X4,14145
|
38
|
+
toolkits/system/dict_helper.py,sha256=BSwB87zlKKfvmRDmq2h7u2f25_VoylbxmhyzMLafpIQ,788
|
39
|
+
toolkits/system/excel_helper.py,sha256=H_A91BLTHoXuS59ggUadtRLMCsB7Dr3CxgF5uM4BDLo,3319
|
40
|
+
toolkits/system/file_helper.py,sha256=5-JDgXU2MD9GTeFhjfyatzOocAQGfpNwlloaVbLoI8Q,1569
|
41
|
+
toolkits/system/load_module.py,sha256=UXNfE9drsfz9oOBq9EGdV4gjmiIYlJQECIt3PFaLxwk,1419
|
42
|
+
toolkits/system/priority_tasks.py,sha256=eUtKQatAnYcV3f3V3XY4FQ0u3afX8NHuqiBRcuv0d0I,7632
|
43
|
+
toolkits/system/process_monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
44
|
+
toolkits/system/process_monitor/process_monitor.py,sha256=_tXmWSOzAqAlHeGiBPrMGFU1Wjdr4zUAKNSPucqGRO8,12887
|
45
|
+
toolkits/system/shell_helper.py,sha256=RwMw-t98IFjqo1DiqUL4GK0tRM4vSDj-Jqo623mEyRM,7999
|
46
|
+
toolkits/system/str_helper.py,sha256=89qw0jF6Pf19uuMhBh7XFoCxHvEiZztPwlCDdYuPBEQ,5096
|
47
|
+
toolkits/system/tasks_deamon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
48
|
+
toolkits/system/tasks_deamon/tasks_controller.py,sha256=aobnJ0vJmvHQs94-yXasHqyxGsBHrv8Vqd1lq6JtmOo,2380
|
49
|
+
toolkits/system/tasks_deamon/tasks_multiprocessing.py,sha256=cECLT0eqzFlpJoUfX_-QH4-TllLOWmx1YhEnZusimho,4167
|
50
|
+
toolkits/system/tasks_deamon/tasks_process.py,sha256=b_RUlZ9Fy3XN079IzoBIjMM5UKaXn-Z6GWUob0WIqgI,5095
|
51
|
+
toolkits/system/test_shell_helper.py,sha256=zOCe-2BER1_x8t2NtiySWcd7K-PJEqL_zcIiwSzBQAo,48
|
52
|
+
toolkits/system/time_helper.py,sha256=_nxy97rnnagCn_m5sppZTDsx2sdn5debsTfXPEjoWzU,5644
|
53
|
+
toolkits/system/win32_env.py,sha256=KYkZ-clZYkSQnSvPvpydeOVepaCrPH0egkC-MY-Bs0w,1829
|
54
|
+
toolkits/tookits_app.py,sha256=2qz6V5XkEGc6NjxafFvFavE1hpt-tE0PoGonLAvZNXs,380
|
55
|
+
toolkits/tookits_cli.py,sha256=f8u2IXYwk0QguQwY39aNjgVwJFiAmfukGGumWT6s07I,5273
|
56
|
+
toolkits-0.2.7.dist-info/RECORD,,
|