toolkits 0.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. toolkits/3des/3des.py +93 -0
  2. toolkits/3des/__init__.py +0 -0
  3. toolkits/__init__.py +2 -0
  4. toolkits/basic/__init__.py +0 -0
  5. toolkits/basic/list_helper.py +26 -0
  6. toolkits/config/__init__.py +0 -0
  7. toolkits/config/config_demo.py +43 -0
  8. toolkits/databases/__init__.py +0 -0
  9. toolkits/databases/database_client_util.py +143 -0
  10. toolkits/databases/es_client.py +88 -0
  11. toolkits/databases/hive_client.py +72 -0
  12. toolkits/databases/hive_cmd.py +113 -0
  13. toolkits/databases/hive_helper.py +220 -0
  14. toolkits/databases/redis_mgmt.py +95 -0
  15. toolkits/databases/sql_helper.py +291 -0
  16. toolkits/databases/sqlalchemy_helper.py +71 -0
  17. toolkits/databases/status_check.py +162 -0
  18. toolkits/db_query_demo.py +72 -0
  19. toolkits/libs_core/__init__.py +0 -0
  20. toolkits/libs_core/config_groups_helper.py +60 -0
  21. toolkits/libs_core/config_helper.py +22 -0
  22. toolkits/libs_core/env_prepare.py +145 -0
  23. toolkits/libs_core/load_module.py +46 -0
  24. toolkits/libs_core/mysql_helper.py +151 -0
  25. toolkits/network/__init__.py +0 -0
  26. toolkits/network/ip_helper.py +32 -0
  27. toolkits/network/pdi_helper.py +206 -0
  28. toolkits/network/send_mail.py +105 -0
  29. toolkits/system/__init__.py +0 -0
  30. toolkits/system/aes_cipher.py +44 -0
  31. toolkits/system/basic_utils.py +20 -0
  32. toolkits/system/collections_helper.py +72 -0
  33. toolkits/system/crpyt_helper.py +39 -0
  34. toolkits/system/dict2xml.py +416 -0
  35. toolkits/system/dict_helper.py +29 -0
  36. toolkits/system/excel_helper.py +101 -0
  37. toolkits/system/file_helper.py +52 -0
  38. toolkits/system/load_module.py +47 -0
  39. toolkits/system/priority_tasks.py +199 -0
  40. toolkits/system/process_monitor/__init__.py +0 -0
  41. toolkits/system/process_monitor/process_monitor.py +349 -0
  42. toolkits/system/shell_helper.py +263 -0
  43. toolkits/system/str_helper.py +187 -0
  44. toolkits/system/tasks_deamon/__init__.py +0 -0
  45. toolkits/system/tasks_deamon/tasks_controller.py +70 -0
  46. toolkits/system/tasks_deamon/tasks_multiprocessing.py +134 -0
  47. toolkits/system/tasks_deamon/tasks_process.py +137 -0
  48. toolkits/system/test_shell_helper.py +2 -0
  49. toolkits/system/time_helper.py +175 -0
  50. toolkits/system/win32_env.py +49 -0
  51. toolkits/tookits_app.py +17 -0
  52. toolkits/tookits_cli.py +126 -0
  53. toolkits-0.2.7.dist-info/METADATA +35 -0
  54. toolkits-0.2.7.dist-info/RECORD +56 -0
  55. toolkits-0.2.7.dist-info/WHEEL +4 -0
  56. toolkits-0.2.7.dist-info/entry_points.txt +5 -0
@@ -0,0 +1,137 @@
1
+ # -*- coding: utf-8 -*-
2
+ import json
3
+ import os
4
+ import pprint
5
+ import sys
6
+ import arrow
7
+ import fire
8
+ from log4python.Log4python import log
9
+ import traceback
10
+ import importlib
11
+
12
+ path_cur = os.path.dirname(os.path.realpath(__file__))
13
+ path_parent = "%s/../../" % path_cur
14
+ sys.path.append(path_parent)
15
+
16
+ from system.priority_tasks import PriorityTasks
17
+ from system.tasks_deamon.tasks_controller import TasksController
18
+ from system.tasks_deamon.tasks_multiprocessing import TasksMultiprocessing
19
+ from system.time_helper import get_time_range_interval
20
+
21
+ importlib.reload(sys)
22
+ logger = log("TasksProcess")
23
+ worker_func = None
24
+
25
+
26
+ def load_json(task_data):
27
+ task_obj = None
28
+ try:
29
+ if type(task_data) is str:
30
+ task_obj = json.loads(task_data)
31
+ else:
32
+ logger.error("Error: wrong parameters[%s]" % pprint.pformat(task_data))
33
+ except Exception as ex:
34
+ logger.error("Task[%s] Process Error" % pprint.pformat(task_data))
35
+ logger.error("Error: %s" % ex)
36
+ logger.error(traceback.format_exc())
37
+ return task_obj
38
+
39
+
40
+ def install_worker(exec_func):
41
+ def tasks_worker(task_data):
42
+ global worker_func
43
+ worker_func = exec_func
44
+ task_obj = load_json(task_data)
45
+ if task_obj:
46
+ time_begin = task_obj['time_begin']
47
+ time_end = task_obj['time_end']
48
+ logger.debug("Before ExecTimeRange:[%s]" % json.dumps(task_obj, ensure_ascii=False))
49
+ # do some work
50
+ worker_func(time_begin, time_end)
51
+ logger.debug("After ExecTimeRange:[%s]" % json.dumps(task_obj, ensure_ascii=False))
52
+ return tasks_worker
53
+
54
+
55
+ class TasksRework(TasksMultiprocessing):
56
+ def __init__(self, tasks_queue_name, tasks_switch):
57
+ TasksMultiprocessing.__init__(self, tasks_queue_name, tasks_switch)
58
+ self.__task_queue_name = tasks_queue_name
59
+ self.__tasks_switch = tasks_switch
60
+ self.__date_white_list = None
61
+ self.__database_config = None
62
+ self.__task_worker = None
63
+ self.__init_tasks_flag = None
64
+
65
+ def init_tasks(self, database_config, tasks_worker_func, task_worker_number=10):
66
+ """do not resort the order of below method call !!!
67
+ :param task_worker_number:
68
+ :param database_config:
69
+ :param tasks_worker_func:
70
+ """
71
+ self.__database_config = database_config
72
+ self.__task_worker = tasks_worker_func
73
+ self.init_db(database_config, task_worker_number)
74
+ self.set_worker(tasks_worker_func)
75
+ self.__init_tasks_flag = True
76
+
77
+ def set_date_white_list(self, date_list):
78
+ self.__date_white_list = date_list
79
+
80
+ def __get_process_white_list(self):
81
+ date_list = []
82
+ if not self.__date_white_list:
83
+ date_list = self.__date_white_list
84
+ return date_list
85
+
86
+ def tasks_send(self, time_begin, time_end):
87
+ if not self.__init_tasks_flag:
88
+ msg = "Please first call init_tasks function!!!"
89
+ logger.debug(msg)
90
+ raise Exception(msg)
91
+
92
+ task = PriorityTasks(self.__database_config['redis'],
93
+ self.__task_queue_name,
94
+ work_status_queue_name=self.__tasks_switch)
95
+ time_range_list = get_time_range_interval(time_begin, time_end, "1d")
96
+ date_white_process_list = self.__get_process_white_list()
97
+
98
+ for time_range in time_range_list:
99
+ ts_begin = arrow.get(time_range['time_begin']).format('YYYY-MM-DD HH:mm:ss')
100
+ ts_end = arrow.get(time_range['time_end']).format('YYYY-MM-DD HH:mm:ss')
101
+
102
+ if date_white_process_list and arrow.get(time_range['time_begin']).format('YYYY-MM-DD') in date_white_process_list:
103
+ continue
104
+
105
+ task_info = {
106
+ "time_begin": ts_begin,
107
+ "time_end": ts_end
108
+ }
109
+
110
+ print((json.dumps(task_info, ensure_ascii=False)))
111
+ task.send_high_task(json.dumps(task_info, ensure_ascii=False))
112
+
113
+
114
+ class TasksProcess:
115
+ def __init__(self, db_config, task_queue_name, task_switch_name):
116
+ self.__database_config = db_config
117
+ self.__task_controller = TasksController()
118
+ self.__task_rework = TasksRework(task_queue_name, task_switch_name)
119
+
120
+ def worker_schedule(self, worker_exec, time_daily="06:33"):
121
+ self.__task_controller.run_schedule(worker_exec, time_daily)
122
+
123
+ def worker_redo(self, exec_func, time_begin, time_end):
124
+ self.__task_rework.init_tasks(self.__database_config, exec_func)
125
+ self.__task_rework.tasks_send(time_begin, time_end)
126
+
127
+ def worker_daemon(self, exec_func, task_worker_number=5):
128
+ self.__task_rework.init_tasks(self.__database_config, exec_func, task_worker_number=task_worker_number)
129
+ self.__task_rework.daemon_start("start")
130
+
131
+
132
+ if __name__ == '__main__':
133
+ try:
134
+ fire.Fire(TasksProcess)
135
+ except Exception as ex:
136
+ logger.error("Error: %s" % ex)
137
+ logger.error(traceback.format_exc())
@@ -0,0 +1,2 @@
1
+ def test_exec_shell_async():
2
+ assert False
@@ -0,0 +1,175 @@
1
+ # -*- coding: utf-8 -*-
2
+ import argparse
3
+ import re
4
+ import sys
5
+ import arrow
6
+ from log4python.Log4python import log
7
+ import traceback
8
+ import importlib
9
+ importlib.reload(sys)
10
+ logger = log("timeHelper")
11
+
12
+
13
+ class TimeCheck:
14
+ def __init__(self):
15
+ self.time_begin = 0
16
+ self.time_end = 0
17
+
18
+ def time_audit_begin(self):
19
+ self.time_begin = arrow.now().timestamp
20
+
21
+ def time_audit_end(self):
22
+ self.time_end = arrow.now().timestamp
23
+ return self.time_end - self.time_begin
24
+
25
+
26
+ def get_arrow_time(time_str):
27
+ ret_time = None
28
+ try:
29
+ ret = re.search("([0-9]{4})([0-9]{2})([0-9]{2})", time_str, re.MULTILINE)
30
+ if not ret:
31
+ ret = arrow.get(time_str)
32
+ if ret:
33
+ ret_time = ret.timestamp
34
+ else:
35
+ ret_time = arrow.get("%s-%s-%s 00:00:00" % (ret.group(1), ret.group(2), ret.group(3))).timestamp
36
+ except Exception as ex:
37
+ logger.debug("Error: %s" % ex)
38
+ logger.debug(traceback.format_exc())
39
+ finally:
40
+ return ret_time
41
+
42
+
43
+ def get_month_end(timestamp):
44
+ arr = arrow.get(timestamp)
45
+ return arrow.get(arr.shift(months=1).format("YYYY-MM-01 00:00:00")).timestamp
46
+
47
+
48
+ def get_local_timestamp(datetime_str, time_zone="Asia/Shanghai"):
49
+ """
50
+ :param datetime_str: "2018-08-17 00:00:00"
51
+ :param time_zone: ""Asia/Shanghai"
52
+ :return: unix timestamp
53
+ """
54
+ return arrow.get(datetime_str).replace(tzinfo=time_zone).timestamp
55
+
56
+
57
+ def get_time_range_interval(time_begin, time_end, interval="1d", month_split=False):
58
+ """ usage:
59
+ time_day_begin = arrow.get(ret["time_begin"]).format('YYYY-MM-DD')
60
+ time_day_end = arrow.get(ret["time_end"]).format('YYYY-MM-DD')
61
+ :param month_split: split time by month, default is False
62
+ :param interval:
63
+ :param time_end:
64
+ :param time_begin: """
65
+ ts_begin = get_arrow_time(time_begin)
66
+ ts_end = get_arrow_time(time_end)
67
+ ts_query_end = ts_begin
68
+
69
+ time_range = []
70
+ unit = interval[-1:]
71
+ number = interval[:-1]
72
+ secs_unit = {
73
+ "h": 60 * 60,
74
+ "m": 60,
75
+ "d": 60 * 60 * 24
76
+ }
77
+ time_interval = 1
78
+
79
+ if unit in list(secs_unit.keys()):
80
+ time_interval = secs_unit[unit] * int(number)
81
+ else:
82
+ logger.error("Err: interval format error[%s]!!" % interval)
83
+ exit(-1)
84
+ if ts_begin is None or ts_end is None:
85
+ logger.error("time format error..")
86
+ exit(-1)
87
+ if ts_begin > ts_end:
88
+ logger.error("Err: time_begin bigger than time_end!!")
89
+ exit(-1)
90
+ if ((ts_end - ts_begin) < time_interval) and (ts_end != ts_begin):
91
+ logger.error("Err: 'interval' is smaller than the gap of between time_begin and time_end !!")
92
+ exit(-1)
93
+
94
+ while True:
95
+ ts_query_begin = ts_query_end
96
+ ts_query_end = ts_query_begin + time_interval
97
+ ts_month_end = get_month_end(ts_query_begin)
98
+
99
+ if month_split:
100
+ if ts_query_end > ts_month_end:
101
+ if ts_query_end > ts_end:
102
+ ts_query_end = ts_end
103
+ else:
104
+ ts_query_end = ts_month_end
105
+ else:
106
+ if ts_query_end > ts_end:
107
+ ts_query_end = ts_end
108
+ else:
109
+ if ts_query_end > ts_end:
110
+ ts_query_end = ts_end
111
+
112
+ time_range.append({
113
+ "time_begin": ts_query_begin,
114
+ "time_end": ts_query_end
115
+ })
116
+
117
+ if ts_query_end >= ts_end:
118
+ break
119
+ return time_range
120
+
121
+
122
+ def get_time_range(time_begin, time_end, days_interval):
123
+ ts_begin = get_arrow_time(time_begin)
124
+ ts_end = get_arrow_time(time_end)
125
+ time_range = []
126
+
127
+ if ts_begin is None or ts_end is None:
128
+ print("time format error..")
129
+ exit(-1)
130
+ ts_query_begin = ts_begin
131
+ day_secs = 60 * 60 * 24
132
+ ts_query_end = ts_query_begin - day_secs * 1
133
+
134
+ while True:
135
+ if ts_query_end >= ts_end:
136
+ break
137
+
138
+ time_interval = day_secs * 1
139
+ if (ts_end - ts_begin) > (day_secs * days_interval):
140
+ time_interval = day_secs * days_interval
141
+ ts_query_begin = ts_query_end + day_secs * 1
142
+ ts_query_end = ts_query_begin + time_interval
143
+ ts_month_end = get_month_end(ts_query_begin)
144
+
145
+ if ts_query_end > ts_month_end:
146
+ if ts_query_end > ts_end:
147
+ ts_query_end = ts_end
148
+ else:
149
+ ts_query_end = ts_month_end
150
+ else:
151
+ if ts_query_end > ts_end:
152
+ ts_query_end = ts_end
153
+
154
+ time_day_begin = arrow.get(ts_query_begin).format('YYYY-MM-DD')
155
+ time_day_end = arrow.get(ts_query_end).format('YYYY-MM-DD')
156
+
157
+ time_range.append({
158
+ "time_begin": ts_query_begin,
159
+ "time_end": ts_query_end
160
+ })
161
+
162
+ return time_range
163
+
164
+
165
+ if __name__ == '__main__':
166
+ try:
167
+ # parser = argparse.ArgumentParser()
168
+ # parser.add_argument("logFile", type=str, help="specify the log file's path")
169
+ # args = parser.parse_args()
170
+ time_range = get_time_range_interval("2018-01-02 00:00:00", "2018-01-02 03:00:00", "1h")
171
+ for time_item in time_range:
172
+ print(("TimeRange:%s-%s" % (arrow.get(time_item['time_begin']).format('YYYY-MM-DD HH:mm:ss'), arrow.get(time_item['time_end']).format('YYYY-MM-DD HH:mm:ss'))))
173
+ except Exception as ex:
174
+ logger.debug("Error: %s" % ex)
175
+ logger.debug(traceback.format_exc())
@@ -0,0 +1,49 @@
1
+ # coding=utf-8
2
+ import os
3
+ import sys
4
+ from subprocess import check_call
5
+ if sys.hexversion > 0x03000000:
6
+ import winreg
7
+ else:
8
+ import winreg as winreg
9
+
10
+
11
+ class Win32Env:
12
+ """Utility class to get/set windows environment variable"""
13
+
14
+ def __init__(self, scope):
15
+ assert scope in ('user', 'system')
16
+ self.scope = scope
17
+ if scope == 'user':
18
+ self.root = winreg.HKEY_CURRENT_USER
19
+ self.subkey = 'Environment'
20
+ else:
21
+ self.root = winreg.HKEY_LOCAL_MACHINE
22
+ self.subkey = r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment'
23
+
24
+ def getenv(self, name):
25
+ key = winreg.OpenKey(self.root, self.subkey, 0, winreg.KEY_READ)
26
+ try:
27
+ value, _ = winreg.QueryValueEx(key, name)
28
+ except WindowsError:
29
+ value = ''
30
+ return value
31
+
32
+ def setenv(self, name, value):
33
+ # Note: for 'system' scope, you must run this as Administrator
34
+ key = winreg.OpenKey(self.root, self.subkey, 0, winreg.KEY_ALL_ACCESS)
35
+ winreg.SetValueEx(key, name, 0, winreg.REG_EXPAND_SZ, value)
36
+ winreg.CloseKey(key)
37
+ # For some strange reason, calling SendMessage from the current process
38
+ # doesn't propagate environment changes at all.
39
+ # TODO: handle CalledProcessError (for assert)
40
+ check_call('''\
41
+ "%s" -c "import win32api, win32con; assert win32api.SendMessage(win32con.HWND_BROADCAST, win32con.WM_SETTINGCHANGE, 0, 'Environment')"''' % sys.executable)
42
+
43
+
44
+ if __name__ == "__main__":
45
+ # 程序将显示PATH变量的值,并设置用户的PATHONPATH变量的值。
46
+ e1 = Win32Env(scope="system")
47
+ print((e1.getenv('PATH')))
48
+ e2 = Win32Env(scope="user")
49
+ e2.setenv('PYTHONPATH', os.path.expanduser(r'~\mymodules'))
@@ -0,0 +1,17 @@
1
+ # -*- coding: utf-8 -*-
2
+ from log4python.Log4python import log
3
+ from toolkits.system.basic_utils import get_script_directory, add_relative_search_path
4
+
5
+ logger = log("tookitsApp")
6
+
7
+
8
+ class tookitsApp:
9
+ app_name = 'tookits'
10
+ config_file_path = None
11
+
12
+ def __init__(self):
13
+ self.__base_path = get_script_directory()
14
+
15
+
16
+ if __name__ == '__main__':
17
+ pass
@@ -0,0 +1,126 @@
1
+ # -*- coding: utf-8 -*-
2
+ import sys
3
+ import traceback
4
+
5
+ import fire
6
+ from log4python.Log4python import log
7
+ from toolkits.system.basic_utils import get_script_directory
8
+
9
+ from libs_core.env_prepare import EnvPrepare
10
+ from tookits_app import tookitsApp
11
+ from libs_core.process_admin import ProcessAdmin
12
+ from db_query_demo import DbQueryDemo
13
+
14
+ logger = log("tookits")
15
+
16
+
17
+ class tookits:
18
+ def __init__(self, config_path=None):
19
+ self.__base_path = get_script_directory()
20
+ if config_path is None:
21
+ self.__env = EnvPrepare(tookitsApp.app_name)
22
+ self.__env.check_env()
23
+ self.__config_path = self.__env.get_config_path("config.py")
24
+ self.__env.check_config_ready(self.__config_path, "请先初始化配置文件、在数据库创建相应的数据表")
25
+ else:
26
+ tookitsApp.config_file_path = config_path
27
+ self.db_query = DbQueryDemo
28
+ self.__db_list = self.__env.get_config("db_list")
29
+ # self.auto_stat_stock = AutoStatStock(self.__db_list['dev_ops'])
30
+ self.__configChatGroups = ConfigGroupsHelper(self.__env.get_config('db_list'), 'dict')
31
+ self.__process_admin = ProcessAdmin()
32
+
33
+ def show_db_list(self):
34
+ """
35
+ 列表查看数据库信息
36
+ :return:
37
+ """
38
+ self.__configChatGroups.list_config_groups("RDS-连接")
39
+
40
+ def init_config(self, path_target):
41
+ src_path = "%s/config/" % self.__base_path
42
+ self.__env.init_default_config("directory", src_path, path_target)
43
+
44
+ def init_database(self):
45
+ ddl_sql_list = self.__get_db_ddl_list("policy_status_monitor")
46
+ for ddl_sql in ddl_sql_list:
47
+ database_config = self.__env.get_config(self.__config_path, "database_config")
48
+ self.__env.init_db(database_config['mysql'], ddl_sql)
49
+
50
+ def __schedule_tasks(self):
51
+ self.__process_admin.loop_check_exit()
52
+ # self.auto_stat_stock.schedule_start()
53
+
54
+ def __recall_tasks(self):
55
+ self.__process_admin.loop_check_exit()
56
+ logger.info("DBinfo: %s", json.dumps(self.__db_list['dev_ops'], ensure_ascii=True))
57
+ # app = StrategyRecall(mysql_conn_info=self.__db_list['dev_ops'])
58
+ # app.process_buy_sale_data("stock_buy_sale_data", "group_test_01")
59
+
60
+ def start_all_tasks(self):
61
+ tasks_list = [
62
+ self.__schedule_tasks,
63
+ self.__recall_tasks
64
+ ]
65
+ self.__process_admin.processes_start(tasks_list)
66
+
67
+ @staticmethod
68
+ def __get_db_ddl_list(table_name):
69
+ ddl_sql = ["""CREATE TABLE `%s` (
70
+ `task_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
71
+ `task_name` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
72
+ `task_status` varchar(10) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '0:关闭,1:启用,8:暂停',
73
+ `desc` varchar(500) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
74
+ `task_type` varchar(30) COLLATE utf8mb4_unicode_ci NOT NULL,
75
+ `task_params` text COLLATE utf8mb4_unicode_ci NOT NULL,
76
+ `task_crontab` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
77
+ `action_type` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
78
+ `action_params` text COLLATE utf8mb4_unicode_ci NOT NULL,
79
+ `event_id` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
80
+ `check_condition` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
81
+ `db_config` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
82
+ `extra_data` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
83
+ PRIMARY KEY (`task_id`)
84
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;""" % table_name,
85
+ """INSERT INTO """ + table_name + """
86
+ (task_id, task_name, task_status, `desc`, task_type, task_params, task_crontab, action_type, action_params, event_id, check_condition, db_config, extra_data)
87
+ VALUES('00065df8eb99442db2808ee19592000x', 'task_50031_dlp_dchat', '0', '策略产出监控-DLP-Dchat安装状态监控', 'sql', 'select mc_rule as event_id, FROM_UNIXTIME(occur_ts/1000,''%%Y-%%m-%%d'') as event_time, count(*) as event_value, "" as event_detail
88
+ from online_alarm oa
89
+ where occur_ts >= unix_timestamp(concat(CURDATE(), " 00:00:00")) * 1000
90
+ and occur_ts < unix_timestamp(concat(ADDDATE(CURDATE(), 1), " 00:00:00")) * 1000
91
+ and mc_rule = ''20071''
92
+ group by FROM_UNIXTIME(occur_ts/1000,''%%Y-%%m-%%d''),mc_rule
93
+ limit 1', '15 20 * * *', 'email', '{
94
+ "event_id": "50031",
95
+ "event_action": {
96
+ "ding_talk": {
97
+ "group_info": {
98
+ "name": "long_term_query_talk",
99
+ "desc": "长期查询监控群",
100
+ "web_hook": "https://oapi.dingtalk.com/robot/send?access_token=69b8e2bae810c9f392b197072699a8241e0a25552291789128cbe5c805cfd5da"
101
+ },
102
+ "group_target": "long_term_query_talk"
103
+ }
104
+ }
105
+ }', '50031', 'event_value < 5000', '{
106
+ "user": "db_admin",
107
+ "pwd": "gYrmHFNOpkIb/UNM5zCuiDScMdCPwsneOoV38RbEYOs=",
108
+ "host": "10.83.16.15",
109
+ "port": 8012,
110
+ "db_name": "sec_admin"
111
+ }', NULL);
112
+ ;"""
113
+ ]
114
+ return ddl_sql
115
+
116
+
117
+ def main():
118
+ try:
119
+ fire.Fire(tookits)
120
+ except Exception as ex:
121
+ logger.error("Error: %s" % ex)
122
+ logger.error(traceback.format_exc())
123
+
124
+
125
+ if __name__ == '__main__':
126
+ main()
@@ -0,0 +1,35 @@
1
+ Metadata-Version: 2.1
2
+ Name: toolkits
3
+ Version: 0.2.7
4
+ Summary: toolkits for quickly reference
5
+ Author-Email: li_jia_yue <59727816@qq.com>
6
+ License: MIT
7
+ Requires-Python: >=3.9
8
+ Requires-Dist: fire>=0.6.0
9
+ Requires-Dist: arrow>=1.3.0
10
+ Requires-Dist: pytest>=8.2.2
11
+ Requires-Dist: icecream>=2.1.3
12
+ Requires-Dist: schedule>=1.2.2
13
+ Requires-Dist: jmespath>=1.0.1
14
+ Requires-Dist: unicodecsv>=0.14.1
15
+ Requires-Dist: log4python>=1.0.11
16
+ Requires-Dist: unipath>=1.1
17
+ Description-Content-Type: text/markdown
18
+
19
+ # toolkits
20
+ ## 迁移PDM项目
21
+ 1, Create virtual env
22
+ pdm venv create
23
+
24
+ 2, Install dependency
25
+ pdm install
26
+
27
+ 3, basic python libs
28
+ pdm add arrow toolkits requests dingtalkchatbot exchangelib redis PyYAML sqlalchemy kafka-python-ng urllib3==1.26.15 wheel jmespath log4python PyMySQL pytest crython icecream
29
+
30
+ ## 使用包中的Python脚本
31
+ ### 设置Python环境变量
32
+ export PYTHONPATH="/home/pythonDemo/toolkits/src/:$PYTHONPATH"
33
+ ### 源码目录中不要有[与包名一致的Python脚本文件]
34
+ ### 直接调用Python脚本
35
+ python toolkits/db_query_demo.py --config_path="/home/pythonDemo/toolkits/src/toolkits/config/config_demo.py" worker test_01.txt
@@ -0,0 +1,56 @@
1
+ toolkits-0.2.7.dist-info/METADATA,sha256=UAc6iTuOM6I0yDPPPMP6Y7AwYSLtKhRoytG4jyl3ytw,1157
2
+ toolkits-0.2.7.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
3
+ toolkits-0.2.7.dist-info/entry_points.txt,sha256=ioUpExuekI3N4XaBZ8NKV1mx0DZnKWLkg2PoGICq6OY,72
4
+ toolkits/3des/3des.py,sha256=DWfAUGtU7U74GrpnIgA8ZEw7R-kKWyKGfv3Ycqachlw,3400
5
+ toolkits/3des/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ toolkits/__init__.py,sha256=CBKKalWuRi-mXuf2iwrAI5zzzGcUU5e8xhQNgqBERmg,50
7
+ toolkits/basic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ toolkits/basic/list_helper.py,sha256=jvD-YFneCC-LSCgsJ4PWKhBd-hxuE8PAFXScmtbg3kc,717
9
+ toolkits/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ toolkits/config/config_demo.py,sha256=F9gfD5o7psX6gBXcYc4PVcavTAGECJ5U8Qv98xysJ3k,1087
11
+ toolkits/databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ toolkits/databases/database_client_util.py,sha256=Jy1d7xbKmbkxfUWNiz1K-BY1OB1vNgJ0WiCBfmEA8fk,4658
13
+ toolkits/databases/es_client.py,sha256=08VYRLMIKXz8FK9E_NDDdsKVrFLZdNwWLu4GzCAfQik,2904
14
+ toolkits/databases/hive_client.py,sha256=3qPpGP2nlMOCULkvwvxbVLfgE1dVKztUmsuJGQaL79w,2154
15
+ toolkits/databases/hive_cmd.py,sha256=pPr5-6z4y1Yx0J5GoCom2wBFD7x9HOSk8nDIYfZK7Pc,3652
16
+ toolkits/databases/hive_helper.py,sha256=LCt4rff5meLJ1uCozPzL_S7rpGC8wOWvpe2pk0JXlsQ,8233
17
+ toolkits/databases/redis_mgmt.py,sha256=f02qbYNGfvNGvmVfau5QMK6R92YusbZ-XcZnaRMpgWg,3386
18
+ toolkits/databases/sql_helper.py,sha256=JHB9UtvXVhcZNVmuM9teWjsNoLYNxRn0RxQaZwi2N-8,9707
19
+ toolkits/databases/sqlalchemy_helper.py,sha256=26lgOiJDNmuJpX0YWAzZkNylBFGlmaY4jnbcQJIY4Rg,2027
20
+ toolkits/databases/status_check.py,sha256=12oeY94n5ZRnPoNSrY_P8uB6JMWMStAnuRO6gmTrm9w,6474
21
+ toolkits/db_query_demo.py,sha256=0xPiOaAwY4ulZ91x4IhQ1gNRoOIHVk558FKGMAjnHGY,2577
22
+ toolkits/libs_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
23
+ toolkits/libs_core/config_groups_helper.py,sha256=k3ji_4iZVqTL5jB6xrIUKNbATM7Aoc0gAi7rJ1HyJeA,1987
24
+ toolkits/libs_core/config_helper.py,sha256=t4j10aGcrOR6Zb6-qFZUgELO9tShPJs1NNIB3uVCuVg,494
25
+ toolkits/libs_core/env_prepare.py,sha256=1zrxrdKcXR9eouEXuqfH03n1z8LSJ10QZ7HEogtA1_4,6103
26
+ toolkits/libs_core/load_module.py,sha256=AAFPDx4_N6e3KdaI_LKciSqAD0zuHNfZxyu1FImk4o4,1325
27
+ toolkits/libs_core/mysql_helper.py,sha256=XI5b5oAkcqDQ8_r3cBDFjjHIv3ufACq22Ls9dq8XdKQ,6153
28
+ toolkits/network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
+ toolkits/network/ip_helper.py,sha256=VgSnQyXOAViAZuTnXeOZXtPiL38kbf3rD3-SUxRqNsM,665
30
+ toolkits/network/pdi_helper.py,sha256=RR8Jdqyd-g1rnrf98UaZgOOks7Z1jcZFaWXzUbf4fQo,7406
31
+ toolkits/network/send_mail.py,sha256=bHxS2zOQOQXMLMYOOup_KITWecYzMzFIiD0eNb2lzz0,3985
32
+ toolkits/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
+ toolkits/system/aes_cipher.py,sha256=2O-xpT3B1I-od7lczVz9z7kp8Q1OB6-DMT9dqeRVkPU,1143
34
+ toolkits/system/basic_utils.py,sha256=yv22VyuHEFVAE9oQsycb3Lzl_cB5YlgzwjThvlmKuHQ,548
35
+ toolkits/system/collections_helper.py,sha256=esFbjEjRrPL6f2MjahEKUB4uTLLEnxw-felpMXuFsaI,2186
36
+ toolkits/system/crpyt_helper.py,sha256=fobpWDS0uDzgSPeXYhxJN2yhIe9Vu8OXLJjG0WjXKdw,1173
37
+ toolkits/system/dict2xml.py,sha256=ofjrkVYlARgXWv4n_lS6_n5oh7cmFsN3JdREsLeB6X4,14145
38
+ toolkits/system/dict_helper.py,sha256=BSwB87zlKKfvmRDmq2h7u2f25_VoylbxmhyzMLafpIQ,788
39
+ toolkits/system/excel_helper.py,sha256=H_A91BLTHoXuS59ggUadtRLMCsB7Dr3CxgF5uM4BDLo,3319
40
+ toolkits/system/file_helper.py,sha256=5-JDgXU2MD9GTeFhjfyatzOocAQGfpNwlloaVbLoI8Q,1569
41
+ toolkits/system/load_module.py,sha256=UXNfE9drsfz9oOBq9EGdV4gjmiIYlJQECIt3PFaLxwk,1419
42
+ toolkits/system/priority_tasks.py,sha256=eUtKQatAnYcV3f3V3XY4FQ0u3afX8NHuqiBRcuv0d0I,7632
43
+ toolkits/system/process_monitor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
44
+ toolkits/system/process_monitor/process_monitor.py,sha256=_tXmWSOzAqAlHeGiBPrMGFU1Wjdr4zUAKNSPucqGRO8,12887
45
+ toolkits/system/shell_helper.py,sha256=RwMw-t98IFjqo1DiqUL4GK0tRM4vSDj-Jqo623mEyRM,7999
46
+ toolkits/system/str_helper.py,sha256=89qw0jF6Pf19uuMhBh7XFoCxHvEiZztPwlCDdYuPBEQ,5096
47
+ toolkits/system/tasks_deamon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
48
+ toolkits/system/tasks_deamon/tasks_controller.py,sha256=aobnJ0vJmvHQs94-yXasHqyxGsBHrv8Vqd1lq6JtmOo,2380
49
+ toolkits/system/tasks_deamon/tasks_multiprocessing.py,sha256=cECLT0eqzFlpJoUfX_-QH4-TllLOWmx1YhEnZusimho,4167
50
+ toolkits/system/tasks_deamon/tasks_process.py,sha256=b_RUlZ9Fy3XN079IzoBIjMM5UKaXn-Z6GWUob0WIqgI,5095
51
+ toolkits/system/test_shell_helper.py,sha256=zOCe-2BER1_x8t2NtiySWcd7K-PJEqL_zcIiwSzBQAo,48
52
+ toolkits/system/time_helper.py,sha256=_nxy97rnnagCn_m5sppZTDsx2sdn5debsTfXPEjoWzU,5644
53
+ toolkits/system/win32_env.py,sha256=KYkZ-clZYkSQnSvPvpydeOVepaCrPH0egkC-MY-Bs0w,1829
54
+ toolkits/tookits_app.py,sha256=2qz6V5XkEGc6NjxafFvFavE1hpt-tE0PoGonLAvZNXs,380
55
+ toolkits/tookits_cli.py,sha256=f8u2IXYwk0QguQwY39aNjgVwJFiAmfukGGumWT6s07I,5273
56
+ toolkits-0.2.7.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: pdm-backend (2.4.4)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,5 @@
1
+ [console_scripts]
2
+ toolkits = toolkits.toolkits_cli:main
3
+
4
+ [gui_scripts]
5
+