gomyck-tools 1.0.0__py3-none-any.whl → 1.4.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctools/__init__.py +21 -0
- ctools/ai/__init__.py +4 -0
- ctools/ai/llm_chat.py +184 -0
- ctools/ai/llm_client.py +163 -0
- ctools/ai/llm_exception.py +17 -0
- ctools/ai/mcp/__init__.py +4 -0
- ctools/ai/mcp/mcp_client.py +326 -0
- ctools/ai/tools/__init__.py +4 -0
- ctools/ai/tools/json_extract.py +202 -0
- ctools/ai/tools/quick_tools.py +149 -0
- ctools/ai/tools/think_process.py +11 -0
- ctools/ai/tools/tool_use_xml_parse.py +40 -0
- ctools/ai/tools/xml_extract.py +15 -0
- ctools/application.py +50 -47
- ctools/aspect.py +65 -0
- ctools/auto/__init__.py +4 -0
- ctools/{browser_element_tools.py → auto/browser_element.py} +18 -8
- ctools/{plan_area_tools.py → auto/plan_area.py} +5 -7
- ctools/{pty_tools.py → auto/pty_process.py} +6 -3
- ctools/{resource_bundle_tools.py → auto/resource_bundle.py} +4 -4
- ctools/{screenshot_tools.py → auto/screenshot.py} +7 -6
- ctools/{win_canvas.py → auto/win_canvas.py} +10 -4
- ctools/{win_control.py → auto/win_control.py} +14 -5
- ctools/call.py +34 -49
- ctools/cdate.py +84 -0
- ctools/cdebug.py +146 -0
- ctools/cid.py +20 -0
- ctools/cipher/__init__.py +4 -0
- ctools/{aes_tools.py → cipher/aes_util.py} +10 -0
- ctools/{b64.py → cipher/b64.py} +2 -0
- ctools/cipher/czip.py +133 -0
- ctools/cipher/rsa.py +75 -0
- ctools/{sign.py → cipher/sign.py} +2 -1
- ctools/{sm_tools.py → cipher/sm_util.py} +20 -4
- ctools/cjson.py +10 -10
- ctools/cron_lite.py +109 -97
- ctools/database/__init__.py +4 -0
- ctools/{database.py → database/database.py} +93 -26
- ctools/dict_wrapper.py +21 -0
- ctools/ex.py +4 -0
- ctools/geo/__init__.py +4 -0
- ctools/geo/coord_trans.py +127 -0
- ctools/geo/douglas_rarefy.py +139 -0
- ctools/metrics.py +56 -63
- ctools/office/__init__.py +4 -0
- ctools/office/cword.py +30 -0
- ctools/{word_fill.py → office/word_fill.py} +3 -6
- ctools/patch.py +88 -0
- ctools/{work_path.py → path_info.py} +34 -2
- ctools/pkg/__init__.py +4 -0
- ctools/pkg/dynamic_imp.py +38 -0
- ctools/pools/__init__.py +4 -0
- ctools/pools/process_pool.py +41 -0
- ctools/{thread_pool.py → pools/thread_pool.py} +13 -4
- ctools/similar.py +25 -0
- ctools/stream/__init__.py +4 -0
- ctools/stream/ckafka.py +164 -0
- ctools/stream/credis.py +127 -0
- ctools/{mqtt_utils.py → stream/mqtt_utils.py} +20 -12
- ctools/sys_info.py +36 -13
- ctools/sys_log.py +46 -27
- ctools/util/__init__.py +4 -0
- ctools/util/cftp.py +76 -0
- ctools/util/cklock.py +118 -0
- ctools/util/config_util.py +52 -0
- ctools/util/env_config.py +63 -0
- ctools/{html_soup.py → util/html_soup.py} +0 -7
- ctools/{http_utils.py → util/http_util.py} +4 -2
- ctools/{images_tools.py → util/image_process.py} +10 -1
- ctools/util/jb_cut.py +54 -0
- ctools/{id_worker_tools.py → util/snow_id.py} +8 -23
- ctools/web/__init__.py +4 -0
- ctools/web/aio_web_server.py +186 -0
- ctools/web/api_result.py +56 -0
- ctools/web/bottle_web_base.py +239 -0
- ctools/web/bottle_webserver.py +191 -0
- ctools/web/bottle_websocket.py +79 -0
- ctools/web/ctoken.py +103 -0
- ctools/{download_tools.py → web/download_util.py} +14 -12
- ctools/web/params_util.py +46 -0
- ctools/{upload_tools.py → web/upload_util.py} +3 -2
- gomyck_tools-1.4.7.dist-info/METADATA +70 -0
- gomyck_tools-1.4.7.dist-info/RECORD +88 -0
- {gomyck_tools-1.0.0.dist-info → gomyck_tools-1.4.7.dist-info}/WHEEL +1 -1
- gomyck_tools-1.4.7.dist-info/licenses/LICENSE +13 -0
- ctools/bashPath.py +0 -13
- ctools/bottle_server.py +0 -49
- ctools/console.py +0 -55
- ctools/date_utils.py +0 -44
- ctools/enums.py +0 -4
- ctools/excelOpt.py +0 -36
- ctools/imgDialog.py +0 -44
- ctools/license.py +0 -37
- ctools/log.py +0 -28
- ctools/mvc.py +0 -56
- ctools/obj.py +0 -20
- ctools/pacth.py +0 -73
- ctools/ssh.py +0 -9
- ctools/strDiff.py +0 -20
- ctools/string_tools.py +0 -101
- ctools/token_tools.py +0 -13
- ctools/wordFill.py +0 -24
- gomyck_tools-1.0.0.dist-info/METADATA +0 -20
- gomyck_tools-1.0.0.dist-info/RECORD +0 -54
- /ctools/{word_fill_entity.py → office/word_fill_entity.py} +0 -0
- /ctools/{compile_tools.py → util/compile_util.py} +0 -0
- {gomyck_tools-1.0.0.dist-info → gomyck_tools-1.4.7.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
import math
|
|
3
|
+
|
|
4
|
+
x_pi = 3.14159265358979324 * 3000.0 / 180.0
|
|
5
|
+
pi = 3.1415926535897932384626 # π
|
|
6
|
+
a = 6378245.0 # 长半轴
|
|
7
|
+
ee = 0.00669342162296594323 # 偏心率平方
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def gcj02_to_bd09(lng, lat):
|
|
11
|
+
"""
|
|
12
|
+
火星坐标系(GCJ-02)转百度坐标系(BD-09)
|
|
13
|
+
谷歌、高德——>百度
|
|
14
|
+
:param lng:火星坐标经度
|
|
15
|
+
:param lat:火星坐标纬度
|
|
16
|
+
:return:
|
|
17
|
+
"""
|
|
18
|
+
z = math.sqrt(lng * lng + lat * lat) + 0.00002 * math.sin(lat * x_pi)
|
|
19
|
+
theta = math.atan2(lat, lng) + 0.000003 * math.cos(lng * x_pi)
|
|
20
|
+
bd_lng = z * math.cos(theta) + 0.0065
|
|
21
|
+
bd_lat = z * math.sin(theta) + 0.006
|
|
22
|
+
return [bd_lng, bd_lat]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def bd09_to_gcj02(bd_lon, bd_lat):
|
|
26
|
+
"""
|
|
27
|
+
百度坐标系(BD-09)转火星坐标系(GCJ-02)
|
|
28
|
+
百度——>谷歌、高德
|
|
29
|
+
:param bd_lat:百度坐标纬度
|
|
30
|
+
:param bd_lon:百度坐标经度
|
|
31
|
+
:return:转换后的坐标列表形式
|
|
32
|
+
"""
|
|
33
|
+
x = bd_lon - 0.0065
|
|
34
|
+
y = bd_lat - 0.006
|
|
35
|
+
z = math.sqrt(x * x + y * y) - 0.00002 * math.sin(y * x_pi)
|
|
36
|
+
theta = math.atan2(y, x) - 0.000003 * math.cos(x * x_pi)
|
|
37
|
+
gg_lng = z * math.cos(theta)
|
|
38
|
+
gg_lat = z * math.sin(theta)
|
|
39
|
+
return [gg_lng, gg_lat]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def wgs84_to_gcj02(lng, lat):
|
|
43
|
+
"""
|
|
44
|
+
WGS84转GCJ02(火星坐标系)
|
|
45
|
+
:param lng:WGS84坐标系的经度
|
|
46
|
+
:param lat:WGS84坐标系的纬度
|
|
47
|
+
:return:
|
|
48
|
+
"""
|
|
49
|
+
if out_of_china(lng, lat): # 判断是否在国内
|
|
50
|
+
return [lng, lat]
|
|
51
|
+
dlat = _transformlat(lng - 105.0, lat - 35.0)
|
|
52
|
+
dlng = _transformlng(lng - 105.0, lat - 35.0)
|
|
53
|
+
radlat = lat / 180.0 * pi
|
|
54
|
+
magic = math.sin(radlat)
|
|
55
|
+
magic = 1 - ee * magic * magic
|
|
56
|
+
sqrtmagic = math.sqrt(magic)
|
|
57
|
+
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
|
|
58
|
+
dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
|
|
59
|
+
mglat = lat + dlat
|
|
60
|
+
mglng = lng + dlng
|
|
61
|
+
return [mglng, mglat]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def gcj02_to_wgs84(lng, lat):
|
|
65
|
+
"""
|
|
66
|
+
GCJ02(火星坐标系)转GPS84
|
|
67
|
+
:param lng:火星坐标系的经度
|
|
68
|
+
:param lat:火星坐标系纬度
|
|
69
|
+
:return:
|
|
70
|
+
"""
|
|
71
|
+
if out_of_china(lng, lat):
|
|
72
|
+
return [lng, lat]
|
|
73
|
+
dlat = _transformlat(lng - 105.0, lat - 35.0)
|
|
74
|
+
dlng = _transformlng(lng - 105.0, lat - 35.0)
|
|
75
|
+
radlat = lat / 180.0 * pi
|
|
76
|
+
magic = math.sin(radlat)
|
|
77
|
+
magic = 1 - ee * magic * magic
|
|
78
|
+
sqrtmagic = math.sqrt(magic)
|
|
79
|
+
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
|
|
80
|
+
dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
|
|
81
|
+
mglat = lat + dlat
|
|
82
|
+
mglng = lng + dlng
|
|
83
|
+
return [lng * 2 - mglng, lat * 2 - mglat]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def bd09_to_wgs84(bd_lon, bd_lat):
|
|
87
|
+
lon, lat = bd09_to_gcj02(bd_lon, bd_lat)
|
|
88
|
+
return gcj02_to_wgs84(lon, lat)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def wgs84_to_bd09(lon, lat):
|
|
92
|
+
lon, lat = wgs84_to_gcj02(lon, lat)
|
|
93
|
+
return gcj02_to_bd09(lon, lat)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _transformlat(lng, lat):
|
|
97
|
+
ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
|
|
98
|
+
0.1 * lng * lat + 0.2 * math.sqrt(math.fabs(lng))
|
|
99
|
+
ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
|
|
100
|
+
math.sin(2.0 * lng * pi)) * 2.0 / 3.0
|
|
101
|
+
ret += (20.0 * math.sin(lat * pi) + 40.0 *
|
|
102
|
+
math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
|
|
103
|
+
ret += (160.0 * math.sin(lat / 12.0 * pi) + 320 *
|
|
104
|
+
math.sin(lat * pi / 30.0)) * 2.0 / 3.0
|
|
105
|
+
return ret
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _transformlng(lng, lat):
|
|
109
|
+
ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
|
|
110
|
+
0.1 * lng * lat + 0.1 * math.sqrt(math.fabs(lng))
|
|
111
|
+
ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
|
|
112
|
+
math.sin(2.0 * lng * pi)) * 2.0 / 3.0
|
|
113
|
+
ret += (20.0 * math.sin(lng * pi) + 40.0 *
|
|
114
|
+
math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
|
|
115
|
+
ret += (150.0 * math.sin(lng / 12.0 * pi) + 300.0 *
|
|
116
|
+
math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
|
|
117
|
+
return ret
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def out_of_china(lng, lat):
|
|
121
|
+
"""
|
|
122
|
+
判断是否在国内,不在国内不做偏移
|
|
123
|
+
:param lng:
|
|
124
|
+
:param lat:
|
|
125
|
+
:return:
|
|
126
|
+
"""
|
|
127
|
+
return not (lng > 73.66 and lng < 135.05 and lat > 3.86 and lat < 53.55)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: UTF-8 -*-
|
|
3
|
+
__author__ = 'haoyang'
|
|
4
|
+
__date__ = '2024/9/19 14:02'
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
|
|
8
|
+
from jsonpath_ng import parser
|
|
9
|
+
|
|
10
|
+
from ctools import cjson
|
|
11
|
+
from ctools.sys_log import flog as log
|
|
12
|
+
|
|
13
|
+
"""
|
|
14
|
+
douglas_rarefy.DouglasRarefy(res, level=3).sparse_points()
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class THIN_LEVEL:
|
|
19
|
+
L1 = 0.00001
|
|
20
|
+
L2 = 0.00003
|
|
21
|
+
L3 = 0.00009
|
|
22
|
+
L4 = 0.0002
|
|
23
|
+
L5 = 0.0004
|
|
24
|
+
L6 = 0.0007
|
|
25
|
+
L7 = 0.0011
|
|
26
|
+
L8 = 0.0017
|
|
27
|
+
L9 = 0.0022
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Point:
|
|
31
|
+
def __init__(self, lng, lat, origin_data):
|
|
32
|
+
self.lng = lng
|
|
33
|
+
self.lat = lat
|
|
34
|
+
self.origin_data = origin_data
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _get_line_by_point(xy1, xy2):
|
|
38
|
+
"""
|
|
39
|
+
根据两个点求直线方程 ax + by + c = 0
|
|
40
|
+
:param xy1: 点1, 例如 {'lat': 1, 'lng': 1}
|
|
41
|
+
:param xy2: 点2, 例如 {'lat': 2, 'lng': 2}
|
|
42
|
+
:return: 直线方程的三个参数 [a, b, c]
|
|
43
|
+
"""
|
|
44
|
+
x1 = xy1.lng
|
|
45
|
+
y1 = xy1.lat
|
|
46
|
+
x2 = xy2.lng
|
|
47
|
+
y2 = xy2.lat
|
|
48
|
+
a = y2 - y1
|
|
49
|
+
b = x1 - x2
|
|
50
|
+
c = (y1 - y2) * x1 - y1 * (x1 - x2)
|
|
51
|
+
return [a, b, c]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _get_distance_from_point_to_line(a, b, c, xy):
|
|
55
|
+
"""
|
|
56
|
+
点到直线的距离,直线方程为 ax + by + c = 0
|
|
57
|
+
:param a: 直线参数a
|
|
58
|
+
:param b: 直线参数b
|
|
59
|
+
:param c: 直线参数c
|
|
60
|
+
:param xy: 点坐标,例如 {'lat': 2, 'lng': 2}
|
|
61
|
+
:return: 距离
|
|
62
|
+
"""
|
|
63
|
+
x = xy.lng
|
|
64
|
+
y = xy.lat
|
|
65
|
+
return abs((a * x + b * y + c) / math.sqrt(a * a + b * b))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class DouglasRarefy:
|
|
69
|
+
"""
|
|
70
|
+
DouglasRarefy Use Guide:
|
|
71
|
+
points must be arrays, element can be dict or arrays, when element is arrays, index 0 must be lng, index 1 must be lat, and element can be use max column num is 2 (lng, lat) in ret_tpl
|
|
72
|
+
level default is L2, this level can be hold most of the points detail
|
|
73
|
+
ret_tpl is the result tpl, can be arrays and json or some can be json loads, exp: [{lng}, {lat}] OR {{"lng": {lng}, "lat": {lat}}}
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
def __init__(self, points: [], level=THIN_LEVEL.L2, ret_tpl=None, get_lng=None, get_lat=None):
|
|
77
|
+
if not isinstance(points, list): raise Exception('points must be list obj !!')
|
|
78
|
+
if len(points) < 3: raise Exception('points length must be gt 2 !!')
|
|
79
|
+
self.points = points
|
|
80
|
+
self.threshold = THIN_LEVEL.L2 if level is None else (getattr(THIN_LEVEL, "L{}".format(int(level))) if int(level) >= 1 else level)
|
|
81
|
+
log.debug("threshold is: {}".format(self.threshold))
|
|
82
|
+
self.is_json = isinstance(points[0], dict)
|
|
83
|
+
self.get_lng = get_lng
|
|
84
|
+
self.get_lat = get_lat
|
|
85
|
+
if self.is_json:
|
|
86
|
+
if not self.get_lng: self.get_lng = '$.lng'
|
|
87
|
+
if not self.get_lat: self.get_lat = '$.lat'
|
|
88
|
+
else:
|
|
89
|
+
if not self.get_lng: self.get_lng = '$.[0]'
|
|
90
|
+
if not self.get_lat: self.get_lat = '$.[1]'
|
|
91
|
+
log.debug("get_lng is: {}, get_lat is: {}".format(self.get_lng, self.get_lat))
|
|
92
|
+
self.lng_parser = parser.parse(self.get_lng)
|
|
93
|
+
self.lat_parser = parser.parse(self.get_lat)
|
|
94
|
+
log.debug("is_json is: {}".format(self.is_json))
|
|
95
|
+
self.ret_tpl = ret_tpl
|
|
96
|
+
log.debug("ret_tpl is: {}".format(self.ret_tpl))
|
|
97
|
+
self.data = [Point(self.lng_parser.find(p)[0].value, self.lat_parser.find(p)[0].value, p) for p in self.points]
|
|
98
|
+
|
|
99
|
+
def _sparse_points(self, points):
|
|
100
|
+
"""
|
|
101
|
+
点位压缩
|
|
102
|
+
:return: 稀疏后的点集
|
|
103
|
+
"""
|
|
104
|
+
if len(points) < 3:
|
|
105
|
+
if not self.ret_tpl:
|
|
106
|
+
return [points[0].origin_data, points[-1].origin_data]
|
|
107
|
+
else:
|
|
108
|
+
if self.is_json:
|
|
109
|
+
return [cjson.loads(self.ret_tpl.format(**points[0].origin_data)), cjson.loads(self.ret_tpl.format(**points[-1].origin_data))]
|
|
110
|
+
else:
|
|
111
|
+
return [cjson.loads(self.ret_tpl.format(lng=points[0].lng, lat=points[0].lat)), cjson.loads(self.ret_tpl.format(lng=points[-1].lng, lat=points[-1].lat))]
|
|
112
|
+
|
|
113
|
+
xy_first = points[0] # 第一个点
|
|
114
|
+
xy_end = points[-1] # 最后一个点
|
|
115
|
+
a, b, c = _get_line_by_point(xy_first, xy_end) # 获取直线方程的 a, b, c 值
|
|
116
|
+
d_max = 0 # 记录点到直线的最大距离
|
|
117
|
+
split = 0 # 分割位置
|
|
118
|
+
for i in range(1, len(points) - 1):
|
|
119
|
+
d = _get_distance_from_point_to_line(a, b, c, points[i])
|
|
120
|
+
if d > d_max:
|
|
121
|
+
split = i
|
|
122
|
+
d_max = d
|
|
123
|
+
if d_max > self.threshold:
|
|
124
|
+
# 如果存在点到首位点连成直线的距离大于 max_distance 的, 即需要再次划分
|
|
125
|
+
child_left = self._sparse_points(points[:split + 1]) # 递归处理左边部分
|
|
126
|
+
child_right = self._sparse_points(points[split:]) # 递归处理右边部分
|
|
127
|
+
# 合并结果,避免重复
|
|
128
|
+
return child_left + child_right[1:]
|
|
129
|
+
else:
|
|
130
|
+
if not self.ret_tpl:
|
|
131
|
+
return [points[0].origin_data, points[-1].origin_data]
|
|
132
|
+
else:
|
|
133
|
+
if self.is_json:
|
|
134
|
+
return [cjson.loads(self.ret_tpl.format(**points[0].origin_data)), cjson.loads(self.ret_tpl.format(**points[-1].origin_data))]
|
|
135
|
+
else:
|
|
136
|
+
return [cjson.loads(self.ret_tpl.format(lng=points[0].lng, lat=points[0].lat)), cjson.loads(self.ret_tpl.format(lng=points[-1].lng, lat=points[-1].lat))]
|
|
137
|
+
|
|
138
|
+
def sparse_points(self):
|
|
139
|
+
return self._sparse_points(self.data)
|
ctools/metrics.py
CHANGED
|
@@ -2,22 +2,28 @@ import os
|
|
|
2
2
|
import threading
|
|
3
3
|
from enum import Enum
|
|
4
4
|
|
|
5
|
-
from prometheus_client import Counter, Gauge, Summary, Histogram
|
|
5
|
+
from prometheus_client import Counter, Gauge, Summary, Histogram, start_http_server
|
|
6
6
|
|
|
7
|
-
from
|
|
8
|
-
from ctools import call, cjson, sys_log
|
|
9
|
-
from ctools.application import Server
|
|
7
|
+
from ctools import call, cjson, sys_log, path_info
|
|
10
8
|
|
|
11
9
|
log = sys_log.flog
|
|
12
10
|
|
|
13
|
-
|
|
11
|
+
_metrics_port = 8889
|
|
12
|
+
_persistent_json = {}
|
|
14
13
|
_metrics_initial = {}
|
|
15
|
-
persistent_json = {}
|
|
16
|
-
temp_metrics_json = {}
|
|
17
|
-
is_metrics_init: bool = True
|
|
18
14
|
_lock = threading.Lock()
|
|
15
|
+
_metrics_persistent_path = os.path.join(path_info.get_user_work_path('metrics', True), 'persistent.json')
|
|
19
16
|
|
|
20
17
|
|
|
18
|
+
# 认证中间件
|
|
19
|
+
# @app.before_request
|
|
20
|
+
# def check_authentication():
|
|
21
|
+
# auth = request.authorization
|
|
22
|
+
# if not auth or auth.username != USERNAME or auth.password != PASSWORD:
|
|
23
|
+
# return Response(
|
|
24
|
+
# "Unauthorized", 401, {"WWW-Authenticate": 'Basic realm="Login Required"'}
|
|
25
|
+
# )
|
|
26
|
+
|
|
21
27
|
class MetricType(Enum):
|
|
22
28
|
COUNTER = 'counter'
|
|
23
29
|
GAUGE = 'gauge'
|
|
@@ -26,8 +32,7 @@ class MetricType(Enum):
|
|
|
26
32
|
|
|
27
33
|
|
|
28
34
|
class Metric:
|
|
29
|
-
def __init__(self, metric_type: MetricType, metric_key: str, metric_labels: [],
|
|
30
|
-
persistent: bool = False, buckets: [] = None, reset: bool = False, desc: str = ""):
|
|
35
|
+
def __init__(self, metric_type: MetricType, metric_key: str, metric_labels: [], persistent: bool = False, buckets: [] = None, reset: bool = False, desc: str = ""):
|
|
31
36
|
self.metric_type = metric_type
|
|
32
37
|
self.metric_key = metric_key
|
|
33
38
|
self.metric_labels = metric_labels
|
|
@@ -50,34 +55,33 @@ class Metric:
|
|
|
50
55
|
|
|
51
56
|
|
|
52
57
|
@call.once
|
|
53
|
-
def init():
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
if os.path.exists(persistent_path):
|
|
58
|
-
with open(persistent_path, 'r') as persistent_file:
|
|
59
|
-
global persistent_json
|
|
58
|
+
def init(reset_persistent: bool = False):
|
|
59
|
+
if os.path.exists(_metrics_persistent_path) and not reset_persistent:
|
|
60
|
+
with open(_metrics_persistent_path, 'r') as persistent_file:
|
|
61
|
+
global _persistent_json
|
|
60
62
|
try:
|
|
61
63
|
content = persistent_file.readline()
|
|
62
|
-
|
|
63
|
-
|
|
64
|
+
log.info("persistent初始化: %s" % content)
|
|
65
|
+
_persistent_json = cjson.loads(content)
|
|
64
66
|
except Exception:
|
|
65
67
|
log.error('persistent.json is not valid json!!!!!')
|
|
66
|
-
|
|
68
|
+
_persistent_json = {}
|
|
67
69
|
_init_all_metrics()
|
|
68
|
-
|
|
70
|
+
_persistent_json = _persistent_json or {}
|
|
71
|
+
for key, item in _persistent_json.items():
|
|
69
72
|
metrics_key = key.split("-")[0]
|
|
70
73
|
if '_labels' in key or metrics_key not in _metrics_initial: continue
|
|
71
|
-
opt(metrics_key,
|
|
74
|
+
opt(metrics_key, _persistent_json[key + '_labels'], _persistent_json[key])
|
|
72
75
|
persistent_metrics()
|
|
73
|
-
|
|
76
|
+
start_http_server(port=_metrics_port)
|
|
74
77
|
|
|
75
78
|
|
|
76
|
-
@call.schd(
|
|
79
|
+
@call.schd(5, start_by_call=True)
|
|
77
80
|
def persistent_metrics():
|
|
78
|
-
if
|
|
79
|
-
|
|
80
|
-
|
|
81
|
+
if _persistent_json and not _lock.locked():
|
|
82
|
+
log.info('begin persistent metrics to file...')
|
|
83
|
+
with open(_metrics_persistent_path, 'w') as persistent_file:
|
|
84
|
+
persistent_file.write(cjson.dumps(_persistent_json))
|
|
81
85
|
persistent_file.flush()
|
|
82
86
|
|
|
83
87
|
|
|
@@ -85,34 +89,31 @@ def opt(metric_key: str, label_values: [], metric_value: int):
|
|
|
85
89
|
_lock.acquire(timeout=5)
|
|
86
90
|
try:
|
|
87
91
|
persistent_key = "%s-%s" % (metric_key, "_".join(map(str, label_values)))
|
|
88
|
-
metric_entity: Metric =
|
|
89
|
-
if not metric_entity:
|
|
90
|
-
metric_entity = metrics[persistent_key] = _metrics_initial[metric_key]
|
|
91
|
-
|
|
92
|
+
metric_entity: Metric = _metrics_initial[metric_key]
|
|
92
93
|
if metric_entity.persistent:
|
|
93
|
-
if not
|
|
94
|
-
|
|
94
|
+
if not metric_entity.reset and persistent_key in _persistent_json:
|
|
95
|
+
_persistent_json[persistent_key] += metric_value
|
|
95
96
|
else:
|
|
96
|
-
|
|
97
|
-
|
|
97
|
+
_persistent_json[persistent_key] = metric_value
|
|
98
|
+
_persistent_json[persistent_key + '_labels'] = label_values
|
|
98
99
|
|
|
99
|
-
if
|
|
100
|
-
|
|
100
|
+
if _persistent_json[persistent_key] < 0:
|
|
101
|
+
_persistent_json[persistent_key] = 0
|
|
101
102
|
metric_value = 0
|
|
102
103
|
|
|
103
|
-
temp_metrics_json[persistent_key] = metric_value
|
|
104
|
-
|
|
105
104
|
if metric_entity.metric_type == MetricType.COUNTER or metric_entity.metric_type == MetricType.GAUGE:
|
|
106
105
|
if label_values is None or len(label_values) == 0:
|
|
107
106
|
if metric_entity.metric_type == MetricType.COUNTER and metric_entity.reset:
|
|
108
|
-
metric_entity.metric.reset()
|
|
109
|
-
metric_entity.
|
|
107
|
+
metric_entity.metric.labels('').reset()
|
|
108
|
+
if metric_entity.metric_type == MetricType.GAUGE and metric_entity.reset:
|
|
109
|
+
metric_entity.metric.labels('').set(0)
|
|
110
|
+
metric_entity.metric.labels('').inc(metric_value)
|
|
110
111
|
else:
|
|
111
112
|
if metric_entity.reset:
|
|
112
|
-
|
|
113
|
-
metric_entity.metric.
|
|
114
|
-
|
|
115
|
-
|
|
113
|
+
if metric_entity.metric_type == MetricType.COUNTER and metric_entity.reset:
|
|
114
|
+
metric_entity.metric.labels(*label_values).reset()
|
|
115
|
+
if metric_entity.metric_type == MetricType.GAUGE and metric_entity.reset:
|
|
116
|
+
metric_entity.metric.labels(*label_values).set(0)
|
|
116
117
|
metric_entity.metric.labels(*label_values).inc(metric_value)
|
|
117
118
|
else:
|
|
118
119
|
if label_values is None or len(label_values) == 0:
|
|
@@ -125,21 +126,13 @@ def opt(metric_key: str, label_values: [], metric_value: int):
|
|
|
125
126
|
|
|
126
127
|
|
|
127
128
|
def _init_all_metrics():
|
|
128
|
-
Metric(MetricType.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
Metric(MetricType.COUNTER, MetricKey.WORKLOAD_PROCESS_HOURS.value, ['process_id'], persistent=True, desc="全部流程工作时长")
|
|
139
|
-
Metric(MetricType.COUNTER, MetricKey.WORKLOAD_PROCESS_HOURS_SINGLE.value, ['process_id'], reset=True, desc="单次流程工作时长")
|
|
140
|
-
Metric(MetricType.COUNTER, MetricKey.WORKLOAD_PROCESS_COLLECT_DATA_TOTAL.value, ['process_id'], persistent=True, desc="全部流程采集数据总量")
|
|
141
|
-
Metric(MetricType.COUNTER, MetricKey.WORKLOAD_PROCESS_COLLECT_DATA_TOTAL_SINGLE.value, ['process_id'], reset=True, desc="单次流程采集数据总量")
|
|
142
|
-
Metric(MetricType.COUNTER, MetricKey.WORKLOAD_PROCESS_SEND_DATA_TOTAL.value, ['process_id'], persistent=True, desc="全部流程发送数据总量")
|
|
143
|
-
Metric(MetricType.COUNTER, MetricKey.WORKLOAD_PROCESS_SEND_DATA_TOTAL_SINGLE.value, ['process_id'], reset=True, desc="单次流程发送数据总量")
|
|
144
|
-
|
|
145
|
-
Metric(MetricType.COUNTER, MetricKey.ERROR_SCHEDULE_COUNT.value, ['process_id'], persistent=True, desc="出错任务数")
|
|
129
|
+
Metric(MetricType.GAUGE, 'gomyck', ['g_label1', 'g_label2'], persistent=True, reset=True)
|
|
130
|
+
|
|
131
|
+
# if __name__ == '__main__':
|
|
132
|
+
# init()
|
|
133
|
+
# import random
|
|
134
|
+
# while True:
|
|
135
|
+
# opt('data_reported_time', ['123', '123'], random.randint(1, 10))
|
|
136
|
+
# opt('data_received_time', ['123', '123'], random.randint(1, 10))
|
|
137
|
+
# opt('data_insert_time', ['123', '123'], random.randint(1, 10))
|
|
138
|
+
# time.sleep(1)
|
ctools/office/cword.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# from docxtpl import DocxTemplate
|
|
2
|
+
#
|
|
3
|
+
# # tpl = DocxTemplate('/Users/haoyang/Desktop/xxx.docx')
|
|
4
|
+
# tpl = DocxTemplate('/Users/haoyang/Desktop/123.doc')
|
|
5
|
+
#
|
|
6
|
+
# # 设置好各标签需要填写的内容
|
|
7
|
+
# context = {'xxxx': '计算机科学与技术', 'cccc': '2022050513'}
|
|
8
|
+
# # 将标签内容填入模板中
|
|
9
|
+
# tpl.render(context)
|
|
10
|
+
# # 保存
|
|
11
|
+
# tpl.save('/Users/haoyang/Desktop/new_test2.docx')
|
|
12
|
+
|
|
13
|
+
from docx import Document
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def merge_word_files(input_files: [], output_file: str):
|
|
17
|
+
merged_doc = Document()
|
|
18
|
+
for file in input_files:
|
|
19
|
+
doc = Document(file)
|
|
20
|
+
for element in doc.element.body:
|
|
21
|
+
merged_doc.element.body.append(element)
|
|
22
|
+
merged_doc.save(output_file)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_word_file(input_file: str):
|
|
26
|
+
doc = Document(input_file)
|
|
27
|
+
text = []
|
|
28
|
+
for paragraph in doc.paragraphs:
|
|
29
|
+
text.append(paragraph.text)
|
|
30
|
+
return "\n".join(text)
|
|
@@ -16,7 +16,7 @@ from xlsxtpl import writerx as xlsx_writerx
|
|
|
16
16
|
from xlutils.copy import copy
|
|
17
17
|
|
|
18
18
|
from ctools import cjson
|
|
19
|
-
from ctools.word_fill_entity import WordTable, WordCell, Style, Font, Alignment
|
|
19
|
+
from ctools.office.word_fill_entity import WordTable, WordCell, Style, Font, Alignment
|
|
20
20
|
|
|
21
21
|
EXCEL_WORD_ALIGNMENT = {"left": WD_TABLE_ALIGNMENT.LEFT,
|
|
22
22
|
"center": WD_TABLE_ALIGNMENT.CENTER,
|
|
@@ -469,6 +469,7 @@ def check_format(input_path: str):
|
|
|
469
469
|
res = False
|
|
470
470
|
return res
|
|
471
471
|
|
|
472
|
+
|
|
472
473
|
def excel_optimize(input_path: str):
|
|
473
474
|
"""
|
|
474
475
|
优化excel内容清除无法使用的隐患
|
|
@@ -492,9 +493,8 @@ def excel_optimize(input_path: str):
|
|
|
492
493
|
except Exception as e:
|
|
493
494
|
print("优化模板内容清除无法使用的隐患异常: %s" % e)
|
|
494
495
|
|
|
495
|
-
|
|
496
496
|
# 示例用法:从第1行到第5行、从第1列到第3列的区域复制到Word文档中
|
|
497
|
-
# excel_file = r'C:\Users\DELL\
|
|
497
|
+
# excel_file = r'C:\Users\DELL\xxx/xxx-rpa/document\test-2024-04-01_09-05-26.xlsx' # Excel文件名
|
|
498
498
|
# excel_file_1 = r'E:\test\c.xlsx' # Excel文件名
|
|
499
499
|
# excel_file_2 = r'E:\test\b.xlsx' # Excel文件名
|
|
500
500
|
# word_file = r'E:\test\test.docx' # 输出Word文件名
|
|
@@ -557,6 +557,3 @@ def excel_optimize(input_path: str):
|
|
|
557
557
|
# print(check_format(input_path))
|
|
558
558
|
|
|
559
559
|
# from docxtpl import DocxTemplate, RichText
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
ctools/patch.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from sqlalchemy.sql import text
|
|
4
|
+
|
|
5
|
+
from ctools import path_info
|
|
6
|
+
from ctools.database import database
|
|
7
|
+
|
|
8
|
+
"""
|
|
9
|
+
from ctools import patch
|
|
10
|
+
def xx():
|
|
11
|
+
print('hello world')
|
|
12
|
+
def xx1():
|
|
13
|
+
print('hello world1')
|
|
14
|
+
def xx2():
|
|
15
|
+
print('hello world2')
|
|
16
|
+
patch_funcs = {
|
|
17
|
+
'V1.0.2': xx,
|
|
18
|
+
'V1.0.3': xx1,
|
|
19
|
+
'V1.1.4': xx2
|
|
20
|
+
}
|
|
21
|
+
patch.sync_version("kwc", "V1.1.5", patch_funcs)
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
class Patch:
|
|
25
|
+
|
|
26
|
+
def __init__(self, oldVersion, newVersion, patch_func: dict) -> None:
|
|
27
|
+
super().__init__()
|
|
28
|
+
if oldVersion:
|
|
29
|
+
self.oldV = version_to_int(oldVersion)
|
|
30
|
+
else:
|
|
31
|
+
self.oldV = 0
|
|
32
|
+
self.currentV = version_to_int(newVersion)
|
|
33
|
+
self.snapshot = '-snapshot' in newVersion or (oldVersion is not None and '-snapshot' in oldVersion)
|
|
34
|
+
self.patch_func = patch_func
|
|
35
|
+
|
|
36
|
+
def apply_patch(self):
|
|
37
|
+
patch_methods = [method for method in self.patch_func.keys() if method and (method.startswith('V') or method.startswith('v'))]
|
|
38
|
+
patch_methods.sort(key=lambda x: version_to_int(x))
|
|
39
|
+
max_method_name = patch_methods[-1]
|
|
40
|
+
exec_max_method = False
|
|
41
|
+
for method_name in patch_methods:
|
|
42
|
+
slVersion = version_to_int(method_name)
|
|
43
|
+
if self.currentV > slVersion >= self.oldV:
|
|
44
|
+
if max_method_name == method_name: exec_max_method = True
|
|
45
|
+
method = self.patch_func[method_name]
|
|
46
|
+
print('start exec patch {}'.format(method_name))
|
|
47
|
+
method()
|
|
48
|
+
print('patch {} update success'.format(method_name))
|
|
49
|
+
if self.snapshot and not exec_max_method:
|
|
50
|
+
print('start exec snapshot patch {}'.format(max_method_name))
|
|
51
|
+
method = self.patch_func[max_method_name]
|
|
52
|
+
method()
|
|
53
|
+
print('snapshot patch {} update success'.format(max_method_name))
|
|
54
|
+
|
|
55
|
+
def version_to_int(version):
|
|
56
|
+
return int(version.replace('V', '').replace('v', '').replace('.', '').replace('-snapshot', ''))
|
|
57
|
+
|
|
58
|
+
def run_sqls(sqls):
|
|
59
|
+
with database.get_session() as s:
|
|
60
|
+
for sql in sqls.split(";"):
|
|
61
|
+
try:
|
|
62
|
+
s.execute(text(sql.strip()))
|
|
63
|
+
s.commit()
|
|
64
|
+
except Exception as e:
|
|
65
|
+
print('结构升级错误, 请检查!!! {}'.format(e.__cause__))
|
|
66
|
+
|
|
67
|
+
def sync_version(app_name, new_version, patch_func: dict):
|
|
68
|
+
destFilePath = os.path.join(path_info.get_user_work_path(".ck/{}".format(app_name), mkdir=True), "version")
|
|
69
|
+
if not os.path.exists(destFilePath):
|
|
70
|
+
patch = Patch(oldVersion=None, newVersion=new_version, patch_func=patch_func)
|
|
71
|
+
patch.apply_patch()
|
|
72
|
+
with open(destFilePath, 'w') as nv:
|
|
73
|
+
nv.write(new_version)
|
|
74
|
+
print('初始化安装, 版本信息为: {}'.format(new_version))
|
|
75
|
+
nv.flush()
|
|
76
|
+
else:
|
|
77
|
+
with open(destFilePath, 'r') as oldVersion:
|
|
78
|
+
oldV = oldVersion.readline()
|
|
79
|
+
print('本地版本信息为: {}, 程序版本信息为: {}'.format(oldV, new_version))
|
|
80
|
+
oldVersion.close()
|
|
81
|
+
if oldV >= new_version and '-snapshot' not in oldV: return
|
|
82
|
+
print('开始升级本地程序..')
|
|
83
|
+
patch = Patch(oldVersion=oldV, newVersion=new_version, patch_func=patch_func)
|
|
84
|
+
patch.apply_patch()
|
|
85
|
+
with open(destFilePath, 'w') as newVersion:
|
|
86
|
+
newVersion.write(new_version)
|
|
87
|
+
print('程序升级成功, 更新版本信息为: {}'.format(new_version))
|
|
88
|
+
newVersion.flush()
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import os
|
|
3
3
|
import sys
|
|
4
|
+
import importlib
|
|
5
|
+
from types import ModuleType
|
|
6
|
+
from pathlib import Path
|
|
4
7
|
|
|
5
8
|
|
|
6
9
|
def get_current_path(subPath: str = '') -> str:
|
|
@@ -33,13 +36,16 @@ def get_app_path(subPath: str = '') -> str:
|
|
|
33
36
|
return os.path.join(base_path, subPath)
|
|
34
37
|
|
|
35
38
|
|
|
36
|
-
def get_user_work_path(subPath: str = ''
|
|
39
|
+
def get_user_work_path(subPath: str = '', mkdir: bool = False):
|
|
37
40
|
"""
|
|
38
41
|
获取用户工作目录
|
|
39
42
|
:param subPath: 拼接的子路径
|
|
43
|
+
:param mkdir: 是否创建目录
|
|
40
44
|
:return: 用户工作目录
|
|
41
45
|
"""
|
|
42
|
-
|
|
46
|
+
path = os.path.join(os.path.expanduser("~"), subPath)
|
|
47
|
+
if mkdir and not os.path.exists(path): os.makedirs(path, exist_ok=True)
|
|
48
|
+
return path
|
|
43
49
|
|
|
44
50
|
|
|
45
51
|
def get_user_temp_path(subPath: str = '') -> str:
|
|
@@ -64,3 +70,29 @@ def get_install_path(subPath: str = '') -> str:
|
|
|
64
70
|
:return: 安装包安装的路径
|
|
65
71
|
"""
|
|
66
72
|
return os.path.join(os.getcwd(), subPath)
|
|
73
|
+
|
|
74
|
+
def get_package_path(package: str | ModuleType) -> Path:
|
|
75
|
+
"""
|
|
76
|
+
获取包所在目录
|
|
77
|
+
:param package: 包名字符串或已导入模块
|
|
78
|
+
:return: 包目录的 Path 对象
|
|
79
|
+
"""
|
|
80
|
+
if isinstance(package, str):
|
|
81
|
+
module = importlib.import_module(package)
|
|
82
|
+
elif isinstance(package, ModuleType):
|
|
83
|
+
module = package
|
|
84
|
+
else:
|
|
85
|
+
raise TypeError("package 必须是字符串或模块对象")
|
|
86
|
+
|
|
87
|
+
return Path(module.__file__).resolve().parent
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def get_resource_path(package: str | ModuleType, *subpaths: str) -> Path:
|
|
91
|
+
"""
|
|
92
|
+
获取包内资源文件路径(自动拼接子路径)
|
|
93
|
+
:param package: 包名或模块
|
|
94
|
+
:param subpaths: 可变参数,表示路径中的子目录或文件名
|
|
95
|
+
:return: 资源的绝对路径
|
|
96
|
+
"""
|
|
97
|
+
base = get_package_path(package)
|
|
98
|
+
return base.joinpath(*subpaths)
|
ctools/pkg/__init__.py
ADDED