gomyck-tools 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctools/__init__.py +21 -0
- ctools/ai/env_config.py +18 -1
- ctools/ai/llm_chat.py +8 -8
- ctools/ai/llm_client.py +26 -24
- ctools/ai/mcp/mcp_client.py +33 -17
- ctools/ai/tools/json_extract.py +3 -2
- ctools/ai/tools/quick_tools.py +71 -22
- ctools/ai/tools/tool_use_xml_parse.py +2 -1
- ctools/ai/tools/xml_extract.py +3 -0
- ctools/application.py +21 -19
- ctools/aspect.py +65 -0
- ctools/auto/browser_element.py +11 -3
- ctools/auto/plan_area.py +2 -2
- ctools/auto/pty_process.py +0 -1
- ctools/auto/screenshot.py +3 -4
- ctools/auto/win_canvas.py +10 -4
- ctools/auto/win_control.py +8 -4
- ctools/call.py +32 -47
- ctools/cdate.py +43 -2
- ctools/cid.py +6 -4
- ctools/cipher/aes_util.py +2 -2
- ctools/cipher/b64.py +2 -0
- ctools/cipher/czip.py +3 -1
- ctools/cipher/rsa.py +6 -1
- ctools/cipher/sign.py +1 -0
- ctools/cipher/sm_util.py +3 -0
- ctools/cjson.py +5 -0
- ctools/cron_lite.py +10 -4
- ctools/database/database.py +52 -22
- ctools/dict_wrapper.py +1 -0
- ctools/ex.py +4 -0
- ctools/geo/coord_trans.py +94 -94
- ctools/geo/douglas_rarefy.py +13 -9
- ctools/metrics.py +6 -0
- ctools/office/cword.py +7 -7
- ctools/office/word_fill.py +1 -4
- ctools/patch.py +88 -0
- ctools/path_info.py +29 -0
- ctools/pkg/__init__.py +4 -0
- ctools/pkg/dynamic_imp.py +38 -0
- ctools/pools/process_pool.py +6 -1
- ctools/pools/thread_pool.py +6 -2
- ctools/similar.py +3 -0
- ctools/stream/ckafka.py +11 -5
- ctools/stream/credis.py +37 -23
- ctools/stream/mqtt_utils.py +2 -2
- ctools/sys_info.py +8 -0
- ctools/sys_log.py +4 -1
- ctools/util/cftp.py +4 -2
- ctools/util/cklock.py +118 -0
- ctools/util/config_util.py +52 -0
- ctools/util/http_util.py +1 -0
- ctools/util/image_process.py +8 -0
- ctools/util/jb_cut.py +53 -0
- ctools/util/snow_id.py +3 -2
- ctools/web/__init__.py +2 -2
- ctools/web/aio_web_server.py +19 -9
- ctools/web/api_result.py +3 -2
- ctools/web/bottle_web_base.py +134 -70
- ctools/web/bottle_webserver.py +41 -35
- ctools/web/bottle_websocket.py +4 -0
- ctools/web/ctoken.py +81 -13
- ctools/web/download_util.py +1 -1
- ctools/web/params_util.py +4 -0
- ctools/web/upload_util.py +1 -1
- {gomyck_tools-1.4.1.dist-info → gomyck_tools-1.4.3.dist-info}/METADATA +9 -11
- gomyck_tools-1.4.3.dist-info/RECORD +88 -0
- ctools/auto/pacth.py +0 -74
- gomyck_tools-1.4.1.dist-info/RECORD +0 -82
- {gomyck_tools-1.4.1.dist-info → gomyck_tools-1.4.3.dist-info}/WHEEL +0 -0
- {gomyck_tools-1.4.1.dist-info → gomyck_tools-1.4.3.dist-info}/licenses/LICENSE +0 -0
- {gomyck_tools-1.4.1.dist-info → gomyck_tools-1.4.3.dist-info}/top_level.txt +0 -0
ctools/cjson.py
CHANGED
|
@@ -6,6 +6,7 @@ jsonpickle.set_preferred_backend('json')
|
|
|
6
6
|
jsonpickle.set_encoder_options('json', ensure_ascii=False)
|
|
7
7
|
jsonpickle.set_decoder_options('json')
|
|
8
8
|
|
|
9
|
+
|
|
9
10
|
def dumps(obj, **kwargs) -> str:
|
|
10
11
|
"""
|
|
11
12
|
将对象转换为json字符串
|
|
@@ -17,6 +18,7 @@ def dumps(obj, **kwargs) -> str:
|
|
|
17
18
|
if type(obj) == str: return obj
|
|
18
19
|
return f'{jsonpickle.encode(obj, unpicklable=False, make_refs=False, **kwargs)}'
|
|
19
20
|
|
|
21
|
+
|
|
20
22
|
def loads(json_str: str, **kwargs) -> dict:
|
|
21
23
|
"""
|
|
22
24
|
将json字符串转换为对象
|
|
@@ -25,6 +27,7 @@ def loads(json_str: str, **kwargs) -> dict:
|
|
|
25
27
|
"""
|
|
26
28
|
return jsonpickle.decode(json_str, **kwargs)
|
|
27
29
|
|
|
30
|
+
|
|
28
31
|
def unify_to_str(json_str: str) -> str:
|
|
29
32
|
if not str_value_keys and len(str_value_keys) == 0: return json_str
|
|
30
33
|
obj = loads(json_str)
|
|
@@ -34,6 +37,7 @@ def unify_to_str(json_str: str) -> str:
|
|
|
34
37
|
_handle_dict(obj)
|
|
35
38
|
return dumps(obj)
|
|
36
39
|
|
|
40
|
+
|
|
37
41
|
def _handle_list(data):
|
|
38
42
|
for o in data:
|
|
39
43
|
if isinstance(o, list):
|
|
@@ -41,6 +45,7 @@ def _handle_list(data):
|
|
|
41
45
|
elif isinstance(o, dict):
|
|
42
46
|
_handle_dict(o)
|
|
43
47
|
|
|
48
|
+
|
|
44
49
|
def _handle_dict(data):
|
|
45
50
|
for k, v in data.items():
|
|
46
51
|
if isinstance(v, list):
|
ctools/cron_lite.py
CHANGED
|
@@ -29,6 +29,7 @@ print(123123)
|
|
|
29
29
|
cron_lite.start_all()
|
|
30
30
|
"""
|
|
31
31
|
|
|
32
|
+
|
|
32
33
|
class SchedulerMeta:
|
|
33
34
|
timer_task_name: str = None
|
|
34
35
|
switch: bool = True
|
|
@@ -40,7 +41,7 @@ class SchedulerMeta:
|
|
|
40
41
|
|
|
41
42
|
scheduler_map: Dict[str, SchedulerMeta] = {} # {timer_task_name: SchedulerMeta}
|
|
42
43
|
_switch = False
|
|
43
|
-
_info_handler
|
|
44
|
+
_info_handler = print
|
|
44
45
|
_error_handler = print
|
|
45
46
|
_time_zone: Optional[pytz.BaseTzInfo] = pytz.timezone("Asia/Shanghai")
|
|
46
47
|
|
|
@@ -49,6 +50,7 @@ def set_time_zone(time_zone_name: str):
|
|
|
49
50
|
global _time_zone
|
|
50
51
|
_time_zone = pytz.timezone(time_zone_name)
|
|
51
52
|
|
|
53
|
+
|
|
52
54
|
# @annotation
|
|
53
55
|
def cron_task(cron_expr: str, task_name: str = None, till_time_stamp: int = None):
|
|
54
56
|
"""
|
|
@@ -92,8 +94,9 @@ def reg_cron_task(cron_expr, func, params, timer_task_name=None, till_time_stamp
|
|
|
92
94
|
:return: the real decorator
|
|
93
95
|
"""
|
|
94
96
|
cron_expr = _convert_cron(cron_expr)
|
|
95
|
-
assert len(cron_expr.split(" ")) in (5, 6),
|
|
97
|
+
assert len(cron_expr.split(" ")) in (5, 6), "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
|
|
96
98
|
task_name = func.__name__ if timer_task_name is None else timer_task_name
|
|
99
|
+
|
|
97
100
|
@wraps(func)
|
|
98
101
|
def wrapper(*args, **kwargs):
|
|
99
102
|
try:
|
|
@@ -107,6 +110,7 @@ def reg_cron_task(cron_expr, func, params, timer_task_name=None, till_time_stamp
|
|
|
107
110
|
|
|
108
111
|
_register_next(task_name, wrapper, cron_expr, till_time_stamp, init=True)
|
|
109
112
|
|
|
113
|
+
|
|
110
114
|
def start_all(spawn: bool = True, daemon: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
|
|
111
115
|
"""
|
|
112
116
|
start_all starts all cron tasks registered before.
|
|
@@ -144,6 +148,7 @@ def active(timer_task_name):
|
|
|
144
148
|
if timer_task_name in scheduler_map:
|
|
145
149
|
scheduler_map.get(timer_task_name).status = True
|
|
146
150
|
|
|
151
|
+
|
|
147
152
|
def get_switch(timer_task_name):
|
|
148
153
|
switch = True
|
|
149
154
|
if timer_task_name in scheduler_map:
|
|
@@ -215,6 +220,7 @@ def _run_sched(scheduler_meta: SchedulerMeta):
|
|
|
215
220
|
return
|
|
216
221
|
time.sleep(0.5)
|
|
217
222
|
|
|
223
|
+
|
|
218
224
|
def _start(taskName: str = None):
|
|
219
225
|
global _switch
|
|
220
226
|
_switch = True
|
|
@@ -223,7 +229,7 @@ def _start(taskName: str = None):
|
|
|
223
229
|
for timer_task_name, scheduler_meta in scheduler_map.items():
|
|
224
230
|
if taskName is not None and timer_task_name != taskName: continue
|
|
225
231
|
print("register job: ", timer_task_name, ", cron: ", scheduler_meta.cron_str)
|
|
226
|
-
thread = threading.Thread(target=_run_sched, args=(scheduler_meta,
|
|
232
|
+
thread = threading.Thread(target=_run_sched, args=(scheduler_meta,), daemon=True)
|
|
227
233
|
thread.start()
|
|
228
234
|
taskList.append(thread)
|
|
229
235
|
for task in taskList: task.join()
|
|
@@ -231,6 +237,7 @@ def _start(taskName: str = None):
|
|
|
231
237
|
_switch = False
|
|
232
238
|
scheduler_map.clear()
|
|
233
239
|
|
|
240
|
+
|
|
234
241
|
def _convert_cron(cron_expr):
|
|
235
242
|
res_cron = ""
|
|
236
243
|
cron_list = cron_expr.split(" ")
|
|
@@ -242,4 +249,3 @@ def _convert_cron(cron_expr):
|
|
|
242
249
|
else:
|
|
243
250
|
res_cron = cron_expr
|
|
244
251
|
return res_cron
|
|
245
|
-
|
ctools/database/database.py
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
import contextlib
|
|
2
2
|
import datetime
|
|
3
|
-
|
|
4
3
|
import math
|
|
5
|
-
|
|
4
|
+
import threading
|
|
5
|
+
|
|
6
|
+
from sqlalchemy import create_engine, BigInteger, Column, event
|
|
6
7
|
from sqlalchemy.ext.declarative import declarative_base
|
|
7
8
|
from sqlalchemy.orm import sessionmaker, Session
|
|
8
9
|
from sqlalchemy.sql import text
|
|
9
10
|
|
|
10
11
|
from ctools import call
|
|
11
|
-
from ctools.pools.thread_pool import thread_local
|
|
12
12
|
from ctools import cid
|
|
13
|
+
from ctools.pools.thread_pool import thread_local
|
|
14
|
+
from ctools.web.bottle_web_base import PageInfo
|
|
13
15
|
|
|
14
16
|
"""
|
|
15
17
|
class XXXX(BaseMixin):
|
|
@@ -30,14 +32,17 @@ inited_db = {}
|
|
|
30
32
|
engines = {}
|
|
31
33
|
sessionMakers = {}
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
|
|
36
|
+
def getEngine(db_key: str = 'default'):
|
|
34
37
|
return engines[db_key]
|
|
35
38
|
|
|
39
|
+
|
|
36
40
|
@call.init
|
|
37
41
|
def _init():
|
|
38
42
|
global Base
|
|
39
43
|
Base = declarative_base()
|
|
40
44
|
|
|
45
|
+
|
|
41
46
|
"""
|
|
42
47
|
The string form of the URL is
|
|
43
48
|
dialect[+driver]://user:password@host/dbname[?key=value..]
|
|
@@ -45,6 +50,7 @@ where ``dialect`` is a database name such as ``mysql``, ``oracle``, ``postgresql
|
|
|
45
50
|
and ``driver`` the name of a DBAPI such as ``psycopg2``, ``pyodbc``, ``cx_oracle``, etc. Alternatively
|
|
46
51
|
"""
|
|
47
52
|
|
|
53
|
+
|
|
48
54
|
# 密码里的@ 要替换成 %40
|
|
49
55
|
|
|
50
56
|
# sqlite connect_args={"check_same_thread": False} db_url=sqlite:///{}.format(db_url)
|
|
@@ -53,41 +59,47 @@ and ``driver`` the name of a DBAPI such as ``psycopg2``, ``pyodbc``, ``cx_oracle
|
|
|
53
59
|
# > PRAGMA journal_mode=WAL; 设置事务的模式, wal 允许读写并发, 但是会额外创建俩文件
|
|
54
60
|
# > PRAGMA synchronous=NORMAL; 设置写盘策略, 默认是 FULL, 日志,数据都落, 设置成 NORMAL, 日志写完就算事务完成
|
|
55
61
|
|
|
56
|
-
def init_db(db_url: str, db_key: str='default', connect_args: dict={}, default_schema: str=None, pool_size: int=5, max_overflow: int=25, echo: bool=False):
|
|
62
|
+
def init_db(db_url: str, db_key: str = 'default', connect_args: dict = {}, default_schema: str = None, pool_size: int = 5, max_overflow: int = 25, echo: bool = False, auto_gen_table: bool = False):
|
|
57
63
|
if db_url.startswith('mysql'):
|
|
58
64
|
import pymysql
|
|
59
65
|
pymysql.install_as_MySQLdb()
|
|
60
66
|
if inited_db.get(db_key): raise Exception('db {} already init!!!'.format(db_key))
|
|
61
67
|
global engines, sessionMakers
|
|
68
|
+
if default_schema: connect_args.update({'options': '-csearch_path={}'.format(default_schema)})
|
|
62
69
|
engine, sessionMaker = _create_connection(db_url=db_url, connect_args=connect_args, pool_size=pool_size, max_overflow=max_overflow, echo=echo)
|
|
63
70
|
engines[db_key] = engine
|
|
64
71
|
sessionMakers[db_key] = sessionMaker
|
|
65
72
|
inited_db[db_key] = True
|
|
66
|
-
|
|
67
|
-
|
|
73
|
+
# 这个有并发问题, 高并发会导致卡顿, 可以考虑去做一些别的事儿
|
|
74
|
+
#if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: _set_search_path(dbapi_connection, default_schema))
|
|
75
|
+
if auto_gen_table: Base.metadata.create_all(engine)
|
|
76
|
+
|
|
68
77
|
|
|
69
78
|
def _set_search_path(dbapi_connection, default_schema):
|
|
70
79
|
with dbapi_connection.cursor() as cursor:
|
|
71
80
|
cursor.execute(f'SET search_path TO {default_schema}')
|
|
72
81
|
|
|
73
|
-
|
|
82
|
+
|
|
83
|
+
def _create_connection(db_url: str, pool_size: int = 5, max_overflow: int = 25, connect_args={}, echo: bool = False):
|
|
74
84
|
engine = create_engine('{}'.format(db_url),
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
sm = sessionmaker(bind=engine)
|
|
85
|
+
echo=echo,
|
|
86
|
+
future=True,
|
|
87
|
+
pool_size=pool_size,
|
|
88
|
+
max_overflow=max_overflow,
|
|
89
|
+
pool_pre_ping=True,
|
|
90
|
+
pool_recycle=3600,
|
|
91
|
+
connect_args=connect_args)
|
|
92
|
+
sm = sessionmaker(bind=engine, expire_on_commit=False)
|
|
83
93
|
return engine, sm
|
|
84
94
|
|
|
95
|
+
|
|
85
96
|
def generate_custom_id():
|
|
86
97
|
return str(cid.get_snowflake_id())
|
|
87
98
|
|
|
99
|
+
|
|
88
100
|
class BaseMixin(Base):
|
|
89
101
|
__abstract__ = True
|
|
90
|
-
obj_id = Column(
|
|
102
|
+
obj_id = Column(BigInteger, primary_key=True, default=generate_custom_id)
|
|
91
103
|
|
|
92
104
|
# ext1 = Column(String)
|
|
93
105
|
# ext2 = Column(String)
|
|
@@ -112,10 +124,16 @@ class BaseMixin(Base):
|
|
|
112
124
|
ret_state[key] = state[key]
|
|
113
125
|
return ret_state
|
|
114
126
|
|
|
127
|
+
@classmethod
|
|
128
|
+
def init(cls, data: dict):
|
|
129
|
+
valid_keys = cls.__table__.columns.keys()
|
|
130
|
+
filtered = {k: v for k, v in data.items() if k in valid_keys}
|
|
131
|
+
return cls(**filtered)
|
|
132
|
+
|
|
115
133
|
@contextlib.contextmanager
|
|
116
|
-
def get_session(db_key: str='default') -> Session:
|
|
134
|
+
def get_session(db_key: str = 'default') -> Session:
|
|
117
135
|
thread_local.db_key = db_key
|
|
118
|
-
if sm:=sessionMakers.get(db_key):
|
|
136
|
+
if sm := sessionMakers.get(db_key):
|
|
119
137
|
s = sm()
|
|
120
138
|
else:
|
|
121
139
|
raise ValueError("Invalid db_key: {}".format(db_key))
|
|
@@ -127,22 +145,32 @@ def get_session(db_key: str='default') -> Session:
|
|
|
127
145
|
finally:
|
|
128
146
|
s.close()
|
|
129
147
|
|
|
148
|
+
|
|
130
149
|
class PageInfoBuilder:
|
|
131
150
|
|
|
132
|
-
def __init__(self, pageInfo, total_count, records):
|
|
151
|
+
def __init__(self, pageInfo: PageInfo, total_count, records):
|
|
133
152
|
self.page_size = pageInfo.page_size
|
|
134
153
|
self.page_index = pageInfo.page_index
|
|
135
154
|
self.total_count = total_count
|
|
136
155
|
self.total_page = math.ceil(total_count / int(pageInfo.page_size))
|
|
137
156
|
self.records = records
|
|
138
157
|
|
|
139
|
-
|
|
158
|
+
|
|
159
|
+
def query_by_page(query, pageInfo) -> PageInfoBuilder:
|
|
160
|
+
"""
|
|
161
|
+
使用方法:
|
|
162
|
+
with database.get_session() as s:
|
|
163
|
+
query = s.query(AppInfoEntity).filter(AppInfoEntity.app_name.contains(params.app_name))
|
|
164
|
+
result = database.query_by_page(query, params.page_info)
|
|
165
|
+
return R.ok(result)
|
|
166
|
+
"""
|
|
140
167
|
records = query.offset((pageInfo.page_index - 1) * pageInfo.page_size).limit(pageInfo.page_size).all()
|
|
141
168
|
rs = []
|
|
142
169
|
for r in records:
|
|
143
170
|
rs.append(r)
|
|
144
171
|
return PageInfoBuilder(pageInfo, query.count(), rs)
|
|
145
172
|
|
|
173
|
+
|
|
146
174
|
def query4_crd_sql(session, sql: str, params: dict) -> []:
|
|
147
175
|
records = session.execute(text(sql), params).fetchall()
|
|
148
176
|
rs = []
|
|
@@ -153,6 +181,7 @@ def query4_crd_sql(session, sql: str, params: dict) -> []:
|
|
|
153
181
|
rs.append(data)
|
|
154
182
|
return rs
|
|
155
183
|
|
|
184
|
+
|
|
156
185
|
sqlite_and_pg_page_sql = """
|
|
157
186
|
limit :limit offset :offset
|
|
158
187
|
"""
|
|
@@ -160,7 +189,8 @@ mysql_page_sql = """
|
|
|
160
189
|
limit :offset, :limit
|
|
161
190
|
"""
|
|
162
191
|
|
|
163
|
-
|
|
192
|
+
|
|
193
|
+
def query_by_page4_crd_sql(session, sql: str, params: dict, pageInfo: PageInfo) -> []:
|
|
164
194
|
db_name = engines[thread_local.db_key].name
|
|
165
195
|
if db_name == 'postgresql' or db_name == 'sqlite':
|
|
166
196
|
page_sql = sqlite_and_pg_page_sql
|
ctools/dict_wrapper.py
CHANGED
ctools/ex.py
CHANGED
|
@@ -2,6 +2,10 @@ import time
|
|
|
2
2
|
import traceback
|
|
3
3
|
from functools import wraps
|
|
4
4
|
|
|
5
|
+
"""
|
|
6
|
+
@exception_handler(fail_return=['解析错误'], print_exc=True)
|
|
7
|
+
"""
|
|
8
|
+
|
|
5
9
|
# annotation
|
|
6
10
|
def exception_handler(fail_return, retry_num=0, delay=3, catch_e=Exception, print_exc=False):
|
|
7
11
|
def decorator(func):
|
ctools/geo/coord_trans.py
CHANGED
|
@@ -6,122 +6,122 @@ pi = 3.1415926535897932384626 # π
|
|
|
6
6
|
a = 6378245.0 # 长半轴
|
|
7
7
|
ee = 0.00669342162296594323 # 偏心率平方
|
|
8
8
|
|
|
9
|
+
|
|
9
10
|
def gcj02_to_bd09(lng, lat):
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
11
|
+
"""
|
|
12
|
+
火星坐标系(GCJ-02)转百度坐标系(BD-09)
|
|
13
|
+
谷歌、高德——>百度
|
|
14
|
+
:param lng:火星坐标经度
|
|
15
|
+
:param lat:火星坐标纬度
|
|
16
|
+
:return:
|
|
17
|
+
"""
|
|
18
|
+
z = math.sqrt(lng * lng + lat * lat) + 0.00002 * math.sin(lat * x_pi)
|
|
19
|
+
theta = math.atan2(lat, lng) + 0.000003 * math.cos(lng * x_pi)
|
|
20
|
+
bd_lng = z * math.cos(theta) + 0.0065
|
|
21
|
+
bd_lat = z * math.sin(theta) + 0.006
|
|
22
|
+
return [bd_lng, bd_lat]
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
def bd09_to_gcj02(bd_lon, bd_lat):
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
26
|
+
"""
|
|
27
|
+
百度坐标系(BD-09)转火星坐标系(GCJ-02)
|
|
28
|
+
百度——>谷歌、高德
|
|
29
|
+
:param bd_lat:百度坐标纬度
|
|
30
|
+
:param bd_lon:百度坐标经度
|
|
31
|
+
:return:转换后的坐标列表形式
|
|
32
|
+
"""
|
|
33
|
+
x = bd_lon - 0.0065
|
|
34
|
+
y = bd_lat - 0.006
|
|
35
|
+
z = math.sqrt(x * x + y * y) - 0.00002 * math.sin(y * x_pi)
|
|
36
|
+
theta = math.atan2(y, x) - 0.000003 * math.cos(x * x_pi)
|
|
37
|
+
gg_lng = z * math.cos(theta)
|
|
38
|
+
gg_lat = z * math.sin(theta)
|
|
39
|
+
return [gg_lng, gg_lat]
|
|
39
40
|
|
|
40
41
|
|
|
41
42
|
def wgs84_to_gcj02(lng, lat):
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
43
|
+
"""
|
|
44
|
+
WGS84转GCJ02(火星坐标系)
|
|
45
|
+
:param lng:WGS84坐标系的经度
|
|
46
|
+
:param lat:WGS84坐标系的纬度
|
|
47
|
+
:return:
|
|
48
|
+
"""
|
|
49
|
+
if out_of_china(lng, lat): # 判断是否在国内
|
|
50
|
+
return [lng, lat]
|
|
51
|
+
dlat = _transformlat(lng - 105.0, lat - 35.0)
|
|
52
|
+
dlng = _transformlng(lng - 105.0, lat - 35.0)
|
|
53
|
+
radlat = lat / 180.0 * pi
|
|
54
|
+
magic = math.sin(radlat)
|
|
55
|
+
magic = 1 - ee * magic * magic
|
|
56
|
+
sqrtmagic = math.sqrt(magic)
|
|
57
|
+
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
|
|
58
|
+
dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
|
|
59
|
+
mglat = lat + dlat
|
|
60
|
+
mglng = lng + dlng
|
|
61
|
+
return [mglng, mglat]
|
|
61
62
|
|
|
62
63
|
|
|
63
64
|
def gcj02_to_wgs84(lng, lat):
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
65
|
+
"""
|
|
66
|
+
GCJ02(火星坐标系)转GPS84
|
|
67
|
+
:param lng:火星坐标系的经度
|
|
68
|
+
:param lat:火星坐标系纬度
|
|
69
|
+
:return:
|
|
70
|
+
"""
|
|
71
|
+
if out_of_china(lng, lat):
|
|
72
|
+
return [lng, lat]
|
|
73
|
+
dlat = _transformlat(lng - 105.0, lat - 35.0)
|
|
74
|
+
dlng = _transformlng(lng - 105.0, lat - 35.0)
|
|
75
|
+
radlat = lat / 180.0 * pi
|
|
76
|
+
magic = math.sin(radlat)
|
|
77
|
+
magic = 1 - ee * magic * magic
|
|
78
|
+
sqrtmagic = math.sqrt(magic)
|
|
79
|
+
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
|
|
80
|
+
dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
|
|
81
|
+
mglat = lat + dlat
|
|
82
|
+
mglng = lng + dlng
|
|
83
|
+
return [lng * 2 - mglng, lat * 2 - mglat]
|
|
83
84
|
|
|
84
85
|
|
|
85
86
|
def bd09_to_wgs84(bd_lon, bd_lat):
|
|
86
|
-
|
|
87
|
-
|
|
87
|
+
lon, lat = bd09_to_gcj02(bd_lon, bd_lat)
|
|
88
|
+
return gcj02_to_wgs84(lon, lat)
|
|
88
89
|
|
|
89
90
|
|
|
90
91
|
def wgs84_to_bd09(lon, lat):
|
|
91
|
-
|
|
92
|
-
|
|
92
|
+
lon, lat = wgs84_to_gcj02(lon, lat)
|
|
93
|
+
return gcj02_to_bd09(lon, lat)
|
|
93
94
|
|
|
94
95
|
|
|
95
96
|
def _transformlat(lng, lat):
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
97
|
+
ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
|
|
98
|
+
0.1 * lng * lat + 0.2 * math.sqrt(math.fabs(lng))
|
|
99
|
+
ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
|
|
100
|
+
math.sin(2.0 * lng * pi)) * 2.0 / 3.0
|
|
101
|
+
ret += (20.0 * math.sin(lat * pi) + 40.0 *
|
|
102
|
+
math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
|
|
103
|
+
ret += (160.0 * math.sin(lat / 12.0 * pi) + 320 *
|
|
104
|
+
math.sin(lat * pi / 30.0)) * 2.0 / 3.0
|
|
105
|
+
return ret
|
|
105
106
|
|
|
106
107
|
|
|
107
108
|
def _transformlng(lng, lat):
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
109
|
+
ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
|
|
110
|
+
0.1 * lng * lat + 0.1 * math.sqrt(math.fabs(lng))
|
|
111
|
+
ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
|
|
112
|
+
math.sin(2.0 * lng * pi)) * 2.0 / 3.0
|
|
113
|
+
ret += (20.0 * math.sin(lng * pi) + 40.0 *
|
|
114
|
+
math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
|
|
115
|
+
ret += (150.0 * math.sin(lng / 12.0 * pi) + 300.0 *
|
|
116
|
+
math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
|
|
117
|
+
return ret
|
|
117
118
|
|
|
118
119
|
|
|
119
120
|
def out_of_china(lng, lat):
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
121
|
+
"""
|
|
122
|
+
判断是否在国内,不在国内不做偏移
|
|
123
|
+
:param lng:
|
|
124
|
+
:param lat:
|
|
125
|
+
:return:
|
|
126
|
+
"""
|
|
127
|
+
return not (lng > 73.66 and lng < 135.05 and lat > 3.86 and lat < 53.55)
|
ctools/geo/douglas_rarefy.py
CHANGED
|
@@ -4,6 +4,7 @@ __author__ = 'haoyang'
|
|
|
4
4
|
__date__ = '2024/9/19 14:02'
|
|
5
5
|
|
|
6
6
|
import math
|
|
7
|
+
|
|
7
8
|
from jsonpath_ng import parser
|
|
8
9
|
|
|
9
10
|
from ctools import cjson
|
|
@@ -13,6 +14,7 @@ from ctools.sys_log import flog as log
|
|
|
13
14
|
douglas_rarefy.DouglasRarefy(res, level=3).sparse_points()
|
|
14
15
|
"""
|
|
15
16
|
|
|
17
|
+
|
|
16
18
|
class THIN_LEVEL:
|
|
17
19
|
L1 = 0.00001
|
|
18
20
|
L2 = 0.00003
|
|
@@ -24,12 +26,14 @@ class THIN_LEVEL:
|
|
|
24
26
|
L8 = 0.0017
|
|
25
27
|
L9 = 0.0022
|
|
26
28
|
|
|
29
|
+
|
|
27
30
|
class Point:
|
|
28
31
|
def __init__(self, lng, lat, origin_data):
|
|
29
32
|
self.lng = lng
|
|
30
33
|
self.lat = lat
|
|
31
34
|
self.origin_data = origin_data
|
|
32
35
|
|
|
36
|
+
|
|
33
37
|
def _get_line_by_point(xy1, xy2):
|
|
34
38
|
"""
|
|
35
39
|
根据两个点求直线方程 ax + by + c = 0
|
|
@@ -68,7 +72,8 @@ class DouglasRarefy:
|
|
|
68
72
|
level default is L2, this level can be hold most of the points detail
|
|
69
73
|
ret_tpl is the result tpl, can be arrays and json or some can be json loads, exp: [{lng}, {lat}] OR {{"lng": {lng}, "lat": {lat}}}
|
|
70
74
|
"""
|
|
71
|
-
|
|
75
|
+
|
|
76
|
+
def __init__(self, points: [], level=THIN_LEVEL.L2, ret_tpl=None, get_lng=None, get_lat=None):
|
|
72
77
|
if not isinstance(points, list): raise Exception('points must be list obj !!')
|
|
73
78
|
if len(points) < 3: raise Exception('points length must be gt 2 !!')
|
|
74
79
|
self.points = points
|
|
@@ -105,11 +110,11 @@ class DouglasRarefy:
|
|
|
105
110
|
else:
|
|
106
111
|
return [cjson.loads(self.ret_tpl.format(lng=points[0].lng, lat=points[0].lat)), cjson.loads(self.ret_tpl.format(lng=points[-1].lng, lat=points[-1].lat))]
|
|
107
112
|
|
|
108
|
-
xy_first = points[0]
|
|
109
|
-
xy_end
|
|
110
|
-
a, b, c
|
|
111
|
-
d_max = 0
|
|
112
|
-
split = 0
|
|
113
|
+
xy_first = points[0] # 第一个点
|
|
114
|
+
xy_end = points[-1] # 最后一个点
|
|
115
|
+
a, b, c = _get_line_by_point(xy_first, xy_end) # 获取直线方程的 a, b, c 值
|
|
116
|
+
d_max = 0 # 记录点到直线的最大距离
|
|
117
|
+
split = 0 # 分割位置
|
|
113
118
|
for i in range(1, len(points) - 1):
|
|
114
119
|
d = _get_distance_from_point_to_line(a, b, c, points[i])
|
|
115
120
|
if d > d_max:
|
|
@@ -117,8 +122,8 @@ class DouglasRarefy:
|
|
|
117
122
|
d_max = d
|
|
118
123
|
if d_max > self.threshold:
|
|
119
124
|
# 如果存在点到首位点连成直线的距离大于 max_distance 的, 即需要再次划分
|
|
120
|
-
child_left
|
|
121
|
-
child_right = self._sparse_points(points[split:])
|
|
125
|
+
child_left = self._sparse_points(points[:split + 1]) # 递归处理左边部分
|
|
126
|
+
child_right = self._sparse_points(points[split:]) # 递归处理右边部分
|
|
122
127
|
# 合并结果,避免重复
|
|
123
128
|
return child_left + child_right[1:]
|
|
124
129
|
else:
|
|
@@ -132,4 +137,3 @@ class DouglasRarefy:
|
|
|
132
137
|
|
|
133
138
|
def sparse_points(self):
|
|
134
139
|
return self._sparse_points(self.data)
|
|
135
|
-
|
ctools/metrics.py
CHANGED
|
@@ -14,6 +14,7 @@ _metrics_initial = {}
|
|
|
14
14
|
_lock = threading.Lock()
|
|
15
15
|
_metrics_persistent_path = os.path.join(path_info.get_user_work_path('metrics', True), 'persistent.json')
|
|
16
16
|
|
|
17
|
+
|
|
17
18
|
# 认证中间件
|
|
18
19
|
# @app.before_request
|
|
19
20
|
# def check_authentication():
|
|
@@ -29,6 +30,7 @@ class MetricType(Enum):
|
|
|
29
30
|
SUMMARY = 'summary'
|
|
30
31
|
HISTOGRAM = 'histogram'
|
|
31
32
|
|
|
33
|
+
|
|
32
34
|
class Metric:
|
|
33
35
|
def __init__(self, metric_type: MetricType, metric_key: str, metric_labels: [], persistent: bool = False, buckets: [] = None, reset: bool = False, desc: str = ""):
|
|
34
36
|
self.metric_type = metric_type
|
|
@@ -51,6 +53,7 @@ class Metric:
|
|
|
51
53
|
raise Exception('metric type not found')
|
|
52
54
|
_metrics_initial[metric_key] = self
|
|
53
55
|
|
|
56
|
+
|
|
54
57
|
@call.once
|
|
55
58
|
def init(reset_persistent: bool = False):
|
|
56
59
|
if os.path.exists(_metrics_persistent_path) and not reset_persistent:
|
|
@@ -72,6 +75,7 @@ def init(reset_persistent: bool = False):
|
|
|
72
75
|
persistent_metrics()
|
|
73
76
|
start_http_server(port=_metrics_port)
|
|
74
77
|
|
|
78
|
+
|
|
75
79
|
@call.schd(5, start_by_call=True)
|
|
76
80
|
def persistent_metrics():
|
|
77
81
|
if _persistent_json and not _lock.locked():
|
|
@@ -80,6 +84,7 @@ def persistent_metrics():
|
|
|
80
84
|
persistent_file.write(cjson.dumps(_persistent_json))
|
|
81
85
|
persistent_file.flush()
|
|
82
86
|
|
|
87
|
+
|
|
83
88
|
def opt(metric_key: str, label_values: [], metric_value: int):
|
|
84
89
|
_lock.acquire(timeout=5)
|
|
85
90
|
try:
|
|
@@ -119,6 +124,7 @@ def opt(metric_key: str, label_values: [], metric_value: int):
|
|
|
119
124
|
log.error("添加指标信息异常: %s" % e)
|
|
120
125
|
_lock.release()
|
|
121
126
|
|
|
127
|
+
|
|
122
128
|
def _init_all_metrics():
|
|
123
129
|
Metric(MetricType.GAUGE, 'gomyck', ['g_label1', 'g_label2'], persistent=True, reset=True)
|
|
124
130
|
|