gomyck-tools 1.0.0__py3-none-any.whl → 1.4.7__py3-none-any.whl

ctools/{sm_tools.py → cipher/sm_util.py}

@@ -1,24 +1,40 @@
 import base64
 
-from gmssl import sm2, func
+from gmssl import sm2
 from gmssl.sm4 import CryptSM4, SM4_ENCRYPT, SM4_DECRYPT
 
-sm2_crypt: sm2.CryptSM2
+sm2_crypt: sm2.CryptSM2 = None
+
+
+def init(private_key: str, public_key: str):
+  global sm2_crypt
+  if sm2_crypt is not None:
+    print('sm2 is already init!!!')
+    return
+  sm2_crypt = sm2.CryptSM2(private_key=private_key, public_key=public_key, asn1=True, mode=1)
 
 
 def sign_with_sm2(sign_data: str) -> str:
-  return sm2_crypt.sign(sign_data.encode('UTF-8'), func.random_hex(sm2_crypt.para_len))
+  if sm2_crypt is None: raise Exception('sm2 is not init!!!')
+  return sm2_crypt.sign_with_sm3(sign_data.encode('UTF-8'))
 
 
 def verify_with_sm2(sign_val: str, sign_data: str) -> bool:
-  return sm2_crypt.verify(sign_val, sign_data.encode('UTF-8'))
+  if sm2_crypt is None: raise Exception('sm2 is not init!!!')
+  try:
+    return sm2_crypt.verify_with_sm3(sign_val, sign_data.encode('UTF-8'))
+  except Exception as e:
+    print('签名验证失败: {}'.format(e))
+    return False
 
 
 def encrypt_with_sm2(encrypt_data: str) -> str:
+  if sm2_crypt is None: raise Exception('sm2 is not init!!!')
   return base64.b64encode(sm2_crypt.encrypt(encrypt_data.encode('UTF-8'))).decode('UTF-8')
 
 
 def decrypt_with_sm2(encrypt_data: str) -> str:
+  if sm2_crypt is None: raise Exception('sm2 is not init!!!')
   return sm2_crypt.decrypt(base64.b64decode(encrypt_data.encode('UTF-8'))).decode('UTF-8')
 
 

ctools/cjson.py

@@ -1,35 +1,35 @@
 import jsonpickle
 
+# 需要转换成str的属性
+str_value_keys = []
 jsonpickle.set_preferred_backend('json')
 jsonpickle.set_encoder_options('json', ensure_ascii=False)
-jsonpickle.set_decoder_options('json', encoding='utf-8')
+jsonpickle.set_decoder_options('json')
 
 
-def dumps(obj) -> str:
+def dumps(obj, **kwargs) -> str:
   """
   将对象转换为json字符串
   :param obj: 对象
   :return: json 字符串
   """
+  # indent = 2 可以美化输出
   if obj is None: return None
-  return f'{jsonpickle.encode(obj, unpicklable=False)}'
+  if type(obj) == str: return obj
+  return f'{jsonpickle.encode(obj, unpicklable=False, make_refs=False, **kwargs)}'
 
 
-def loads(json_str: str) -> dict:
+def loads(json_str: str, **kwargs) -> dict:
   """
   将json字符串转换为对象
   :param json_str: json 字符串
   :return: 对象
   """
-  return jsonpickle.decode(json_str)
-
-
-# 需要转换成str的属性
-str_value_keys = []
+  return jsonpickle.decode(json_str, **kwargs)
 
 
 def unify_to_str(json_str: str) -> str:
-  if not str_value_keys: return json_str
+  if not str_value_keys and len(str_value_keys) == 0: return json_str
   obj = loads(json_str)
   if isinstance(obj, list):
     _handle_list(obj)

ctools/cron_lite.py

@@ -11,6 +11,24 @@ from typing import Optional, Dict
 import pytz
 from croniter import croniter
 
+"""
+@cron_lite.cron_task('0/1 * * * * ? *')
+def demo():
+  print('hello world')
+
+@cron_lite.cron_task('0/1 * * * * ? *')
+def demo1():
+  print('hello world111')
+
+def demo2(xx, fff):
+  print('hello world222', xx, fff)
+
+cron_lite.reg_cron_task('0/1 * * * * ? *', demo2, (123123123, 34534534))
+print(123123)
+
+cron_lite.start_all()
+"""
+
 
 class SchedulerMeta:
   timer_task_name: str = None
@@ -18,13 +36,14 @@ class SchedulerMeta:
   status: bool = False
   event: sched.Event = None
   scheduler: sched.scheduler = None
+  cron_str: str = None
 
 
 scheduler_map: Dict[str, SchedulerMeta] = {}  # {timer_task_name: SchedulerMeta}
 _switch = False
-_error_handler = print
 _info_handler = print
-_time_zone: Optional[pytz.BaseTzInfo] = None
+_error_handler = print
+_time_zone: Optional[pytz.BaseTzInfo] = pytz.timezone("Asia/Shanghai")
 
 
 def set_time_zone(time_zone_name: str):
@@ -32,74 +51,8 @@ def set_time_zone(time_zone_name: str):
   _time_zone = pytz.timezone(time_zone_name)
 
 
-def _register_next(timer_task_name, base_func, cron_expr, till_time_stamp):
-  cron_obj = croniter(cron_expr)
-  if _time_zone:
-    cron_obj.set_current(datetime.now(tz=_time_zone))
-  next_time = int(cron_obj.get_next())
-  if scheduler_map.get(timer_task_name) is None:
-    scheduler_meta = SchedulerMeta()
-    scheduler_meta.timer_task_name = timer_task_name
-    scheduler_meta.switch = True
-    scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
-    scheduler_map[timer_task_name] = scheduler_meta
-  if till_time_stamp is None or next_time <= till_time_stamp:
-    scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
-
-
-def _run_sched(scheduler_meta: SchedulerMeta):
-  active(scheduler_meta.timer_task_name)
-  while True:
-    scheduler = scheduler_meta.scheduler
-    if not _switch or not scheduler_meta.switch:
-      scheduler.empty()
-      inactive(scheduler_meta.timer_task_name)
-      return
-    t = scheduler.run(False)
-    if t is None:
-      inactive(scheduler_meta.timer_task_name)
-      return
-    st = time.time()
-    while time.time() - st < t:
-      if not _switch or not scheduler_meta.switch:
-        scheduler.empty()
-        inactive(scheduler_meta.timer_task_name)
-        return
-      time.sleep(0.5)
-
-
-def _start():
-  global _switch
-  _info_handler("cron started")
-  tl = []
-  for timer_task_name, scheduler_meta in scheduler_map.items():
-    print("Registering Job:", timer_task_name)
-    t = threading.Thread(target=_run_sched, args=(scheduler_meta,), daemon=True)
-    # 有些task非常耗时，会影响退出。目前设计改为退出时不保证task完成
-    t.start()
-    tl.append(t)
-
-  for t in tl:
-    t.join()
-  _info_handler("cron finished")
-  _switch = False  # ensure close when there are no more tasks with switch open
-  scheduler_map.clear()
-
-
-def convert_cron(cron_expr):
-  res_cron = ""
-  cron_list = cron_expr.split(" ")
-  if len(cron_list) > 6:
-    for cron in cron_list[1:]:
-      if cron != "?":
-        res_cron += "%s " % cron
-    res_cron += "%s" % cron_list[0]
-  else:
-    res_cron = cron_expr
-  return res_cron
-
-
-def cron_task(cron_expr: str, till_time_stamp: int = None):
+# @annotation
+def cron_task(cron_expr: str, task_name: str = None, till_time_stamp: int = None):
   """
   cron_task decorator to register a function as crontab task
   :param cron_expr: the croniter accepted cron_expression. NOTICE: the default timezone is UTC and can be changed by
@@ -107,7 +60,7 @@ def cron_task(cron_expr: str, till_time_stamp: int = None):
   :param till_time_stamp: run this jog till when. None means forever
   :return: the real decorator
   """
-  cron_expr = convert_cron(cron_expr)
+  cron_expr = _convert_cron(cron_expr)
   assert len(cron_expr.split(" ")) in (5, 6), \
     "only supported <min hour day month weekday> and <min hour day month weekday sec>"
 
@@ -121,15 +74,15 @@ def cron_task(cron_expr: str, till_time_stamp: int = None):
           _error_handler(f"run {func.__name__} failed\n" + traceback.format_exc())
         except Exception:
           _error_handler(f"run {func.__name__} failed\n")
-      _register_next(inner, inner, cron_expr, till_time_stamp)
+      _register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp)
 
-    _register_next(inner, inner, cron_expr, till_time_stamp)
+    _register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp, init=True)
     return inner
 
   return deco
 
 
-def apply_cron_task(timer_task_name, func, params, cron_expr, till_time_stamp=None):
+def reg_cron_task(cron_expr, func, params, timer_task_name=None, till_time_stamp=None):
   """
   cron_task decorator to register a function as crontab task
   :param func: task callback function
@@ -140,37 +93,29 @@ def apply_cron_task(timer_task_name, func, params, cron_expr, till_time_stamp=No
   :param till_time_stamp: run this jog till when. None means forever
   :return: the real decorator
   """
-  cron_expr = convert_cron(cron_expr)
-  assert len(cron_expr.split(" ")) in (5, 6), \
-    "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
+  cron_expr = _convert_cron(cron_expr)
+  assert len(cron_expr.split(" ")) in (5, 6), "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
+  task_name = func.__name__ if timer_task_name is None else timer_task_name
 
   @wraps(func)
   def wrapper(*args, **kwargs):
     try:
-      func(timer_task_name, params, *args, **kwargs)
+      nonlocal params
+      func.__taskName__ = task_name
+      func(*params, *args, **kwargs)
     except Exception as exc:
       _error_handler(f"Run {func.__name__} failed with error: {str(exc)}")
     finally:
-      _register_next(timer_task_name, wrapper, cron_expr, till_time_stamp)
+      _register_next(task_name, wrapper, cron_expr, till_time_stamp)
 
-  _register_next(timer_task_name, wrapper, cron_expr, till_time_stamp)
+  _register_next(task_name, wrapper, cron_expr, till_time_stamp, init=True)
 
-  global _switch
-  _switch = True
-
-  scheduler = scheduler_map.get(timer_task_name)
-  if scheduler:
-    scheduler.switch = True
-    t = threading.Thread(target=_run_sched, name=timer_task_name, args=(scheduler,), daemon=True)
-    # 有些task非常耗时，会影响退出。目前设计改为退出时不保证task完成
-    t.start()
-  return wrapper
 
-
-def start_all(spawn: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
+def start_all(spawn: bool = True, daemon: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
   """
   start_all starts all cron tasks registered before.
   :param spawn: whether to start a new thread for scheduler. If not, the action will block the current thread
+  :param daemon: the new thread is daemon if True
   :param info_handler: handle info output (scheduler start / stop), default = print, can use logging.info
   :param error_handler: handle error output (task execute exception), default = print, can use logging.error
   :raise RuntimeError: if the tasks are already started and still running we cannot start again. The feature is not
@@ -179,16 +124,13 @@ def start_all(spawn: bool = True, info_handler=None, error_handler=None) -> Opti
   """
   global _switch, _info_handler, _error_handler
   if _switch:
-    raise RuntimeError("the crontab was already started")
+    raise RuntimeError("the crontab was already started...")
   if info_handler:
     _info_handler = info_handler
   if error_handler:
     _error_handler = error_handler
-
-  _switch = True
   if spawn:
-    t = threading.Thread(target=_start)
-    t.setDaemon(True)
+    t = threading.Thread(target=_start, daemon=daemon)
     t.start()
     return t
   else:
@@ -206,6 +148,7 @@ def active(timer_task_name):
   if timer_task_name in scheduler_map:
     scheduler_map.get(timer_task_name).status = True
 
+
 def get_switch(timer_task_name):
   switch = True
   if timer_task_name in scheduler_map:
@@ -237,3 +180,72 @@ def stop_all(wait_thread: Optional[threading.Thread] = None):
     scheduler_map.get(timer_task_name).switch = False
   if wait_thread:
     wait_thread.join()
+
+
+def _register_next(timer_task_name, base_func, cron_expr, till_time_stamp, init: bool = False):
+  cron_obj = croniter(cron_expr)
+  if _time_zone:
+    cron_obj.set_current(datetime.now(tz=_time_zone))
+  next_time = int(cron_obj.get_next())
+  if scheduler_map.get(timer_task_name) is None:
+    scheduler_meta = SchedulerMeta()
+    scheduler_meta.timer_task_name = timer_task_name
+    scheduler_meta.switch = True
+    scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
+    scheduler_meta.cron_str = cron_expr
+    scheduler_map[timer_task_name] = scheduler_meta
+  elif init:
+    raise ValueError(f"task name: {timer_task_name} already exists!!!!!")
+  if till_time_stamp is None or next_time <= till_time_stamp:
+    scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
+
+
+def _run_sched(scheduler_meta: SchedulerMeta):
+  active(scheduler_meta.timer_task_name)
+  while True:
+    scheduler = scheduler_meta.scheduler
+    if not _switch or not scheduler_meta.switch:
+      scheduler.empty()
+      inactive(scheduler_meta.timer_task_name)
+      return
+    t = scheduler.run(False)
+    if t is None:
+      inactive(scheduler_meta.timer_task_name)
+      return
+    st = time.time()
+    while time.time() - st < t:
+      if not _switch or not scheduler_meta.switch:
+        scheduler.empty()
+        inactive(scheduler_meta.timer_task_name)
+        return
+      time.sleep(0.5)
+
+
+def _start(taskName: str = None):
+  global _switch
+  _switch = True
+  _info_handler("cron job begin start...")
+  taskList = []
+  for timer_task_name, scheduler_meta in scheduler_map.items():
+    if taskName is not None and timer_task_name != taskName: continue
+    print("register job: ", timer_task_name, ", cron: ", scheduler_meta.cron_str)
+    thread = threading.Thread(target=_run_sched, args=(scheduler_meta,), daemon=True)
+    thread.start()
+    taskList.append(thread)
+  for task in taskList: task.join()
+  _info_handler("cron job execute finished...")
+  _switch = False
+  scheduler_map.clear()
+
+
+def _convert_cron(cron_expr):
+  res_cron = ""
+  cron_list = cron_expr.split(" ")
+  if len(cron_list) > 6:
+    for cron in cron_list[1:]:
+      if cron != "?":
+        res_cron += "%s " % cron
+    res_cron += "%s" % cron_list[0]
+  else:
+    res_cron = cron_expr
+  return res_cron

ctools/database/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:22'

ctools/{database.py → database/database.py}

@@ -1,57 +1,106 @@
 import contextlib
 import datetime
 import math
-from sqlalchemy import create_engine, Integer, Column, event
+import threading
+
+from sqlalchemy import create_engine, BigInteger, Column, event
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, Session
 from sqlalchemy.sql import text
-from ctools import call, string_tools
-from ctools.thread_pool import thread_local
+
+from ctools import call
+from ctools import cid
+from ctools.pools.thread_pool import thread_local
+from ctools.web.bottle_web_base import PageInfo
+
+"""
+class XXXX(BaseMixin):
+  __tablename__ = 't_xxx_info'
+  __table_args__ = {'comment': 'xxx信息表'}
+  server_content: Column = Column(String(50), nullable=True, default='', comment='123123')
+  server_ip: Column = Column(String(30), index=True)
+  user_id: Column = Column(BigInteger)
+
+database.init_db('postgresql://postgres:123456@192.168.3.107:32566/abc', default_schema='public', db_key='source', pool_size=100)
+with database.get_session('source') as s:
+  s.execute(text('insert into xxx (name) values (:name)'), {'name': string_tools.get_random_str(5)})
+  s.commit()
+"""
 
 Base = None
 inited_db = {}
 engines = {}
 sessionMakers = {}
 
-def getEngine(db_key: str='default'):
+
+def getEngine(db_key: str = 'default'):
   return engines[db_key]
 
+
 @call.init
 def _init():
   global Base
   Base = declarative_base()
 
-def set_search_path(dbapi_connection, default_schema):
-  with dbapi_connection.cursor() as cursor:
-    cursor.execute(f'SET search_path TO {default_schema}')
+
+"""
+The string form of the URL is
+dialect[+driver]://user:password@host/dbname[?key=value..]
+where ``dialect`` is a database name such as ``mysql``, ``oracle``, ``postgresql``, etc.
+and ``driver`` the name of a DBAPI such as ``psycopg2``, ``pyodbc``, ``cx_oracle``, etc.  Alternatively
+"""
+
 
 # 密码里的@  要替换成 %40
-def init_db(db_url: str, db_key: str='default', connect_args: dict={}, default_schema: str=None, pool_size: int=5, max_overflow: int=25, echo: bool=False):
+
+# sqlite connect_args={"check_same_thread": False}  db_url=sqlite:///{}.format(db_url)
+# sqlite 数据库, 初始化之后, 优化一下配置
+# $ sqlite3 app.db
+# > PRAGMA journal_mode=WAL; 设置事务的模式, wal 允许读写并发, 但是会额外创建俩文件
+# > PRAGMA synchronous=NORMAL; 设置写盘策略, 默认是 FULL, 日志,数据都落, 设置成 NORMAL, 日志写完就算事务完成
+
+def init_db(db_url: str, db_key: str = 'default', connect_args: dict = {}, default_schema: str = None, pool_size: int = 5, max_overflow: int = 25, echo: bool = False, auto_gen_table: bool = False):
+  if db_url.startswith('mysql'):
+    import pymysql
+    pymysql.install_as_MySQLdb()
   if inited_db.get(db_key): raise Exception('db {} already init!!!'.format(db_key))
   global engines, sessionMakers
+  if default_schema: connect_args.update({'options': '-csearch_path={}'.format(default_schema)})
   engine, sessionMaker = _create_connection(db_url=db_url, connect_args=connect_args, pool_size=pool_size, max_overflow=max_overflow, echo=echo)
   engines[db_key] = engine
   sessionMakers[db_key] = sessionMaker
   inited_db[db_key] = True
-  if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: set_search_path(dbapi_connection, default_schema))
+  # 这个有并发问题, 高并发会导致卡顿, 可以考虑去做一些别的事儿
+  #if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: _set_search_path(dbapi_connection, default_schema))
+  if auto_gen_table: Base.metadata.create_all(engine)
 
-def _create_connection(db_url: str, pool_size: int=5, max_overflow: int=25, connect_args={}, echo: bool=False):
+
+def _set_search_path(dbapi_connection, default_schema):
+  with dbapi_connection.cursor() as cursor:
+    cursor.execute(f'SET search_path TO {default_schema}')
+
+
+def _create_connection(db_url: str, pool_size: int = 5, max_overflow: int = 25, connect_args={}, echo: bool = False):
   engine = create_engine('{}'.format(db_url),
-                               echo=echo,
-                               future=True,
-                               pool_size=pool_size,
-                               max_overflow=max_overflow,
-                               pool_pre_ping=True,
-                               pool_recycle=3600,
-                               connect_args=connect_args)
-  sm = sessionmaker(bind=engine)
+                         echo=echo,
+                         future=True,
+                         pool_size=pool_size,
+                         max_overflow=max_overflow,
+                         pool_pre_ping=True,
+                         pool_recycle=3600,
+                         connect_args=connect_args)
+  sm = sessionmaker(bind=engine, expire_on_commit=False)
   return engine, sm
 
+
 def generate_custom_id():
-  return str(string_tools.get_snowflake_id())
+  return str(cid.get_snowflake_id())
+
+
+class BaseMixin(Base):
+  __abstract__ = True
+  obj_id = Column(BigInteger, primary_key=True, default=generate_custom_id)
 
-class BaseMixin:
-  obj_id = Column(Integer, primary_key=True, default=generate_custom_id)
   # ext1 = Column(String)
   # ext2 = Column(String)
   # ext3 = Column(String)
@@ -75,10 +124,16 @@ class BaseMixin:
           ret_state[key] = state[key]
     return ret_state
 
+  @classmethod
+  def init(cls, data: dict):
+    valid_keys = cls.__table__.columns.keys()
+    filtered = {k: v for k, v in data.items() if k in valid_keys}
+    return cls(**filtered)
+
 @contextlib.contextmanager
-def get_session(db_key: str='default') -> Session:
+def get_session(db_key: str = 'default') -> Session:
   thread_local.db_key = db_key
-  if sm:=sessionMakers.get(db_key):
+  if sm := sessionMakers.get(db_key):
     s = sm()
   else:
     raise ValueError("Invalid db_key: {}".format(db_key))
@@ -90,22 +145,32 @@ def get_session(db_key: str='default') -> Session:
   finally:
     s.close()
 
+
 class PageInfoBuilder:
 
-  def __init__(self, pageInfo, total_count, records):
+  def __init__(self, pageInfo: PageInfo, total_count, records):
     self.page_size = pageInfo.page_size
     self.page_index = pageInfo.page_index
     self.total_count = total_count
     self.total_page = math.ceil(total_count / int(pageInfo.page_size))
     self.records = records
 
-def query_by_page(query, pageInfo):
+
+def query_by_page(query, pageInfo) -> PageInfoBuilder:
+  """
+  使用方法:
+    with database.get_session() as s:
+      query = s.query(AppInfoEntity).filter(AppInfoEntity.app_name.contains(params.app_name))
+      result = database.query_by_page(query, params.page_info)
+      return R.ok(result)
+  """
   records = query.offset((pageInfo.page_index - 1) * pageInfo.page_size).limit(pageInfo.page_size).all()
   rs = []
   for r in records:
     rs.append(r)
   return PageInfoBuilder(pageInfo, query.count(), rs)
 
+
 def query4_crd_sql(session, sql: str, params: dict) -> []:
   records = session.execute(text(sql), params).fetchall()
   rs = []
@@ -116,6 +181,7 @@ def query4_crd_sql(session, sql: str, params: dict) -> []:
     rs.append(data)
   return rs
 
+
 sqlite_and_pg_page_sql = """
  limit :limit offset :offset
 """
@@ -123,7 +189,8 @@ mysql_page_sql = """
  limit :offset, :limit
 """
 
-def query_by_page4_crd_sql(session, sql: str, params: dict, pageInfo) -> []:
+
+def query_by_page4_crd_sql(session, sql: str, params: dict, pageInfo: PageInfo) -> []:
   db_name = engines[thread_local.db_key].name
   if db_name == 'postgresql' or db_name == 'sqlite':
     page_sql = sqlite_and_pg_page_sql

ctools/dict_wrapper.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/10/25 09:42'
+
+
+class DictWrapper(dict):
+
+  def __getattr__(self, key):
+    res = self.get(key)
+    if res is None:
+      raise AttributeError(f" ==>> {key} <<== Not Found In This Entity!!!")
+    if isinstance(res, dict):
+      return DictWrapper(res)
+    return res
+
+  def __setattr__(self, key, value):
+    self[key] = value
+
+  def __delattr__(self, key):
+    del self[key]

ctools/ex.py

@@ -2,7 +2,11 @@ import time
 import traceback
 from functools import wraps
 
+"""
+@exception_handler(fail_return=['解析错误'], print_exc=True)
+"""
 
+# annotation
 def exception_handler(fail_return, retry_num=0, delay=3, catch_e=Exception, print_exc=False):
   def decorator(func):
     @wraps(func)

ctools/geo/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:20'