gomyck-tools 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

ctools/util/cklock.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/18 15:46'
+
+import contextvars
+import threading
+from contextlib import contextmanager
+from functools import wraps
+
+from ctools.stream.credis import get_redis, add_lock, remove_lock
+from ctools.web import ctoken
+from ctools.web.api_result import R
+
+# 全局锁容器
+_lock_dict = {}
+_lock_dict_lock = threading.Lock()
+
+def try_acquire_lock(key: str) -> bool:
+  with _lock_dict_lock:
+    if key not in _lock_dict:
+      _lock_dict[key] = threading.Lock()
+    return _lock_dict[key].acquire(blocking=False)
+
+def try_acquire_lock_block(key: str):
+  with _lock_dict_lock:
+    if key not in _lock_dict:
+      _lock_dict[key] = threading.Lock()
+    _lock = _lock_dict[key]
+  _lock.acquire()  # 这里是阻塞的
+
+def release_lock(key: str):
+  with _lock_dict_lock:
+    _lock = _lock_dict.get(key)
+    if _lock and _lock.locked():
+      _lock.release()
+    if _lock and not _lock.locked():
+      _lock_dict.pop(key, None)
+
+@contextmanager
+def try_lock(key: str="sys_lock", block=False):
+  if not block:
+    acquired = try_acquire_lock(key)
+    try:
+      yield acquired
+    finally:
+      if acquired:
+        release_lock(key)
+  else:
+    try_acquire_lock_block(key)
+    try:
+      yield
+    finally:
+      release_lock(key)
+
+#annotation
+"""
+@lock("params.attr")
+"""
+# 上下文保存锁key集合
+current_locks = contextvars.ContextVar("current_locks", default=set())
+
+def lock(lock_attrs=None):
+  def decorator(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+      lock_key = ""
+      nonlocal lock_attrs
+      user_level_lock = False
+
+      if not lock_attrs:
+        user_id = ctoken.get_user_id()
+        if user_id:
+          user_level_lock = True
+          lock_key = f"USER_ID_LOCK_{user_id}"
+        else:
+          raise ValueError("请设置 lock_attrs 或使用 token！")
+
+      if not user_level_lock:
+        if isinstance(lock_attrs, str): lock_attrs = [lock_attrs]
+        try:
+          for attr in lock_attrs:
+            parts = attr.split(".")
+            if len(parts) != 2:
+              raise ValueError(f"lock_attr: {attr} 格式错误")
+            obj = kwargs.get(parts[0]) or args[0]
+            if obj is None:
+              raise ValueError(f"参数 {parts[0]} 不存在")
+            lock_key += f"_{getattr(obj, parts[1], None)}"
+        except Exception as e:
+          raise ValueError(f"生成锁键失败: {e}")
+
+      lock_set = current_locks.get()
+      if lock_key in lock_set:
+        return func(*args, **kwargs)
+      token = current_locks.set(lock_set | {lock_key})
+
+      try:
+        if not get_redis():
+          with try_lock(lock_key) as locked:
+            if not locked:
+              return R.error("操作过于频繁, 请稍后再试")
+            return func(*args, **kwargs)
+        else:
+          locked = add_lock(get_redis(), lock_key)
+          try:
+            if locked:
+              return func(*args, **kwargs)
+            else:
+              return R.error("操作过于频繁, 请稍后再试")
+          finally:
+            if locked:
+              remove_lock(get_redis(), lock_key)
+      finally:
+        current_locks.reset(token)
+    return wrapper
+  return decorator
+

ctools/util/config_util.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/16 14:19'
+
+"""
+config = load_config("application.ini")
+print(config)
+print(config.base.app_name)
+print(config.base.version)
+"""
+from configparser import ConfigParser
+
+cache = {}
+
+class AttrNoneNamespace:
+  def __init__(self):
+    pass
+  def __setattr__(self, key, value):
+    super().__setattr__(key, value)
+  def __getattr__(self, item):
+    return None
+
+def _convert_value(value: str):
+  val = value.strip()
+  if val.lower() in ('true', 'yes', 'on'):
+    return True
+  if val.lower() in ('false', 'no', 'off'):
+    return False
+  if val.isdigit():
+    return int(val)
+  try:
+    return float(val)
+  except ValueError:
+    return val
+
+def _config_to_object(config: ConfigParser):
+  result = AttrNoneNamespace()
+  for section in config.sections():
+    section_obj = AttrNoneNamespace()
+    for key, value in config.items(section):
+      setattr(section_obj, key, _convert_value(value))
+    setattr(result, section, section_obj)
+  return result
+
+def load_config(file_path):
+  if file_path in cache: return cache[file_path]
+  config = ConfigParser()
+  config.read(file_path)
+  cf = _config_to_object(config)
+  cache[file_path] = cf
+  return cf

ctools/util/http_util.py

@@ -1,5 +1,6 @@
 import requests
 
+
 def get(url, params=None, headers=None):
   result = ""
   try:

ctools/util/image_process.py

@@ -1,3 +1,4 @@
+import base64
 from io import BytesIO
 
 from PIL import Image
@@ -6,6 +7,7 @@ from PIL import Image
 def get_size(image_path):
   return Image.open(image_path).size
 
+
 def change_color(image_path, area=None, rgb_color=None):
   """
   修改图片指定区域颜色
@@ -25,3 +27,9 @@ def change_color(image_path, area=None, rgb_color=None):
     img_binary = img_bytes.getvalue()
   return img_binary
 
+
+def img2b64(img: Image, fmt="PNG"):
+  buf = BytesIO()
+  img.save(buf, format=fmt.upper())
+  return base64.b64encode(buf.getvalue()).decode("utf-8")
+

ctools/util/jb_cut.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/15 13:08'
+
+import sys
+
+import jieba
+from jieba import posseg as pseg
+
+jqfx_exclude = ('ul', 'uj', 'uz', 'a', 'c', 'm', 'f', 'ad', 'an', 'r', 'q', 'u', 't', 'd', 'p', 'x')
+
+def add_dict(dic_word: str = None, dic_path: str = None):
+  """
+  添加自定义词库(自定义词库添加之后, 如果不适用全模式切词, 有时也不好使, 因为权重没有默认的高)
+  :param dic_word: 一个单词
+  :param dic_path: 单词表文件地址
+  单词表文件格式:
+  单词  词频  标签
+  单词1  3    i
+  单词2  3    i
+  单词3  3    i
+  """
+  if dic_word: jieba.add_word(dic_word)
+  if dic_path: jieba.load_userdict(dic_path)
+
+def add_freq(word: ()):
+  """
+  添加词频
+  :param word: 一个单词
+  """
+  jieba.suggest_freq(word, True)
+
+def get_declare_type_word(word: str, word_flag=(), use_paddle=False, exclude_flag="x"):
+  #import paddle
+  #paddle.enable_static()
+  #sys.stdout = sys.__stdout__
+  #sys.stderr = sys.__stderr__
+  """
+  获取声明类型的单词
+  :param word: 单词
+  :param word_flag: 标签
+  :param use_paddle: 是否使用 paddle
+  :param exclude_flag: 排除的标签
+  :return:  筛选之后的单词(数组)
+  """
+  # linux 可以开放下面的语句, 并发执行分词, windows 不支持
+  # if platform.system() == 'Linux': jieba.enable_parallel(4)
+  ret = []
+  for w, flag in pseg.cut(word, use_paddle=use_paddle):
+    if (flag in word_flag or len(word_flag) == 0) and flag not in exclude_flag:
+      ret.append((w, flag))
+  return ret

ctools/util/snow_id.py

@@ -16,10 +16,12 @@ SEQUENCE_MASK = -1 ^ (-1 << SEQUENCE_BITS)
 # Twitter元年时间戳
 TWEPOCH = 1288834974657
 
+
 class SnowId(object):
   """
   用于生成IDs
   """
+
   def __init__(self, datacenter_id=0, worker_id=0, sequence=0):
     """
     初始化
@@ -53,8 +55,7 @@ class SnowId(object):
     timestamp = self._gen_timestamp()
     # 时钟回拨
     if timestamp < self.last_timestamp:
-      print('clock is moving backwards. Rejecting requests until {}'.format(self.last_timestamp))
-      raise
+      timestamp = self._til_next_millis(self.last_timestamp)
     if timestamp == self.last_timestamp:
       self.sequence = (self.sequence + 1) & SEQUENCE_MASK
       if self.sequence == 0:

ctools/web/__init__.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 # -*- coding: UTF-8 -*-
-__author__='haoyang'
-__date__='2025/6/11 09:12'
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:12'

ctools/web/aio_web_server.py

@@ -5,12 +5,12 @@
 __author__ = 'haoyang'
 __date__ = '2025/5/30 09:54'
 
+import asyncio
 import sys
 from pathlib import Path
 from typing import Optional, Dict, Any
 
 from aiohttp import web
-
 from ctools import sys_info, cjson
 from ctools.sys_log import flog as log
 from ctools.web.api_result import R
@@ -30,13 +30,14 @@ async def response_wrapper_middleware(request, handler):
     else:
       return result
   except web.HTTPException as http_exc:
-      raise http_exc
+    raise http_exc
   except Exception as e:
     log.error(f"Error in response_wrapper_middleware: {e}", exc_info=True)
     return web.json_response(text=R.error(str(e)), status=500, content_type='application/json')
 
+
 class AioHttpServer:
-  def __init__(self, port: int = DEFAULT_PORT, app: Optional[web.Application] = None, routes: Optional[web.RouteTableDef] = None, async_func = None):
+  def __init__(self, port: int = DEFAULT_PORT, app: Optional[web.Application] = None, routes: Optional[web.RouteTableDef] = None, async_func=None):
     """
     Initialize the HTTP server.
 
@@ -142,26 +143,35 @@ class AioHttpServer:
     """Run the server."""
     if self.async_func:
       await self.async_func()
+
     print(
       'Server running at:\n'
       f'\tLocal: http://localhost:{self.port}\n'
       f'\tNetwork: http://{sys_info.get_local_ipv4()}:{self.port}',
       file=sys.stderr
     )
-    await web._run_app(
-      self.app,
-      port=self.port,
-      host='0.0.0.0'
-    )
+    runner = None
+    try:
+      runner = web.AppRunner(self.app)
+      await runner.setup()
+      site = web.TCPSite(runner, host='0.0.0.0', port=self.port)
+      await site.start()
+      while True: await asyncio.sleep(3600)
+    except Exception as e:
+      print(f"Server failed to start: {e}", file=sys.stderr)
+    finally:
+      await runner.cleanup()
 
 
 def init_routes() -> web.RouteTableDef:
   return web.RouteTableDef()
 
-def init_server(routes: Optional[web.RouteTableDef] = None, app: Optional[web.Application] = None, port: int = DEFAULT_PORT, async_func = None) -> AioHttpServer:
+
+def init_server(routes: Optional[web.RouteTableDef] = None, app: Optional[web.Application] = None, port: int = DEFAULT_PORT, async_func=None) -> AioHttpServer:
   """Initialize and return a new AioHttpServer instance."""
   return AioHttpServer(port=port, app=app, routes=routes, async_func=async_func)
 
+
 async def get_stream_resp(request, content_type: str = 'text/event-stream') -> web.StreamResponse:
   resp = web.StreamResponse(
     status=200,

ctools/web/api_result.py

@@ -13,6 +13,7 @@ class _ResEnum(object):
   def __eq__(self, o: object) -> bool:
     return self.code == o
 
+
 class R(object):
   class Code:
 
@@ -21,8 +22,8 @@ class R(object):
       return _ResEnum(code, msg)
 
     SUCCESS = _ResEnum(200, "成功")
-    FAIL    = _ResEnum(400, "失败")
-    ERROR   = _ResEnum(500, "异常")
+    FAIL = _ResEnum(400, "失败")
+    ERROR = _ResEnum(500, "异常")
 
   def __init__(self, code: int, message: str, data=""):
     self.code = code

ctools/web/bottle_web_base.py

@@ -5,74 +5,103 @@ from functools import wraps
 import bottle
 from bottle import response, Bottle, request
 
+from ctools import call
 from ctools.dict_wrapper import DictWrapper
 from ctools.sys_log import flog as log
+from ctools.util.cklock import try_lock
 from ctools.web import ctoken
 from ctools.web.api_result import R
 
 bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 50
 func_has_params = {}
+app_cache = {}
 
 class GlobalState:
   lock = threading.Lock()
-  withOutLoginURI = [
+  withOutLoginURI = {
     '/',
     '/index',
-    '/login'
-  ]
-  allowRemoteCallURI = [
-
-  ]
+    '/login',
+    '/favicon.ico',
+  }
+  allowRemoteCallURI = set()
+  auth_ignore_func = set()
   token = {}
   interceptors = []
 
-def init_app(context_path=None):
-  app = Bottle()
-  app.context_path = context_path
-
-  @app.hook('before_request')
-  def before_request():
-    for interceptor in GlobalState.interceptors:
-      res: R = interceptor['func']()
-      if res.code != 200: bottle.abort(res.code, res.message)
-
-  @app.error(401)
-  def unauthorized(error):
-    response.status = 401
-    log.error("系统未授权: {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(401, "系统未授权! {}".format(error.body)))
-
-  @app.error(403)
-  def unauthorized(error):
-    response.status = 403
-    log.error("访问受限: {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(403, "访问受限: {}".format(error.body)))
-
-  @app.error(404)
-  def not_found(error):
-    response.status = 404
-    log.error("404 not found : {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(404, "资源未找到: {}".format(error.body)))
-
-  @app.error(405)
-  def method_not_allow(error):
-    response.status = 405
-    log.error("请求方法错误: {} {} {}".format(error.status_line, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(405, '请求方法错误: {}'.format(error.status_line)))
-
-  @app.error(500)
-  def internal_error(error):
-    response.status = 500
-    log.error("系统发生错误: {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(msg='系统发生错误: {}'.format(error.exception))
-
-  @app.hook('after_request')
-  def after_request():
-    enable_cors()
-
+def join_path(*parts):
+  """拼接 url"""
+  cleaned_parts = [p.strip('/') for p in parts if p and p.strip('/')]
+  return '/' + '/'.join(cleaned_parts)
+
+@call.once
+def cache_white_list(app):
+  """缓存白名单"""
+  for route in app.routes:
+    real_func = route.config.get('mountpoint.target')
+    if not real_func: continue
+    for method, routes in real_func.router.static.items():
+      for path, tuples in routes.items():
+        req_func = inspect.getmodule(tuples[0].callback).__name__ + "." + tuples[0].callback.__name__
+        if req_func in GlobalState.auth_ignore_func:
+          print("add white list: {}".format(join_path(app.context_path, real_func.context_path, path)))
+          GlobalState.withOutLoginURI.add(join_path(app.context_path, real_func.context_path, path))
+
+def init_app(context_path="/", main_app=False):
+  with try_lock(block=True):
+    cache_app = app_cache.get(context_path)
+    if cache_app: return cache_app
+    app = Bottle()
+    app_cache[context_path] = app
+    app.context_path = context_path if context_path else "/"
+
+  def init_main_app():
+    @app.hook('before_request')
+    def before_request():
+      if request.path.startswith('/static') or request.path in GlobalState.withOutLoginURI: return
+      for interceptor in GlobalState.interceptors:
+        res: R = interceptor['func']()
+        if res.code != 200: bottle.abort(res.code, res.message)
+
+    @app.error(401)
+    def unauthorized(error):
+      response.status = 401
+      log.error("系统未授权: {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(401, "系统未授权! {}".format(error.body)))
+
+    @app.error(403)
+    def unauthorized(error):
+      response.status = 403
+      log.error("访问受限: {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(403, "访问受限: {}".format(error.body)))
+
+    @app.error(404)
+    def not_found(error):
+      response.status = 404
+      log.error("404 not found : {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(404, "资源未找到: {}".format(error.body)))
+
+    @app.error(405)
+    def method_not_allow(error):
+      response.status = 405
+      log.error("请求方法错误: {} {} {}".format(error.status_line, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(405, '请求方法错误: {}'.format(error.status_line)))
+
+    @app.error(500)
+    def internal_error(error):
+      response.status = 500
+      log.error("系统发生错误: {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(msg='系统发生错误: {}'.format(error.exception))
+
+    @app.hook('after_request')
+    def after_request():
+      enable_cors()
+
+  if main_app: init_main_app()
   app.install(params_resolve)
   return app
 
+
 def enable_cors():
   response.headers['Access-Control-Allow-Origin'] = '*'
   response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE, OPTIONS'
@@ -80,26 +109,50 @@ def enable_cors():
   response.headers['Access-Control-Allow-Headers'] = request_headers if request_headers else ''
   response.headers['Access-Control-Expose-Headers'] = '*'
 
+
 # annotation
 def before_intercept(order=0):
   def decorator(func):
+    for interceptor in GlobalState.interceptors:
+      if interceptor['func'].__name__ == func.__name__:
+        log.info("duplicate interceptor: {}".format(func.__name__))
+        return
     log.info("add before interceptor: {}".format(func.__name__))
     GlobalState.interceptors.append({'order': order, 'func': func})
     GlobalState.interceptors = sorted(GlobalState.interceptors, key=lambda x: x['order'])
+
   return decorator
 
+# annotation
+def auth_ignore(func):
+  """忽略登录验证的接口"""
+  ignore_req_func = inspect.getmodule(func).__name__ + "." + func.__name__
+  if ignore_req_func in GlobalState.auth_ignore_func: raise Exception("duplicate ignore func: {}".format(ignore_req_func))
+  GlobalState.auth_ignore_func.add(ignore_req_func)
+  @wraps(func)
+  def decorated(*args, **kwargs):
+    return func(*args, **kwargs)
+  return decorated
+
 # annotation
 def rule(key):
   def return_func(func):
     @wraps(func)
     def decorated(*args, **kwargs):
-      # if GlobalState.licenseInfo is not None and key not in GlobalState.licenseInfo.access_module:
-      #   log.error("系统未授权! {} {}".format(request.fullpath, '当前请求的模块未授权!请联系管理员!'))
-      #   return R.error(resp=R.Code.cus_code(9999, "系统未授权! {}".format('当前请求的模块未授权!请联系管理员!')))
-      return func(*args, **kwargs)
+      rules = ctoken.get_token_attr("rules") or []
+      if _match_rule_by_prefix(key, rules):
+        return func(*args, **kwargs)
+      else:
+        return R.error("权限不足, 请联系管理员: {}".format(key))
     return decorated
   return return_func
 
+def _match_rule_by_prefix(key, rules):
+  for r in rules:
+    if key.startswith(r):
+      return True
+  return False
+
 # annotation or plugins, has auto install, don't need to call
 def params_resolve(func):
   @wraps(func)
@@ -114,56 +167,67 @@ def params_resolve(func):
         return func(*args, **kwargs)
       else:
         func_has_params[request.fullpath] = True
-    if request.method == 'GET':
+    if request.method == 'GET' or request.method == 'DELETE':
       queryStr = request.query.decode('utf-8')
       page_info = PageInfo(
         page_size=10 if request.headers.get('page_size') is None else int(request.headers.get('page_size')),
         page_index=1 if request.headers.get('page_index') is None else int(request.headers.get('page_index'))
       )
+      queryStr = auto_exchange(func, queryStr)
       queryStr.page_info = page_info
       return func(params=queryStr, *args, **kwargs)
-    elif request.method == 'POST':
+    elif request.method == 'POST' or request.method == 'PUT':
       query_params = request.query.decode('utf-8')
       content_type = request.get_header('content-type')
       if content_type == 'application/json':
         params = request.json or {}
         dict_wrapper = DictWrapper(params)
         dict_wrapper.update(query_params.dict)
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
       elif 'multipart/form-data' in content_type:
         form_data = request.forms.decode()
         form_files = request.files.decode()
         dict_wrapper = DictWrapper(form_data)
         dict_wrapper.update(query_params.dict)
         dict_wrapper.files = form_files
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
       elif 'application/x-www-form-urlencoded' in content_type:
         params = request.forms.decode()
         dict_wrapper = DictWrapper(params.dict)
         dict_wrapper.update(query_params.dict)
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
       elif 'text/plain' in content_type:
         params = request.body.read().decode('utf-8')
         dict_wrapper = DictWrapper({'body': params})
         dict_wrapper.update(query_params.dict)
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
     else:
       return func(*args, **kwargs)
+
   return decorated
 
+# 自动转换参数类型
+def auto_exchange(func, dict_wrapper):
+  model_class = func.__annotations__.get('params')
+  if model_class:
+    try:
+      model_instance = model_class(**dict_wrapper)
+      return model_instance
+    except Exception as e:
+      log.exception(e)
+      return dict_wrapper
+  else:
+    return dict_wrapper
+
+# 分页信息对象
 class PageInfo:
   def __init__(self, page_size, page_index):
     self.page_size = page_size
     self.page_index = page_index
 
+
 # 通用的鉴权方法
-def common_auth_verify(aes_key):
-  if request.path.startswith('/static') or request.path in GlobalState.withOutLoginURI:
-    return R.ok(to_json_str=False)
-  auth_token = request.get_header('Authorization')
-  if auth_token is None:
-    auth_token = request.get_cookie('Authorization')
-  payload = ctoken.get_payload(auth_token, aes_key)
-  if payload:
-    return R.ok(to_json_str=False)
-  return R.error(resp=R.Code.cus_code(401, "未授权"), to_json_str=False)
+def common_auth_verify() -> R:
+  valid = ctoken.is_valid()
+  if valid: return R.ok(to_json_str=False)
+  return R.error(resp=R.Code.cus_code(401, "请登录!"), to_json_str=False)