gomyck-tools 1.4.2__py3-none-any.whl → 1.4.4__py3-none-any.whl

ctools/util/config_util.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/16 14:19'
+
+"""
+config = load_config("application.ini")
+print(config)
+print(config.base.app_name)
+print(config.base.version)
+"""
+from configparser import ConfigParser
+
+cache = {}
+
+class AttrNoneNamespace:
+  def __init__(self):
+    pass
+  def __setattr__(self, key, value):
+    super().__setattr__(key, value)
+  def __getattr__(self, item):
+    return None
+
+def _convert_value(value: str):
+  val = value.strip()
+  if val.lower() in ('true', 'yes', 'on'):
+    return True
+  if val.lower() in ('false', 'no', 'off'):
+    return False
+  if val.isdigit():
+    return int(val)
+  try:
+    return float(val)
+  except ValueError:
+    return val
+
+def _config_to_object(config: ConfigParser):
+  result = AttrNoneNamespace()
+  for section in config.sections():
+    section_obj = AttrNoneNamespace()
+    for key, value in config.items(section):
+      setattr(section_obj, key, _convert_value(value))
+    setattr(result, section, section_obj)
+  return result
+
+def load_config(file_path):
+  if file_path in cache: return cache[file_path]
+  config = ConfigParser()
+  config.read(file_path)
+  cf = _config_to_object(config)
+  cache[file_path] = cf
+  return cf

ctools/util/image_process.py

@@ -1,3 +1,4 @@
+import base64
 from io import BytesIO
 
 from PIL import Image
@@ -25,3 +26,10 @@ def change_color(image_path, area=None, rgb_color=None):
     img.save(img_bytes, format='JPEG')
     img_binary = img_bytes.getvalue()
   return img_binary
+
+
+def img2b64(img: Image, fmt="PNG"):
+  buf = BytesIO()
+  img.save(buf, format=fmt.upper())
+  return base64.b64encode(buf.getvalue()).decode("utf-8")
+

ctools/util/jb_cut.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/15 13:08'
+
+import sys
+
+import jieba
+from jieba import posseg as pseg
+
+jqfx_exclude = ('ul', 'uj', 'uz', 'a', 'c', 'm', 'f', 'ad', 'an', 'r', 'q', 'u', 't', 'd', 'p', 'x')
+
+def add_dict(dic_word: str = None, dic_path: str = None):
+  """
+  添加自定义词库(自定义词库添加之后, 如果不适用全模式切词, 有时也不好使, 因为权重没有默认的高)
+  :param dic_word: 一个单词
+  :param dic_path: 单词表文件地址
+  单词表文件格式:
+  单词  词频  标签
+  单词1  3    i
+  单词2  3    i
+  单词3  3    i
+  """
+  if dic_word: jieba.add_word(dic_word)
+  if dic_path: jieba.load_userdict(dic_path)
+
+def add_freq(word: ()):
+  """
+  添加词频
+  :param word: 一个单词
+  """
+  jieba.suggest_freq(word, True)
+
+def get_declare_type_word(word: str, word_flag=(), use_paddle=False, exclude_flag="x"):
+  #import paddle
+  #paddle.enable_static()
+  #sys.stdout = sys.__stdout__
+  #sys.stderr = sys.__stderr__
+  """
+  获取声明类型的单词
+  :param word: 单词
+  :param word_flag: 标签
+  :param use_paddle: 是否使用 paddle
+  :param exclude_flag: 排除的标签
+  :return:  筛选之后的单词(数组)
+  """
+  # linux 可以开放下面的语句, 并发执行分词, windows 不支持
+  # if platform.system() == 'Linux': jieba.enable_parallel(4)
+  ret = []
+  for w, flag in pseg.cut(word, use_paddle=use_paddle):
+    if (flag in word_flag or len(word_flag) == 0) and flag not in exclude_flag:
+      ret.append((w, flag))
+  return ret

ctools/web/bottle_web_base.py

@@ -5,74 +5,99 @@ from functools import wraps
 import bottle
 from bottle import response, Bottle, request
 
+from ctools import call
 from ctools.dict_wrapper import DictWrapper
 from ctools.sys_log import flog as log
+from ctools.util.cklock import try_lock
 from ctools.web import ctoken
 from ctools.web.api_result import R
 
 bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 50
 func_has_params = {}
-
+app_cache = {}
 
 class GlobalState:
   lock = threading.Lock()
-  withOutLoginURI = [
+  withOutLoginURI = {
     '/',
     '/index',
     '/login',
     '/favicon.ico',
-  ]
-  allowRemoteCallURI = [
-
-  ]
+  }
+  allowRemoteCallURI = set()
+  auth_ignore_func = set()
   token = {}
   interceptors = []
 
-
-def init_app(context_path=None):
-  app = Bottle()
-  app.context_path = context_path
-
-  @app.hook('before_request')
-  def before_request():
-    for interceptor in GlobalState.interceptors:
-      res: R = interceptor['func']()
-      if res.code != 200: bottle.abort(res.code, res.message)
-
-  @app.error(401)
-  def unauthorized(error):
-    response.status = 401
-    log.error("系统未授权: {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(401, "系统未授权! {}".format(error.body)))
-
-  @app.error(403)
-  def unauthorized(error):
-    response.status = 403
-    log.error("访问受限: {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(403, "访问受限: {}".format(error.body)))
-
-  @app.error(404)
-  def not_found(error):
-    response.status = 404
-    log.error("404 not found : {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(404, "资源未找到: {}".format(error.body)))
-
-  @app.error(405)
-  def method_not_allow(error):
-    response.status = 405
-    log.error("请求方法错误: {} {} {}".format(error.status_line, request.method, request.fullpath))
-    return R.error(resp=R.Code.cus_code(405, '请求方法错误: {}'.format(error.status_line)))
-
-  @app.error(500)
-  def internal_error(error):
-    response.status = 500
-    log.error("系统发生错误: {} {} {}".format(error.body, request.method, request.fullpath))
-    return R.error(msg='系统发生错误: {}'.format(error.exception))
-
-  @app.hook('after_request')
-  def after_request():
-    enable_cors()
-
+def join_path(*parts):
+  """拼接 url"""
+  cleaned_parts = [p.strip('/') for p in parts if p and p.strip('/')]
+  return '/' + '/'.join(cleaned_parts)
+
+@call.once
+def cache_white_list(app):
+  """缓存白名单"""
+  for route in app.routes:
+    real_func = route.config.get('mountpoint.target')
+    if not real_func: continue
+    for method, routes in real_func.router.static.items():
+      for path, tuples in routes.items():
+        req_func = inspect.getmodule(tuples[0].callback).__name__ + "." + tuples[0].callback.__name__
+        if req_func in GlobalState.auth_ignore_func:
+          print("add white list: {}".format(join_path(app.context_path, real_func.context_path, path)))
+          GlobalState.withOutLoginURI.add(join_path(app.context_path, real_func.context_path, path))
+
+def init_app(context_path="/", main_app=False):
+  with try_lock(block=True):
+    cache_app = app_cache.get(context_path)
+    if cache_app: return cache_app
+    app = Bottle()
+    app_cache[context_path] = app
+    app.context_path = context_path if context_path else "/"
+
+  def init_main_app():
+    @app.hook('before_request')
+    def before_request():
+      if request.path.startswith('/static') or request.path in GlobalState.withOutLoginURI: return
+      for interceptor in GlobalState.interceptors:
+        res: R = interceptor['func']()
+        if res.code != 200: bottle.abort(res.code, res.message)
+
+    @app.error(401)
+    def unauthorized(error):
+      response.status = 401
+      log.error("系统未授权: {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(401, "系统未授权! {}".format(error.body)))
+
+    @app.error(403)
+    def unauthorized(error):
+      response.status = 403
+      log.error("访问受限: {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(403, "访问受限: {}".format(error.body)))
+
+    @app.error(404)
+    def not_found(error):
+      response.status = 404
+      log.error("404 not found : {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(404, "资源未找到: {}".format(error.body)))
+
+    @app.error(405)
+    def method_not_allow(error):
+      response.status = 405
+      log.error("请求方法错误: {} {} {}".format(error.status_line, request.method, request.fullpath))
+      return R.error(resp=R.Code.cus_code(405, '请求方法错误: {}'.format(error.status_line)))
+
+    @app.error(500)
+    def internal_error(error):
+      response.status = 500
+      log.error("系统发生错误: {} {} {}".format(error.body, request.method, request.fullpath))
+      return R.error(msg='系统发生错误: {}'.format(error.exception))
+
+    @app.hook('after_request')
+    def after_request():
+      enable_cors()
+
+  if main_app: init_main_app()
   app.install(params_resolve)
   return app
 
@@ -88,27 +113,45 @@ def enable_cors():
 # annotation
 def before_intercept(order=0):
   def decorator(func):
+    for interceptor in GlobalState.interceptors:
+      if interceptor['func'].__name__ == func.__name__:
+        log.info("duplicate interceptor: {}".format(func.__name__))
+        return
     log.info("add before interceptor: {}".format(func.__name__))
     GlobalState.interceptors.append({'order': order, 'func': func})
     GlobalState.interceptors = sorted(GlobalState.interceptors, key=lambda x: x['order'])
 
   return decorator
 
+# annotation
+def auth_ignore(func):
+  """忽略登录验证的接口"""
+  ignore_req_func = inspect.getmodule(func).__name__ + "." + func.__name__
+  if ignore_req_func in GlobalState.auth_ignore_func: raise Exception("duplicate ignore func: {}".format(ignore_req_func))
+  GlobalState.auth_ignore_func.add(ignore_req_func)
+  @wraps(func)
+  def decorated(*args, **kwargs):
+    return func(*args, **kwargs)
+  return decorated
 
 # annotation
 def rule(key):
   def return_func(func):
     @wraps(func)
     def decorated(*args, **kwargs):
-      # if GlobalState.licenseInfo is not None and key not in GlobalState.licenseInfo.access_module:
-      #   log.error("系统未授权! {} {}".format(request.fullpath, '当前请求的模块未授权!请联系管理员!'))
-      #   return R.error(resp=R.Code.cus_code(9999, "系统未授权! {}".format('当前请求的模块未授权!请联系管理员!')))
-      return func(*args, **kwargs)
-
+      rules = ctoken.get_token_attr("rules") or []
+      if _match_rule_by_prefix(key, rules):
+        return func(*args, **kwargs)
+      else:
+        return R.error("权限不足, 请联系管理员: {}".format(key))
     return decorated
-
   return return_func
 
+def _match_rule_by_prefix(key, rules):
+  for r in rules:
+    if key.startswith(r):
+      return True
+  return False
 
 # annotation or plugins, has auto install, don't need to call
 def params_resolve(func):
@@ -124,45 +167,59 @@ def params_resolve(func):
         return func(*args, **kwargs)
       else:
         func_has_params[request.fullpath] = True
-    if request.method == 'GET':
+    if request.method == 'GET' or request.method == 'DELETE':
       queryStr = request.query.decode('utf-8')
       page_info = PageInfo(
         page_size=10 if request.headers.get('page_size') is None else int(request.headers.get('page_size')),
         page_index=1 if request.headers.get('page_index') is None else int(request.headers.get('page_index'))
       )
+      queryStr = auto_exchange(func, queryStr)
       queryStr.page_info = page_info
       return func(params=queryStr, *args, **kwargs)
-    elif request.method == 'POST':
+    elif request.method == 'POST' or request.method == 'PUT':
       query_params = request.query.decode('utf-8')
       content_type = request.get_header('content-type')
       if content_type == 'application/json':
         params = request.json or {}
         dict_wrapper = DictWrapper(params)
         dict_wrapper.update(query_params.dict)
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
       elif 'multipart/form-data' in content_type:
         form_data = request.forms.decode()
         form_files = request.files.decode()
         dict_wrapper = DictWrapper(form_data)
         dict_wrapper.update(query_params.dict)
         dict_wrapper.files = form_files
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
       elif 'application/x-www-form-urlencoded' in content_type:
         params = request.forms.decode()
         dict_wrapper = DictWrapper(params.dict)
         dict_wrapper.update(query_params.dict)
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
       elif 'text/plain' in content_type:
         params = request.body.read().decode('utf-8')
         dict_wrapper = DictWrapper({'body': params})
         dict_wrapper.update(query_params.dict)
-        return func(params=dict_wrapper, *args, **kwargs)
+        return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
     else:
       return func(*args, **kwargs)
 
   return decorated
 
-
+# 自动转换参数类型
+def auto_exchange(func, dict_wrapper):
+  model_class = func.__annotations__.get('params')
+  if model_class:
+    try:
+      model_instance = model_class(**dict_wrapper)
+      return model_instance
+    except Exception as e:
+      log.exception(e)
+      return dict_wrapper
+  else:
+    return dict_wrapper
+
+# 分页信息对象
 class PageInfo:
   def __init__(self, page_size, page_index):
     self.page_size = page_size
@@ -170,13 +227,7 @@ class PageInfo:
 
 
 # 通用的鉴权方法
-def common_auth_verify(aes_key):
-  if request.path.startswith('/static') or request.path in GlobalState.withOutLoginURI:
-    return R.ok(to_json_str=False)
-  auth_token = request.get_header('Authorization')
-  if auth_token is None:
-    auth_token = request.get_cookie('Authorization')
-  payload = ctoken.get_payload(auth_token, aes_key)
-  if payload:
-    return R.ok(to_json_str=False)
+def common_auth_verify() -> R:
+  valid = ctoken.is_valid()
+  if valid: return R.ok(to_json_str=False)
   return R.error(resp=R.Code.cus_code(401, "请登录!"), to_json_str=False)

ctools/web/bottle_webserver.py

@@ -5,48 +5,36 @@ from wsgiref.simple_server import WSGIServer, WSGIRequestHandler, make_server
 from bottle import ServerAdapter, Bottle, template, static_file, abort, redirect, response
 
 from ctools import sys_info
+from ctools.pkg.dynamic_imp import load_modules_from_package
+from ctools.web.bottle_web_base import cache_white_list
 
-"""
-module_names = list(globals().keys())
-def get_modules():
-  mods = []
-  for modname in module_names:
-    if modname == 'base' or modname == 'online' or modname.startswith('__') or modname == 'importlib': continue
-    module = globals()[modname]
-    mods.append(module)
-  return mods
-
-def get_ws_modules():
-  from . import websocket
-  return [websocket]
-"""
 
 """
-from ctools.web import bottle_web_base, ctoken, bottle_webserver
-from ctools.web.api_result import R
+import controllers
+from ctools import patch
+from ctools.database import database
+from ctools.util.config_util import load_config
+from ctools.web import bottle_web_base, bottle_webserver
+from key_word_cloud.db_core.db_init import init_partitions
+from patch_manager import patch_funcs
 
-secret_key = "xxx"
-app = bottle_web_base.init_app('子模块写 context_path, 主模块就不用写任何东西')
+database.init_db('postgresql://postgres:123123@192.168.xx.xx:5432/xxx', default_schema='xxx', auto_gen_table=False, echo=False)
 
-# 通用的鉴权方法
-@bottle_web_base.before_intercept(0)
-def token_check():
-  return bottle_web_base.common_auth_verify(secret_key)
+config = load_config('application.ini')
 
-@app.post('/login')
-def login(params):
-  return R.ok(ctoken.gen_token({'username': 'xxx'}, secret_key, 3600))
+patch.sync_version(config.base.app_name, config.base.version, patch_funcs)
+init_partitions()
 
-@app.get('/demo')
-@bottle_web_base.rule('DOC:DOWNLOAD')
-def demo(params):
-  print(params)
+app = bottle_web_base.init_app("/api", True)
+
+@bottle_web_base.before_intercept(0)
+def token_check():
+    return bottle_web_base.common_auth_verify(config.base.secret_key)
 
 if __name__ == '__main__':
-  main_app = bottle_webserver.init_bottle() # 这里可以传 APP 当做主模块, 但是 context_path 就不好使了, 上下文必须是 /
-  main_app.mount(app.context_path, app)
-  main_app.set_index(r'index.html')
-  main_app.run()
+  main_server = bottle_webserver.init_bottle(app)
+  main_server.auto_mount(controllers)
+  main_server.run()
 """
 
 _default_port = 8888
@@ -101,6 +89,7 @@ class CBottle:
   def run(self):
     http_server = WSGIRefServer(port=self.port)
     print('Click the link below to open the service homepage %s' % '\n \t\t http://localhost:%s \n \t\t http://%s:%s' % (self.port, sys_info.get_local_ipv4(), self.port), file=sys.stderr)
+    cache_white_list(self.bottle)
     self.bottle.run(server=http_server, quiet=self.quiet)
 
   def set_index(self, filename='index.html', root='./', is_tpl=False, redirect_url=None, **kwargs):
@@ -120,6 +109,17 @@ class CBottle:
     if not context_path: return
     self.bottle.mount(context_path, app, **kwargs)
 
+  def auto_mount(self, package, exclude=None, recursive=True):
+    for module in load_modules_from_package(package, exclude, recursive):
+      if self.bottle.context_path != '/':
+        if module.app.context_path == '/':
+          ctx_path = self.bottle.context_path
+        else:
+          ctx_path = self.bottle.context_path + module.app.context_path
+      else:
+        ctx_path = module.app.context_path
+      print("mount: %s on %s" % (ctx_path, module.__name__))
+      self.bottle.mount(ctx_path, module.app)
 
 def init_bottle(app: Bottle = None, port=_default_port, quiet=False) -> CBottle:
   bottle = app or Bottle()

ctools/web/ctoken.py

@@ -8,32 +8,96 @@ import time
 import jwt
 from bottle import request
 
+from ctools import cid
 from ctools.dict_wrapper import DictWrapper
 
-token_header = 'Authorization'
 
+class CToken:
+  token_audience = ["gomyck"]
+  token_secret_key = 'gomyck123'
+  token_header = 'Authorization'
 
-def gen_token(payload: {}, secret_key, expired: int = 3600) -> str:
-  payload.update({'exp': time.time() + expired})
-  return jwt.encode(payload, secret_key, algorithm='HS256')
+def gen_token(userid, username, payload={}, expired: int = 3600, rules= []) -> str:
+  if expired and expired > 0: payload.update({'exp': time.time() + expired})
+  payload.update({'iss': "gomyck", 'jti': cid.get_snowflake_id_str(), 'nbf': time.time(), 'iat': time.time()})
+  payload.update({'rules': rules})
+  payload.update({'uid': userid})
+  payload.update({'sub': username})
+  payload.update({'aud': CToken.token_audience})
+  payload.update({'rules': rules})
+  return jwt.encode(payload, CToken.token_secret_key, algorithm='HS256')
 
-
-def get_payload(token, secret_key):
+def get_payload(token=None):
   try:
-    payload = jwt.decode(token, secret_key, algorithms=['HS256'])
+    if token is None: token = get_token()
+    payload = jwt.decode(token, options={"verify_signature": False}, algorithms=['HS256'])
     return DictWrapper(payload)
   except Exception as e:
     return None
 
+def get_token_payload(token=None):
+  if token is None: token = get_token()
+  return get_payload(token)
+
+def is_valid(token=None):
+  """
+  判断token是否有效
+  :param token: token 信息
+  :return: 是否有效
+  """
+  if token is None: token = get_token()
+  if token is None: return False
+  try:
+    jwt.decode(token, CToken.token_secret_key, algorithms=['HS256'], audience=CToken.token_audience)
+  except Exception as e:
+    print(e)
+    return False
+  return True
+
+def get_user_name():
+  """
+  获取用户名称
+  :return: 用户ID
+  """
+  token_obj = get_token_attr("sub")
+  return token_obj
+
+def get_user_id():
+  """
+  获取用户ID
+  :return: 用户ID
+  """
+  token_obj = get_token_attr("uid")
+  return token_obj
+
+def get_token_id():
+  """
+  获取token id
+  :return: token id
+  """
+  token_obj = get_token_attr("jti")
+  return token_obj
 
-def get_token(key):
-  return get_payload(request.get_header(token_header), key)
+def get_app_flag() -> []:
+  """
+  获取服务标识
+  :return: 服务标识 []
+  """
+  token_obj = get_token_attr("aud")
+  return token_obj
 
+def get_token_attr(attr):
+    token_obj = get_token_payload()
+    return getattr(token_obj, attr, None)
 
-def is_valid(key):
-  return get_payload(request.get_header(token_header), key) is not None
+def get_token():
+  auth_token = request.get_header(CToken.token_header)
+  if auth_token is None:
+    auth_token = request.get_cookie(CToken.token_header)
+  return auth_token
 
 # if __name__ == '__main__':
-#   token = gen_token({"xx": 123}, '123')
-#   xx = get_payload(token, '123')
-#   print(xx.xx)
+#   token = gen_token(123, username='xxx', expired=0)
+#   print(token)
+#   print(get_payload(token))
+#   print(is_valid(token))

{gomyck_tools-1.4.2.dist-info → gomyck_tools-1.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gomyck-tools
-Version: 1.4.2
+Version: 1.4.4
 Summary: A tools collection for python development by hao474798383
 Author-email: gomyck <hao474798383@163.com>
 License-Expression: Apache-2.0
@@ -15,7 +15,7 @@ Requires-Dist: croniter~=5.0.1
 Requires-Dist: gmssl~=3.2.2
 Requires-Dist: psutil~=6.1.0
 Requires-Dist: jsonpath_ng~=1.7.0
-Requires-Dist: bottle~=0.13.2
+Requires-Dist: bottle~=0.13.4
 Requires-Dist: requests~=2.32.3
 Requires-Dist: urllib3~=1.26.20
 Requires-Dist: kafka-python~=2.0.2
@@ -30,15 +30,16 @@ Requires-Dist: pyjwt==2.10.1
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: redis==5.2.1
 Requires-Dist: uvloop>=0.21.0
-Provides-Extra: db
-Requires-Dist: sqlalchemy>=2.0; extra == "db"
-Requires-Dist: asyncpg>=0.28; extra == "db"
-Provides-Extra: office
-Requires-Dist: pandas>=2.2.3; extra == "office"
-Requires-Dist: xlrd>=2.0.1; extra == "office"
-Requires-Dist: python-docx>=1.1.2; extra == "office"
-Provides-Extra: auto-ui
-Requires-Dist: pynput==1.7.7; extra == "auto-ui"
+Provides-Extra: kwc
+Requires-Dist: imageio>=2.37.0; extra == "kwc"
+Requires-Dist: wordcloud>=1.9.4; extra == "kwc"
+Requires-Dist: jieba==0.42.1; extra == "kwc"
+Requires-Dist: paddlepaddle==2.6.1; extra == "kwc"
+Provides-Extra: cut
+Requires-Dist: jieba==0.42.1; extra == "cut"
+Requires-Dist: paddlepaddle==2.6.1; extra == "cut"
+Provides-Extra: ml
+Requires-Dist: scikit-learn>=1.7.1; extra == "ml"
 Dynamic: license-file
 
 # Gomyck-Tools