gomyck-tools 1.4.2__py3-none-any.whl → 1.4.3__py3-none-any.whl

ctools/ai/tools/quick_tools.py

@@ -10,6 +10,7 @@ import mimetypes
 import sys
 import uuid
 
+from ctools.ai.env_config import bool_env
 from ctools.web.aio_web_server import get_stream_resp
 
 
@@ -60,7 +61,7 @@ def build_image_message(content: str, file: bytes = None, file_path: str = None)
   return build_message(ROLE.USER, img_content)
 
 
-async def build_call_back(debug=False, request=None, SSE=True):
+async def build_call_back(debug=None, request=None, SSE=True):
   """
   快速构建回调函数
   Parameters
@@ -71,6 +72,7 @@ async def build_call_back(debug=False, request=None, SSE=True):
   Returns 响应对象, 消息队列, 回调函数
   -------
   """
+  if not debug: debug = bool_env("LLM_DEBUG", False)
   response = None
   if request: response = await get_stream_resp(request)
   call_id = uuid.uuid4()
@@ -89,7 +91,7 @@ async def build_call_back(debug=False, request=None, SSE=True):
 
   async def on_final(cid, is_final, msg):
     nonlocal response
-    if debug: print(cid, is_final, msg, file=sys.__stdout__, flush=True)
+    if debug: print("\n", cid, "\n", is_final, "\n", msg, "\n", file=sys.__stdout__, flush=True)
     if is_final:
       await message_queue.put("[DONE]")
       if response:

ctools/application.py

@@ -159,7 +159,7 @@ def sync_version(callFunc):
       shutil.rmtree(taguiPath)
     except Exception:
       pass
-    from ctools.auto.pacth import Patch
+    import Patch
     patch = Patch(oldVersion='V1.0.0', newVersion=Server.version, pythonPath=pythonPath, playwrightPath=msPlayPath, driverPath=driverPath)
     patch.apply_patch()
     if callFunc: callFunc()
@@ -174,7 +174,7 @@ def sync_version(callFunc):
       oldVersion.close()
     if oldV == Server.version and '-snapshot' not in oldV: return
     print('开始升级本地程序..')
-    from ctools.auto.pacth import Patch
+    import Patch
     patch = Patch(oldVersion=oldV, newVersion=Server.version, pythonPath=pythonPath, playwrightPath=msPlayPath, driverPath=driverPath)
     patch.apply_patch()
     if callFunc: callFunc()

ctools/aspect.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/23 08:33'
+
+import functools
+import sys
+
+from ctools.pools import thread_pool
+
+def _ensure_list(funcs):
+  if callable(funcs):
+    return [funcs]
+  if isinstance(funcs, (list, tuple, set)):
+    return list(funcs)
+  raise TypeError("必须是可调用对象或可迭代对象")
+
+def before(before_funcs):
+  """
+  用于将无参函数注入目标函数的调用前
+  支持多个函数
+  """
+  before_funcs = _ensure_list(before_funcs)
+  def decorator(func, sync=True):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for bf in before_funcs:
+        if callable(bf):
+          if sync:
+            bf()
+          else:
+            thread_pool.submit(bf)
+      return func(*args, **kwargs)
+    _replace_func_binding(func, wrapper)
+    return wrapper
+  return decorator
+
+def after(after_funcs, sync=True):
+  """
+  用于将无参函数注入目标函数的调用后
+  支持多个函数
+  """
+  after_funcs = _ensure_list(after_funcs)
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      result = func(*args, **kwargs)
+      for af in after_funcs:
+        if callable(af):
+          if sync:
+            af()
+          else:
+            thread_pool.submit(af)
+      return result
+    _replace_func_binding(func, wrapper)
+    return wrapper
+  return decorator
+
+def _replace_func_binding(old_func, new_func):
+  """
+  替换函数在其模块中的绑定，确保所有使用点都生效
+  """
+  mod = sys.modules.get(old_func.__module__)
+  if mod and hasattr(mod, old_func.__name__):
+    setattr(mod, old_func.__name__, new_func)

ctools/call.py

@@ -1,3 +1,4 @@
+import os
 import sched
 import threading
 import time
@@ -5,63 +6,51 @@ from functools import wraps
 
 
 # annotation
+_global_once_cache = {}
 def once(func):
-  """
-  decorator to initialize a function once
-  :param func: function to be initialized
-  :return: the real decorator for return the result
-  """
-  initialized = False
-  res = None
-
+  code = func.__code__
+  key = f"{os.path.abspath(code.co_filename)}:{code.co_firstlineno}"
   def wrapper(*args, **kwargs):
-    nonlocal initialized, res
-    if not initialized:
-      res = func(*args, **kwargs)
-      initialized = True
-      return res
-    else:
-      return res
-
+    if key not in _global_once_cache:
+      _global_once_cache[key] = func(*args, **kwargs)
+    return _global_once_cache[key]
   return wrapper
 
-
 # annotation
+_cache = {}
 def init(func):
-  """
-  decorator to initialize a function automic
-  :param func: function to be initialized
-  :return: the real decorator for return the result
-  """
-  res = func()
-
+  code = func.__code__
+  key = f"{os.path.abspath(code.co_filename)}:{code.co_firstlineno}"
+  if key not in _cache:
+    _cache[key] = func()
   def wrapper():
-    return res
-
+    return _cache[key]
   return wrapper
 
-
 # annotation
+_scheduler_cache = {}
 def schd(interval_seconds, start_by_call=False, run_now=False):
-  scheduler = sched.scheduler(time.time, time.sleep)
-  lock = threading.Lock()
-  started = [False]  # 可变对象，线程可见
-  print("schd delay is: ", interval_seconds)
   def decorator(func):
+    key = f"{os.path.abspath(func.__code__.co_filename)}:{func.__code__.co_firstlineno}"
+    lock = threading.Lock()
     @wraps(func)
     def wrapper(*args, **kwargs):
+      if key in _scheduler_cache:
+        return  # 已经调度过
+      scheduler = sched.scheduler(time.time, time.sleep)
       def job():
         func(*args, **kwargs)
         scheduler.enter(interval_seconds, 1, job)
       def start_scheduler():
         with lock:
-          if started[0]: return
-          started[0] = True
-          if run_now: func(*args, **kwargs)
+          if _scheduler_cache.get(key): return
+          _scheduler_cache[key] = True
+          if run_now:
+            func(*args, **kwargs)
           scheduler.enter(interval_seconds, 1, job)
           scheduler.run()
       threading.Thread(target=start_scheduler, daemon=True).start()
-    # 如果不是手动触发，则自动启动一次（无参数）
-    if not start_by_call: wrapper()
-    return wrapper  # 如果是 start_by_call=True，返回 wrapper 让用户手动调用时带参
+    if not start_by_call:
+      wrapper()
+    return wrapper
   return decorator

ctools/cid.py

@@ -7,6 +7,9 @@ def get_snowflake_id():
   return idWorker.get_id()
 
 
+def get_snowflake_id_str():
+  return str(get_snowflake_id())
+
 def get_random_str(size: int = 10) -> str:
   import random
   return "".join(random.sample('abcdefghjklmnpqrstuvwxyz123456789', size))

ctools/database/database.py

@@ -1,8 +1,9 @@
 import contextlib
 import datetime
 import math
+import threading
 
-from sqlalchemy import create_engine, Integer, Column, event
+from sqlalchemy import create_engine, BigInteger, Column, event
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, Session
 from sqlalchemy.sql import text
@@ -58,17 +59,19 @@ and ``driver`` the name of a DBAPI such as ``psycopg2``, ``pyodbc``, ``cx_oracle
 # > PRAGMA journal_mode=WAL; 设置事务的模式, wal 允许读写并发, 但是会额外创建俩文件
 # > PRAGMA synchronous=NORMAL; 设置写盘策略, 默认是 FULL, 日志,数据都落, 设置成 NORMAL, 日志写完就算事务完成
 
-def init_db(db_url: str, db_key: str = 'default', connect_args: dict = {}, default_schema: str = None, pool_size: int = 5, max_overflow: int = 25, echo: bool = False, auto_gen_table: bool = True):
+def init_db(db_url: str, db_key: str = 'default', connect_args: dict = {}, default_schema: str = None, pool_size: int = 5, max_overflow: int = 25, echo: bool = False, auto_gen_table: bool = False):
   if db_url.startswith('mysql'):
     import pymysql
     pymysql.install_as_MySQLdb()
   if inited_db.get(db_key): raise Exception('db {} already init!!!'.format(db_key))
   global engines, sessionMakers
+  if default_schema: connect_args.update({'options': '-csearch_path={}'.format(default_schema)})
   engine, sessionMaker = _create_connection(db_url=db_url, connect_args=connect_args, pool_size=pool_size, max_overflow=max_overflow, echo=echo)
   engines[db_key] = engine
   sessionMakers[db_key] = sessionMaker
   inited_db[db_key] = True
-  if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: _set_search_path(dbapi_connection, default_schema))
+  # 这个有并发问题, 高并发会导致卡顿, 可以考虑去做一些别的事儿
+  #if default_schema: event.listen(engine, 'connect', lambda dbapi_connection, connection_record: _set_search_path(dbapi_connection, default_schema))
   if auto_gen_table: Base.metadata.create_all(engine)
 
 
@@ -86,7 +89,7 @@ def _create_connection(db_url: str, pool_size: int = 5, max_overflow: int = 25,
                          pool_pre_ping=True,
                          pool_recycle=3600,
                          connect_args=connect_args)
-  sm = sessionmaker(bind=engine)
+  sm = sessionmaker(bind=engine, expire_on_commit=False)
   return engine, sm
 
 
@@ -96,7 +99,7 @@ def generate_custom_id():
 
 class BaseMixin(Base):
   __abstract__ = True
-  obj_id = Column(Integer, primary_key=True, default=generate_custom_id)
+  obj_id = Column(BigInteger, primary_key=True, default=generate_custom_id)
 
   # ext1 = Column(String)
   # ext2 = Column(String)
@@ -153,7 +156,14 @@ class PageInfoBuilder:
     self.records = records
 
 
-def query_by_page(query, pageInfo):
+def query_by_page(query, pageInfo) -> PageInfoBuilder:
+  """
+  使用方法:
+    with database.get_session() as s:
+      query = s.query(AppInfoEntity).filter(AppInfoEntity.app_name.contains(params.app_name))
+      result = database.query_by_page(query, params.page_info)
+      return R.ok(result)
+  """
   records = query.offset((pageInfo.page_index - 1) * pageInfo.page_size).limit(pageInfo.page_size).all()
   rs = []
   for r in records:

ctools/ex.py

@@ -2,6 +2,9 @@ import time
 import traceback
 from functools import wraps
 
+"""
+@exception_handler(fail_return=['解析错误'], print_exc=True)
+"""
 
 # annotation
 def exception_handler(fail_return, retry_num=0, delay=3, catch_e=Exception, print_exc=False):

ctools/patch.py

@@ -0,0 +1,88 @@
+import os
+
+from sqlalchemy.sql import text
+
+from ctools import path_info
+from ctools.database import database
+
+"""
+from ctools import patch
+def xx():
+  print('hello world')
+def xx1():
+  print('hello world1')
+def xx2():
+  print('hello world2')
+patch_funcs = {
+  'V1.0.2': xx,
+  'V1.0.3': xx1,
+  'V1.1.4': xx2
+}
+patch.sync_version("kwc", "V1.1.5", patch_funcs)
+"""
+
+class Patch:
+
+  def __init__(self, oldVersion, newVersion, patch_func: dict) -> None:
+    super().__init__()
+    if oldVersion:
+      self.oldV = version_to_int(oldVersion)
+    else:
+      self.oldV = 0
+    self.currentV = version_to_int(newVersion)
+    self.snapshot = '-snapshot' in newVersion or (oldVersion is not None and '-snapshot' in oldVersion)
+    self.patch_func = patch_func
+
+  def apply_patch(self):
+    patch_methods = [method for method in self.patch_func.keys() if method and (method.startswith('V') or method.startswith('v'))]
+    patch_methods.sort(key=lambda x: version_to_int(x))
+    max_method_name = patch_methods[-1]
+    exec_max_method = False
+    for method_name in patch_methods:
+      slVersion = version_to_int(method_name)
+      if self.currentV > slVersion >= self.oldV:
+        if max_method_name == method_name: exec_max_method = True
+        method = self.patch_func[method_name]
+        print('start exec patch {}'.format(method_name))
+        method()
+        print('patch {} update success'.format(method_name))
+    if self.snapshot and not exec_max_method:
+      print('start exec snapshot patch {}'.format(max_method_name))
+      method = self.patch_func[max_method_name]
+      method()
+      print('snapshot patch {} update success'.format(max_method_name))
+
+def version_to_int(version):
+  return int(version.replace('V', '').replace('v', '').replace('.', '').replace('-snapshot', ''))
+
+def run_sqls(sqls):
+  with database.get_session() as s:
+    for sql in sqls.split(";"):
+      try:
+        s.execute(text(sql.strip()))
+        s.commit()
+      except Exception as e:
+        print('结构升级错误, 请检查!!! {}'.format(e.__cause__))
+
+def sync_version(app_name, new_version, patch_func: dict):
+  destFilePath = os.path.join(path_info.get_user_work_path(".ck/{}".format(app_name), mkdir=True), "version")
+  if not os.path.exists(destFilePath):
+    patch = Patch(oldVersion=None, newVersion=new_version, patch_func=patch_func)
+    patch.apply_patch()
+    with open(destFilePath, 'w') as nv:
+      nv.write(new_version)
+      print('初始化安装, 版本信息为: {}'.format(new_version))
+      nv.flush()
+  else:
+    with open(destFilePath, 'r') as oldVersion:
+      oldV = oldVersion.readline()
+      print('本地版本信息为: {}, 程序版本信息为: {}'.format(oldV, new_version))
+      oldVersion.close()
+    if oldV >= new_version and '-snapshot' not in oldV: return
+    print('开始升级本地程序..')
+    patch = Patch(oldVersion=oldV, newVersion=new_version, patch_func=patch_func)
+    patch.apply_patch()
+    with open(destFilePath, 'w') as newVersion:
+      newVersion.write(new_version)
+      print('程序升级成功, 更新版本信息为: {}'.format(new_version))
+      newVersion.flush()

ctools/pkg/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/15 11:02'

ctools/pkg/dynamic_imp.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/15 11:03'
+
+import importlib
+import pkgutil
+
+
+def load_modules_from_package(package, exclude=None, recursive=True):
+  """
+  递归加载指定包下所有模块（不包括包本身）
+
+  :param package: 要加载模块的包对象（如 mypkg.plugins）
+  :param exclude: 排除的模块完整路径列表（如 ['mypkg.plugins.demo.mod2']）
+  :param recursive: 是否递归子包
+  :return: 模块列表（不含子包本身，只包含模块）
+  """
+  if exclude is None: exclude = []
+  modules = []
+  for finder, modname, ispkg in pkgutil.iter_modules(package.__path__):
+    full_modname = f"{package.__name__}.{modname}"
+    if ispkg and recursive:
+      try:
+        subpkg = importlib.import_module(full_modname)
+        modules.extend(load_modules_from_package(subpkg, exclude, recursive))
+      except Exception as e:
+        print(f"递归子包 {full_modname} 失败：{e}")
+      continue
+    if full_modname in exclude:
+      continue
+    try:
+      module = importlib.import_module(full_modname)
+      modules.append(module)
+    except Exception as e:
+      print(f"!!!!!!加载模块 {full_modname} 失败：{e}!!!!!!")
+      continue
+  return modules

ctools/stream/credis.py

@@ -9,6 +9,10 @@ from redis import Redis
 from ctools import cdate, cid
 from ctools.pools import thread_pool
 
+# 最后一次连接的redis
+_ck_redis: Redis = None
+
+def get_redis(): return _ck_redis
 
 def init_pool(host: str = 'localhost', port: int = 6379, db: int = 0, password: str = None,
               username: str = None, decode_responses: bool = True, max_connections: int = 75,
@@ -27,7 +31,9 @@ def init_pool(host: str = 'localhost', port: int = 6379, db: int = 0, password:
       )
       if r.ping():
         print('CRedis connect {} {} success!'.format(host, port))
-        return r
+        global _ck_redis
+        _ck_redis = r
+        return _ck_redis
     except redis.ConnectionError as e:
       if attempt == retry_count - 1:
         raise Exception(f"Failed to connect to Redis after {retry_count} attempts: {str(e)}")
@@ -37,8 +43,8 @@ def init_pool(host: str = 'localhost', port: int = 6379, db: int = 0, password:
 def add_lock(r: Redis, key: str, timeout: int = 30):
   if r.exists(key):
     expire_time = r.get(key)
-    if cdate.time_diff_in_seconds(expire_time, cdate.get_date_time()) > 0:
-      return True
+    if expire_time and cdate.time_diff_in_seconds(expire_time, cdate.get_date_time()) > 0:
+      return False
     else:
       r.delete(key)
   return r.set(key, cdate.opt_time(seconds=timeout), nx=True, ex=timeout) is not None

ctools/sys_log.py

@@ -85,7 +85,7 @@ class StreamToLogger(io.StringIO):
 @call.init
 def _init_log() -> None:
   global flog, clog
-  flog = _file_log(sys_log_path='{}/ck-py-log/'.format(path_info.get_user_work_path()), mixin=True, log_level=logging.DEBUG)
+  flog = _file_log(path_info.get_user_work_path(".ck/ck-py-log", mkdir=True), mixin=True, log_level=logging.DEBUG)
   clog = _console_log()
   sys.stdout = StreamToLogger(flog, level=logging.INFO)
   sys.stderr = StreamToLogger(flog, level=logging.ERROR)

ctools/util/cklock.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/18 15:46'
+
+import contextvars
+import threading
+from contextlib import contextmanager
+from functools import wraps
+
+from ctools.stream.credis import get_redis, add_lock, remove_lock
+from ctools.web import ctoken
+from ctools.web.api_result import R
+
+# 全局锁容器
+_lock_dict = {}
+_lock_dict_lock = threading.Lock()
+
+def try_acquire_lock(key: str) -> bool:
+  with _lock_dict_lock:
+    if key not in _lock_dict:
+      _lock_dict[key] = threading.Lock()
+    return _lock_dict[key].acquire(blocking=False)
+
+def try_acquire_lock_block(key: str):
+  with _lock_dict_lock:
+    if key not in _lock_dict:
+      _lock_dict[key] = threading.Lock()
+    _lock = _lock_dict[key]
+  _lock.acquire()  # 这里是阻塞的
+
+def release_lock(key: str):
+  with _lock_dict_lock:
+    _lock = _lock_dict.get(key)
+    if _lock and _lock.locked():
+      _lock.release()
+    if _lock and not _lock.locked():
+      _lock_dict.pop(key, None)
+
+@contextmanager
+def try_lock(key: str="sys_lock", block=False):
+  if not block:
+    acquired = try_acquire_lock(key)
+    try:
+      yield acquired
+    finally:
+      if acquired:
+        release_lock(key)
+  else:
+    try_acquire_lock_block(key)
+    try:
+      yield
+    finally:
+      release_lock(key)
+
+#annotation
+"""
+@lock("params.attr")
+"""
+# 上下文保存锁key集合
+current_locks = contextvars.ContextVar("current_locks", default=set())
+
+def lock(lock_attrs=None):
+  def decorator(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+      lock_key = ""
+      nonlocal lock_attrs
+      user_level_lock = False
+
+      if not lock_attrs:
+        user_id = ctoken.get_user_id()
+        if user_id:
+          user_level_lock = True
+          lock_key = f"USER_ID_LOCK_{user_id}"
+        else:
+          raise ValueError("请设置 lock_attrs 或使用 token！")
+
+      if not user_level_lock:
+        if isinstance(lock_attrs, str): lock_attrs = [lock_attrs]
+        try:
+          for attr in lock_attrs:
+            parts = attr.split(".")
+            if len(parts) != 2:
+              raise ValueError(f"lock_attr: {attr} 格式错误")
+            obj = kwargs.get(parts[0]) or args[0]
+            if obj is None:
+              raise ValueError(f"参数 {parts[0]} 不存在")
+            lock_key += f"_{getattr(obj, parts[1], None)}"
+        except Exception as e:
+          raise ValueError(f"生成锁键失败: {e}")
+
+      lock_set = current_locks.get()
+      if lock_key in lock_set:
+        return func(*args, **kwargs)
+      token = current_locks.set(lock_set | {lock_key})
+
+      try:
+        if not get_redis():
+          with try_lock(lock_key) as locked:
+            if not locked:
+              return R.error("操作过于频繁, 请稍后再试")
+            return func(*args, **kwargs)
+        else:
+          locked = add_lock(get_redis(), lock_key)
+          try:
+            if locked:
+              return func(*args, **kwargs)
+            else:
+              return R.error("操作过于频繁, 请稍后再试")
+          finally:
+            if locked:
+              remove_lock(get_redis(), lock_key)
+      finally:
+        current_locks.reset(token)
+    return wrapper
+  return decorator
+

ctools/util/config_util.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/16 14:19'
+
+"""
+config = load_config("application.ini")
+print(config)
+print(config.base.app_name)
+print(config.base.version)
+"""
+from configparser import ConfigParser
+
+cache = {}
+
+class AttrNoneNamespace:
+  def __init__(self):
+    pass
+  def __setattr__(self, key, value):
+    super().__setattr__(key, value)
+  def __getattr__(self, item):
+    return None
+
+def _convert_value(value: str):
+  val = value.strip()
+  if val.lower() in ('true', 'yes', 'on'):
+    return True
+  if val.lower() in ('false', 'no', 'off'):
+    return False
+  if val.isdigit():
+    return int(val)
+  try:
+    return float(val)
+  except ValueError:
+    return val
+
+def _config_to_object(config: ConfigParser):
+  result = AttrNoneNamespace()
+  for section in config.sections():
+    section_obj = AttrNoneNamespace()
+    for key, value in config.items(section):
+      setattr(section_obj, key, _convert_value(value))
+    setattr(result, section, section_obj)
+  return result
+
+def load_config(file_path):
+  if file_path in cache: return cache[file_path]
+  config = ConfigParser()
+  config.read(file_path)
+  cf = _config_to_object(config)
+  cache[file_path] = cf
+  return cf

ctools/util/image_process.py

@@ -1,3 +1,4 @@
+import base64
 from io import BytesIO
 
 from PIL import Image
@@ -25,3 +26,10 @@ def change_color(image_path, area=None, rgb_color=None):
     img.save(img_bytes, format='JPEG')
     img_binary = img_bytes.getvalue()
   return img_binary
+
+
+def img2b64(img: Image, fmt="PNG"):
+  buf = BytesIO()
+  img.save(buf, format=fmt.upper())
+  return base64.b64encode(buf.getvalue()).decode("utf-8")
+