gomyck-tools 1.0.0__py3-none-any.whl → 1.4.7__py3-none-any.whl

ctools/sys_log.py

@@ -3,20 +3,13 @@ import os
 import sys
 import time
 
-from ctools import application, call
+from ctools import call, path_info
 
-application.sync_config()
-
-clog, flog = None, None
+clog: logging.Logger = None
+flog: logging.Logger = None
 
 neglect_keywords = [
   "OPTIONS",
-  '/scheduleInfo/list',
-  '/sys/get_sys_state',
-  '/scheduleInfo/tmpList',
-  '/sys/getSysLog',
-  '/downloadManage/static_file',
-  '/downloadManage/static_images'
 ]
 
 
@@ -27,13 +20,13 @@ def _file_log(sys_log_path: str = './', log_level: int = logging.INFO, mixin: bo
     os.mkdir(sys_log_path)
   except Exception:
     pass
-  log_file = sys_log_path + os.path.sep + "log-" + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime(time.time())) + ".log"
+  log_file = sys_log_path + os.path.sep + "log-" + time.strftime("%Y-%m-%d-%H", time.localtime(time.time())) + ".log"
   if mixin:
     handlers = [logging.FileHandler(filename=log_file, encoding='utf-8'), logging.StreamHandler()]
   else:
     handlers = [logging.FileHandler(filename=log_file, encoding='utf-8')]
   logging.basicConfig(level=log_level,
-                      format='%(asctime)s-%(levelname)s-%(thread)d-%(module)s(%(funcName)s:%(lineno)d) %(message)s',
+                      format='%(asctime)s | %(levelname)-5s | T%(thread)d | %(module)s.%(funcName)s:%(lineno)d: %(message)s',
                       datefmt='%Y%m%d%H%M%S',
                       handlers=handlers)
   logger = logging.getLogger('ck-flog')
@@ -45,33 +38,59 @@ def _file_log(sys_log_path: str = './', log_level: int = logging.INFO, mixin: bo
 def _console_log(log_level: int = logging.INFO) -> logging:
   handler = logging.StreamHandler()
   logging.basicConfig(level=log_level,
-                      format='%(asctime)s-%(levelname)s-%(thread)d-%(module)s(%(funcName)s:%(lineno)d) %(message)s',
+                      format='%(asctime)s | %(levelname)-5s | T%(thread)d | %(name)s | %(module)s.%(funcName)s:%(lineno)d: %(message)s',
                       datefmt='%Y%m%d%H%M%S',
                       handlers=[handler])
   logger = logging.getLogger('ck-clog')
   return logger
 
 
-class GlobalLogger(object):
-  def __init__(self, logger):
-    sys.stdout = self
-    sys.stderr = self
-    self.log = logger
+import io
+import logging
+
+
+class StreamToLogger(io.StringIO):
+  def __init__(self, logger: logging.Logger, level: int = logging.INFO):
+    super().__init__()
+    self.logger = logger
+    self.level = level
+    self._buffer = ''
 
-  def write(self, message):
-    if message == '\n': return
-    global neglect_keywords
-    for neglect_keyword in neglect_keywords:
-      if neglect_keyword in message: return
-    self.log.info(message)
+  def write(self, message: str):
+    if not message:
+      return
+    self._buffer += message
+    if '\n' in self._buffer:
+      lines = self._buffer.splitlines(keepends=False)
+      for line in lines:
+        if line.strip():
+          try:
+            self.logger.log(self.level, line.strip(), stacklevel=3)
+          except Exception:
+            self.logger.log(self.level, line.strip())
+      self._buffer = ''
 
   def flush(self):
-    pass
+    if self._buffer.strip():
+      try:
+        self.logger.log(self.level, self._buffer.strip(), stacklevel=3)
+      except Exception:
+        self.logger.log(self.level, self._buffer.strip())
+    self._buffer = ''
+
+  def fileno(self):
+    return sys.__stdout__.fileno()
 
 
 @call.init
 def _init_log() -> None:
   global flog, clog
-  flog = _file_log(sys_log_path=application.Server.sysLogPath, mixin=True, log_level=logging.INFO if application.Authorization.enabled else logging.DEBUG)
+  flog = _file_log(path_info.get_user_work_path(".ck/ck-py-log", mkdir=True), mixin=True, log_level=logging.DEBUG)
   clog = _console_log()
-  GlobalLogger(flog)
+  sys.stdout = StreamToLogger(flog, level=logging.INFO)
+  sys.stderr = StreamToLogger(flog, level=logging.ERROR)
+
+
+def setLevel(log_level=logging.INFO):
+  flog.setLevel(log_level)
+  clog.setLevel(log_level)

ctools/util/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:29'

ctools/util/cftp.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/1/24 08:53'
+
+import io
+import time
+from ftplib import FTP, error_perm, error_temp
+
+
+class CFTP:
+  """
+  with open('xx/xx.md', 'rb') as file:
+    ftp_host = 'x.x.x.x'
+    ftp_username = 'x'
+    ftp_password = 'x'
+    CFTP(ftp_host, ftp_username, ftp_password).upload_file_to_ftp('xx.md', file)
+  """
+
+  def __init__(self, host, username, password, timeout=30, max_retries=3, retry_delay=5):
+    self.host = host
+    self.username = username
+    self.password = password
+    self.ftp: FTP = None
+    self.timeout = timeout
+    self.max_retries = max_retries
+    self.retry_delay = retry_delay
+
+  def upload_file_to_ftp(self, file_name, file: io.BytesIO, ftp_directory='/'):
+    if not file_name: raise Exception('文件名不能为空')
+    if not file: raise Exception('文件不能为空')
+    for attempt in range(self.max_retries):
+      try:
+        if not self.ftp:
+          ftp = FTP(self.host, timeout=self.timeout)
+          ftp.login(self.username, self.password)
+          ftp.set_pasv(True)
+          self.ftp = ftp
+        try:
+          self.ftp.cwd(ftp_directory)
+        except error_perm:
+          print(f"FTP 目录不存在：{ftp_directory}")
+          self.ftp.mkd(ftp_directory)
+          print(f"FTP 目录创建成功：{ftp_directory}, 正在切换到目录：{ftp_directory}")
+          self.ftp.cwd(ftp_directory)
+        print(f"正在上传文件：{file_name}")
+        self.ftp.storbinary(f"STOR {file_name}", file)
+        self.ftp.quit()
+        print(f"文件成功上传到 FTP: {file_name}")
+        return
+      except (error_perm, error_temp) as e:
+        try:
+          self.ftp.quit()
+        except Exception:
+          pass
+        self.ftp = None
+        print(f"FTP 错误：{e}")
+        if attempt < self.max_retries - 1:
+          print(f"正在尝试重连... 第 {attempt + 1} 次重试...")
+          time.sleep(self.retry_delay)
+        else:
+          print("重试次数已用尽，上传失败。")
+          raise
+      except Exception as e:
+        try:
+          self.ftp.quit()
+        except Exception:
+          pass
+        self.ftp = None
+        print(f"连接或上传出现异常：{e}")
+        if attempt < self.max_retries - 1:
+          print(f"正在尝试重连... 第 {attempt + 1} 次重试...")
+          time.sleep(self.retry_delay)
+        else:
+          print("重试次数已用尽，上传失败。")
+          raise

ctools/util/cklock.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/18 15:46'
+
+import contextvars
+import threading
+from contextlib import contextmanager
+from functools import wraps
+
+from ctools.stream.credis import get_redis, add_lock, remove_lock
+from ctools.web import ctoken
+from ctools.web.api_result import R
+
+# 全局锁容器
+_lock_dict = {}
+_lock_dict_lock = threading.Lock()
+
+def try_acquire_lock(key: str) -> bool:
+  with _lock_dict_lock:
+    if key not in _lock_dict:
+      _lock_dict[key] = threading.Lock()
+    return _lock_dict[key].acquire(blocking=False)
+
+def try_acquire_lock_block(key: str):
+  with _lock_dict_lock:
+    if key not in _lock_dict:
+      _lock_dict[key] = threading.Lock()
+    _lock = _lock_dict[key]
+  _lock.acquire()  # 这里是阻塞的
+
+def release_lock(key: str):
+  with _lock_dict_lock:
+    _lock = _lock_dict.get(key)
+    if _lock and _lock.locked():
+      _lock.release()
+    if _lock and not _lock.locked():
+      _lock_dict.pop(key, None)
+
+@contextmanager
+def try_lock(key: str="sys_lock", block=False):
+  if not block:
+    acquired = try_acquire_lock(key)
+    try:
+      yield acquired
+    finally:
+      if acquired:
+        release_lock(key)
+  else:
+    try_acquire_lock_block(key)
+    try:
+      yield
+    finally:
+      release_lock(key)
+
+#annotation
+"""
+@lock("params.attr")
+"""
+# 上下文保存锁key集合
+current_locks = contextvars.ContextVar("current_locks", default=set())
+
+def lock(lock_attrs=None):
+  def decorator(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+      lock_key = ""
+      nonlocal lock_attrs
+      user_level_lock = False
+
+      if not lock_attrs:
+        user_id = ctoken.get_user_id()
+        if user_id:
+          user_level_lock = True
+          lock_key = f"USER_ID_LOCK_{user_id}"
+        else:
+          raise ValueError("请设置 lock_attrs 或使用 token！")
+
+      if not user_level_lock:
+        if isinstance(lock_attrs, str): lock_attrs = [lock_attrs]
+        try:
+          for attr in lock_attrs:
+            parts = attr.split(".")
+            if len(parts) != 2:
+              raise ValueError(f"lock_attr: {attr} 格式错误")
+            obj = kwargs.get(parts[0]) or args[0]
+            if obj is None:
+              raise ValueError(f"参数 {parts[0]} 不存在")
+            lock_key += f"_{getattr(obj, parts[1], None)}"
+        except Exception as e:
+          raise ValueError(f"生成锁键失败: {e}")
+
+      lock_set = current_locks.get()
+      if lock_key in lock_set:
+        return func(*args, **kwargs)
+      token = current_locks.set(lock_set | {lock_key})
+
+      try:
+        if not get_redis():
+          with try_lock(lock_key) as locked:
+            if not locked:
+              return R.error("操作过于频繁, 请稍后再试")
+            return func(*args, **kwargs)
+        else:
+          locked = add_lock(get_redis(), lock_key)
+          try:
+            if locked:
+              return func(*args, **kwargs)
+            else:
+              return R.error("操作过于频繁, 请稍后再试")
+          finally:
+            if locked:
+              remove_lock(get_redis(), lock_key)
+      finally:
+        current_locks.reset(token)
+    return wrapper
+  return decorator
+

ctools/util/config_util.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/16 14:19'
+
+"""
+config = load_config("application.ini")
+print(config)
+print(config.base.app_name)
+print(config.base.version)
+"""
+from configparser import ConfigParser
+
+cache = {}
+
+class AttrNoneNamespace:
+  def __init__(self):
+    pass
+  def __setattr__(self, key, value):
+    super().__setattr__(key, value)
+  def __getattr__(self, item):
+    return None
+
+def _convert_value(value: str):
+  val = value.strip()
+  if val.lower() in ('true', 'yes', 'on'):
+    return True
+  if val.lower() in ('false', 'no', 'off'):
+    return False
+  if val.isdigit():
+    return int(val)
+  try:
+    return float(val)
+  except ValueError:
+    return val
+
+def _config_to_object(config: ConfigParser):
+  result = AttrNoneNamespace()
+  for section in config.sections():
+    section_obj = AttrNoneNamespace()
+    for key, value in config.items(section):
+      setattr(section_obj, key, _convert_value(value))
+    setattr(result, section, section_obj)
+  return result
+
+def load_config(file_path):
+  if file_path in cache: return cache[file_path]
+  config = ConfigParser()
+  config.read(file_path)
+  cf = _config_to_object(config)
+  cache[file_path] = cf
+  return cf

ctools/util/env_config.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/5/16 16:42'
+
+import json
+import os
+from typing import Any, Optional
+
+from dotenv.main import DotEnv
+
+
+class Configuration:
+  """Manages configuration and environment variables for the MCP client."""
+
+  def __init__(self, dotenv_path: str = ".env") -> None:
+    """Initialize configuration with environment variables."""
+    if not os.path.exists(dotenv_path): raise FileNotFoundError(f"Could not find .env file at {dotenv_path}")
+    self.env = DotEnv(dotenv_path=dotenv_path)
+    for key, value in self.env.dict().items():
+      if value: os.environ[key] = value
+
+  def get_env(self, key: str, default: Optional[Any] = None) -> Any:
+    value = self.env.get(key)
+    if value:
+      val = value.strip().lower()
+      if val == "true": return True
+      if val == "false": return False
+      return value
+    value = os.getenv(key)
+    if value:
+      val = value.strip().lower()
+      if val == "true": return True
+      if val == "false": return False
+      return value
+    return default
+
+  def get_llm_api_key(self) -> str:
+    api_key = self.get_env("LLM_API_KEY")
+    if not api_key: raise ValueError("LLM_API_KEY not found in environment variables")
+    return api_key
+
+  def get_mcp_server_config(self) -> dict[str, Any]:
+    with open(self.get_env("MCP_CONFIG_PATH"), "r") as f:
+      return json.load(f)
+
+def bool_env(key, default):
+  value = os.getenv(key)
+  if value:
+    val = value.strip().lower()
+    if val == "true": return True
+    if val == "false": return False
+  return default
+
+def float_env(key, default):
+  value = os.getenv(key)
+  if value: return float(value)
+  return default
+
+def int_env(key, default):
+  value = os.getenv(key)
+  if value: return int(value)
+  return default

ctools/{html_soup.py → util/html_soup.py}

@@ -33,10 +33,3 @@ def table2list(html, include_header=True, recursive_find=True,
     row = [i.text for i in td]
     rows.append(row)
   return rows
-
-
-for e in table2list(open(r'C:\Users\hylink\Desktop\A-DESC.txt', encoding='utf-8'), table_attrs={'class_': '123'}):
-  print(e)
-
-for e in table2list(open(r'C:\Users\hylink\Desktop\DEMO.txt', encoding='utf-8'), recursive_find=False, header_cell_tag='div', table_tag='div', table_class='table', row_tag='div', cell_tag='div'):
-  print(e)

ctools/{http_utils.py → util/http_util.py}

@@ -4,19 +4,21 @@ import requests
 def get(url, params=None, headers=None):
   result = ""
   try:
-    response = requests.get(url, params=params, headers=headers, timeout=5)
+    response = requests.get(url, params=params, headers=headers, timeout=60, verify=False)
     response.raise_for_status()
     if response.status_code == 200:
       result = response.content
   except Exception as e:
     print("GET请求异常:", e)
+  if isinstance(result, bytes): return result.decode('utf-8')
   return result
 
 
 def post(url, data=None, json=None, headers=None, files=None):
   result = ""
-  response = requests.post(url, data=data, json=json, files=files, headers=headers, timeout=5)
+  response = requests.post(url, data=data, json=json, files=files, headers=headers, timeout=600, verify=False)
   response.raise_for_status()
   if response.status_code == 200:
     result = response.content
+  if isinstance(result, bytes): return result.decode('utf-8')
   return result

ctools/{images_tools.py → util/image_process.py}

@@ -1,10 +1,13 @@
+import base64
+from io import BytesIO
 
 from PIL import Image
-from io import BytesIO
+
 
 def get_size(image_path):
   return Image.open(image_path).size
 
+
 def change_color(image_path, area=None, rgb_color=None):
   """
   修改图片指定区域颜色
@@ -24,3 +27,9 @@ def change_color(image_path, area=None, rgb_color=None):
     img_binary = img_bytes.getvalue()
   return img_binary
 
+
+def img2b64(img: Image, fmt="PNG"):
+  buf = BytesIO()
+  img.save(buf, format=fmt.upper())
+  return base64.b64encode(buf.getvalue()).decode("utf-8")
+

ctools/util/jb_cut.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/15 13:08'
+
+import sys
+
+import jieba
+from jieba import posseg as pseg
+
+jqfx_exclude = ('ul', 'uj', 'uz', 'a', 'c', 'm', 'f', 'ad', 'an', 'r', 'q', 'u', 't', 'd', 'p', 'x')
+
+def add_dict(dic_word: str = None, dic_path: str = None):
+  """
+  添加自定义词库(自定义词库添加之后, 如果不适用全模式切词, 有时也不好使, 因为权重没有默认的高)
+  :param dic_word: 一个单词
+  :param dic_path: 单词表文件地址
+  单词表文件格式:
+  单词  词频  标签
+  单词1  3    i
+  单词2  3    i
+  单词3  3    i
+  """
+  if dic_word: jieba.add_word(dic_word)
+  if dic_path: jieba.load_userdict(dic_path)
+
+def add_freq(word: ()):
+  """
+  添加词频
+  :param word: 一个单词
+  """
+  jieba.suggest_freq(word, True)
+
+def get_declare_type_word(word: str, word_flag=(), use_paddle=False, exclude_flag=jqfx_exclude, exclude_word=()):
+  #import paddle
+  #paddle.enable_static()
+  #sys.stdout = sys.__stdout__
+  #sys.stderr = sys.__stderr__
+  """
+  获取声明类型的单词
+  :param word: 单词
+  :param word_flag: 标签
+  :param use_paddle: 是否使用 paddle
+  :param exclude_flag: 排除的标签
+  :param exclude_word: 排除的词
+  :return:  筛选之后的单词(数组)
+  """
+  # linux 可以开放下面的语句, 并发执行分词, windows 不支持
+  # if platform.system() == 'Linux': jieba.enable_parallel(4)
+  ret = []
+  for w, flag in pseg.cut(word, use_paddle=use_paddle):
+    if (flag in word_flag or len(word_flag) == 0) and flag not in exclude_flag and w not in exclude_word:
+      ret.append((w, flag))
+  return ret

ctools/{id_worker_tools.py → util/snow_id.py}

@@ -4,49 +4,43 @@ import time
 WORKER_ID_BITS = 5
 DATACENTER_ID_BITS = 5
 SEQUENCE_BITS = 12
-
 # 最大取值计算
 MAX_WORKER_ID = -1 ^ (-1 << WORKER_ID_BITS)  # 2**5-1 0b11111
 MAX_DATACENTER_ID = -1 ^ (-1 << DATACENTER_ID_BITS)
-
 # 移位偏移计算
-WOKER_ID_SHIFT = SEQUENCE_BITS
+WORKER_ID_SHIFT = SEQUENCE_BITS
 DATACENTER_ID_SHIFT = SEQUENCE_BITS + WORKER_ID_BITS
 TIMESTAMP_LEFT_SHIFT = SEQUENCE_BITS + WORKER_ID_BITS + DATACENTER_ID_BITS
-
 # 序号循环掩码
 SEQUENCE_MASK = -1 ^ (-1 << SEQUENCE_BITS)
-
 # Twitter元年时间戳
 TWEPOCH = 1288834974657
 
 
-class IdWorker(object):
+class SnowId(object):
   """
   用于生成IDs
   """
 
-  def __init__(self, datacenter_id, worker_id, sequence=0):
+  def __init__(self, datacenter_id=0, worker_id=0, sequence=0):
     """
     初始化
     :param datacenter_id: 数据中心（机器区域）ID
     :param worker_id: 机器ID
-    :param sequence: 其实序号
+    :param sequence: 起始序号
     """
     # sanity check
     if worker_id > MAX_WORKER_ID or worker_id < 0:
       raise ValueError('worker_id值越界')
-
     if datacenter_id > MAX_DATACENTER_ID or datacenter_id < 0:
       raise ValueError('datacenter_id值越界')
-
     self.worker_id = worker_id
     self.datacenter_id = datacenter_id
     self.sequence = sequence
-
     self.last_timestamp = -1  # 上次计算的时间戳
 
-  def _gen_timestamp(self):
+  @staticmethod
+  def _gen_timestamp():
     """
     生成整数时间戳
     :return:int timestamp
@@ -59,23 +53,18 @@ class IdWorker(object):
     :return:
     """
     timestamp = self._gen_timestamp()
-
     # 时钟回拨
     if timestamp < self.last_timestamp:
-      print('clock is moving backwards. Rejecting requests until {}'.format(self.last_timestamp))
-      raise
-
+      timestamp = self._til_next_millis(self.last_timestamp)
     if timestamp == self.last_timestamp:
       self.sequence = (self.sequence + 1) & SEQUENCE_MASK
       if self.sequence == 0:
         timestamp = self._til_next_millis(self.last_timestamp)
     else:
       self.sequence = 0
-
     self.last_timestamp = timestamp
-
     new_id = ((timestamp - TWEPOCH) << TIMESTAMP_LEFT_SHIFT) | (self.datacenter_id << DATACENTER_ID_SHIFT) | \
-             (self.worker_id << WOKER_ID_SHIFT) | self.sequence
+             (self.worker_id << WORKER_ID_SHIFT) | self.sequence
     return new_id
 
   def _til_next_millis(self, last_timestamp):
@@ -86,7 +75,3 @@ class IdWorker(object):
     while timestamp <= last_timestamp:
       timestamp = self._gen_timestamp()
     return timestamp
-
-# if __name__ == '__main__':
-#   worker = IdWorker(1, 2, 0)
-#   print(worker.get_id())

ctools/web/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:12'