gomyck-tools 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

ctools/credis.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/2/14 11:09'
+
+import redis
+from redis import Redis
+
+from ctools import date_utils, thread_pool, string_tools
+
+def init_pool(host: str = 'localhost', port: int = 6379, db: int = 0, password: str = None,
+              username: str = None, decode_responses: bool = True, max_connections: int = 75,
+              health_check_interval: int = 30, retry_count: int = 3) -> Redis:
+    for attempt in range(retry_count):
+        try:
+            r: Redis = redis.StrictRedis(
+                host=host, port=port, db=db,
+                username=username, password=password,
+                retry_on_timeout=True,
+                max_connections=max_connections,
+                decode_responses=decode_responses,
+                health_check_interval=health_check_interval,
+                socket_connect_timeout=5,
+                socket_timeout=5
+            )
+            if r.ping():
+                print('CRedis connect {} {} success!'.format(host, port))
+                return r
+        except redis.ConnectionError as e:
+            if attempt == retry_count - 1:
+                raise Exception(f"Failed to connect to Redis after {retry_count} attempts: {str(e)}")
+            print(f"Connection attempt {attempt + 1} failed, retrying...")
+
+def add_lock(r: Redis, key: str, timeout: int = 30):
+  if r.exists(key):
+    expire_time = r.get(key)
+    if date_utils.time_diff_in_seconds(expire_time, date_utils.get_date_time()) > 0:
+      return True
+    else:
+      r.delete(key)
+  return r.set(key, date_utils.opt_time(seconds=timeout), nx=True, ex=timeout) is not None
+
+def remove_lock(r: Redis, key: str):
+  r.delete(key)
+
+def subscribe(r: Redis, channel_name, callback):
+  def thread_func():
+    pubsub = r.pubsub()
+    pubsub.subscribe(channel_name)
+    for message in pubsub.listen():
+      callback(message)
+  thread_pool.submit(thread_func)
+
+def _process_pending_messages(r: Redis, stream_name: str, group_name: str, consumer_name: str, callback):
+  """
+  处理未确认的消息
+  :param r: Redis 连接
+  :param stream_name: 流名称
+  :param group_name: 消费者组名称
+  :param consumer_name: 消费者名称
+  :param callback: 消息处理回调函数
+  """
+  # 检查未确认的消息
+  pending_messages = r.xpending(stream_name, group_name)
+  if pending_messages['pending'] > 0:
+    print(f"Found {pending_messages['pending']} pending messages.")
+    # 获取未确认的消息列表
+    pending_list = r.xpending_range(stream_name, group_name, min='-', max='+', count=pending_messages['pending'])
+    for message in pending_list:
+      message_id = message['message_id']
+      claimed_messages = r.xclaim(stream_name, group_name, consumer_name, min_idle_time=0, message_ids=[message_id])
+      if claimed_messages:
+        # 处理消息
+        for claimed_message in claimed_messages:
+          message_id, data = claimed_message
+          print(f"Processing pending message: {message_id}, data: {data}")
+          try:
+            if callback(message_id, data):
+              r.xack(stream_name, group_name, message_id)
+          except Exception as e:
+            print(f"Error processing message {message_id}: {e}")
+  else:
+    print("No pending messages found.")
+
+def stream_subscribe(r: Redis, stream_name, group_name, callback, from_id: str='$', noack: bool = False):
+  def thread_func():
+    try:
+      # $表示从最后面消费, 0表示从开始消费
+      r.xgroup_create(name=stream_name, groupname=group_name, id=from_id, mkstream=True)
+      print(f"Consumer group '{group_name}' created successfully.")
+    except Exception as e:
+      if "already exists" in str(e):
+        print(f"Consumer group '{group_name}' already exists.")
+      else:
+        print(f"Error creating consumer group '{group_name}': {e}")
+    consumer_name = 'consumer-{}'.format(string_tools.get_uuid())
+    # 处理未确认的消息
+    _process_pending_messages(r, stream_name, group_name, consumer_name, callback)
+    while True:
+      messages = r.xreadgroup(group_name, consumer_name, {stream_name: '>'}, block=1000, noack=noack)
+      for message in messages:
+        try:
+          message_id, data = message[1][0]
+          res = callback(message_id, data)
+          if res: r.xack(stream_name, group_name, message_id)
+        except Exception as e:
+          print('stream_subscribe error: ', e)
+  thread_pool.submit(thread_func)
+
+def stream_publish(r: Redis, stream_name, message):
+  r.xadd(stream_name, message)

ctools/cron_lite.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3.6
+# coding: utf-8
+import sched
+import threading
+import time
+import traceback
+from datetime import datetime
+from functools import wraps
+from typing import Optional, Dict
+
+import pytz
+from croniter import croniter
+
+"""
+@cron_lite.cron_task('0/1 * * * * ? *')
+def demo():
+  print('hello world')
+
+@cron_lite.cron_task('0/1 * * * * ? *')
+def demo1():
+  print('hello world111')
+
+def demo2(xx, fff):
+  print('hello world222', xx, fff)
+
+cron_lite.reg_cron_task('0/1 * * * * ? *', demo2, (123123123, 34534534))
+print(123123)
+
+cron_lite.start_all()
+"""
+
+class SchedulerMeta:
+  timer_task_name: str = None
+  switch: bool = True
+  status: bool = False
+  event: sched.Event = None
+  scheduler: sched.scheduler = None
+  cron_str: str = None
+
+
+scheduler_map: Dict[str, SchedulerMeta] = {}  # {timer_task_name: SchedulerMeta}
+_switch = False
+_info_handler  = print
+_error_handler = print
+_time_zone: Optional[pytz.BaseTzInfo] = pytz.timezone("Asia/Shanghai")
+
+
+def set_time_zone(time_zone_name: str):
+  global _time_zone
+  _time_zone = pytz.timezone(time_zone_name)
+
+# @annotation
+def cron_task(cron_expr: str, task_name: str = None, till_time_stamp: int = None):
+  """
+  cron_task decorator to register a function as crontab task
+  :param cron_expr: the croniter accepted cron_expression. NOTICE: the default timezone is UTC and can be changed by
+  `set_time_zone`. The format is `min hour day month weekday [sec]`
+  :param till_time_stamp: run this jog till when. None means forever
+  :return: the real decorator
+  """
+  cron_expr = _convert_cron(cron_expr)
+  assert len(cron_expr.split(" ")) in (5, 6), \
+    "only supported <min hour day month weekday> and <min hour day month weekday sec>"
+
+  def deco(func):
+    @wraps(func)
+    def inner():
+      try:
+        func()
+      except Exception:
+        try:
+          _error_handler(f"run {func.__name__} failed\n" + traceback.format_exc())
+        except Exception:
+          _error_handler(f"run {func.__name__} failed\n")
+      _register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp)
+
+    _register_next(inner.__name__ if task_name is None else task_name, inner, cron_expr, till_time_stamp, init=True)
+    return inner
+
+  return deco
+
+
+def reg_cron_task(cron_expr, func, params, timer_task_name=None, till_time_stamp=None):
+  """
+  cron_task decorator to register a function as crontab task
+  :param func: task callback function
+  :param params: transparent parameters
+  :param cron_expr: the croniter accepted cron_expression. NOTICE: the default timezone is UTC and can be changed by
+  `set_time_zone`. The format is `min hour day month weekday [sec]`
+  :param timer_task_name: task name
+  :param till_time_stamp: run this jog till when. None means forever
+  :return: the real decorator
+  """
+  cron_expr = _convert_cron(cron_expr)
+  assert len(cron_expr.split(" ")) in (5, 6),  "Only supported <minute hour day month weekday> and <minute hour day month weekday second>"
+  task_name = func.__name__ if timer_task_name is None else timer_task_name
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    try:
+      nonlocal params
+      func.__taskName__ = task_name
+      func(*params, *args, **kwargs)
+    except Exception as exc:
+      _error_handler(f"Run {func.__name__} failed with error: {str(exc)}")
+    finally:
+      _register_next(task_name, wrapper, cron_expr, till_time_stamp)
+
+  _register_next(task_name, wrapper, cron_expr, till_time_stamp, init=True)
+
+def start_all(spawn: bool = True, daemon: bool = True, info_handler=None, error_handler=None) -> Optional[threading.Thread]:
+  """
+  start_all starts all cron tasks registered before.
+  :param spawn: whether to start a new thread for scheduler. If not, the action will block the current thread
+  :param daemon: the new thread is daemon if True
+  :param info_handler: handle info output (scheduler start / stop), default = print, can use logging.info
+  :param error_handler: handle error output (task execute exception), default = print, can use logging.error
+  :raise RuntimeError: if the tasks are already started and still running we cannot start again. The feature is not
+  concurrent-safe
+  :return: the new thread if spawn = True
+  """
+  global _switch, _info_handler, _error_handler
+  if _switch:
+    raise RuntimeError("the crontab was already started...")
+  if info_handler:
+    _info_handler = info_handler
+  if error_handler:
+    _error_handler = error_handler
+  if spawn:
+    t = threading.Thread(target=_start, daemon=daemon)
+    t.start()
+    return t
+  else:
+    _start()
+
+
+def is_active(timer_task_name):
+  res = False
+  if timer_task_name in scheduler_map:
+    res = scheduler_map.get(timer_task_name).switch or scheduler_map.get(timer_task_name).status
+  return res
+
+
+def active(timer_task_name):
+  if timer_task_name in scheduler_map:
+    scheduler_map.get(timer_task_name).status = True
+
+def get_switch(timer_task_name):
+  switch = True
+  if timer_task_name in scheduler_map:
+    switch = scheduler_map.get(timer_task_name).switch
+  return switch
+
+
+def inactive(timer_task_name):
+  if timer_task_name in scheduler_map:
+    scheduler_map.get(timer_task_name).status = False
+    if not scheduler_map.get(timer_task_name).switch:
+      scheduler_map.get(timer_task_name).scheduler.cancel(scheduler_map[timer_task_name].event)
+
+
+def stop(timer_task_name):
+  if timer_task_name in scheduler_map:
+    scheduler_map.get(timer_task_name).switch = False
+    time.sleep(1)
+
+
+def stop_all(wait_thread: Optional[threading.Thread] = None):
+  """
+  stop_all turns off the switch to stop the scheduler. Running jobs will be wait till finished.
+  :param wait_thread: join() the spawned scheduler thread (if you started it as spawn and you want) to ensure all jobs
+  to finish
+  :return:
+  """
+  for timer_task_name in scheduler_map:
+    scheduler_map.get(timer_task_name).switch = False
+  if wait_thread:
+    wait_thread.join()
+
+
+def _register_next(timer_task_name, base_func, cron_expr, till_time_stamp, init: bool = False):
+  cron_obj = croniter(cron_expr)
+  if _time_zone:
+    cron_obj.set_current(datetime.now(tz=_time_zone))
+  next_time = int(cron_obj.get_next())
+  if scheduler_map.get(timer_task_name) is None:
+    scheduler_meta = SchedulerMeta()
+    scheduler_meta.timer_task_name = timer_task_name
+    scheduler_meta.switch = True
+    scheduler_meta.scheduler = sched.scheduler(time.time, time.sleep)
+    scheduler_meta.cron_str = cron_expr
+    scheduler_map[timer_task_name] = scheduler_meta
+  elif init:
+    raise ValueError(f"task name: {timer_task_name} already exists!!!!!")
+  if till_time_stamp is None or next_time <= till_time_stamp:
+    scheduler_map[timer_task_name].event = scheduler_map[timer_task_name].scheduler.enterabs(next_time, 0, base_func)
+
+
+def _run_sched(scheduler_meta: SchedulerMeta):
+  active(scheduler_meta.timer_task_name)
+  while True:
+    scheduler = scheduler_meta.scheduler
+    if not _switch or not scheduler_meta.switch:
+      scheduler.empty()
+      inactive(scheduler_meta.timer_task_name)
+      return
+    t = scheduler.run(False)
+    if t is None:
+      inactive(scheduler_meta.timer_task_name)
+      return
+    st = time.time()
+    while time.time() - st < t:
+      if not _switch or not scheduler_meta.switch:
+        scheduler.empty()
+        inactive(scheduler_meta.timer_task_name)
+        return
+      time.sleep(0.5)
+
+def _start(taskName: str = None):
+  global _switch
+  _switch = True
+  _info_handler("cron job begin start...")
+  taskList = []
+  for timer_task_name, scheduler_meta in scheduler_map.items():
+    if taskName is not None and timer_task_name != taskName: continue
+    print("register job: ", timer_task_name, ", cron: ", scheduler_meta.cron_str)
+    thread = threading.Thread(target=_run_sched, args=(scheduler_meta, ), daemon=True)
+    thread.start()
+    taskList.append(thread)
+  for task in taskList: task.join()
+  _info_handler("cron job execute finished...")
+  _switch = False
+  scheduler_map.clear()
+
+def _convert_cron(cron_expr):
+  res_cron = ""
+  cron_list = cron_expr.split(" ")
+  if len(cron_list) > 6:
+    for cron in cron_list[1:]:
+      if cron != "?":
+        res_cron += "%s " % cron
+    res_cron += "%s" % cron_list[0]
+  else:
+    res_cron = cron_expr
+  return res_cron
+

ctools/ctoken.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/1/21 16:01'
+
+import time
+import jwt
+from bottle import request
+
+from ctools.dict_wrapper import DictWrapper
+
+token_header = 'Authorization'
+
+def gen_token(payload: {}, secret_key, expired: int=3600) -> str:
+  payload.update({'exp': time.time() + expired})
+  return jwt.encode(payload, secret_key, algorithm='HS256')
+
+def get_payload(token, secret_key):
+  try:
+    payload = jwt.decode(token, secret_key, algorithms=['HS256'])
+    return DictWrapper(payload)
+  except Exception as e:
+    return None
+
+def get_token(key):
+  return get_payload(request.get_header(token_header), key)
+
+def is_valid(key):
+  return get_payload(request.get_header(token_header), key) is not None
+
+# if __name__ == '__main__':
+#   token = gen_token({"xx": 123}, '123')
+#   xx = get_payload(token, '123')
+#   print(xx.xx)

ctools/cword.py

@@ -0,0 +1,30 @@
+# from docxtpl import DocxTemplate
+#
+# # tpl = DocxTemplate('/Users/haoyang/Desktop/xxx.docx')
+# tpl = DocxTemplate('/Users/haoyang/Desktop/123.doc')
+#
+# # 设置好各标签需要填写的内容
+# context = {'xxxx': '计算机科学与技术', 'cccc': '2022050513'}
+# # 将标签内容填入模板中
+# tpl.render(context)
+# # 保存
+# tpl.save('/Users/haoyang/Desktop/new_test2.docx')
+
+from docx import Document
+
+def merge_word_files(input_files: [], output_file: str):
+    merged_doc = Document()
+    for file in input_files:
+        doc = Document(file)
+        for element in doc.element.body:
+            merged_doc.element.body.append(element)
+    merged_doc.save(output_file)
+
+
+def read_word_file(input_file: str):
+  doc = Document(input_file)
+  text = []
+  for paragraph in doc.paragraphs:
+    text.append(paragraph.text)
+  return "\n".join(text)
+

ctools/czip.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/1/24 08:48'
+
+import io
+import os
+import time
+import pyzipper
+
+"""
+  target_directory = '/Users/haoyang/Desktop/知识库文件'
+  zip_password = None
+  process_directory_to_single_zip(target_directory, zip_password, "knowledge_base")
+
+  files_to_compress = [
+    '/path/to/file1.txt',
+    '/path/to/file2.pdf',
+    '/path/to/file3.jpg'
+  ]
+  output_directory = '/Users/haoyang/Desktop'
+  compress_specific_files(files_to_compress, output_directory, zip_password, "my_files")
+"""
+def create_zip_with_files(file_dict, password=None) -> io.BytesIO:
+  """Compress multiple files into a single password-protected ZIP archive in memory.
+  Args:
+      file_dict: Dictionary of {filename: file_content} pairs
+                 filename = os.path.relpath(file_path, start=root_dir) # 相对路径获取, 用于在 zip 内的路径定位
+      password: Optional password for the ZIP file
+  Returns:
+      BytesIO object containing the ZIP file
+  """
+  zip_buffer = io.BytesIO()
+  try:
+    if password:
+      with pyzipper.AESZipFile(zip_buffer, 'w', compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES) as zipf:
+        zipf.setpassword(password.encode('utf-8'))
+        for filename, content in file_dict.items():
+          zipf.writestr(filename, content)
+    else:
+      with pyzipper.ZipFile(zip_buffer, 'w', compression=pyzipper.ZIP_DEFLATED) as zipf:
+        for filename, content in file_dict.items():
+          zipf.writestr(filename, content)
+    zip_buffer.seek(0)
+    return zip_buffer
+  except Exception as e:
+    zip_buffer.close()
+    raise e
+
+
+def process_directory_to_single_zip(root_dir, password=None, zip_name=None):
+  """Walk through directory and compress all files into a single ZIP.
+  Args:
+      root_dir: Root directory to scan for files
+      password: Optional password for the ZIP file
+      zip_name: Base name for the ZIP file (without extension)
+  """
+  file_dict = {}
+  for dirpath, _, filenames in os.walk(root_dir):
+    for filename in filenames:
+      file_path = os.path.join(dirpath, filename)
+      try:
+        with open(file_path, 'rb') as f:
+          rel_path = os.path.relpath(file_path, start=root_dir)
+          file_dict[rel_path] = f.read()
+      except Exception as e:
+        print(f"Error reading {file_path}: {str(e)}")
+  if not file_dict:
+    print("No files found to compress.")
+    return
+
+  try:
+    zip_buffer = create_zip_with_files(file_dict, password)
+    timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
+    if zip_name:
+      base_name = f"{zip_name}_{timestamp}.zip"
+    else:
+      base_name = f"archive_{timestamp}.zip"
+    output_path = os.path.join(root_dir, base_name)
+    with open(output_path, 'wb') as out_file:
+      out_file.write(zip_buffer.read())
+    print(f"Created single archive: {output_path}")
+  except Exception as e:
+    print(f"Error creating ZIP archive: {str(e)}")
+  finally:
+    if 'zip_buffer' in locals(): zip_buffer.close()
+
+
+def compress_specific_files(file_paths:[], output_dir:str, password=None, zip_name=None):
+  """Compress multiple specified files into a single ZIP archive.
+  Args:
+      file_paths: List of absolute file paths to compress
+      output_dir: Directory where the ZIP file will be saved
+      password: Optional password for the ZIP file
+      zip_name: Base name for the ZIP file (without extension)
+  """
+  if not file_paths:
+    print("No files specified to compress.")
+    return
+  file_dict = {}
+  for file_path in file_paths:
+    if not os.path.isfile(file_path):
+      print(f"Warning: {file_path} is not a file or doesn't exist. Skipping.")
+      continue
+    try:
+      with open(file_path, 'rb') as f:
+        filename_in_zip = os.path.basename(file_path)
+        file_dict[filename_in_zip] = f.read()
+    except Exception as e:
+      print(f"Error reading {file_path}: {str(e)}")
+  if not file_dict:
+    print("No valid files found to compress.")
+    return
+  try:
+    zip_buffer = create_zip_with_files(file_dict, password)
+    timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
+    if zip_name:
+      base_name = f"{zip_name}_{timestamp}.zip"
+    else:
+      first_file = os.path.basename(file_paths[0])
+      base_name = f"{os.path.splitext(first_file)[0]}_{timestamp}.zip"
+    output_path = os.path.join(output_dir, base_name)
+    os.makedirs(output_dir, exist_ok=True)
+    with open(output_path, 'wb') as out_file:
+      out_file.write(zip_buffer.read())
+    print(f"Created archive: {output_path}")
+  except Exception as e:
+    print(f"Error creating ZIP archive: {str(e)}")
+  finally:
+    if 'zip_buffer' in locals(): zip_buffer.close()