gomyck-tools 1.0.0__py3-none-any.whl → 1.4.7__py3-none-any.whl

ctools/pkg/dynamic_imp.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/7/15 11:03'
+
+import importlib
+import pkgutil
+
+
+def load_modules_from_package(package, exclude=None, recursive=True):
+  """
+  递归加载指定包下所有模块（不包括包本身）
+
+  :param package: 要加载模块的包对象（如 mypkg.plugins）
+  :param exclude: 排除的模块完整路径列表（如 ['mypkg.plugins.demo.mod2']）
+  :param recursive: 是否递归子包
+  :return: 模块列表（不含子包本身，只包含模块）
+  """
+  if exclude is None: exclude = []
+  modules = []
+  for finder, modname, ispkg in pkgutil.iter_modules(package.__path__):
+    full_modname = f"{package.__name__}.{modname}"
+    if ispkg and recursive:
+      try:
+        subpkg = importlib.import_module(full_modname)
+        modules.extend(load_modules_from_package(subpkg, exclude, recursive))
+      except Exception as e:
+        print(f"递归子包 {full_modname} 失败：{e}")
+      continue
+    if full_modname in exclude:
+      continue
+    try:
+      module = importlib.import_module(full_modname)
+      modules.append(module)
+    except Exception as e:
+      print(f"!!!!!!加载模块 {full_modname} 失败：{e}!!!!!!")
+      continue
+  return modules

ctools/pools/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:18'

ctools/pools/process_pool.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/9/20 12:00'
+
+import os
+import time
+from concurrent.futures import ProcessPoolExecutor
+
+from ctools import call
+
+_process_pool: ProcessPoolExecutor = None
+
+
+@call.init
+def init():
+  global _process_pool
+  max_workers = min(32, os.cpu_count())  # 最多 32 个
+  _process_pool = ProcessPoolExecutor(max_workers=max_workers)
+
+
+def cb(f, callback):
+  exc = f.exception()
+  if exc:
+    if callback: callback(exc)
+    raise exc
+  else:
+    if callback: callback(f.result())
+
+
+def submit(func, *args, callback=None, **kwargs):
+  if _process_pool is None: raise Exception('process pool is not init')
+  future = _process_pool.submit(func, *args, **kwargs)
+  future.add_done_callback(lambda f: cb(f, callback))
+  time.sleep(0.01)
+  return future
+
+
+def shutdown(wait=True):
+  if _process_pool is None: raise Exception('process pool is not init')
+  _process_pool.shutdown(wait=wait)

ctools/{thread_pool.py → pools/thread_pool.py}

@@ -1,30 +1,39 @@
 import os
+import threading
 import time
 from concurrent.futures import ThreadPoolExecutor
-import threading
+
 from ctools import call
 
 thread_local = threading.local()
 
 _threadPool: ThreadPoolExecutor = None
 
+
 @call.init
 def init():
   global _threadPool
-  max_work_num = min(32, (os.cpu_count() or 1) + 4) # 最多 32 个
-  _threadPool = ThreadPoolExecutor(max_workers= max_work_num, thread_name_prefix='ck-')
+  max_work_num = min(32, (os.cpu_count() or 1) + 4)  # 最多 32 个
+  _threadPool = ThreadPoolExecutor(max_workers=max_work_num, thread_name_prefix='ck-')
+
 
 def cb(f, callback):
   exc = f.exception()
   if exc:
+    print(f"Task failed: {exc}")
     if callback: callback(exc)
-    raise exc
   else:
     if callback: callback(f.result())
 
+
 def submit(func, *args, callback=None, **kwargs):
   if _threadPool is None: raise Exception('thread pool is not init')
   future = _threadPool.submit(func, *args, **kwargs)
   future.add_done_callback(lambda f: cb(f, callback))
   time.sleep(0.01)
   return future
+
+
+def shutdown(wait=True):
+  if _threadPool is None: raise Exception('thread pool is not init')
+  _threadPool.shutdown(wait=wait)

ctools/similar.py

@@ -0,0 +1,25 @@
+from fuzzywuzzy import fuzz, process
+
+
+def best_match(query: str, choices: list[str], score_cutoff: int = 70):
+  """
+  获取最接近 query 的匹配项
+  :return: (匹配项, 相似度得分) 或 None
+  """
+  return process.extractOne(query, choices, scorer=fuzz.ratio, score_cutoff=score_cutoff)
+
+
+def all_matches(query: str, choices: list[str], limit: int = 5):
+  """
+  获取多个相似匹配项
+  :return: [(匹配项, 相似度得分), ...]
+  """
+  return process.extract(query, choices, scorer=fuzz.ratio, limit=limit)
+
+
+def is_similar(s1: str, s2: str, threshold: int = 85):
+  """
+  判断两个字符串是否相似
+  :return: True / False
+  """
+  return fuzz.ratio(s1, s2) >= threshold

ctools/stream/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/6/11 09:21'

ctools/stream/ckafka.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/9/5 10:39'
+
+import time
+from threading import Thread
+
+from kafka import KafkaProducer, errors, KafkaConsumer
+from kafka.producer.future import FutureRecordMetadata
+
+from ctools.cjson import dumps
+
+"""
+import time
+from datetime import datetime
+
+from ctools import thread_pool, string_tools
+from ctools.ckafka import CKafka
+
+c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
+
+producer = c.init_producer()
+consumer = c.init_consumer(enable_auto_commit=False)
+
+def send_msg():
+  while True:
+    command = input('发送消息: Y/n \n')
+    if command.strip() not in ['N', 'n']:
+      producer.send_msg('jqxx', '{{"jqid": "{}", "xxxx": "{}"}}'.format(string_tools.get_snowflake_id(), datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')))
+    else:
+      break
+
+thread_pool.submit(send_msg)
+
+def consumer_callback(msg):
+  print(msg)
+  return True
+
+consumer.receive_msg('jqxx', callBack=consumer_callback)
+
+while True: time.sleep(1)
+"""
+
+
+class KafkaInstance:
+  def __init__(self, producer: KafkaProducer, consumer: KafkaConsumer):
+    self.start_consumer = False
+    self.quited = False
+    self.producer = producer
+    self.consumer = consumer
+    self.consumer_callback = {"topic_key": []}
+
+  # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
+  # r.add_callback(lambda x: print(x))
+  # r.get() 可以同步获取结果
+  def send_msg(self, topic, msg, key: str = None, partition: int = None) -> FutureRecordMetadata:
+    if self.producer is None: raise RuntimeError("Producer is not initialized")
+    if self.quited: return
+    return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
+
+  def receive_msg(self, topics: str, callBack=print):
+    if self.consumer is None: raise RuntimeError("Consumer is not initialized")
+    for topic in topics.split(','):
+      if topic not in self.consumer_callback.keys():
+        self.consumer_callback[topic] = []
+        self.consumer.subscribe(self.consumer_callback.keys())
+      self.consumer_callback[topic].append(callBack)
+    if not self.start_consumer:
+      t = Thread(target=self._start_consumer_poll, daemon=True)
+      t.start()
+
+  def _start_consumer_poll(self):
+    self.start_consumer = True
+    for msg in self.consumer:
+      if self.quited: break
+      funcList = []
+      begin_time = time.time()
+      for func in self.consumer_callback[msg.topic]:
+        if self.quited: break
+        res = func(msg)
+        if not self.consumer.config['enable_auto_commit'] and res: self.consumer.commit()
+        funcList.append(func.__name__)
+      end_time = time.time()
+      if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
+      funcList.clear()
+
+  def shutdown(self):
+    self.quited = True
+    try:
+      self.consumer.close()
+    except Exception:
+      pass
+    try:
+      self.producer.close()
+    except Exception:
+      pass
+
+
+class CKafka:
+
+  def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password'):
+    self.kafka_url = kafka_url
+    self.secure = secure
+    self.username = username
+    self.password = password
+
+  def init_producer(self, acks=1) -> KafkaInstance:
+    print("[ Producer ] Connecting to Kafka [{}]".format(self.kafka_url))
+    for i in range(0, 6):
+      try:
+        if self.secure:
+          producer = KafkaProducer(
+            acks=acks,
+            bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8'),
+            sasl_plain_username=self.username,
+            sasl_plain_password=self.password,
+            security_protocol='SASL_PLAINTEXT',
+            sasl_mechanism='PLAIN'
+          )
+        else:
+          producer = KafkaProducer(
+            acks=acks,
+            bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8')
+          )
+        print("[ Producer ] Success Connected to Kafka [{}]".format(self.kafka_url))
+        return KafkaInstance(producer=producer, consumer=None)
+      except errors.NoBrokersAvailable:
+        print("[ Producer ] Waiting for Kafka [{}] to become available...".format(self.kafka_url))
+        time.sleep(3)
+    raise RuntimeError("[ Producer ] Failed to connect to Kafka [{}] within 60 seconds".format(self.kafka_url))
+
+  def init_consumer(self, client_id: str = 'ck-py-kafka-consumer', consumer_group: str = 'ck-py-kafka-consumer', enable_auto_commit: bool = True) -> KafkaInstance:
+    print("[ Consumer ] Connecting to Kafka [{}]".format(self.kafka_url))
+    for i in range(0, 6):
+      try:
+        if self.secure:
+          consumer = KafkaConsumer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
+            bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8'),
+            sasl_plain_username=self.username,
+            sasl_plain_password=self.password,
+            security_protocol='SASL_PLAINTEXT',
+            sasl_mechanism='PLAIN'
+          )
+        else:
+          consumer = KafkaProducer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
+            bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8')
+          )
+        print("[ Consumer ] Success Connected to Kafka [{}]".format(self.kafka_url))
+        return KafkaInstance(producer=None, consumer=consumer)
+      except errors.NoBrokersAvailable:
+        print("[ Consumer ] Waiting for Kafka [{}] to become available...".format(self.kafka_url))
+        time.sleep(3)
+    raise RuntimeError("[ Consumer ] Failed to connect to Kafka [{}] within 60 seconds".format(self.kafka_url))

ctools/stream/credis.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/2/14 11:09'
+
+import redis
+from redis import Redis
+
+from ctools import cdate, cid
+from ctools.pools import thread_pool
+
+# 最后一次连接的redis
+_ck_redis: Redis = None
+
+def get_redis(): return _ck_redis
+
+def init_pool(host: str = 'localhost', port: int = 6379, db: int = 0, password: str = None,
+              username: str = None, decode_responses: bool = True, max_connections: int = 75,
+              health_check_interval: int = 30, retry_count: int = 3) -> Redis:
+  for attempt in range(retry_count):
+    try:
+      r: Redis = redis.StrictRedis(
+        host=host, port=port, db=db,
+        username=username, password=password,
+        retry_on_timeout=True,
+        max_connections=max_connections,
+        decode_responses=decode_responses,
+        health_check_interval=health_check_interval,
+        socket_connect_timeout=5,
+        socket_timeout=5
+      )
+      if r.ping():
+        print('CRedis connect {} {} success!'.format(host, port))
+        global _ck_redis
+        _ck_redis = r
+        return _ck_redis
+    except redis.ConnectionError as e:
+      if attempt == retry_count - 1:
+        raise Exception(f"Failed to connect to Redis after {retry_count} attempts: {str(e)}")
+      print(f"Connection attempt {attempt + 1} failed, retrying...")
+
+
+def add_lock(r: Redis, key: str, timeout: int = 30):
+  if r.exists(key):
+    expire_time = r.get(key)
+    if expire_time and cdate.time_diff_in_seconds(expire_time, cdate.get_date_time()) > 0:
+      return False
+    else:
+      r.delete(key)
+  return r.set(key, cdate.opt_time(seconds=timeout), nx=True, ex=timeout) is not None
+
+
+def remove_lock(r: Redis, key: str):
+  r.delete(key)
+
+
+def subscribe(r: Redis, channel_name, callback):
+  def thread_func():
+    pubsub = r.pubsub()
+    pubsub.subscribe(channel_name)
+    for message in pubsub.listen():
+      callback(message)
+
+  thread_pool.submit(thread_func)
+
+
+def _process_pending_messages(r: Redis, stream_name: str, group_name: str, consumer_name: str, callback):
+  """
+  处理未确认的消息
+  :param r: Redis 连接
+  :param stream_name: 流名称
+  :param group_name: 消费者组名称
+  :param consumer_name: 消费者名称
+  :param callback: 消息处理回调函数
+  """
+  # 检查未确认的消息
+  pending_messages = r.xpending(stream_name, group_name)
+  if pending_messages['pending'] > 0:
+    print(f"Found {pending_messages['pending']} pending messages.")
+    # 获取未确认的消息列表
+    pending_list = r.xpending_range(stream_name, group_name, min='-', max='+', count=pending_messages['pending'])
+    for message in pending_list:
+      message_id = message['message_id']
+      claimed_messages = r.xclaim(stream_name, group_name, consumer_name, min_idle_time=0, message_ids=[message_id])
+      if claimed_messages:
+        # 处理消息
+        for claimed_message in claimed_messages:
+          message_id, data = claimed_message
+          print(f"Processing pending message: {message_id}, data: {data}")
+          try:
+            if callback(message_id, data):
+              r.xack(stream_name, group_name, message_id)
+          except Exception as e:
+            print(f"Error processing message {message_id}: {e}")
+  else:
+    print("No pending messages found.")
+
+
+def stream_subscribe(r: Redis, stream_name, group_name, callback, from_id: str = '$', noack: bool = False):
+  def thread_func():
+    try:
+      # $表示从最后面消费, 0表示从开始消费
+      r.xgroup_create(name=stream_name, groupname=group_name, id=from_id, mkstream=True)
+      print(f"Consumer group '{group_name}' created successfully.")
+    except Exception as e:
+      if "already exists" in str(e):
+        print(f"Consumer group '{group_name}' already exists.")
+      else:
+        print(f"Error creating consumer group '{group_name}': {e}")
+    consumer_name = 'consumer-{}'.format(cid.get_uuid())
+    # 处理未确认的消息
+    _process_pending_messages(r, stream_name, group_name, consumer_name, callback)
+    while True:
+      messages = r.xreadgroup(group_name, consumer_name, {stream_name: '>'}, block=1000, noack=noack)
+      for message in messages:
+        try:
+          message_id, data = message[1][0]
+          res = callback(message_id, data)
+          if res: r.xack(stream_name, group_name, message_id)
+        except Exception as e:
+          print('stream_subscribe error: ', e)
+
+  thread_pool.submit(thread_func)
+
+
+def stream_publish(r: Redis, stream_name, message):
+  r.xadd(stream_name, message)

ctools/{mqtt_utils.py → stream/mqtt_utils.py}

@@ -1,12 +1,20 @@
 import time
+from enum import Enum
 from typing import Dict
 
 from paho.mqtt import client as mqtt
 from paho.mqtt.enums import CallbackAPIVersion
 
-from business.common import DictToObj
-from business.common.constant import MQTTEvent
-from ctools import sys_log, cjson, string_tools, sys_info, date_utils, sm_tools, thread_pool
+from ctools import cid, cdate
+from ctools import sys_log, cjson, sys_info
+from ctools.cipher import sm_util
+from ctools.dict_wrapper import DictWrapper as DictToObj
+from ctools.pools import thread_pool
+
+
+class MQTTEvent(Enum):
+  pass
+
 
 '''
 MQTT服务使用示例:
@@ -37,9 +45,9 @@ class MQTTRequest:
 
   def __init__(self, **kwargs):
     self.event: str = kwargs.get('event')
-    self.trace_id: str = kwargs.get('trace_id') if kwargs.get('trace_id') else string_tools.get_uuid()
+    self.trace_id: str = kwargs.get('trace_id') if kwargs.get('trace_id') else cid.get_uuid()
     self.client_id: str = kwargs.get('client_id')
-    self.time: str = kwargs.get('time') if kwargs.get('time') else date_utils.get_date_time()
+    self.time: str = kwargs.get('time') if kwargs.get('time') else cdate.get_date_time()
     self.body = kwargs.get('body')
 
   def __str__(self):
@@ -55,7 +63,7 @@ class MQTTResponse:
     self.event: str = kwargs.get('event')
     self.trace_id: str = kwargs.get('trace_id')
     self.status: int = kwargs.get('status')
-    self.time: str = kwargs.get('time') if kwargs.get('time') else date_utils.get_date_time()
+    self.time: str = kwargs.get('time') if kwargs.get('time') else cdate.get_date_time()
     self.msg: str = kwargs.get('msg')
     self.body = kwargs.get('body')
 
@@ -89,8 +97,8 @@ class MQTTUtils:
       will_msg = {
         "event": "offline",
         "client_id": self.client_id,
-        "trace_id": string_tools.get_uuid(),
-        "time": date_utils.get_date_time(),
+        "trace_id": cid.get_uuid(),
+        "time": cdate.get_date_time(),
         "body": {"type": "will_msg"}
       }
       self.client.will_set(self.publish_topic, cjson.dumps(will_msg), 2, False)
@@ -145,16 +153,16 @@ class MQTTUtils:
   def encrypt_body(self, body):
     if self.broker_encrypt_key:
       try:
-        return sm_tools.encrypt_with_sm4(self.broker_encrypt_key, cjson.dumps(body))
+        return sm_util.encrypt_with_sm4(self.broker_encrypt_key, cjson.dumps(body))
       except Exception:
-        return sm_tools.encrypt_with_sm4(self.broker_encrypt_key, body)
+        return sm_util.encrypt_with_sm4(self.broker_encrypt_key, body)
 
   def decrypt_body(self, body):
     if self.broker_encrypt_key:
       try:
-        return cjson.loads(sm_tools.decrypt_with_sm4(self.broker_encrypt_key, body))
+        return cjson.loads(sm_util.decrypt_with_sm4(self.broker_encrypt_key, body))
       except Exception:
-        sm_tools.decrypt_with_sm4(self.broker_encrypt_key, body)
+        sm_util.decrypt_with_sm4(self.broker_encrypt_key, body)
 
   def connect(self, username=None, password=None, timeout=10):
     if username and password:

ctools/sys_info.py

@@ -1,13 +1,9 @@
 import hashlib
 import os
 import platform
-import socket
-import uuid
 
-import psutil
-from bottle import request
-import getpass
-from ctools import cjson, sm_tools, work_path
+from ctools import cjson, path_info
+from ctools.cipher import sm_util
 
 MACHINE_KEY = b'EnrGffoorbFyTYoS0902YyT1Fhehj4InpbezIDUuPOg='
 
@@ -15,12 +11,15 @@ MACHINE_KEY = b'EnrGffoorbFyTYoS0902YyT1Fhehj4InpbezIDUuPOg='
 class MachineInfo:
   machine_code = None
 
+
 def get_user():
+  import getpass
   return getpass.getuser()
 
+
 def get_machine_code():
   if MachineInfo.machine_code: return MachineInfo.machine_code
-  destPath = os.path.join(work_path.get_user_work_path(), "AppData/Local/machine")
+  destPath = os.path.join(path_info.get_user_work_path(), "AppData/Local/machine")
   machine_file = os.path.join(destPath, 'machine_code.mc')
   origin_machine_code = get_origin_machine_code()
   if os.path.exists(machine_file):
@@ -28,7 +27,7 @@ def get_machine_code():
       file_code = f.readline()
       dec_code = ''
       try:
-        dec_code = cjson.loads(sm_tools.decrypt_with_sm4(MACHINE_KEY, file_code))
+        dec_code = cjson.loads(sm_util.decrypt_with_sm4(MACHINE_KEY, file_code))
         origin_code = dec_code.get("origin_code")
         if origin_code == origin_machine_code:
           MachineInfo.machine_code = dec_code.get("hash_code")
@@ -38,7 +37,7 @@ def get_machine_code():
         print('machine code file is error: {} {}'.format(file_code, dec_code))
   hash_machine_code = get_hash_machine_code(origin_machine_code)
   machine_code = {"origin_code": origin_machine_code, "hash_code": hash_machine_code}
-  enc_code = sm_tools.encrypt_with_sm4(MACHINE_KEY, cjson.dumps(machine_code))
+  enc_code = sm_util.encrypt_with_sm4(MACHINE_KEY, cjson.dumps(machine_code))
   os.makedirs(destPath, exist_ok=True)
   with open(machine_file, 'w') as f:
     f.write(enc_code)
@@ -68,26 +67,49 @@ def get_origin_machine_code():
       0
     )
     disk_serial = str(volume_serial.value)
-  combined_info = cpu_serial + disk_serial + '-hylink'
+  combined_info = cpu_serial + disk_serial + '-gomyck'
   return hashlib.md5(combined_info.encode()).hexdigest()
 
 
 def get_hash_machine_code(origin_code):
+  import uuid
   code = origin_code + uuid.uuid1().hex
   machine_code = hashlib.md5(code.encode()).hexdigest()
   return machine_code.upper()
 
 
+def get_public_ip():
+  import requests
+  try:
+    response = requests.get("https://api.ipify.org?format=json")
+    ip = response.json()["ip"]
+    return ip
+  except Exception as e:
+    return f"Failed to get public IP: {e}"
+
+
 def get_local_ipv4():
-  ip = socket.gethostname()
+  import psutil
+  import socket
+  interfaces = psutil.net_if_addrs()
+  for interface, addresses in interfaces.items():
+    for address in addresses:
+      if address.family == socket.AF_INET and not address.address.startswith("127."):
+        return address.address
+  print("Failed to get local IPv4 address, try another way...")
+  s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
   try:
-    ip = socket.gethostbyname_ex(ip)[2][-1]
+    s.connect(("8.8.8.8", 80))
+    ip = s.getsockname()[0]
   except Exception:
-    pass
+    ip = '127.0.0.1'
+  finally:
+    s.close()
   return ip
 
 
 def get_remote_ipv4():
+  from bottle import request
   try:
     return request.remote_route[0]
   except:
@@ -95,6 +117,7 @@ def get_remote_ipv4():
 
 
 def get_proc_pid_by(cmdline):
+  import psutil
   """
   根据命令行信息获取进程pid
   :param cmdline: