gomyck-tools 1.3.1__py3-none-any.whl → 1.3.2__py3-none-any.whl

ctools/cjson.py

@@ -0,0 +1,54 @@
+import jsonpickle
+
+# 需要转换成str的属性
+str_value_keys = []
+jsonpickle.set_preferred_backend('json')
+jsonpickle.set_encoder_options('json', ensure_ascii=False)
+jsonpickle.set_decoder_options('json')
+
+def dumps(obj, **kwargs) -> str:
+  """
+  将对象转换为json字符串
+  :param obj: 对象
+  :return: json 字符串
+  """
+  # indent = 2 可以美化输出
+  if obj is None: return None
+  if type(obj) == str: return obj
+  return f'{jsonpickle.encode(obj, unpicklable=False, make_refs=False, **kwargs)}'
+
+def loads(json_str: str, **kwargs) -> dict:
+  """
+  将json字符串转换为对象
+  :param json_str: json 字符串
+  :return: 对象
+  """
+  return jsonpickle.decode(json_str, **kwargs)
+
+def unify_to_str(json_str: str) -> str:
+  if not str_value_keys and len(str_value_keys) == 0: return json_str
+  obj = loads(json_str)
+  if isinstance(obj, list):
+    _handle_list(obj)
+  elif isinstance(obj, dict):
+    _handle_dict(obj)
+  return dumps(obj)
+
+def _handle_list(data):
+  for o in data:
+    if isinstance(o, list):
+      _handle_list(o)
+    elif isinstance(o, dict):
+      _handle_dict(o)
+
+def _handle_dict(data):
+  for k, v in data.items():
+    if isinstance(v, list):
+      _handle_list(v)
+    elif isinstance(v, dict):
+      _handle_dict(v)
+    elif k in str_value_keys:
+      try:
+        data[k] = str(v)
+      except Exception:
+        pass

ctools/ckafka.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/9/5 10:39'
+
+import time
+from threading import Thread, Lock
+
+from kafka import KafkaProducer, errors, KafkaConsumer
+from kafka.producer.future import FutureRecordMetadata
+
+from ctools import thread_pool
+from ctools.cjson import dumps
+
+"""
+import time
+from datetime import datetime
+
+from ctools import thread_pool, string_tools
+from ctools.ckafka import CKafka
+
+c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
+
+producer = c.init_producer()
+consumer = c.init_consumer(enable_auto_commit=False)
+
+def send_msg():
+  while True:
+    command = input('发送消息: Y/n \n')
+    if command.strip() not in ['N', 'n']:
+      producer.send_msg('jqxx', '{{"jqid": "{}", "xxxx": "{}"}}'.format(string_tools.get_snowflake_id(), datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')))
+    else:
+      break
+
+thread_pool.submit(send_msg)
+
+def consumer_callback(msg):
+  print(msg)
+  return True
+
+consumer.receive_msg('jqxx', callBack=consumer_callback)
+
+while True: time.sleep(1)
+"""
+class KafkaInstance:
+  def __init__(self, producer: KafkaProducer, consumer: KafkaConsumer):
+    self.start_consumer = False
+    self.quited = False
+    self.producer = producer
+    self.consumer = consumer
+    self.consumer_callback = {"topic_key": []}
+
+  # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
+  # r.add_callback(lambda x: print(x))
+  # r.get() 可以同步获取结果
+  def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
+    if self.producer is None: raise RuntimeError("Producer is not initialized")
+    if self.quited: return
+    return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
+
+  def receive_msg(self, topics: str, callBack=print):
+    if self.consumer is None: raise RuntimeError("Consumer is not initialized")
+    for topic in topics.split(','):
+      if topic not in self.consumer_callback.keys():
+        self.consumer_callback[topic] = []
+        self.consumer.subscribe(self.consumer_callback.keys())
+      self.consumer_callback[topic].append(callBack)
+    if not self.start_consumer:
+      t = Thread(target=self._start_consumer_poll, daemon=True)
+      t.start()
+
+  def _start_consumer_poll(self):
+    self.start_consumer = True
+    for msg in self.consumer:
+      if self.quited: break
+      funcList = []
+      begin_time = time.time()
+      for func in self.consumer_callback[msg.topic]:
+        if self.quited: break
+        res = func(msg)
+        if not self.consumer.config['enable_auto_commit'] and res: self.consumer.commit()
+        funcList.append(func.__name__)
+      end_time = time.time()
+      if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
+      funcList.clear()
+
+  def shutdown(self):
+    self.quited = True
+    try: self.consumer.close()
+    except Exception: pass
+    try: self.producer.close()
+    except Exception: pass
+
+
+class CKafka:
+
+  def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password'):
+    self.kafka_url = kafka_url
+    self.secure = secure
+    self.username = username
+    self.password = password
+
+  def init_producer(self, acks=1) -> KafkaInstance:
+    print("[ Producer ] Connecting to Kafka [{}]".format(self.kafka_url))
+    for i in range(0, 6):
+      try:
+        if self.secure:
+          producer = KafkaProducer(
+            acks=acks,
+            bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8'),
+            sasl_plain_username=self.username,
+            sasl_plain_password=self.password,
+            security_protocol='SASL_PLAINTEXT',
+            sasl_mechanism='PLAIN'
+          )
+        else:
+          producer = KafkaProducer(
+            acks=acks,
+            bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8')
+          )
+        print("[ Producer ] Success Connected to Kafka [{}]".format(self.kafka_url))
+        return KafkaInstance(producer=producer, consumer=None)
+      except errors.NoBrokersAvailable:
+        print("[ Producer ] Waiting for Kafka [{}] to become available...".format(self.kafka_url))
+        time.sleep(3)
+    raise RuntimeError("[ Producer ] Failed to connect to Kafka [{}] within 60 seconds".format(self.kafka_url))
+
+  def init_consumer(self, client_id: str = 'ck-py-kafka-consumer', consumer_group: str = 'ck-py-kafka-consumer', enable_auto_commit: bool = True) -> KafkaInstance:
+    print("[ Consumer ] Connecting to Kafka [{}]".format(self.kafka_url))
+    for i in range(0, 6):
+      try:
+        if self.secure:
+          consumer = KafkaConsumer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
+            bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8'),
+            sasl_plain_username=self.username,
+            sasl_plain_password=self.password,
+            security_protocol='SASL_PLAINTEXT',
+            sasl_mechanism='PLAIN'
+          )
+        else:
+          consumer = KafkaProducer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
+            bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8')
+          )
+        print("[ Consumer ] Success Connected to Kafka [{}]".format(self.kafka_url))
+        return KafkaInstance(producer=None, consumer=consumer)
+      except errors.NoBrokersAvailable:
+        print("[ Consumer ] Waiting for Kafka [{}] to become available...".format(self.kafka_url))
+        time.sleep(3)
+    raise RuntimeError("[ Consumer ] Failed to connect to Kafka [{}] within 60 seconds".format(self.kafka_url))

ctools/compile_tools.py

@@ -0,0 +1,18 @@
+import importlib
+import os
+import time
+
+
+def code_to_pyc(code: str, out_file_path: str):
+  file_name = os.path.split(out_file_path)[-1]
+  compiled_code = compile(code, file_name, 'exec')
+  bytecode = importlib._bootstrap_external._code_to_timestamp_pyc(compiled_code, time.time(), len(code))
+  with open(out_file_path, 'wb') as f:
+    f.write(bytecode)
+
+
+def file_to_pyc(file_path: str, out_file_path: str):
+  with open(file_path, 'r') as f:
+    code = f.read()
+  if code:
+    code_to_pyc(code, out_file_path)

ctools/console.py

@@ -0,0 +1,55 @@
+import logging
+import sys
+import tkinter as tk
+
+
+class Console:
+
+  def __init__(self, master):
+    self.master = master
+
+    # 创建文本框和滚动条
+    self.textbox = tk.Text(self.master, wrap=tk.NONE)
+
+    self.vertical_scrollbar = tk.Scrollbar(self.textbox, command=self.textbox.yview)
+    self.horizontal_scrollbar = tk.Scrollbar(self.textbox, command=self.textbox.xview, orient=tk.HORIZONTAL)
+
+    self.textbox.configure(yscrollcommand=self.vertical_scrollbar.set, xscrollcommand=self.horizontal_scrollbar.set)
+    self.textbox.pack(side=tk.LEFT, pady=10, padx=10, ipadx=10, ipady=10, fill=tk.BOTH, expand=True)
+
+    self.vertical_scrollbar.pack(side=tk.RIGHT, fill=tk.Y)
+    self.horizontal_scrollbar.pack(side=tk.BOTTOM, fill=tk.X)
+
+    # 将标准输出和标准错误输出重定向到文本框中
+    sys.stdout = self
+    sys.stderr = self
+
+    # # 创建输入框和按钮
+    # self.entry = tk.Entry(self.master)
+    # self.entry.pack(side=tk.BOTTOM, fill=tk.X, expand=True)
+    # self.button = tk.Button(self.master, text="Send", command=self.send)
+    # self.button.pack(side=tk.BOTTOM)
+
+    # 将日志输出到文本框中
+    self.log_handler = logging.StreamHandler(self)
+    self.log_handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
+    logging.getLogger().addHandler(self.log_handler)
+    # logging.getLogger().setLevel(logging.INFO)
+
+  def write(self, message):
+    # 在文本框中输出消息
+    self.textbox.insert(tk.END, message + '\n')
+    self.textbox.see(tk.END)
+
+  def flush(self):
+    pass
+
+  def send(self):
+    # 获取输入框中的文本并打印到控制台
+    text = self.entry.get()
+    print(text)
+    self.entry.delete(0, tk.END)
+
+  def __del__(self):
+    # 关闭日志处理器
+    logging.getLogger().removeHandler(self.log_handler)

ctools/coord_trans.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+import math
+
+x_pi = 3.14159265358979324 * 3000.0 / 180.0
+pi = 3.1415926535897932384626  # π
+a = 6378245.0  # 长半轴
+ee = 0.00669342162296594323  # 偏心率平方
+
+def gcj02_to_bd09(lng, lat):
+    """
+    火星坐标系(GCJ-02)转百度坐标系(BD-09)
+    谷歌、高德——>百度
+    :param lng:火星坐标经度
+    :param lat:火星坐标纬度
+    :return:
+    """
+    z = math.sqrt(lng * lng + lat * lat) + 0.00002 * math.sin(lat * x_pi)
+    theta = math.atan2(lat, lng) + 0.000003 * math.cos(lng * x_pi)
+    bd_lng = z * math.cos(theta) + 0.0065
+    bd_lat = z * math.sin(theta) + 0.006
+    return [bd_lng, bd_lat]
+
+
+def bd09_to_gcj02(bd_lon, bd_lat):
+    """
+    百度坐标系(BD-09)转火星坐标系(GCJ-02)
+    百度——>谷歌、高德
+    :param bd_lat:百度坐标纬度
+    :param bd_lon:百度坐标经度
+    :return:转换后的坐标列表形式
+    """
+    x = bd_lon - 0.0065
+    y = bd_lat - 0.006
+    z = math.sqrt(x * x + y * y) - 0.00002 * math.sin(y * x_pi)
+    theta = math.atan2(y, x) - 0.000003 * math.cos(x * x_pi)
+    gg_lng = z * math.cos(theta)
+    gg_lat = z * math.sin(theta)
+    return [gg_lng, gg_lat]
+
+
+def wgs84_to_gcj02(lng, lat):
+    """
+    WGS84转GCJ02(火星坐标系)
+    :param lng:WGS84坐标系的经度
+    :param lat:WGS84坐标系的纬度
+    :return:
+    """
+    if out_of_china(lng, lat):  # 判断是否在国内
+        return [lng, lat]
+    dlat = _transformlat(lng - 105.0, lat - 35.0)
+    dlng = _transformlng(lng - 105.0, lat - 35.0)
+    radlat = lat / 180.0 * pi
+    magic = math.sin(radlat)
+    magic = 1 - ee * magic * magic
+    sqrtmagic = math.sqrt(magic)
+    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
+    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
+    mglat = lat + dlat
+    mglng = lng + dlng
+    return [mglng, mglat]
+
+
+def gcj02_to_wgs84(lng, lat):
+    """
+    GCJ02(火星坐标系)转GPS84
+    :param lng:火星坐标系的经度
+    :param lat:火星坐标系纬度
+    :return:
+    """
+    if out_of_china(lng, lat):
+        return [lng, lat]
+    dlat = _transformlat(lng - 105.0, lat - 35.0)
+    dlng = _transformlng(lng - 105.0, lat - 35.0)
+    radlat = lat / 180.0 * pi
+    magic = math.sin(radlat)
+    magic = 1 - ee * magic * magic
+    sqrtmagic = math.sqrt(magic)
+    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
+    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
+    mglat = lat + dlat
+    mglng = lng + dlng
+    return [lng * 2 - mglng, lat * 2 - mglat]
+
+
+def bd09_to_wgs84(bd_lon, bd_lat):
+    lon, lat = bd09_to_gcj02(bd_lon, bd_lat)
+    return gcj02_to_wgs84(lon, lat)
+
+
+def wgs84_to_bd09(lon, lat):
+    lon, lat = wgs84_to_gcj02(lon, lat)
+    return gcj02_to_bd09(lon, lat)
+
+
+def _transformlat(lng, lat):
+    ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
+          0.1 * lng * lat + 0.2 * math.sqrt(math.fabs(lng))
+    ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
+            math.sin(2.0 * lng * pi)) * 2.0 / 3.0
+    ret += (20.0 * math.sin(lat * pi) + 40.0 *
+            math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
+    ret += (160.0 * math.sin(lat / 12.0 * pi) + 320 *
+            math.sin(lat * pi / 30.0)) * 2.0 / 3.0
+    return ret
+
+
+def _transformlng(lng, lat):
+    ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
+          0.1 * lng * lat + 0.1 * math.sqrt(math.fabs(lng))
+    ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
+            math.sin(2.0 * lng * pi)) * 2.0 / 3.0
+    ret += (20.0 * math.sin(lng * pi) + 40.0 *
+            math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
+    ret += (150.0 * math.sin(lng / 12.0 * pi) + 300.0 *
+            math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
+    return ret
+
+
+def out_of_china(lng, lat):
+    """
+    判断是否在国内，不在国内不做偏移
+    :param lng:
+    :param lat:
+    :return:
+    """
+    return not (lng > 73.66 and lng < 135.05 and lat > 3.86 and lat < 53.55)
+

ctools/credis.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2025/2/14 11:09'
+
+import redis
+from redis import Redis
+
+from ctools import date_utils, thread_pool, string_tools
+
+def init_pool(host: str = 'localhost', port: int = 6379, db: int = 0, password: str = None,
+              username: str = None, decode_responses: bool = True, max_connections: int = 75,
+              health_check_interval: int = 30, retry_count: int = 3) -> Redis:
+    for attempt in range(retry_count):
+        try:
+            r: Redis = redis.StrictRedis(
+                host=host, port=port, db=db,
+                username=username, password=password,
+                retry_on_timeout=True,
+                max_connections=max_connections,
+                decode_responses=decode_responses,
+                health_check_interval=health_check_interval,
+                socket_connect_timeout=5,
+                socket_timeout=5
+            )
+            if r.ping():
+                print('CRedis connect {} {} success!'.format(host, port))
+                return r
+        except redis.ConnectionError as e:
+            if attempt == retry_count - 1:
+                raise Exception(f"Failed to connect to Redis after {retry_count} attempts: {str(e)}")
+            print(f"Connection attempt {attempt + 1} failed, retrying...")
+
+def add_lock(r: Redis, key: str, timeout: int = 30):
+  if r.exists(key):
+    expire_time = r.get(key)
+    if date_utils.time_diff_in_seconds(expire_time, date_utils.get_date_time()) > 0:
+      return True
+    else:
+      r.delete(key)
+  return r.set(key, date_utils.opt_time(seconds=timeout), nx=True, ex=timeout) is not None
+
+def remove_lock(r: Redis, key: str):
+  r.delete(key)
+
+def subscribe(r: Redis, channel_name, callback):
+  def thread_func():
+    pubsub = r.pubsub()
+    pubsub.subscribe(channel_name)
+    for message in pubsub.listen():
+      callback(message)
+  thread_pool.submit(thread_func)
+
+def _process_pending_messages(r: Redis, stream_name: str, group_name: str, consumer_name: str, callback):
+  """
+  处理未确认的消息
+  :param r: Redis 连接
+  :param stream_name: 流名称
+  :param group_name: 消费者组名称
+  :param consumer_name: 消费者名称
+  :param callback: 消息处理回调函数
+  """
+  # 检查未确认的消息
+  pending_messages = r.xpending(stream_name, group_name)
+  if pending_messages['pending'] > 0:
+    print(f"Found {pending_messages['pending']} pending messages.")
+    # 获取未确认的消息列表
+    pending_list = r.xpending_range(stream_name, group_name, min='-', max='+', count=pending_messages['pending'])
+    for message in pending_list:
+      message_id = message['message_id']
+      claimed_messages = r.xclaim(stream_name, group_name, consumer_name, min_idle_time=0, message_ids=[message_id])
+      if claimed_messages:
+        # 处理消息
+        for claimed_message in claimed_messages:
+          message_id, data = claimed_message
+          print(f"Processing pending message: {message_id}, data: {data}")
+          try:
+            if callback(message_id, data):
+              r.xack(stream_name, group_name, message_id)
+          except Exception as e:
+            print(f"Error processing message {message_id}: {e}")
+  else:
+    print("No pending messages found.")
+
+def stream_subscribe(r: Redis, stream_name, group_name, callback, from_id: str='$', noack: bool = False):
+  def thread_func():
+    try:
+      # $表示从最后面消费, 0表示从开始消费
+      r.xgroup_create(name=stream_name, groupname=group_name, id=from_id, mkstream=True)
+      print(f"Consumer group '{group_name}' created successfully.")
+    except Exception as e:
+      if "already exists" in str(e):
+        print(f"Consumer group '{group_name}' already exists.")
+      else:
+        print(f"Error creating consumer group '{group_name}': {e}")
+    consumer_name = 'consumer-{}'.format(string_tools.get_uuid())
+    # 处理未确认的消息
+    _process_pending_messages(r, stream_name, group_name, consumer_name, callback)
+    while True:
+      messages = r.xreadgroup(group_name, consumer_name, {stream_name: '>'}, block=1000, noack=noack)
+      for message in messages:
+        try:
+          message_id, data = message[1][0]
+          res = callback(message_id, data)
+          if res: r.xack(stream_name, group_name, message_id)
+        except Exception as e:
+          print('stream_subscribe error: ', e)
+  thread_pool.submit(thread_func)
+
+def stream_publish(r: Redis, stream_name, message):
+  r.xadd(stream_name, message)