ODtools 2.1.21__tar.gz → 2.2.1__tar.gz

{ODtools-2.1.21 → odtools-2.2.1}/ODtools/monitor_tools.py

@@ -4,17 +4,26 @@
 # @File : test_monitor_class.py
 
 
-import asyncio
 import time
 from datetime import datetime
 from enum import Enum
 
 import requests
-from aiohttp import ClientSession
-from aiosocksy.connector import ProxyClientRequest, ProxyConnector
 from apscheduler.schedulers.blocking import BlockingScheduler
 from redis import Redis
 from rediscluster import RedisCluster
+import asyncio
+from urllib.parse import urlparse
+from aiohttp import ClientSession, ClientResponse
+from typing import List, Dict, Any, Tuple, Optional
+
+# 只在需要 SOCKS 时才导入（避免无代理时依赖 aiohttp-socks）
+try:
+    from aiohttp_socks import ProxyConnector
+    
+    HAS_SOCKS = True
+except ImportError:
+    HAS_SOCKS = False
 
 
 # 把超时时间的设置
@@ -101,7 +110,7 @@ class Task(object):
     """
     任务调度分发类,主要分为定时分发任务、循环分发任务、单次分发任务
     """
-
+    
     def __init__(self, db_client=None, *, db_host: str = None, db_port: int = 6379, db_cluster: bool = False,
                  db_password: str = None):
         """
@@ -117,7 +126,7 @@ class Task(object):
         else:
             redis_type = RedisCluster if db_cluster else Redis
             self.db_client = redis_type(host=db_host, port=db_port, password=db_password)
-
+    
     def cron_job(self, que_name: str, task_data: list, *, hour: str = None, minute: str = None, second: str = None):
         """
         定时分发任务,任务队列类型必须是列表类型
@@ -137,7 +146,7 @@ class Task(object):
             args=[que_name, task_data]
         )
         scheduler.start()
-
+    
     def cyclic_job(self, que_name: str, task_data: list, second: int):
         """
         循环分发任务,任务队列类型必须是列表类型
@@ -150,7 +159,7 @@ class Task(object):
             self.db_client.rpush(que_name, *task_data)
             print('task distribution complete...')
             time.sleep(second)
-
+    
     def single_job(self, que_name: str, task_data: list, que_type: str = 'list', sort: str = 'r'):
         """
         单次任务分发,队列可以是列表或集合
@@ -169,7 +178,7 @@ class Mrequest(object):
     """
     发送请求的类,分为同步和异步
     """
-
+    
     def __init__(self, db_client=None, statistic=False, *, db_host: str = None, db_port: int = 6379,
                  db_cluster: bool = False,
                  db_password: str = None):
@@ -188,7 +197,7 @@ class Mrequest(object):
         else:
             redis_type = RedisCluster if db_cluster else Redis
             self.db_client = redis_type(host=db_host, port=db_port, password=db_password)
-
+    
     def mrequest(self, urls: str, method: str = 'get', record_dict: dict = dict, step: int = 1, **kwargs):
         """
         使用requests同步发送请求,参数分别为目标任务url,请求方法,计数器字典,计数步长
@@ -216,58 +225,92 @@ class Mrequest(object):
                 spider_record(self.db_client, record_dict, step, False)
             print('失败计数加一')
             return Response
-
-    def aio_request(self, urls: list, record_dict: dict = dict, response_model: str = 'html', step: int = 1,
-                    cookies=None, **kwargs):
+    
+    def aio_request(
+            self,
+            urls: List[str],
+            record_dict: Dict[str, Any] = None,
+            response_model: str = 'html',
+            step: int = 1,
+            cookies: Optional[Dict] = None,
+            proxy_url: Optional[str] = None,  # 支持: http://, https://, socks4://, socks5://
+            **kwargs
+    ) -> List[Tuple[Any, str, Optional[ClientResponse]]]:
         """
-        异步发送请求,使用aiohttp.请求参数分别是: 目标任务url列表,计数器字典,请求响应格式,计数器步长
-        :return:
+        异步发送请求，自适应支持 HTTP/HTTPS 和 SOCKS4/SOCKS5 代理。
+        - HTTP/HTTPS 代理：使用 aiohttp 原生 proxy 参数
+        - SOCKS 代理：使用 aiohttp-socks（需安装）
         """
-        if self.statistic:
-            component_name = record_dict['component_name']
-
-        async def async_request(url, cookies=None, **kwargs):
-            if cookies:
-                clientSessionObj = ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest,
-                                                 cookies=cookies)
+        if record_dict is None:
+            record_dict = {}
+        
+        async def async_request(url: str, cookies: Optional[Dict] = None, **kwargs):
+            # 解析代理类型
+            use_socks = False
+            if proxy_url:
+                parsed = urlparse(proxy_url)
+                if parsed.scheme in ('socks4', 'socks5'):
+                    use_socks = True
+                elif parsed.scheme not in ('http', 'https'):
+                    raise ValueError(f"Unsupported proxy scheme: {parsed.scheme}. Use http, https, socks4, or socks5.")
+            
+            try:
+                if use_socks:
+                    # 使用 SOCKS 代理
+                    if not HAS_SOCKS:
+                        raise RuntimeError(
+                            "SOCKS proxy requires 'aiohttp-socks' package. Install with: pip install aiohttp-socks")
+                    connector = ProxyConnector.from_url(proxy_url)
+                    async with ClientSession(connector=connector, cookies=cookies) as session:
+                        async with session.get(url, **kwargs) as response:
+                            return await _process_response(response, url)
+                else:
+                    # 使用 HTTP/HTTPS 代理 或 无代理
+                    async with ClientSession(cookies=cookies) as session:
+                        async with session.get(url, proxy=proxy_url, **kwargs) as response:
+                            return await _process_response(response, url)
+            
+            except Exception as e:
+                if self.statistic and 'component_name' in record_dict:
+                    component_name = record_dict['component_name']
+                    record_dict['record_info'] = f'{component_name}_request_fail'
+                    spider_record(self.db_client, record_dict, step, False)
+                import traceback
+                print(f"Error fetching {url} via {proxy_url or 'direct'}: {e}\n{traceback.format_exc()}")
+                return '', url, None
+        
+        async def _process_response(response: ClientResponse, url: str):
+            """统一处理响应内容"""
+            if self.statistic and 'component_name' in record_dict:
+                component_name = record_dict['component_name']
+                if response.status == 200:
+                    record_dict['record_info'] = f'{component_name}_request_success'
+                else:
+                    record_dict['record_info'] = f'{component_name}_request_fail'
+                spider_record(self.db_client, record_dict, step, False)
+            
+            if response_model == 'json':
+                content = await response.json()
+            elif response_model == 'bytes':
+                content = await response.read()
+            elif response_model == 'html':
+                content = await response.text()
             else:
-                clientSessionObj = ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest)
-            async with clientSessionObj as session:
-                try:
-                    async with session.get(url, **kwargs) as response:
-                        if response.status == 200:
-                            if self.statistic: record_dict['record_info'] = '{}_request_success'.format(component_name)
-                        else:
-                            if self.statistic: record_dict['record_info'] = '{}_request_fail'.format(component_name)
-                        if self.statistic: spider_record(self.db_client, record_dict, step, False)
-                        if response_model == 'json':
-                            return await response.json(), str(response.url), response
-                        elif response_model == 'bytes':
-                            return await response.read(), str(response.url), response
-                        elif response_model == 'html':
-                            return await response.text(), str(response.url), response
-                        else:
-                            return response, str(response.url), response
-                except Exception as e:
-                    if self.statistic:
-                        record_dict['record_info'] = '{}_request_fail'.format(component_name)
-                        spider_record(self.db_client, record_dict, step, False)
-                    import traceback
-                    print(traceback.format_exc(), url)
-                    return '', url, Response
-
-        tasks = [asyncio.ensure_future(async_request(i, cookies=cookies, **kwargs)) for i in urls]
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(asyncio.wait(tasks))
-        del loop
-        return [i.result() for i in tasks]
+                content = response
+            
+            return content, str(response.url), response
+        
+        # 执行所有任务
+        tasks = [async_request(url, cookies=cookies, **kwargs) for url in urls]
+        results = asyncio.run(asyncio.gather(*tasks, return_exceptions=False))
+        return results
 
 
 class Parse(object):
     """
     解析源数据类,使用思路为继承该类并重写analysis_data方法
     """
-
+    
     def __init__(self, db_client=None, statistic=False, *, db_host: str = None, db_port: int = 6379,
                  db_cluster: bool = False,
                  db_password: str = None):
@@ -277,7 +320,7 @@ class Parse(object):
         else:
             redis_type = RedisCluster if db_cluster else Redis
             self.db_client = redis_type(host=db_host, port=db_port, password=db_password)
-
+    
     # @time_counter
     def analysis_data(self, source_code, record_dict: dict = dict, step: int = 1, *args, **kwargs):
         """
@@ -293,7 +336,7 @@ class Parse(object):
 
 
 class Monitor(object):
-
+    
     def __init__(self, rds=None, statistic=False, *, redis_host: str = None, redis_port: int = 6379,
                  redis_cluster: bool = False,
                  redis_password: str = None):
@@ -303,11 +346,11 @@ class Monitor(object):
         else:
             redis_type = RedisCluster if redis_cluster else Redis
             self.rds = redis_type(host=redis_host, port=redis_port, password=redis_password)
-
+    
     def log(self):
         """日志统计"""
         ...
-
+    
     # 在收集的大key前面添加日期，之后删除被收集的大key
     def statistics_count(self, summary_keys: dict, *, hour: str = None, minute: str = None, second: str = None):
         """
@@ -318,7 +361,7 @@ class Monitor(object):
         :param second:
         :return:
         """
-
+        
         def summary_count(summary_keys: dict):
             today_date = get_today_str()
             for summary_key, big_keys in summary_keys.items():
@@ -328,7 +371,7 @@ class Monitor(object):
                         if today_date not in summary_key:
                             summary_key = today_date + '_' + summary_key
                         self.rds.hset(summary_key, k, v)
-
+        
         scheduler = BlockingScheduler(timezone="Asia/Shanghai")
         scheduler.add_job(
             summary_count, 'cron',

{ODtools-2.1.21 → odtools-2.2.1}/ODtools/monitor_tools_new.py

@@ -4,18 +4,26 @@
 # @File : test_monitor_class.py
 
 
-import asyncio
-import json
 import time
 from datetime import datetime
 from enum import Enum
 
 import requests
-from aiohttp import ClientSession
-from aiosocksy.connector import ProxyClientRequest, ProxyConnector
 from apscheduler.schedulers.blocking import BlockingScheduler
 from redis import Redis
 from rediscluster import RedisCluster
+import asyncio
+from urllib.parse import urlparse
+from aiohttp import ClientSession, ClientResponse
+from typing import List, Dict, Any, Tuple, Optional
+
+# 只在需要 SOCKS 时才导入（避免无代理时依赖 aiohttp-socks）
+try:
+    from aiohttp_socks import ProxyConnector
+    
+    HAS_SOCKS = True
+except ImportError:
+    HAS_SOCKS = False
 
 
 # 把超时时间的设置
@@ -54,28 +62,6 @@ def spider_record(client, record_dict: dict = dict, step: int = 1, del_record: b
             if not client.ttl(record_info): client.expireat(record_info, get_tomorrow_timestamp())
 
 
-# redis hash计数器的改变
-def spider_proxy_record(client, record_dict: dict = dict, del_record: bool = False):
-    """
-    增加和较少redis计数器，控制redis的使用个数
-     {"count":0,“bandwidth”:1}
-    :param client: 数据库连接对象
-    :param record_dict: 形式为{'general_info': xxx,"project_name":1,"source_name":"Weibo", 'proxy_info': {"111.114.114.114":1"}},字典的key必须是general_info和record_info
-    :return:
-    """
-    source_name = record_dict['source_name']
-    proxy_info = record_dict['proxy_info']
-    if not isinstance(source_name, Source): raise TypeError('error data type source_name')
-    proxy_key = '{}:Proxy'.format(source_name.value)
-    for k, v in proxy_info.items():
-        key_value = client.hget(proxy_key, k)
-        if not key_value:
-            key_value = """{"count":"0"}"""
-        key_value = json.loads(key_value)
-        key_value.update({"count": int(key_value.get("count")) + v, })
-        client.hset(proxy_key, k, json.dumps(key_value, ensure_ascii=False))
-
-
 def get_tomorrow_timestamp() -> int:
     """
     获取第二天零点的时间戳
@@ -108,27 +94,23 @@ class Source(Enum):
     WECHAT = 'Wechat'
     DOUYIN = 'Douyin'
     NEWS = 'News'
-    SD = 'SD'
     TIEBA = 'Tieba'
     APP = 'App'
     HOTLIST = 'HotList'
     TOUTIAO = 'Toutiao'
-    BAIDU_BAIJIAHAO = 'baidu_baijihao'
     OTHER = 'OtherSource'
 
 
 class Response(Enum):
     """错误的数据链接请求"""
     status_code = 500
-    status = 500
-    headers = {}
 
 
 class Task(object):
     """
     任务调度分发类,主要分为定时分发任务、循环分发任务、单次分发任务
     """
-
+    
     def __init__(self, db_client=None, *, db_host: str = None, db_port: int = 6379, db_cluster: bool = False,
                  db_password: str = None):
         """
@@ -144,7 +126,7 @@ class Task(object):
         else:
             redis_type = RedisCluster if db_cluster else Redis
             self.db_client = redis_type(host=db_host, port=db_port, password=db_password)
-
+    
     def cron_job(self, que_name: str, task_data: list, *, hour: str = None, minute: str = None, second: str = None):
         """
         定时分发任务,任务队列类型必须是列表类型
@@ -164,7 +146,7 @@ class Task(object):
             args=[que_name, task_data]
         )
         scheduler.start()
-
+    
     def cyclic_job(self, que_name: str, task_data: list, second: int):
         """
         循环分发任务,任务队列类型必须是列表类型
@@ -177,7 +159,7 @@ class Task(object):
             self.db_client.rpush(que_name, *task_data)
             print('task distribution complete...')
             time.sleep(second)
-
+    
     def single_job(self, que_name: str, task_data: list, que_type: str = 'list', sort: str = 'r'):
         """
         单次任务分发,队列可以是列表或集合
@@ -196,7 +178,7 @@ class Mrequest(object):
     """
     发送请求的类,分为同步和异步
     """
-
+    
     def __init__(self, db_client=None, statistic=False, *, db_host: str = None, db_port: int = 6379,
                  db_cluster: bool = False,
                  db_password: str = None):
@@ -215,7 +197,7 @@ class Mrequest(object):
         else:
             redis_type = RedisCluster if db_cluster else Redis
             self.db_client = redis_type(host=db_host, port=db_port, password=db_password)
-
+    
     def mrequest(self, urls: str, method: str = 'get', record_dict: dict = dict, step: int = 1, **kwargs):
         """
         使用requests同步发送请求,参数分别为目标任务url,请求方法,计数器字典,计数步长
@@ -228,112 +210,107 @@ class Mrequest(object):
             request_method = getattr(requests, method.lower())
         except AttributeError:
             raise AttributeError('no method named {}'.format(method))
-        """增加请求的单次计数，记录所有请求"""
-        if self.statistic: record_dict['record_info'] = '{}_request_all'.format(component_name)
-        if self.statistic: spider_record(self.db_client, record_dict, step, False)
         try:
             response = request_method(urls, **kwargs)
-            if "proxies" in kwargs.keys():
-                """增加代理正在使用限制"""
-                proxy_ip = kwargs.get("proxies").get("http")
-                if self.statistic: record_dict['proxy_info'] = {proxy_ip.split("//")[-1]: 1} if proxy_ip else {"": 1}
-                if self.statistic: spider_proxy_record(self.db_client, record_dict, False)
-
-                if self.statistic: record_dict['record_info'] = '{}_request_proxy'.format(component_name)
-                if self.statistic: spider_record(self.db_client, record_dict, step, False)
-
             if response.status_code == 200:
                 if self.statistic: record_dict['record_info'] = '{}_request_success'.format(component_name)
-                if self.statistic: spider_record(self.db_client, record_dict, step, False)
-                """增加返回值，结果结果统计，按照bytes为单位统计"""
-                if self.statistic: record_dict['record_info'] = '{}_request_content_length'.format(component_name)
-                content_length = eval(response.headers.get("Content-Length", "0"))
-                if self.statistic: spider_record(self.db_client, record_dict,
-                                                 content_length if content_length else len(response.content),
-                                                 False)
             else:
                 if self.statistic: record_dict['record_info'] = '{}_request_fail'.format(component_name)
-                if self.statistic: spider_record(self.db_client, record_dict, step, False)
+            if self.statistic: spider_record(self.db_client, record_dict, step, False)
             return response
         except Exception as e:
-            import traceback
-            print("请求出错，原因 {}".format(traceback.format_exc()))
+            print(e)
             if self.statistic:
                 record_dict['record_info'] = '{}_request_fail'.format(component_name)
                 spider_record(self.db_client, record_dict, step, False)
-            print('请求出错，失败计数加一', record_dict)
+            print('失败计数加一')
             return Response
-        finally:
-            if "proxies" in kwargs.keys():
-                """增加代理正在使用限制"""
-                proxy_ip = kwargs.get("proxies").get("http")
-                if self.statistic: record_dict['proxy_info'] = {proxy_ip.split("//")[-1]: -1} if proxy_ip else {"": 1}
-                if self.statistic: spider_proxy_record(self.db_client, record_dict, False)
-
-    def aio_request(self, urls: list, record_dict: dict = dict, response_model: str = 'html', step: int = 1,
-                    cookies=None, **kwargs):
+    
+    def aio_request(
+            self,
+            urls: List[str],
+            record_dict: Dict[str, Any] = None,
+            response_model: str = 'html',
+            step: int = 1,
+            cookies: Optional[Dict] = None,
+            proxy_url: Optional[str] = None,  # 支持: http://, https://, socks4://, socks5://
+            **kwargs
+    ) -> List[Tuple[Any, str, Optional[ClientResponse]]]:
         """
-        异步发送请求,使用aiohttp.请求参数分别是: 目标任务url列表,计数器字典,请求响应格式,计数器步长
-        :return:
+        异步发送请求，自适应支持 HTTP/HTTPS 和 SOCKS4/SOCKS5 代理。
+        - HTTP/HTTPS 代理：使用 aiohttp 原生 proxy 参数
+        - SOCKS 代理：使用 aiohttp-socks（需安装）
         """
-        if self.statistic:
-            component_name = record_dict['component_name']
-
-        async def async_request(url, cookies=None, **kwargs):
-            if cookies:
-                clientSessionObj = ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest,
-                                                 cookies=cookies)
+        if record_dict is None:
+            record_dict = {}
+        
+        async def async_request(url: str, cookies: Optional[Dict] = None, **kwargs):
+            # 解析代理类型
+            use_socks = False
+            if proxy_url:
+                parsed = urlparse(proxy_url)
+                if parsed.scheme in ('socks4', 'socks5'):
+                    use_socks = True
+                elif parsed.scheme not in ('http', 'https'):
+                    raise ValueError(f"Unsupported proxy scheme: {parsed.scheme}. Use http, https, socks4, or socks5.")
+            
+            try:
+                if use_socks:
+                    # 使用 SOCKS 代理
+                    if not HAS_SOCKS:
+                        raise RuntimeError(
+                            "SOCKS proxy requires 'aiohttp-socks' package. Install with: pip install aiohttp-socks")
+                    connector = ProxyConnector.from_url(proxy_url)
+                    async with ClientSession(connector=connector, cookies=cookies) as session:
+                        async with session.get(url, **kwargs) as response:
+                            return await _process_response(response, url)
+                else:
+                    # 使用 HTTP/HTTPS 代理 或 无代理
+                    async with ClientSession(cookies=cookies) as session:
+                        async with session.get(url, proxy=proxy_url, **kwargs) as response:
+                            return await _process_response(response, url)
+            
+            except Exception as e:
+                if self.statistic and 'component_name' in record_dict:
+                    component_name = record_dict['component_name']
+                    record_dict['record_info'] = f'{component_name}_request_fail'
+                    spider_record(self.db_client, record_dict, step, False)
+                import traceback
+                print(f"Error fetching {url} via {proxy_url or 'direct'}: {e}\n{traceback.format_exc()}")
+                return '', url, None
+        
+        async def _process_response(response: ClientResponse, url: str):
+            """统一处理响应内容"""
+            if self.statistic and 'component_name' in record_dict:
+                component_name = record_dict['component_name']
+                if response.status == 200:
+                    record_dict['record_info'] = f'{component_name}_request_success'
+                else:
+                    record_dict['record_info'] = f'{component_name}_request_fail'
+                spider_record(self.db_client, record_dict, step, False)
+            
+            if response_model == 'json':
+                content = await response.json()
+            elif response_model == 'bytes':
+                content = await response.read()
+            elif response_model == 'html':
+                content = await response.text()
             else:
-                clientSessionObj = ClientSession(connector=ProxyConnector(), request_class=ProxyClientRequest)
-            async with clientSessionObj as session:
-                try:
-                    async with session.get(url, **kwargs) as response:
-                        if response.status == 200:
-                            if self.statistic: record_dict['record_info'] = '{}_request_success'.format(component_name)
-                        else:
-                            if self.statistic: record_dict['record_info'] = '{}_request_fail'.format(component_name)
-                        if self.statistic: spider_record(self.db_client, record_dict, step, False)
-                        if self.statistic: record_dict['record_info'] = '{}_request_content_length'.format(
-                            component_name)
-                        content_length = eval(response.headers.get("Content-Length", "0"))
-                        res_content = await response.read()
-                        if self.statistic: spider_record(self.db_client, record_dict,
-                                                         content_length if content_length else len(res_content),
-                                                         False)
-                        if response_model == 'json':
-                            return await response.json(), str(response.url), response
-                        elif response_model == 'bytes':
-                            return await response.read(), str(response.url), response
-                        elif response_model == 'html':
-                            return await response.text(), str(response.url), response
-                        else:
-                            return await response, str(response.url), response
-                except Exception as e:
-                    if self.statistic:
-                        record_dict['record_info'] = '{}_request_fail'.format(component_name)
-                        spider_record(self.db_client, record_dict, step, False)
-                    import traceback
-                    print(traceback.format_exc(), url)
-                    if self.statistic: record_dict['record_info'] = '{}_request_content_length'.format(component_name)
-                    content_length = 0
-                    res_content = ""
-                    if self.statistic: spider_record(self.db_client, record_dict,
-                                                     content_length if content_length else len(res_content),
-                                                     False)
-                    return '', url, Response
-
-        tasks = [asyncio.ensure_future(async_request(i, cookies=cookies, **kwargs)) for i in urls]
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(asyncio.wait(tasks))
-        del loop
-        return [i.result() for i in tasks]
+                content = response
+            
+            return content, str(response.url), response
+        
+        # 执行所有任务
+        tasks = [async_request(url, cookies=cookies, **kwargs) for url in urls]
+        results = asyncio.run(asyncio.gather(*tasks, return_exceptions=False))
+        return results
 
 
 class Parse(object):
     """
     解析源数据类,使用思路为继承该类并重写analysis_data方法
     """
-
+    
     def __init__(self, db_client=None, statistic=False, *, db_host: str = None, db_port: int = 6379,
                  db_cluster: bool = False,
                  db_password: str = None):
@@ -343,7 +320,7 @@ class Parse(object):
         else:
             redis_type = RedisCluster if db_cluster else Redis
             self.db_client = redis_type(host=db_host, port=db_port, password=db_password)
-
+    
     # @time_counter
     def analysis_data(self, source_code, record_dict: dict = dict, step: int = 1, *args, **kwargs):
         """
@@ -359,7 +336,7 @@ class Parse(object):
 
 
 class Monitor(object):
-
+    
     def __init__(self, rds=None, statistic=False, *, redis_host: str = None, redis_port: int = 6379,
                  redis_cluster: bool = False,
                  redis_password: str = None):
@@ -369,11 +346,11 @@ class Monitor(object):
         else:
             redis_type = RedisCluster if redis_cluster else Redis
             self.rds = redis_type(host=redis_host, port=redis_port, password=redis_password)
-
+    
     def log(self):
         """日志统计"""
         ...
-
+    
     # 在收集的大key前面添加日期，之后删除被收集的大key
     def statistics_count(self, summary_keys: dict, *, hour: str = None, minute: str = None, second: str = None):
         """
@@ -384,7 +361,7 @@ class Monitor(object):
         :param second:
         :return:
         """
-
+        
         def summary_count(summary_keys: dict):
             today_date = get_today_str()
             for summary_key, big_keys in summary_keys.items():
@@ -394,7 +371,7 @@ class Monitor(object):
                         if today_date not in summary_key:
                             summary_key = today_date + '_' + summary_key
                         self.rds.hset(summary_key, k, v)
-
+        
         scheduler = BlockingScheduler(timezone="Asia/Shanghai")
         scheduler.add_job(
             summary_count, 'cron',

{ODtools-2.1.21 → odtools-2.2.1}/ODtools.egg-info/PKG-INFO

@@ -1,12 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: ODtools
-Version: 2.1.21
+Version: 2.2.1
 Summary: zkrTools
 Home-page: https://github.com/zkr-origin-data-dpt/ODtools
 Author: zkrPython
 Author-email: 178031608@qq.com
 License: Apache License
-Platform: UNKNOWN
 Classifier: Environment :: Web Environment
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
@@ -23,6 +22,28 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
-
-UNKNOWN
-
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: xlrd
+Requires-Dist: xlwt
+Requires-Dist: elasticsearch==8.2.0
+Requires-Dist: thrift
+Requires-Dist: kafka-python
+Requires-Dist: redis-py-cluster
+Requires-Dist: pymysql
+Requires-Dist: loguru
+Requires-Dist: colorlog
+Requires-Dist: aiohttp-socks
+Requires-Dist: apscheduler
+Requires-Dist: wheel
+Requires-Dist: twine
+Requires-Dist: PySocks
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary

{ODtools-2.1.21 → odtools-2.2.1}/ODtools.egg-info/requires.txt

@@ -1,12 +1,14 @@
 xlrd
 xlwt
-elasticsearch==6.1.1
+elasticsearch==8.2.0
 thrift
 kafka-python
-redis-py-cluster==2.1.3
+redis-py-cluster
 pymysql
 loguru
 colorlog
-aiosocksy==0.1.2
+aiohttp-socks
 apscheduler
+wheel
+twine
 PySocks

{ODtools-2.1.21 → odtools-2.2.1}/PKG-INFO

@@ -1,12 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: ODtools
-Version: 2.1.21
+Version: 2.2.1
 Summary: zkrTools
 Home-page: https://github.com/zkr-origin-data-dpt/ODtools
 Author: zkrPython
 Author-email: 178031608@qq.com
 License: Apache License
-Platform: UNKNOWN
 Classifier: Environment :: Web Environment
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
@@ -23,6 +22,28 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
-
-UNKNOWN
-
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: xlrd
+Requires-Dist: xlwt
+Requires-Dist: elasticsearch==8.2.0
+Requires-Dist: thrift
+Requires-Dist: kafka-python
+Requires-Dist: redis-py-cluster
+Requires-Dist: pymysql
+Requires-Dist: loguru
+Requires-Dist: colorlog
+Requires-Dist: aiohttp-socks
+Requires-Dist: apscheduler
+Requires-Dist: wheel
+Requires-Dist: twine
+Requires-Dist: PySocks
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary

{ODtools-2.1.21 → odtools-2.2.1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ODtools",
-    version="2.1.21",
+    version="2.2.1",
     author="zkrPython",
     author_email="178031608@qq.com",
     description="zkrTools",
@@ -15,15 +15,17 @@ setup(
     install_requires=[
         "xlrd",
         "xlwt",
-        "elasticsearch==6.1.1",
+        "elasticsearch==8.2.0",
         "thrift",
         "kafka-python",
-        "redis-py-cluster==2.1.3",
+        "redis-py-cluster",
         "pymysql",
         "loguru",
         "colorlog",
-        "aiosocksy==0.1.2",
+        "aiohttp-socks",
         "apscheduler",
+        "wheel",
+        "twine",
         "PySocks",#增加依赖，requests socket 代理依赖
     ],
     classifiers=[
@@ -43,5 +45,9 @@ setup(
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
     ],
 )