mdbq 4.0.10__py3-none-any.whl → 4.0.11__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.10'
+VERSION = '4.0.11'

mdbq/aggregation/query_data.py

@@ -3676,6 +3676,7 @@ def query3(months=1, download_manager=None):
 
 
 def main(months=3):
+    logger.info('数据聚合任务开始')
     # 1. 更新日期表  更新货品年份基准表， 属性设置 3 - 货品年份基准
     date_table()
     # 2. 数据聚合

mdbq/mysql/deduplicator.py

@@ -6,6 +6,7 @@ import warnings
 import pymysql
 import os
 from mdbq.log import mylogger
+from mdbq.config import config
 from typing import List, Dict, Optional, Any, Tuple
 from dbutils.pooled_db import PooledDB
 import threading
@@ -1348,7 +1349,7 @@ class MySQLDeduplicator:
 
 
 def main():
-    from mdbq.config import config
+    logger.info('去重任务开始')
     dir_path = os.path.expanduser("~")
     my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
     username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
@@ -1401,6 +1402,8 @@ def main():
 
     # 关闭连接
     deduplicator.close()
+    logger.info('去重任务结束')
+
 
 if __name__ == '__main__':
     main()

mdbq/mysql/s_query.py

@@ -35,7 +35,7 @@ class QueryDatas:
     """
 
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4',
-                 maxconnections: int = 20, mincached: int = 2, maxcached: int = 5,
+                 pool_size: int = 20, mincached: int = 2, maxcached: int = 5,
                  connect_timeout: int = 10, read_timeout: int = 30, write_timeout: int = 30,
                  max_retries: int = 3, retry_waiting_time: int = 5, collation: str = 'utf8mb4_0900_ai_ci') -> None:
         """
@@ -47,7 +47,7 @@ class QueryDatas:
             host: 数据库主机
             port: 数据库端口
             charset: 字符集，默认utf8mb4
-            maxconnections: 最大活动连接数，默认20
+            pool_size: 最大活动连接数，默认20
             mincached: 最小缓存连接数，空闲连接数量，默认2
             maxcached: 最大缓存连接数，最大空闲连接数，默认5
             connect_timeout: 连接超时时间，默认10秒
@@ -87,14 +87,14 @@ class QueryDatas:
             'write_timeout': write_timeout,
             'autocommit': True
         }
-        self.pool = self._create_connection_pool(maxconnections, mincached, maxcached)
+        self.pool = self._create_connection_pool(pool_size, mincached, maxcached)
 
-    def _create_connection_pool(self, maxconnections: int, mincached: int, maxcached: int) -> PooledDB:
+    def _create_connection_pool(self, pool_size: int, mincached: int, maxcached: int) -> PooledDB:
         """
         创建数据库连接池
         
         Args:
-            maxconnections: 最大连接数
+            pool_size: 最大连接数
             mincached: 最小缓存连接数
             maxcached: 最大缓存连接数
             
@@ -122,7 +122,7 @@ class QueryDatas:
         }
         pool_params = {
             'creator': pymysql,
-            'maxconnections': maxconnections,
+            'maxconnections': pool_size,
             'mincached': mincached,
             'maxcached': maxcached,
             'blocking': True,
@@ -133,7 +133,7 @@ class QueryDatas:
         try:
             pool = PooledDB(**pool_params, **connection_params)
             logger.debug('连接池创建成功', {
-                '连接池大小': maxconnections,
+                '连接池大小': pool_size,
                 '最小缓存': mincached,
                 '最大缓存': maxcached,
                 '主机': self.host,
@@ -717,7 +717,7 @@ class QueryDatas:
         if hasattr(self, 'pool') and self.pool is not None:
             try:
                 self.pool.close()
-                logger.info('连接池已关闭', {
+                logger.debug('连接池已关闭', {
                     '主机': self.host,
                     '端口': self.port
                 })

mdbq/mysql/uploader.py

@@ -539,6 +539,8 @@ class MySQLUploader:
             is_nan = True
         elif str(value).lower() in ['nan', 'none']:
             is_nan = True
+        elif value == '':
+            is_nan = True
         if is_nan:
             if not allow_null:
                 if 'int' in column_type_lower:

mdbq/spider/aikucun.py

@@ -25,9 +25,7 @@ config_file = os.path.join(dir_path, 'spd.txt')
 content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
-uld = uploader.MySQLUploader(username=username, password=password, host=host, port=int(port), pool_size=10)
 # 实例化一个数据查询类，用来获取 cookies 表数据
-download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 logger = mylogger.MyLogger(
     logging_mode='file',
     log_level='info',
@@ -48,15 +46,15 @@ def keep_connect(_db_name, _config, max_try: int=10):
             connection = pymysql.connect(**_config)  # 连接数据库
             return connection
         except Exception as e:
-            logger.error(f'{_db_name}: 连接失败，正在重试: {host}:{port}  {attempts}/{max_try} {e}')
+            logger.error('连接失败', {'数据库': _db_name, '主机': host, '端口': port, '重试次数': attempts, '最大重试次数': max_try, '错误信息': e})
             attempts += 1
             time.sleep(30)
-    logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+    logger.error('连接失败', {'数据库': _db_name, '主机': host, '端口': port, '重试次数': attempts, '最大重试次数': max_try})
     return None
 
 
 class AikuCun:
-    def __init__(self):
+    def __init__(self, uld_manager, download_manager):
         self.url = 'https://gray-merc.aikucun.com/index.html'
         self.db_name = 'cookie文件'
         self.table_name = 'main_aikucun'
@@ -66,6 +64,8 @@ class AikuCun:
         self.start_date = (self.today - datetime.timedelta(days=7)).strftime('%Y-%m-%d')
         self.end_date = (self.today - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
         self.error_count = 0
+        self.uld = uld_manager
+        self.download = download_manager
 
     def logining(self, shop_name='aikucun', headless=False):
         option = webdriver.ChromeOptions()
@@ -171,7 +171,7 @@ class AikuCun:
 
     def save_token(self):
         if not self.token:
-            print('self.token 不能为空')
+            logger.error('self.token 不能为空')
             return
         set_typ = {
             '日期': 'DATE',
@@ -182,11 +182,11 @@ class AikuCun:
             '更新时间': 'timestamp'
         }
         # 更新至数据库记录
-        uld.upload_data(
+        self.uld.upload_data(
             db_name=self.db_name,
             table_name=self.table_name,
             data=self.token,
-            set_typ={},
+            set_typ=set_typ,
             primary_keys=[],
             check_duplicate=False,
             update_on_duplicate=False,
@@ -209,7 +209,7 @@ class AikuCun:
             self.end_date = end_date
         date_list = otk.dates_between(start_date=self.start_date, end_date=self.end_date)
 
-        df = download.data_to_df(
+        df = self.download.data_to_df(
             db_name=self.db_name,
             table_name=self.table_name,
             start_date='2025-03-07',
@@ -230,7 +230,7 @@ class AikuCun:
             idx = df.groupby(['平台', '店铺名称'])['更新时间'].idxmax()
             df = df.loc[idx][['token']]
             if len(df) == 0:
-                print(f'从数据库获取的 token 不能为空')
+                logger.error(f'从数据库获取的 token 不能为空')
                 return
             self.token = df.iloc[0, 0]
 
@@ -247,7 +247,7 @@ class AikuCun:
         results = []
         for date in date_list:
             if self.error_count > 5:
-                print('已退出请求 -> self.error_count > 5')
+                logger.logger('已退出请求 -> self.error_count > 5')
                 break
             req_date = re.sub('-', '', date)
             data = {
@@ -273,16 +273,15 @@ class AikuCun:
                 # cookies=cookies,
                 data=json.dumps(data)
             )
-            print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
-            # print(res.json())
+            logger.info('获取数据', {'进度': num/len(date_list), '日期': date, '榜单类型': item_type})
             if not res.json().get('success', None):
-                print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
+                logger.error('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
                 continue
             if not res.json().get('data', {}).get('rows', None):
-                print("返回的数据字典异常, ['data']['rows'] 不能为空")
+                logger.error("返回的数据字典异常, ['data']['rows'] 不能为空")
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
@@ -291,7 +290,7 @@ class AikuCun:
             num += 1
             time.sleep(1)
             if num % 32 == 0:
-                print("避免频繁请求, 正在休眠...")
+                logger.info("避免频繁请求, 正在休眠...")
                 # time.sleep(60)
 
         return results
@@ -413,18 +412,18 @@ class AikuCun:
             '尺码': 'varchar(50)',
             '货号': 'varchar(50)',  # 款号 + 颜色编码
         }
-        print(f'{self.shop_name} 正在更新数据库 {db_name} -> {table_name}...')
+        logger.info('更新数据库', {'店铺名称': self.shop_name, '库': db_name, '表': table_name})
         if 'spu' in table_name:
             drop_dup = ['日期', '平台', '店铺名称', '商品款号', '访客量']
         else:
             drop_dup = ['日期', '平台', '店铺名称', '条码']
-        uld.upload_data(
+        self.uld.upload_data(
             db_name=db_name,
             table_name=table_name,
             data=_results,
             set_typ=set_typ,  # 定义列和数据类型
             primary_keys=[],  # 创建唯一主键
-            check_duplicate=True,  # 检查重复数据
+            check_duplicate=False,  # 检查重复数据
             update_on_duplicate=False,  # 遇到重复时更新数据，默认 False 跳过
             duplicate_columns=drop_dup,  # 指定排重的组合键
             allow_null=False,  # 允许插入空值
@@ -470,36 +469,44 @@ class AikuCun:
             headers=headers,
             data=json.dumps(data)
         )
-        print(res.json())
 
 
 def main(start_date, end_date=None, item_type=['spu']):
-    ak = AikuCun()
-    # ak.get_sign()
-    for type_ in item_type:
-        if type_ not in ['spu', 'sku']:
-            print(f'{item_type} 非法参数: {type_}')
-            continue
-        for i in range(2):
-            data_list = ak.get_data_from_bbx(
-                start_date=start_date,
-                end_date=end_date,
-                item_type=type_,
-                page_num=1,
-                page_size=300
-            )
-            if not data_list:
-                ak.logining()
-                ak.save_token()
-                ak.error_count = 0  # 重置错误计数器
-            else:
-                break
+    db_config = {
+        'username': username,
+        'password': password,
+        'host': host,
+        'port': int(port),
+        'pool_size': 3
+    }
+    with uploader.MySQLUploader(**db_config) as uld:
+        with s_query.QueryDatas(**db_config) as download:
+            ak = AikuCun(uld_manager=uld, download_manager=download)
+            # ak.get_sign()
+            for type_ in item_type:
+                if type_ not in ['spu', 'sku']:
+                    logger.error(f'{item_type} 非法参数: {type_}')
+                    continue
+                for i in range(2):
+                    data_list = ak.get_data_from_bbx(
+                        start_date=start_date,
+                        end_date=end_date,
+                        item_type=type_,
+                        page_num=1,
+                        page_size=300
+                    )
+                    if not data_list:
+                        ak.logining()
+                        ak.save_token()
+                        ak.error_count = 0  # 重置错误计数器
+                    else:
+                        break
 
-        ak.insert_datas(
-            data_list=data_list,
-            db_name='爱库存2',
-            table_name=f'{type_}榜单'
-        )
+                ak.insert_datas(
+                    data_list=data_list,
+                    db_name='爱库存2',
+                    table_name=f'{type_}榜单'
+                )
 
 
 
@@ -508,7 +515,7 @@ if __name__ == '__main__':
         start_date='2025-05-13',
         # end_date='2025-04-28',  # 不传则默认到今天
         item_type=[
-            # 'spu',
+            'spu',
             'sku'
         ]
     )

{mdbq-4.0.10.dist-info → mdbq-4.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.10
+Version: 4.0.11
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.10.dist-info → mdbq-4.0.11.dist-info}/RECORD

@@ -1,18 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=DkNrzYKkaEbmtMXcPR8H0qa5y2WJne8kPSNBfTO6mFo,18
+mdbq/__version__.py,sha256=PQJs_Lgx6OvamcsXbLCVuBAvLc7j2xwJDZEWigwyUy8,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=NdhsLJvt6NgZSMHpkMxnmEEXzDhUiR5tRwYwI-PfwIw,166732
+mdbq/aggregation/query_data.py,sha256=SM8cS9lBKmhLBQdwJz-sRu9bl7w1HS0MEq10s6Tqf_0,166777
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=9w_o5mYB3FooIxobq_lSa6oCYTKIhPxDFox-jeLtUHI,21714
-mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
+mdbq/mysql/deduplicator.py,sha256=fS1dSs92vN15tuqmAKrUVdKk6z9dwW_Fe9WHMBYsy2U,73172
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=jGBdGPE4mtB06vccfaWIEWpSAqdY-nWc1s9bzCUh8Gg,42916
+mdbq/mysql/s_query.py,sha256=RPC-KZVuqPlCSmpmtUmYAOJdxJT01i0DvlIbmum4MxM,42882
 mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=wNQE7UjCEyAKri9CnQXO7d6EVXCaYqFze2i2tcGAVpw,81001
+mdbq/mysql/uploader.py,sha256=wX2gHhVQJwGErnjUbLnsljkZ8Yd3YK-HS3P7q8DizAA,81053
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -24,8 +23,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.10.dist-info/METADATA,sha256=AJXVA5kCyLJQiObIW13jNvJAWsXdXJRFN2xCCSdDO78,364
-mdbq-4.0.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.10.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=7oquQ2RIJr6B1xblQMfnmHzteOlvHA7dIcPRaAPfHBc,21546
+mdbq-4.0.11.dist-info/METADATA,sha256=zZh35aA-suJ3B_v39Mw8V_O2GSdOLdylfNPl_E99uqQ,364
+mdbq-4.0.11.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.11.dist-info/RECORD,,

mdbq/log/spider_logging.py

@@ -1,47 +0,0 @@
-import logging
-from logging.handlers import RotatingFileHandler
-import platform
-import os
-import sys
-import getpass
-
-
-def setup_logging(reMoveOldHandler=True, filename='spider_tg.log'):
-    """
-    reMoveOldHandler: 替换根日志记录器的所有现有处理器
-    """
-    dir_path = os.path.expanduser("~")
-    if not os.path.isdir(os.path.join(dir_path, 'logfile')):
-        os.makedirs(os.path.join(dir_path, 'logfile'))
-
-    log_file = os.path.join(dir_path, 'logfile', filename)
-    file_handler = RotatingFileHandler(
-        filename=log_file,
-        maxBytes=3*1024*1024,  # 3MB
-        backupCount=10,
-        encoding='utf-8'  # 明确指定编码（避免Windows乱码）
-    )
-    stream_handler = logging.StreamHandler()  # 终端输出Handler
-    formatter = logging.Formatter(
-        fmt='[%(asctime)s] %(levelname)s %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
-    )
-    file_handler.setFormatter(formatter)
-    stream_handler.setFormatter(formatter)  # 终端使用相同格式
-    file_handler.setLevel(logging.INFO)
-    stream_handler.setLevel(logging.INFO)
-
-    # 获取根日志记录器并添加Handler
-    logger = logging.getLogger()
-    if reMoveOldHandler:
-        # 移除根日志记录器的所有现有处理器
-        for handler in logger.handlers[:]:  # 使用[:]来创建handlers列表的一个副本，因为我们在迭代时修改列表
-            logger.removeHandler(handler)
-    logger.addHandler(file_handler)
-    logger.addHandler(stream_handler)
-    logger.setLevel(logging.INFO)  # 设置根日志级别
-    return logger
-
-
-if __name__ == '__main__':
-    pass