mdbq 4.0.0__py3-none-any.whl → 4.0.1__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.0'
+VERSION = '4.0.1'

mdbq/aggregation/query_data.py

@@ -3,6 +3,7 @@ import re
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import config
+from mdbq.log import mylogger
 import datetime
 from dateutil.relativedelta import relativedelta
 import pandas as pd
@@ -13,7 +14,6 @@ import os
 import time
 import calendar
 import concurrent.futures
-import logging
 
 """
 
@@ -25,9 +25,17 @@ content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 
-logger = logging.getLogger(__name__)
-# from mdbq.log import spider_logging
-# logger = spider_logging.setup_logging()
+logger = mylogger.MyLogger(
+    logging_mode='file',
+    log_level='info',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 class MysqlDatasQuery:
@@ -162,8 +170,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -208,8 +215,7 @@ class MysqlDatasQuery:
             '花费': 'decimal(12,2)',
             '更新时间': 'timestamp',
         }
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) 属性设置3/商品索引表_主推排序调用')
+        logger.info(f'正在更新: mysql ({host}:{port}) 属性设置3/商品索引表_主推排序调用')
         m_engine.df_to_mysql(
             df=df_pic,
             db_name='属性设置3',
@@ -317,8 +323,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -426,8 +431,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -535,8 +539,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -630,8 +633,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -847,8 +849,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -981,8 +982,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1151,8 +1151,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
 
         m_engine.df_to_mysql(
             df=df,
@@ -1263,8 +1262,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1318,8 +1316,7 @@ class MysqlDatasQuery:
         # )  # 制作其他聚合表
         if not self.update_service:
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1373,8 +1370,7 @@ class MysqlDatasQuery:
         # )  # 制作其他聚合表
         if not self.update_service:
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1428,8 +1424,7 @@ class MysqlDatasQuery:
         # )  # 制作其他聚合表
         if not self.update_service:
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1485,8 +1480,7 @@ class MysqlDatasQuery:
         # )
         if not self.update_service:  # 调试加，是否继续执行下面的入库操作
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1532,8 +1526,7 @@ class MysqlDatasQuery:
         df['上市年份'] = df['商品id'].apply(lambda x: check_year(x))
         p = df.pop('上市年份')
         df.insert(loc=5, column='上市年份', value=p)
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         set_typ = {
             '日期': 'date',
             '店铺id': 'bigint',
@@ -1672,8 +1665,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1780,8 +1772,7 @@ class MysqlDatasQuery:
         # df.to_csv('/Users/xigua/Downloads/ll.csv', index=False, header=True, encoding='utf-8_sig')
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1825,8 +1816,7 @@ class MysqlDatasQuery:
             return
         min_date = pd.to_datetime(df['日期'].min()).strftime('%Y-%m-%d')
         max_date = pd.to_datetime(df['日期'].max()).strftime('%Y-%m-%d')
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1939,8 +1929,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2006,8 +1995,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2105,8 +2093,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2182,8 +2169,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2251,8 +2237,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2337,8 +2322,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2473,8 +2457,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         new_dict = {
             '日期': '',
             '店铺名称': '',
@@ -2973,8 +2956,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min().strftime('%Y-%m-%d')
         max_date = df['日期'].max().strftime('%Y-%m-%d')
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3063,8 +3045,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3113,8 +3094,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3223,8 +3203,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3578,7 +3557,6 @@ class MysqlDatasQuery:
                 '退货量_发货后': ('退货量_发货后', np.sum),
             }
         )
-        # logger.info(df.info())
 
         idbm['宝贝id'] = idbm['宝贝id'].astype('int64')
         # 1.  id 编码表合并图片表
@@ -3587,7 +3565,6 @@ class MysqlDatasQuery:
         # 2.  df 合并商品成本表
         df_cb = pd.merge(df_cb, cost, how='left', left_on='商家编码', right_on='款号')
         df_cb = df_cb[['宝贝id', '商家编码', '商品图片', '成本价']]
-        # logger.info(df_cb.info())
         # 3.  合并 df
         df = pd.merge(df, df_cb, how='left', left_on='商品id', right_on='宝贝id')
         df.drop(labels='宝贝id', axis=1, inplace=True)
@@ -3625,8 +3602,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3726,8 +3702,7 @@ class MysqlDatasQuery:
         df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3795,8 +3770,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3856,11 +3830,9 @@ def date_table():
     for name, group in grouped:
         if num > 52:
             num = 1
-        # logger.info(f'Group: {name}')
         group['第n周_new'] = f'第{num}周'
         num += 1
         __res.append(group.copy())
-        # logger.info(group)
         # break
     df = pd.concat(__res, ignore_index=True)
     # df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x))

mdbq/mysql/deduplicator.py

@@ -18,16 +18,15 @@ from contextlib import contextmanager
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
-    name='deduplicator',
     logging_mode='file',
     log_level='info',
-    log_file='deduplicator.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样DEBUG/INFO日志, 0.5表示50%的日志会被采样
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
 )
 
 

mdbq/mysql/mysql.py

@@ -17,16 +17,15 @@ warnings.filterwarnings('ignore')
 建表规范:
 """
 logger = mylogger.MyLogger(
-    name='mysql',
-    logging_mode='both',
+    logging_mode='file',
     log_level='info',
-    log_file='mysql.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=0.5,  # 采样50%的DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
 )
 
 

mdbq/mysql/s_query.py

@@ -4,14 +4,24 @@ import warnings
 import pymysql
 import pandas as pd
 from decimal import Decimal
-import logging
 from contextlib import closing
+from mdbq.log import mylogger
 
 warnings.filterwarnings('ignore')
 """
 程序专门用来下载数据库数据, 并返回 df, 不做清洗数据操作;
 """
-logger = logging.getLogger(__name__)
+logger = mylogger.MyLogger(
+    logging_mode='file',
+    log_level='info',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 class QueryDatas:

mdbq/mysql/uploader.py

@@ -15,16 +15,15 @@ from decimal import Decimal, InvalidOperation
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
-    name='uploader',
     logging_mode='file',
     log_level='info',
-    log_file='uploader.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样DEBUG/INFO日志, 0.5表示50%的日志会被采样
-    sensitive_fields=[],  #  过滤敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
 )
 
 

mdbq/redis/getredis.py

@@ -1,34 +1,26 @@
 # -*- coding: UTF-8 –*-
-import os
 import random
 import pandas as pd
 import numpy as np
 import json
 import datetime
 import threading
-import logging
-from logging.handlers import RotatingFileHandler
+from mdbq.log import mylogger
 from decimal import Decimal
 import orjson
 
 # 获取当前模块的日志记录器
-logger = logging.getLogger(__name__)
-
-# 创建一个文件处理器，用于将日志写入文件
-dir_path = os.path.expanduser("~")
-# config_file = os.path.join(dir_path, 'spd.txt')
-log_file = os.path.join(dir_path, 'logfile', 'redis.log')
-if not os.path.isdir(os.path.join(dir_path, 'logfile')):
-    os.mkdir(os.path.join(dir_path, 'logfile'))
-file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10, encoding='utf-8')  # 保留10个备份文件
-file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
-
-# 创建一个日志格式器，并设置给文件处理器
-formatter = logging.Formatter('[%(asctime)s] %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
-file_handler.setFormatter(formatter)
-
-# 将文件处理器添加到日志记录器
-logger.addHandler(file_handler)
+logger = mylogger.MyLogger(
+    logging_mode='file',
+    log_level='info',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 class RedisData(object):

mdbq/spider/aikucun.py

@@ -29,16 +29,14 @@ uld = uploader.MySQLUploader(username=username, password=password, host=host, po
 # 实例化一个数据查询类，用来获取 cookies 表数据
 download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 logger = mylogger.MyLogger(
-    name='aikucun',
     logging_mode='file',
     log_level='info',
-    log_file='aikucun.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样50%的DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
     enable_metrics=False,  # 是否启用性能指标
 )
 

{mdbq-4.0.0.dist-info → mdbq-4.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.0
+Version: 4.0.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.0.dist-info → mdbq-4.0.1.dist-info}/RECORD

@@ -1,18 +1,18 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=rGzKotWztr8AKlJYeZkowEC43bIAfiOqR7lVganyrJE,17
+mdbq/__version__.py,sha256=WBE7mTxRoSY0T8hh6Mw105SxEvMZncp5QBVgj2EYNrk,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
+mdbq/aggregation/query_data.py,sha256=U6dYK8_gEaNnsGKooEkzfAWnzNA8kt0uomec49e4olE,177536
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=qBOHJK_h6R_SpfQ1yC5fAlJIEm6uro810i-47uA9C_U,22872
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=iMloLJz4i0w5UGyIwya-GiQS1iMq-DMmA7XPU8PP5k4,73138
-mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
-mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
+mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
+mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
+mdbq/mysql/s_query.py,sha256=tSBEbyuVQBeE6tckHSbguAQh9T07tvPhf4J6DlpUBP8,10508
 mdbq/mysql/unique_.py,sha256=eygkSlRda786iwpR1Q-ofnrhDqhZUE4Z0yVZ9LR4EEU,21158
-mdbq/mysql/uploader.py,sha256=Oyq1lbI0DrUFQHCK14qkPlFQswR_Qm4dBMSec_CUAYA,70458
+mdbq/mysql/uploader.py,sha256=sqzJsOTDJuNJeh9oT_lVsReMwSifETSHd-ZwwGshj9I,70421
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -22,10 +22,10 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
+mdbq/redis/getredis.py,sha256=l3zBK7wrZl0oO42-_UGylyatnIp_SBw8wDDvof9fht4,23534
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=GaekqY55pDEgVxbeQzHHshnQMC2YDv3v4mA7cQwjli4,21019
-mdbq-4.0.0.dist-info/METADATA,sha256=oGG1nyu37HUrUS5Tes3s5NeleIiE_neE2-rCvcceMaU,363
-mdbq-4.0.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.0.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
+mdbq-4.0.1.dist-info/METADATA,sha256=8oQgsqmQ4Nf0vF2YyIJ7Z9mgGdt5Pv4mPLu2z9057xQ,363
+mdbq-4.0.1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.1.dist-info/RECORD,,