mdbq 4.0.0__tar.gz → 4.0.1__tar.gz

{mdbq-4.0.0 → mdbq-4.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.0
+Version: 4.0.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.1/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.0.1'

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/aggregation/query_data.py

@@ -3,6 +3,7 @@ import re
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import config
+from mdbq.log import mylogger
 import datetime
 from dateutil.relativedelta import relativedelta
 import pandas as pd
@@ -13,7 +14,6 @@ import os
 import time
 import calendar
 import concurrent.futures
-import logging
 
 """
 
@@ -25,9 +25,17 @@ content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 
-logger = logging.getLogger(__name__)
-# from mdbq.log import spider_logging
-# logger = spider_logging.setup_logging()
+logger = mylogger.MyLogger(
+    logging_mode='file',
+    log_level='info',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 class MysqlDatasQuery:
@@ -162,8 +170,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -208,8 +215,7 @@ class MysqlDatasQuery:
             '花费': 'decimal(12,2)',
             '更新时间': 'timestamp',
         }
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) 属性设置3/商品索引表_主推排序调用')
+        logger.info(f'正在更新: mysql ({host}:{port}) 属性设置3/商品索引表_主推排序调用')
         m_engine.df_to_mysql(
             df=df_pic,
             db_name='属性设置3',
@@ -317,8 +323,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -426,8 +431,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -535,8 +539,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -630,8 +633,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -847,8 +849,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -981,8 +982,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1151,8 +1151,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
 
         m_engine.df_to_mysql(
             df=df,
@@ -1263,8 +1262,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1318,8 +1316,7 @@ class MysqlDatasQuery:
         # )  # 制作其他聚合表
         if not self.update_service:
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1373,8 +1370,7 @@ class MysqlDatasQuery:
         # )  # 制作其他聚合表
         if not self.update_service:
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1428,8 +1424,7 @@ class MysqlDatasQuery:
         # )  # 制作其他聚合表
         if not self.update_service:
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1485,8 +1480,7 @@ class MysqlDatasQuery:
         # )
         if not self.update_service:  # 调试加，是否继续执行下面的入库操作
             return
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1532,8 +1526,7 @@ class MysqlDatasQuery:
         df['上市年份'] = df['商品id'].apply(lambda x: check_year(x))
         p = df.pop('上市年份')
         df.insert(loc=5, column='上市年份', value=p)
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         set_typ = {
             '日期': 'date',
             '店铺id': 'bigint',
@@ -1672,8 +1665,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1780,8 +1772,7 @@ class MysqlDatasQuery:
         # df.to_csv('/Users/xigua/Downloads/ll.csv', index=False, header=True, encoding='utf-8_sig')
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1825,8 +1816,7 @@ class MysqlDatasQuery:
             return
         min_date = pd.to_datetime(df['日期'].min()).strftime('%Y-%m-%d')
         max_date = pd.to_datetime(df['日期'].max()).strftime('%Y-%m-%d')
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1939,8 +1929,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2006,8 +1995,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2105,8 +2093,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2182,8 +2169,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2251,8 +2237,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2337,8 +2322,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2473,8 +2457,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         new_dict = {
             '日期': '',
             '店铺名称': '',
@@ -2973,8 +2956,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min().strftime('%Y-%m-%d')
         max_date = df['日期'].max().strftime('%Y-%m-%d')
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3063,8 +3045,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3113,8 +3094,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3223,8 +3203,7 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3578,7 +3557,6 @@ class MysqlDatasQuery:
                 '退货量_发货后': ('退货量_发货后', np.sum),
             }
         )
-        # logger.info(df.info())
 
         idbm['宝贝id'] = idbm['宝贝id'].astype('int64')
         # 1.  id 编码表合并图片表
@@ -3587,7 +3565,6 @@ class MysqlDatasQuery:
         # 2.  df 合并商品成本表
         df_cb = pd.merge(df_cb, cost, how='left', left_on='商家编码', right_on='款号')
         df_cb = df_cb[['宝贝id', '商家编码', '商品图片', '成本价']]
-        # logger.info(df_cb.info())
         # 3.  合并 df
         df = pd.merge(df, df_cb, how='left', left_on='商品id', right_on='宝贝id')
         df.drop(labels='宝贝id', axis=1, inplace=True)
@@ -3625,8 +3602,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3726,8 +3702,7 @@ class MysqlDatasQuery:
         df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
         min_date = df['日期'].min()
         max_date = df['日期'].max()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3795,8 +3770,7 @@ class MysqlDatasQuery:
             return
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3856,11 +3830,9 @@ def date_table():
     for name, group in grouped:
         if num > 52:
             num = 1
-        # logger.info(f'Group: {name}')
         group['第n周_new'] = f'第{num}周'
         num += 1
         __res.append(group.copy())
-        # logger.info(group)
         # break
     df = pd.concat(__res, ignore_index=True)
     # df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x))

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/mysql/deduplicator.py

@@ -18,16 +18,15 @@ from contextlib import contextmanager
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
-    name='deduplicator',
     logging_mode='file',
     log_level='info',
-    log_file='deduplicator.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样DEBUG/INFO日志, 0.5表示50%的日志会被采样
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
 )
 
 

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/mysql/mysql.py

@@ -17,16 +17,15 @@ warnings.filterwarnings('ignore')
 建表规范:
 """
 logger = mylogger.MyLogger(
-    name='mysql',
-    logging_mode='both',
+    logging_mode='file',
     log_level='info',
-    log_file='mysql.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=0.5,  # 采样50%的DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
 )
 
 

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/mysql/s_query.py

@@ -4,14 +4,24 @@ import warnings
 import pymysql
 import pandas as pd
 from decimal import Decimal
-import logging
 from contextlib import closing
+from mdbq.log import mylogger
 
 warnings.filterwarnings('ignore')
 """
 程序专门用来下载数据库数据, 并返回 df, 不做清洗数据操作;
 """
-logger = logging.getLogger(__name__)
+logger = mylogger.MyLogger(
+    logging_mode='file',
+    log_level='info',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 class QueryDatas:

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/mysql/uploader.py

@@ -15,16 +15,15 @@ from decimal import Decimal, InvalidOperation
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
-    name='uploader',
     logging_mode='file',
     log_level='info',
-    log_file='uploader.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样DEBUG/INFO日志, 0.5表示50%的日志会被采样
-    sensitive_fields=[],  #  过滤敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
 )
 
 

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/redis/getredis.py

@@ -1,34 +1,26 @@
 # -*- coding: UTF-8 –*-
-import os
 import random
 import pandas as pd
 import numpy as np
 import json
 import datetime
 import threading
-import logging
-from logging.handlers import RotatingFileHandler
+from mdbq.log import mylogger
 from decimal import Decimal
 import orjson
 
 # 获取当前模块的日志记录器
-logger = logging.getLogger(__name__)
-
-# 创建一个文件处理器，用于将日志写入文件
-dir_path = os.path.expanduser("~")
-# config_file = os.path.join(dir_path, 'spd.txt')
-log_file = os.path.join(dir_path, 'logfile', 'redis.log')
-if not os.path.isdir(os.path.join(dir_path, 'logfile')):
-    os.mkdir(os.path.join(dir_path, 'logfile'))
-file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10, encoding='utf-8')  # 保留10个备份文件
-file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
-
-# 创建一个日志格式器，并设置给文件处理器
-formatter = logging.Formatter('[%(asctime)s] %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
-file_handler.setFormatter(formatter)
-
-# 将文件处理器添加到日志记录器
-logger.addHandler(file_handler)
+logger = mylogger.MyLogger(
+    logging_mode='file',
+    log_level='info',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 class RedisData(object):

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq/spider/aikucun.py

@@ -29,16 +29,14 @@ uld = uploader.MySQLUploader(username=username, password=password, host=host, po
 # 实例化一个数据查询类，用来获取 cookies 表数据
 download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 logger = mylogger.MyLogger(
-    name='aikucun',
     logging_mode='file',
     log_level='info',
-    log_file='aikucun.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样50%的DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段过滤
     enable_metrics=False,  # 是否启用性能指标
 )
 

{mdbq-4.0.0 → mdbq-4.0.1}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.0
+Version: 4.0.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.0/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '4.0.0'