mdbq 4.0.7__py3-none-any.whl → 4.0.8__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.7'
+VERSION = '4.0.8'

mdbq/aggregation/query_data.py

@@ -3129,7 +3129,7 @@ class MysqlDatasQuery:
             'partition_date_column': '日期',  # 用于分表的日期列名，默认为'日期'
             'indexes': [],  # 普通索引列
             'transaction_mode': 'batch',  # 事务模式
-            'unique_keys': [['日期', '店铺名称', '人群id', '营销渠道', '计划基础信息']],  # 唯一约束列表
+            'unique_keys': [['日期', '店铺名称', '人群id', '营销渠道', '计划基础信息', '推广单元信息']],  # 唯一约束列表
         }
 
     @try_except
@@ -3683,5 +3683,5 @@ def main(months=3):
 
 
 if __name__ == '__main__':
-    # main(months=3)
+    main(months=3)
     pass

mdbq/log/mylogger.py

@@ -11,18 +11,13 @@ from typing import Optional, Dict, Any, List, Callable
 import atexit
 import traceback
 import inspect
+import psutil
 
-try:
-    import psutil
-    HAS_PSUTIL = True
-except ImportError:
-    HAS_PSUTIL = False
 
 def get_caller_filename(default='mylogger'):
     stack = inspect.stack()
     for frame_info in stack:
         filename = frame_info.filename
-        # 跳过本日志库自身
         if not filename.endswith('mylogger.py'):
             return os.path.splitext(os.path.basename(filename))[0]
     return default
@@ -102,7 +97,6 @@ class MyLogger:
         self.name = name
         self.logging_mode = logging_mode.lower()
         self.log_level = log_level.upper()
-        # log_file 自动为 name+'.log'
         if log_file is None:
             self.log_file = os.path.join(log_path, f"{self.name}.log")
         else:
@@ -117,7 +111,7 @@ class MyLogger:
         self.buffer_size = buffer_size
         self.sample_rate = max(0.0, min(1.0, sample_rate))
         self.filters = filters or []
-        self.enable_metrics = enable_metrics and HAS_PSUTIL
+        self.enable_metrics = enable_metrics
         self.metrics_interval = metrics_interval
         self.message_limited = message_limited
         self.flush_interval = flush_interval
@@ -167,17 +161,14 @@ class MyLogger:
             self.old_context = {}
 
         def __enter__(self):
-            # 保存旧上下文并设置新上下文
             self.old_context = getattr(self.logger._context, 'data', {}).copy()
             self.logger._context.data.update(self.context_vars)
             return self.logger
 
         def __exit__(self, exc_type, exc_val, exc_tb):
-            # 恢复旧上下文
             self.logger._context.data = self.old_context
             if exc_type is not None:
-                self.logger.error(f"上下文内异常2: {exc_val}",
-                                  extra={'类型': str(exc_type)})
+                self.logger.error(f"上下文内异常2: {exc_val}", extra={'类型': str(exc_type)})
             return False
 
     def _init_logging(self):
@@ -185,15 +176,10 @@ class MyLogger:
         valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
         if self.log_level not in valid_levels:
             self.log_level = 'INFO'
-
         self.logger.setLevel(self.log_level)
-
-        # 防止重复添加handler
         if self.logger.handlers:
             for handler in self.logger.handlers[:]:
                 self.logger.removeHandler(handler)
-
-        # 定义日志格式
         if self.log_format.lower() == 'simple':
             class SimpleFormatter(logging.Formatter):
                 def format(self, record):
@@ -255,14 +241,12 @@ class MyLogger:
             )
             file_handler.setFormatter(formatter)
             self._handlers.append(file_handler)
-
         if not self.enable_async:
             for handler in self._handlers:
                 self.logger.addHandler(handler)
 
     def _setup_async_logging(self):
-        # 标准库异步日志实现
-        self._log_queue = queue.Queue(maxsize=self.buffer_size)  # 无限长度，绝不阻塞
+        self._log_queue = queue.Queue(maxsize=self.buffer_size)
         queue_handler = logging.handlers.QueueHandler(self._log_queue)
         self.logger.addHandler(queue_handler)
         self._queue_listener = logging.handlers.QueueListener(
@@ -274,7 +258,6 @@ class MyLogger:
         """获取系统资源使用指标"""
         if not self.enable_metrics:
             return {}
-
         try:
             return {
                 '内存': {
@@ -301,8 +284,7 @@ class MyLogger:
                 }
             }
         except Exception as e:
-            self.logger.warning(f"无法采集系统性能指标: {e}",
-                                extra={'extra_data': {'metrics_error': str(e)}})
+            self.logger.warning(f"无法采集系统性能指标: {e}", extra={'extra_data': {'metrics_error': str(e)}})
             return {}
 
     def _apply_filters(self, level: str, message: str, extra: Dict) -> bool:
@@ -310,11 +292,11 @@ class MyLogger:
         for filter_func in self.filters:
             try:
                 if not filter_func(level, message, extra):
-                    return False  # 如果过滤器返回 False，则丢弃该日志
+                    return False
             except Exception as e:
                 self.logger.warning(f"过滤失败: {e}",
                                     extra={'extra_data': {'filter_error': str(e)}})
-        return True  # 所有过滤器都返回 True，则记录该日志
+        return True
 
     def log_error_handler(retry_times=0, fallback_level='error'):
         """
@@ -334,11 +316,10 @@ class MyLogger:
                     except Exception as e:
                         last_exception = e
                         if attempt < retry_times:
-                            time.sleep(0.1 * (attempt + 1))  # 简单的退避策略
+                            time.sleep(0.1 * (attempt + 1))
                             continue
 
                         try:
-                            # 降级处理
                             logging.basicConfig()
                             fallback_logger = logging.getLogger(f"{getattr(self, 'name', 'mylogger')}_fallback")
                             fallback_msg = f"[降级处理] {message}"[:1000]
@@ -359,23 +340,15 @@ class MyLogger:
         """同步日志记录（兼容异步，直接走logger）"""
         if not hasattr(self.logger, level.lower()):
             return
-
-        # message 仅接收字符串类型
         if not isinstance(message, str):
             message = str(message)
-
-        # 简化日志内容，避免过长
         if len(message) > self.message_limited:
             message = message[:self.message_limited] + '...'
-
-        # 定期收集系统指标
         if self.enable_metrics:
             now = time.time()
             if now - self._last_metrics_time > self.metrics_interval:
                 self._metrics_cache = self._get_system_metrics()
                 self._last_metrics_time = now
-
-        # 准备日志额外数据
         log_extra = {}
         if self.enable_metrics:
             log_extra['性能指标'] = self._metrics_cache
@@ -470,11 +443,8 @@ class MyLogger:
         """记录异常信息"""
         if not extra:
             extra = {}
-
-        # 使用inspect获取调用栈
         frame = inspect.currentframe()
         try:
-            # 向上追溯2层（1层是exception方法本身，2层是实际调用位置）
             caller_frame = frame.f_back.f_back
             extra.update({
                 'module': caller_frame.f_globals.get('__name__', ''),
@@ -486,9 +456,7 @@ class MyLogger:
                 '堆栈': self._format_traceback(exc_info)
             })
         finally:
-            del frame  # 避免循环引用
-
-        # 直接使用logger的error方法记录，保留原始调用栈
+            del frame
         self.log('error', message, extra)
 
     def _format_traceback(self, exc_info):
@@ -556,7 +524,6 @@ class MyLogger:
         """关闭日志记录器，确保所有日志被刷新"""
         if self.enable_async and self._queue_listener:
             self._queue_listener.stop()
-        # 关闭所有handler
         for handler in self.logger.handlers:
             try:
                 handler.close()
@@ -569,7 +536,6 @@ class MyLogger:
                 pass
 
 def main():
-    # 创建日志记录器
     logger = MyLogger(
         name='my_app',
         logging_mode='both',
@@ -579,14 +545,11 @@ def main():
         max_log_size=50,
         backup_count=5,
         enable_async=False,  # 是否启用异步日志
-        sample_rate=1,  # 采样50%的DEBUG/INFO日志
+        sample_rate=1,  # 采样DEBUG/INFO日志
         sensitive_fields=[],  #  敏感字段列表
         enable_metrics=False,  # 是否启用性能指标
     )
-
     logger.info('123')
-
-    # 确保所有日志被刷新
     logger.shutdown()
 
 

mdbq/mysql/s_query.py

@@ -68,15 +68,12 @@ class QueryDatas:
         self.connect_timeout = connect_timeout
         self.read_timeout = read_timeout
         self.write_timeout = write_timeout
-        
-        # 连接池状态监控
         self._pool_stats = {
             'last_health_check': None,
             'health_check_interval': 300,  # 5分钟检查一次
             'consecutive_failures': 0,  # 连续失败次数
             'max_consecutive_failures': 3  # 最大连续失败次数
         }
-        
         self.base_config = {
             'host': self.host,
             'port': int(self.port),
@@ -90,8 +87,6 @@ class QueryDatas:
             'write_timeout': write_timeout,
             'autocommit': True
         }
-        
-        # 创建连接池
         self.pool = self._create_connection_pool(maxconnections, mincached, maxcached)
 
     def _create_connection_pool(self, maxconnections: int, mincached: int, maxcached: int) -> PooledDB:
@@ -111,10 +106,7 @@ class QueryDatas:
         """
         if hasattr(self, 'pool') and self.pool is not None and self._check_pool_health():
             return self.pool
-            
         self.pool = None
-        
-        # 连接参数 - 这些参数会传递给底层的连接创建函数
         connection_params = {
             'host': self.host,
             'port': int(self.port),
@@ -128,8 +120,6 @@ class QueryDatas:
             'write_timeout': self.write_timeout,
             'autocommit': True
         }
-        
-        # 连接池参数
         pool_params = {
             'creator': pymysql,
             'maxconnections': maxconnections,
@@ -140,9 +130,7 @@ class QueryDatas:
             'setsession': [],
             'ping': 7
         }
-        
         try:
-            # 创建连接池，将连接参数作为kwargs传递
             pool = PooledDB(**pool_params, **connection_params)
             logger.debug('连接池创建成功', {
                 '连接池大小': maxconnections,
@@ -170,17 +158,11 @@ class QueryDatas:
         """
         if not self.pool:
             return False
-            
         current_time = time.time()
-        # 检查是否需要执行健康检查
         if (self._pool_stats['last_health_check'] is None or 
             current_time - self._pool_stats['last_health_check'] > self._pool_stats['health_check_interval']):
-            
             try:
-                # 更新健康检查时间
                 self._pool_stats['last_health_check'] = current_time
-                
-                # 检查连接是否可用
                 with self.pool.connection() as conn:
                     with conn.cursor() as cursor:
                         cursor.execute('SELECT 1')
@@ -192,12 +174,9 @@ class QueryDatas:
                                     '连续失败次数': self._pool_stats['consecutive_failures']
                                 })
                             return False
-                
-                # 重置连续失败计数
                 self._pool_stats['consecutive_failures'] = 0
                 logger.debug('连接池健康检查通过')
                 return True
-                
             except Exception as e:
                 self._pool_stats['consecutive_failures'] += 1
                 if self._pool_stats['consecutive_failures'] >= self._pool_stats['max_consecutive_failures']:
@@ -207,7 +186,6 @@ class QueryDatas:
                         '连续失败次数': self._pool_stats['consecutive_failures']
                     })
                 return False
-                
         return True
 
     @staticmethod
@@ -292,28 +270,23 @@ class QueryDatas:
             if self._pool_stats['consecutive_failures'] >= self._pool_stats['max_consecutive_failures']:
                 if not self._check_pool_health():
                     logger.warning('连接池不健康，尝试重新创建')
-                    # 使用默认值重新创建连接池
                     self.pool = self._create_connection_pool(10, 2, 5)
-                    # 重置连续失败计数
                     self._pool_stats['consecutive_failures'] = 0
             
             conn = self.pool.connection()
             if db_name:
-                # 使用原生pymysql连接来选择数据库
                 with conn.cursor() as cursor:
                     cursor.execute(f"USE `{db_name}`")
             return conn
         except pymysql.OperationalError as e:
             error_code = e.args[0] if e.args else None
-            if error_code in (2003, 2006, 2013):  # 连接相关错误
+            if error_code in (2003, 2006, 2013):
                 logger.error('数据库连接错误', {
                     '错误代码': error_code,
                     '错误信息': str(e),
                     '数据库': db_name
                 })
-                # 使用默认值重新创建连接池
                 self.pool = self._create_connection_pool(10, 2, 5)
-                # 重置连续失败计数
                 self._pool_stats['consecutive_failures'] = 0
                 raise ConnectionError(f'数据库连接错误: {str(e)}')
             else:
@@ -389,18 +362,14 @@ class QueryDatas:
         """
         if not date_str:
             return default_date
-            
-        # 记录尝试的日期格式
         attempted_formats = []
         try:
-            # 尝试多种日期格式
             for fmt in ['%Y-%m-%d', '%Y/%m/%d', '%Y%m%d', '%Y.%m.%d']:
                 try:
                     attempted_formats.append(fmt)
                     return pd.to_datetime(date_str, format=fmt).strftime('%Y-%m-%d')
                 except ValueError:
                     continue
-                    
             # 如果所有格式都失败，使用pandas的自动解析
             attempted_formats.append('auto')
             return pd.to_datetime(date_str).strftime('%Y-%m-%d')
@@ -429,27 +398,18 @@ class QueryDatas:
             处理后的日期范围元组 (start_date, end_date)，如果处理失败返回 (None, None)
         """
         try:
-            # 如果两个日期都未提供，返回None表示不进行日期过滤
             if start_date is None and end_date is None:
                 return None, None
-            
-            # 如果只提供了开始日期，结束日期设为今天
             if start_date is not None and end_date is None:
                 end_date = datetime.datetime.today().strftime('%Y-%m-%d')
                 logger.debug('未提供结束日期，使用当前日期', {'库': db_name, '表': table_name, '结束日期': end_date})
-            
-            # 如果只提供了结束日期，开始日期设为1970年
             if start_date is None and end_date is not None:
                 start_date = '1970-01-01'
                 logger.debug('未提供开始日期，使用默认日期', {'库': db_name, '表': table_name, '开始日期': start_date})
-            
-            # 格式化日期
             original_start = start_date
             original_end = end_date
             start_date = self.validate_and_format_date(start_date, '1970-01-01')
             end_date = self.validate_and_format_date(end_date, datetime.datetime.today().strftime('%Y-%m-%d'))
-            
-            # 如果日期格式被修改，记录日志
             if original_start != start_date:
                 logger.debug('开始日期格式已调整', {
                     '库': db_name,
@@ -543,12 +503,8 @@ class QueryDatas:
         if not cols_exist:
             logger.warning('表没有可用列')
             return []
-            
-        # 如果 projection 为 None、空字典或空列表，返回所有列
         if projection is None or projection == {} or projection == []:
             return list(cols_exist)
-            
-        # 验证列名是否包含特殊字符
         invalid_chars = set('`\'"\\')
         selected_columns = []
         for col in projection:
@@ -557,11 +513,9 @@ class QueryDatas:
                 continue
             if col in cols_exist and projection[col]:
                 selected_columns.append(col)
-                
         if not selected_columns:
             logger.info('参数不匹配，返回所有列', {'参数': projection})
             return list(cols_exist)
-            
         return selected_columns
 
     def _build_query_sql(self, db_name: str, table_name: str, selected_columns: List[str], 
@@ -960,10 +914,7 @@ def main():
     username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
     host = 'localhost'
     
-    # 创建QueryDatas实例
     qd = QueryDatas(username=username, password=password, host=host, port=port)
-    
-    # 执行查询
     df = qd.data_to_df('聚合数据', '店铺流量来源构成', limit=10)
     print(df)
 

mdbq/mysql/unique_.py

@@ -321,14 +321,14 @@ def main():
             #     "sku榜单": [['日期', '平台', '店铺名称', '条码']],
             #     "spu榜单": [['日期', '平台', '店铺名称', '商品款号', '访客量']],
             # },
-            "生意参谋3": {
-                # "crm成交客户": [['客户id']],
-                # "商品排行": [['日期', '店铺名称', '商品id']],
-                "流量来源构成": [['日期', '店铺名称', '来源构成', '类别', '一级来源', '二级来源', '三级来源']],
-                # "手淘搜索": [['日期', '店铺名称', '搜索词', '词类型', '访客数']],
-                # "新品追踪": [['日期', '店铺名称', '商品id']],
-                # "直播分场次效果": [['场次id']],
-            },
+            # "生意参谋3": {
+            #     "crm成交客户": [['客户id']],
+            #     "商品排行": [['日期', '店铺名称', '商品id']],
+            #     "流量来源构成": [['日期', '店铺名称', '来源构成', '类别', '一级来源', '二级来源', '三级来源']],
+            #     "手淘搜索": [['日期', '店铺名称', '搜索词', '词类型', '访客数']],
+            #     "新品追踪": [['日期', '店铺名称', '商品id']],
+            #     "直播分场次效果": [['场次id']],
+            # },
             # "生意经3": {
             #     "sku销量_按名称": [['日期', '店铺名称', '宝贝id', 'sku名称', '销售额']],
             #     "sku销量_按商家编码": [['日期', '店铺名称', '宝贝id', 'sku编码', '销售额']],

mdbq/redis/getredis.py

@@ -9,7 +9,6 @@ from mdbq.log import mylogger
 from decimal import Decimal
 import orjson
 
-# 获取当前模块的日志记录器
 logger = mylogger.MyLogger(
     logging_mode='file',
     log_level='info',

{mdbq-4.0.7.dist-info → mdbq-4.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.7
+Version: 4.0.8
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.7.dist-info → mdbq-4.0.8.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=P_9fJTmoyR1p3XJm_JFEO3dJnSh6VG8WMZKVuiE0his,17
+mdbq/__version__.py,sha256=ctSY0gLLZnr9k_avSjRnC3qI18wZwhq3iBz504pEU14,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=Y9AC6xJgZmyZAezz_faRxwo80ev1MxvFEfvzLT2mP_U,165403
+mdbq/aggregation/query_data.py,sha256=86S_Iy3NNTkn-tNNLxlnwqDnXv3x3aHD1cCe1mL7jdg,165423
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=qBOHJK_h6R_SpfQ1yC5fAlJIEm6uro810i-47uA9C_U,22872
+mdbq/log/mylogger.py,sha256=9w_o5mYB3FooIxobq_lSa6oCYTKIhPxDFox-jeLtUHI,21714
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=RnVCwMQ_n9PcAimbMWbHe9k8eil8shtCfa3LwLBZi6c,41909
-mdbq/mysql/unique_.py,sha256=LZKa1LXQdy_aO78ekO1Ul2MNA-k3Sz4-4W5nF2gD8AM,21068
+mdbq/mysql/s_query.py,sha256=FSFrFZE5yzEbnpLrN2AmlRZ_VvTvfpIWaQUjZfLIi9g,40342
+mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
 mdbq/mysql/uploader.py,sha256=OtLWoGflmndc7z0A1aK1Qj-XVkWtGyN4Y5YnLsRovJc,78215
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -22,10 +22,10 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=l3zBK7wrZl0oO42-_UGylyatnIp_SBw8wDDvof9fht4,23534
+mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.7.dist-info/METADATA,sha256=wEh7EHOSkdCJOlxYplcNOVPax8CBXEgGM3eKDL-EQaE,363
-mdbq-4.0.7.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.7.dist-info/RECORD,,
+mdbq-4.0.8.dist-info/METADATA,sha256=zmc_fzk3uaGju9Nby1Xvta_2po8NNOG7CM0TvvDIiuU,363
+mdbq-4.0.8.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.8.dist-info/RECORD,,