mdbq 3.6.15__py3-none-any.whl → 3.6.16__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -121,14 +121,14 @@ class MysqlUpload:
             try:
                 return func(*args, **kwargs)
             except Exception as e:
-                logger.info(f'{func.__name__}, {e}')  # 将异常信息返回
+                logger.error(f'{func.__name__}, {e}')  # 将异常信息返回
                 with open(error_file, 'a') as f:
                     now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     f.write(f'\n{now} \n')
                     f.write(f'函数注释内容(用于定位函数): {func.__doc__} \n')
                     # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n')  # 发生异常所在的文件
                 traceback.print_exc(file=open(error_file, 'a'))  # 返回完整的堆栈信息
-                logger.info(f'更多信息请查看日志文件: {error_file}')
+                logger.error(f'更多信息请查看日志文件: {error_file}')
 
         return wrapper
 
@@ -139,10 +139,10 @@ class MysqlUpload:
                 connection = pymysql.connect(**_config)  # 连接数据库
                 return connection
             except Exception as e:
-                logger.info(f'连接失败，正在重试: {attempts}/{max_try} {e}')
+                logger.error(f'连接失败，正在重试: {attempts}/{max_try} {e}')
                 attempts += 1
                 time.sleep(30)
-        logger.info(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+        logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
         return None
 
     def cover_doc_dtypes(self, dict_data):
@@ -343,7 +343,7 @@ class MysqlUpload:
                     else:
                         logger.info(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                 except Exception as e:
-                    logger.info(f'333 {table_name} {e}')
+                    logger.error(f'333 {table_name} {e}')
                     connection.rollback()
             connection.commit()
 
@@ -386,7 +386,7 @@ class MysqlUpload:
                     else:
                         logger.info(f'参数不正确，cut_data应为 year 或 month ')
                 except Exception as e:
-                    logger.info(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
+                    logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
 
         # connection = pymysql.connect(**self.config)  # 连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
@@ -597,7 +597,7 @@ class MysqlUpload:
                     else:
                         logger.info(f'参数不正确，cut_data应为 year 或 month ')
                 except Exception as e:
-                    logger.info(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
+                    logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
 
         # connection = pymysql.connect(**self.config)  # 连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
@@ -959,7 +959,7 @@ class MysqlUpload:
                     else:
                         logger.info(f'参数不正确，cut_data应为 year 或 month ')
                 except Exception as e:
-                    logger.info(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
+                    logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
         # 清理 dataframe 非法值，并转换获取数据类型
         dtypes, df = self.convert_df_dtypes(df)
         if set_typ:
@@ -1070,7 +1070,7 @@ class MysqlUpload:
                         else:
                             logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                     except Exception as e:
-                        logger.info(f'333 {table_name} {e}')
+                        logger.error(f'333 {table_name} {e}')
                         connection.rollback()
                 connection.commit()  # 提交事务
                 connection.close()
@@ -1126,7 +1126,7 @@ class MysqlUpload:
                     else:
                         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                 except Exception as e:
-                    logger.info(f'333 {table_name} {e}')
+                    logger.error(f'333 {table_name} {e}')
                     connection.rollback()
                 connection.close()
                 return
@@ -1216,10 +1216,6 @@ class MysqlUpload:
                         cursor.execute(sql)
                 except Exception as e:
                     pass
-                    # logger.info(data)
-                    # logger.info(values)
-                    # logger.info(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
-                    # breakpoint()
 
             if reset_id:
                 # 6. 重置自增列
@@ -1249,7 +1245,7 @@ class MysqlUpload:
                     else:
                         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                 except Exception as e:
-                    logger.info(f'333 {table_name} {e}')
+                    logger.error(f'333 {table_name} {e}')
                     connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
@@ -1343,7 +1339,7 @@ class MysqlUpload:
                 columns = [desc[0] for desc in cursor.description]
                 df = pd.DataFrame(rows, columns=columns)  # 转为 df
         except Exception as e:
-            logger.info(f'{e} {db_name} -> {table_name} 表不存在')
+            logger.error(f'{e} {db_name} -> {table_name} 表不存在')
             return df
         finally:
             connection.close()
@@ -1437,14 +1433,14 @@ class OptimizeDatas:
             try:
                 return func(*args, **kwargs)
             except Exception as e:
-                logger.info(f'{func.__name__}, {e}')  # 将异常信息返回
+                logger.error(f'{func.__name__}, {e}')  # 将异常信息返回
                 with open(error_file, 'a') as f:
                     now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     f.write(f'\n{now} \n')
                     f.write(f'函数注释内容(用于定位函数): {func.__doc__} \n')
                     # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n')  # 发生异常所在的文件
                 traceback.print_exc(file=open(error_file, 'a'))  # 返回完整的堆栈信息
-                logger.info(f'更多信息请查看日志文件: {error_file}')
+                logger.error(f'更多信息请查看日志文件: {error_file}')
 
         return wrapper
 
@@ -1455,10 +1451,10 @@ class OptimizeDatas:
                 connection = pymysql.connect(**_config)  # 连接数据库
                 return connection
             except Exception as e:
-                logger.info(f'连接失败，正在重试: {attempts}/{max_try} {e}')
+                logger.error(f'连接失败，正在重试: {attempts}/{max_try} {e}')
                 attempts += 1
                 time.sleep(30)
-        logger.info(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+        logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
         return None
 
     def optimize_list(self):
@@ -1572,7 +1568,7 @@ class OptimizeDatas:
                         else:
                             logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                     except Exception as e:
-                        logger.info(f'333 {table_name} {e}')
+                        logger.error(f'333 {table_name} {e}')
                         self.connection.rollback()
                 self.connection.close()
         logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
@@ -1596,7 +1592,7 @@ class OptimizeDatas:
                     continue
                 all_datas.append(data)  # 数据没有重复
             except Exception as e:
-                logger.info(f'{table_name} 函数: mysql - > OptimizeDatas -> delete_duplicate -> {e}')
+                logger.error(f'{table_name} 函数: mysql - > OptimizeDatas -> delete_duplicate -> {e}')
         del all_datas
 
         if not duplicate_id:  # 如果没有重复数据，则跳过该数据表
@@ -1608,10 +1604,10 @@ class OptimizeDatas:
                 # 移除冗余数据
                 sql = f"DELETE FROM `{table_name}` WHERE id IN ({placeholders})"
                 cursor.execute(sql, duplicate_id)
-                logger.info(f"{table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
+                logger.debug(f"{table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
             self.connection.commit()  # 提交事务
         except Exception as e:
-            logger.info(f'{self.db_name}/{table_name}, {e}')
+            logger.error(f'{self.db_name}/{table_name}, {e}')
             self.connection.rollback()  # 异常则回滚
 
     def delete_duplicate2(self, table_name, except_key=['更新时间']):
@@ -1649,7 +1645,7 @@ class OptimizeDatas:
                       f"remove: {cursor.rowcount}")
             self.connection.commit()  # 提交事务
         except Exception as e:
-            logger.info(f'{self.db_name}/{table_name}, {e}')
+            logger.error(f'{self.db_name}/{table_name}, {e}')
             self.connection.rollback()  # 异常则回滚
 
     def database_list(self):
@@ -1678,7 +1674,7 @@ class OptimizeDatas:
                     logger.info(f'{db_name}: 数据表不存在!')
                     return
         except Exception as e:
-            logger.info(f'002 {e}')
+            logger.error(f'002 {e}')
             return
         finally:
             connection.close()  # 断开连接
@@ -1709,7 +1705,7 @@ class OptimizeDatas:
                 cursor.execute(sql)
                 results = cursor.fetchall()
         except Exception as e:
-            logger.info(f'001 {e}')
+            logger.error(f'001 {e}')
         finally:
             connection.close()
         return results
@@ -1763,7 +1759,7 @@ def year_month_day_bak(start_date, end_date):
     try:
         start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
     except Exception as e:
-        logger.info(e)
+        logger.error(e)
         return []
     # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
     date_range = pd.date_range(start=start_date, end=end_date, freq='MS')

mdbq/mysql/s_query.py

@@ -117,7 +117,7 @@ class QueryDatas:
                     df[decimal_cols] = df[decimal_cols].astype(float)
 
         except Exception as e:
-            logger.info(f"Database operation failed: {str(e)}")
+            logger.error(f"Database operation failed: {str(e)}")
         finally:
             if connection:
                 connection.close()
@@ -195,7 +195,7 @@ class QueryDatas:
                     return False
                 return True
         except Exception as e:
-            logger.info(e)
+            logger.error(e)
             return False
         finally:
             connection.close()  # 断开连接

mdbq/redis/getredis.py

@@ -123,7 +123,7 @@ class RedisData(object):
             ttl = self.redis_engine.ttl(cache_key)
             cache_data = self._fetch_redis_data(cache_key)
         except Exception as e:
-            logger.info(f"Redis 连接异常: {e}，直接访问 MySQL")
+            logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
             return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
 
         # 缓存失效处理逻辑
@@ -175,7 +175,7 @@ class RedisData(object):
             return combined_data
 
         except Exception as e:
-            logger.info(f"缓存更新失败: {cache_key} - {str(e)}")
+            logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
             return pd.DataFrame()
 
     # Helper Methods ------------------------------------------------
@@ -197,7 +197,7 @@ class RedisData(object):
                 projection={}
             )
         except Exception as e:
-            logger.info(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
+            logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
             return pd.DataFrame()
 
     def _fetch_redis_data(self, cache_key: str) -> pd.DataFrame:
@@ -210,7 +210,7 @@ class RedisData(object):
             df = pd.DataFrame(json.loads(data.decode("utf-8")))
             return self._convert_date_columns(df)
         except Exception as e:
-            logger.info(f"Redis 数据解析失败 {cache_key}: {e}")
+            logger.error(f"Redis 数据解析失败 {cache_key}: {e}")
             return pd.DataFrame()
 
     def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:

{mdbq-3.6.15.dist-info → mdbq-3.6.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.15
+Version: 3.6.16
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.15.dist-info → mdbq-3.6.16.dist-info}/RECORD

@@ -18,10 +18,10 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=ZB552VAMn7tz-Z0MPZKwY2LL2mAms0ATb5y68OBL9Tg,98712
+mdbq/mysql/mysql.py,sha256=v9nr-tb2L1VN0mKUVqEkfcWP1qXQRPVHIOHs3q89Cos,98531
 mdbq/mysql/mysql_bak.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
 mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
-mdbq/mysql/s_query.py,sha256=CL2Ayo2sL11RbLnh9nE-GXA-NpA815-rrlFo24TipKY,8792
+mdbq/mysql/s_query.py,sha256=pj5ioJfUT81Su9S-km9G49gF5F2MmXXfw_oAIUzhN28,8794
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=GdphR7Q3psXXVuZoyJ4u_6OWn_rWlcbT0iJ-1zPT6O0,45368
@@ -34,11 +34,11 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=4rYk9lMRvvlpY7cV3VNQcSnbDWlZIsZZ-tSq--YqfSQ,26638
+mdbq/redis/getredis.py,sha256=k4fpC-2ziRA_3HvcjX4jBeHCFIRaf3_f4oHMFlW-xjk,26642
 mdbq/redis/getredis_优化hash.py,sha256=q7omKJCPw_6Zr_r6WwTv4RGSXzZzpLPkIaqJ22svJhE,29104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
-mdbq-3.6.15.dist-info/METADATA,sha256=Zb_mvyFxjvFYudvISR7D9HMXnBj34_ZvWJXHDo9_pOY,244
-mdbq-3.6.15.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.6.15.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.15.dist-info/RECORD,,
+mdbq-3.6.16.dist-info/METADATA,sha256=QpjJk7yttYLxg6QzdLaZGeGSNCdqTfxBWd0mVwUbtAY,244
+mdbq-3.6.16.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.6.16.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.6.16.dist-info/RECORD,,