mdbq 3.3.3__py3-none-any.whl → 3.3.5__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -1253,8 +1253,8 @@ class MysqlDatasQuery:
             '三级来源索引': 'smallint',
         }
         # df.to_csv('/Users/xigua/Downloads/ll.csv', index=False, header=True, encoding='utf-8_sig')
-        min_date = df['日期'].min()
-        max_date = df['日期'].max()
+        min_date = df['日期'].min().strftime("%Y-%m-%d")
+        max_date = df['日期'].max().strftime("%Y-%m-%d")
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(

mdbq/mysql/mysql.py

@@ -128,6 +128,19 @@ class MysqlUpload:
 
         return wrapper
 
+    def keep_connect(self, _db_name, _config, max_try: int=5):
+        attempts = 1
+        while attempts <= max_try:
+            try:
+                connection = pymysql.connect(**_config)  # 连接数据库
+                return connection
+            except Exception as e:
+                print(f'连接失败，正在重试: {attempts}/{max_try} {e}')
+                attempts += 1
+                time.sleep(20)
+        print(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+        return None
+
     def cover_doc_dtypes(self, dict_data):
         """ 清理字典键值 并转换数据类型  """
         if not dict_data:
@@ -201,7 +214,10 @@ class MysqlUpload:
             print(f'dict_data 中"数据主体"键不能为空')
             return
 
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
             database_exists = cursor.fetchone()
@@ -219,7 +235,10 @@ class MysqlUpload:
                 print(f"创建Database: {db_name}")
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             # 1. 查询表, 不存在则创建一个空表
             sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
@@ -362,7 +381,10 @@ class MysqlUpload:
                 except Exception as e:
                     print(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
 
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
             database_exists = cursor.fetchone()
@@ -380,7 +402,10 @@ class MysqlUpload:
                 print(f"创建Database: {db_name}")
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             # 1. 查询表, 不存在则创建一个空表
             sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
@@ -723,7 +748,10 @@ class MysqlUpload:
                 # 确保传进来的 set_typ 键存在于实际的 df 列才 update
                 [dtypes.update({k: inside_v}) for inside_k, inside_v in set_typ.items() if k == inside_k]
 
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
             database_exists = cursor.fetchone()
@@ -741,7 +769,10 @@ class MysqlUpload:
                 print(f"创建Database: {db_name}")
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             # 1. 查询表, 不存在则创建一个空表
             sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
@@ -1017,7 +1048,10 @@ class MysqlUpload:
         if not filename:
             print(f'未指定文件名: filename')
             return
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         # try:
         with connection.cursor() as cursor:
             cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
@@ -1026,7 +1060,10 @@ class MysqlUpload:
                 print(f"Database {db_name} 数据库不存在")
                 return
         self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             # 1. 查询表
             sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
@@ -1056,7 +1093,10 @@ class MysqlUpload:
         end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
         df = pd.DataFrame()
 
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         try:
             with connection.cursor() as cursor:
                 cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
@@ -1075,7 +1115,10 @@ class MysqlUpload:
         before_time = time.time()
         # 读取数据
         self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         try:
             with connection.cursor() as cursor:
                 # 获取指定日期范围的数据
@@ -1197,6 +1240,19 @@ class OptimizeDatas:
 
         return wrapper
 
+    def keep_connect(self, _db_name, _config, max_try: int=5):
+        attempts = 1
+        while attempts <= max_try:
+            try:
+                connection = pymysql.connect(**_config)  # 连接数据库
+                return connection
+            except Exception as e:
+                print(f'连接失败，正在重试: {attempts}/{max_try} {e}')
+                attempts += 1
+                time.sleep(20)
+        print(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+        return None
+
     def optimize_list(self):
         """
         更新多个数据库 移除冗余数据
@@ -1243,7 +1299,10 @@ class OptimizeDatas:
                 # if '店铺指标' not in table_name:
                 #     continue
                 self.config.update({'database': self.db_name})  # 添加更新 config 字段
-                self.connection = pymysql.connect(**self.config)
+                # self.connection = pymysql.connect(**self.config)
+                self.connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
+                if not self.connection:
+                    return
                 with self.connection.cursor() as cursor:
                     sql = f"SELECT 1 FROM `{table_name}` LIMIT 1"
                     # print(sql)
@@ -1395,7 +1454,10 @@ class OptimizeDatas:
 
     def database_list(self):
         """ 获取所有数据库 """
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             cursor.execute("SHOW DATABASES")
             databases = cursor.fetchall()  # 获取所有数据库的结果
@@ -1404,7 +1466,10 @@ class OptimizeDatas:
 
     def table_list(self, db_name):
         """ 获取指定数据库的所有数据表 """
-        connection = pymysql.connect(**self.config)  # 连接数据库
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         try:
             with connection.cursor() as cursor:
                 cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
@@ -1420,7 +1485,10 @@ class OptimizeDatas:
             connection.close()  # 断开连接
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         with connection.cursor() as cursor:
             cursor.execute("SHOW TABLES")
             tables = cursor.fetchall()  # 获取所有数据表
@@ -1432,7 +1500,10 @@ class OptimizeDatas:
         获取指定数据表的数据, 按天获取
         """
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)
+        # connection = pymysql.connect(**self.config)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
         try:
             with connection.cursor() as cursor:
                 sql = f"SELECT * FROM `{table_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (date, date)
@@ -1465,7 +1536,10 @@ class OptimizeDatas:
         for table_dict in tables:
             for key, table_name in table_dict.items():
                 self.config.update({'database': self.db_name})  # 添加更新 config 字段
-                self.connection = pymysql.connect(**self.config)
+                # self.connection = pymysql.connect(**self.config)
+                self.connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
+                if not self.connection:
+                    return
                 with self.connection.cursor() as cursor:
                     cursor.execute(f"SHOW FULL COLUMNS FROM `{table_name}`")  # 查询数据表的列信息
                     columns = cursor.fetchall()

{mdbq-3.3.3.dist-info → mdbq-3.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.3.3
+Version: 3.3.5
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.3.3.dist-info → mdbq-3.3.5.dist-info}/RECORD

@@ -5,7 +5,7 @@ mdbq/aggregation/aggregation.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2k
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
-mdbq/aggregation/query_data.py,sha256=4C9BmMUV4x1YvezIySO5Nh8tqGqg1z4GVnuYDGU5OJs,167595
+mdbq/aggregation/query_data.py,sha256=_5mnSFHV6xAFs_1YF_H2zMOdJeMavgga4lZQ_qpqxPQ,167637
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -26,7 +26,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=dpQWQTq8PpSYFPAHZRNinLuHKfrekSNYbyOxtOD0SwA,82319
+mdbq/mysql/mysql.py,sha256=ZK6E-idQWrURtoimc0uG8B1tnhtSFQXDJHfu8sWeJg4,85675
 mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
 mdbq/mysql/s_query.py,sha256=MbIprZ4yJDAZ9AahZPzl7hqS695Vs0P-AJNwAtA_EEc,9287
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -44,7 +44,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=BKVa0xbTkyhIH5kQgOdyPDtwFPScbMNAFYup_-fFF9Y,24809
-mdbq-3.3.3.dist-info/METADATA,sha256=2_AvXdqYGWMuln2wfqXfLKddw2h8Sbr49mK1UjEiiVU,243
-mdbq-3.3.3.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.3.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.3.3.dist-info/RECORD,,
+mdbq-3.3.5.dist-info/METADATA,sha256=ndDUyzoqoEH--Mhw_0Z7xXEPTwWuLyBkT8jsfUx8AxA,243
+mdbq-3.3.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.3.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.3.5.dist-info/RECORD,,

{mdbq-3.3.3.dist-info → mdbq-3.3.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any