mdbq 3.6.14__py3-none-any.whl → 3.6.15__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -302,8 +302,7 @@ class MysqlUpload:
 
             # 插入数据到数据库
             # 有数据格式错误问题，所以分开处理，将数据主体移到最后面用占位符
-            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            logger.info(f'{now} 正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name} -> {filename}')
+            logger.info(f'正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name} -> {filename}')
             if new_dict:
                 cols = ', '.join(f"`{item}`" for item in new_dict.keys())  # 列名需要转义
                 values = ', '.join([f'"{item}"' for item in new_dict.values()])  # 值要加引号
@@ -1031,9 +1030,7 @@ class MysqlUpload:
             connection.commit()  # 提交事务
 
             if df_sql:
-                now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                logger.info(
-                    f'{now} 正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
+                logger.info(f'正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
                 engine = create_engine(
                     f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
                 # df.to_csv('/Users/xigua/Downloads/mysql.csv', index=False, header=True, encoding='utf-8_sig')
@@ -1303,8 +1300,7 @@ class MysqlUpload:
                     # 将二进制数据写入到文件
                     with open(os.path.join(save_path, filename), 'wb') as f:
                         f.write(result['数据主体'])
-                        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                        logger.info(f'{now} 写入本地文件: ({self.host}:{self.port}) {db_name}/{table_name} -> {os.path.join(save_path, filename)}')
+                        logger.info(f'写入本地文件: ({self.host}:{self.port}) {db_name}/{table_name} -> {os.path.join(save_path, filename)}')
         connection.close()
 
     def read_mysql(self, table_name, start_date, end_date, db_name='远程数据源', date_name='日期'):
@@ -1325,8 +1321,7 @@ class MysqlUpload:
                     logger.info(f"Database {db_name} 数据库不存在")
                     return df
                 else:
-                    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                    logger.info(f'{now} mysql 正在查询表: {table_name}, 范围: {start_date}~{end_date}')
+                    logger.info(f'mysql 正在查询表: {table_name}, 范围: {start_date}~{end_date}')
         except:
             return df
         finally:
@@ -1356,11 +1351,10 @@ class MysqlUpload:
         if len(df) == 0:
             logger.info(f'database: {db_name}, table: {table_name} 查询的数据为空')
         else:
-            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
             cost_time = int(time.time() - before_time)
             if cost_time < 1:
                 cost_time = round(time.time() - before_time, 2)
-            logger.info(f'{now} mysql ({self.host}) 表: {table_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
+            logger.info(f'mysql ({self.host}) 表: {table_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
         return df
 
     def upload_pandas(self, update_path, db_name, days=None):
@@ -1387,10 +1381,6 @@ class MysqlUpload:
                     for name in files:
                         if name.endswith('.csv') and 'baidu' not in name:
                             df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                            # if '日期' not in df.columns.tolist():
-                            #     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                            #     logger.info(f'{now} {root_file} 缺少日期列, 不支持上传 mysql')
-                            #     continue
                             if '日期' in df.columns.tolist():
                                 df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x) if x else x)
                                 df = df[df['日期'] >= start_date]
@@ -1400,10 +1390,6 @@ class MysqlUpload:
             elif os.path.isfile(f_path):
                 if f_path.endswith('.csv') and 'baidu' not in f_path:
                     df = pd.read_csv(f_path, encoding='utf-8_sig', header=0, na_filter=False)
-                    # if '日期' not in df.columns.tolist():
-                    #     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                    #     logger.info(f'{now} {root_file} 缺少日期列, 不支持上传 mysql')
-                    #     continue
                     if '日期' not in df.columns.tolist():
                         df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x) if x else x)
                         df = df[df['日期'] >= start_date]
@@ -1481,8 +1467,7 @@ class OptimizeDatas:
         需要设置 self.db_name_lists
         """
         if not self.db_name_lists:
-            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            logger.info(f'{now} 尚未设置参数: self.db_name_lists')
+            logger.info(f'尚未设置参数: self.db_name_lists')
             return
         for db_name in self.db_name_lists:
             self.db_name = db_name
@@ -1491,13 +1476,11 @@ class OptimizeDatas:
     def optimize(self, except_key=['更新时间']):
         """ 更新一个数据库 移除冗余数据 """
         if not self.db_name:
-            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            logger.info(f'{now} 尚未设置参数: self.db_name')
+            logger.info(f'尚未设置参数: self.db_name')
             return
         tables = self.table_list(db_name=self.db_name)
         if not tables:
-            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            logger.info(f'{now} {self.db_name} -> 数据表不存在')
+            logger.info(f'{self.db_name} -> 数据表不存在')
             return
 
         # 日期初始化
@@ -1514,8 +1497,7 @@ class OptimizeDatas:
         start_date_before = self.start_date
         end_date_before = self.end_date
 
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} mysql({self.host}: {self.port}) {self.db_name} 数据库优化中(日期长度: {self.days} 天)...')
+        logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化中(日期长度: {self.days} 天)...')
         for table_dict in tables:
             for key, table_name in table_dict.items():
                 # if '店铺指标' not in table_name:
@@ -1531,8 +1513,7 @@ class OptimizeDatas:
                     cursor.execute(sql)
                     result = cursor.fetchone()
                     if not result:
-                        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                        logger.info(f'{now} 数据表: {table_name}, 数据长度为 0')
+                        logger.info(f'数据表: {table_name}, 数据长度为 0')
                         continue  # 检查数据表是否为空
 
                     cursor.execute(f"SHOW FULL COLUMNS FROM `{table_name}`")  # 查询数据表的列信息
@@ -1594,8 +1575,7 @@ class OptimizeDatas:
                         logger.info(f'333 {table_name} {e}')
                         self.connection.rollback()
                 self.connection.close()
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(f'{now} mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
+        logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
 
     def delete_duplicate(self, table_name, date, except_key=['更新时间']):
         datas = self.table_datas(db_name=self.db_name, table_name=str(table_name), date=date)
@@ -1628,8 +1608,7 @@ class OptimizeDatas:
                 # 移除冗余数据
                 sql = f"DELETE FROM `{table_name}` WHERE id IN ({placeholders})"
                 cursor.execute(sql, duplicate_id)
-                now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                logger.info(f"{now} {table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
+                logger.info(f"{table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
             self.connection.commit()  # 提交事务
         except Exception as e:
             logger.info(f'{self.db_name}/{table_name}, {e}')
@@ -1666,8 +1645,7 @@ class OptimizeDatas:
                 # 移除冗余数据
                 sql = f"DELETE FROM `{table_name}` WHERE id IN ({placeholders})"
                 cursor.execute(sql, duplicate_id)
-                now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                logger.info(f"{now} {table_name} -> before: {len(datas)}, "
+                logger.info(f"{table_name} -> before: {len(datas)}, "
                       f"remove: {cursor.rowcount}")
             self.connection.commit()  # 提交事务
         except Exception as e:
@@ -1697,8 +1675,7 @@ class OptimizeDatas:
                 cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
                 database_exists = cursor.fetchone()
                 if not database_exists:
-                    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                    logger.info(f'{now} {db_name}: 数据表不存在!')
+                    logger.info(f'{db_name}: 数据表不存在!')
                     return
         except Exception as e:
             logger.info(f'002 {e}')

{mdbq-3.6.14.dist-info → mdbq-3.6.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.14
+Version: 3.6.15
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.14.dist-info → mdbq-3.6.15.dist-info}/RECORD

@@ -18,7 +18,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=1HoOvSZKPHc3Bz1YI_A_5x3JmGvjUN4suCM9oHCF-t4,100465
+mdbq/mysql/mysql.py,sha256=ZB552VAMn7tz-Z0MPZKwY2LL2mAms0ATb5y68OBL9Tg,98712
 mdbq/mysql/mysql_bak.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
 mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
 mdbq/mysql/s_query.py,sha256=CL2Ayo2sL11RbLnh9nE-GXA-NpA815-rrlFo24TipKY,8792
@@ -38,7 +38,7 @@ mdbq/redis/getredis.py,sha256=4rYk9lMRvvlpY7cV3VNQcSnbDWlZIsZZ-tSq--YqfSQ,26638
 mdbq/redis/getredis_优化hash.py,sha256=q7omKJCPw_6Zr_r6WwTv4RGSXzZzpLPkIaqJ22svJhE,29104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
-mdbq-3.6.14.dist-info/METADATA,sha256=6nLgVL5HWxyW9-EBob6LnpGnfncM3l8rOL4nKQtqtbg,244
-mdbq-3.6.14.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.6.14.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.14.dist-info/RECORD,,
+mdbq-3.6.15.dist-info/METADATA,sha256=Zb_mvyFxjvFYudvISR7D9HMXnBj34_ZvWJXHDo9_pOY,244
+mdbq-3.6.15.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.6.15.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.6.15.dist-info/RECORD,,