mdbq 3.6.13__py3-none-any.whl → 3.6.14__py3-none-any.whl

mdbq/aggregation/optimize_data.py

@@ -7,6 +7,7 @@ import subprocess
 import psutil
 import time
 import platform
+import logging
 """
 对指定数据库所有冗余数据进行清理
 """
@@ -22,8 +23,9 @@ elif socket.gethostname() in ['company', 'Mac2.local']:
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     service_database = {'company': 'mysql'}
 if not username:
-    print(f'找不到主机：')
+    logger.info(f'找不到主机：')
 
+logger = logging.getLogger(__name__)
 
 
 def restart_mongodb():
@@ -39,7 +41,7 @@ def restart_mongodb():
         return None
 
     if platform.system() == 'Windows':
-        print(f'即将重启 mongodb 服务')
+        logger.info(f'即将重启 mongodb 服务')
         time.sleep(60)
         stop_command = f'net stop MongoDB'
         subprocess.call(stop_command, shell=True)  # 停止MongoDB服务
@@ -50,7 +52,7 @@ def restart_mongodb():
         time.sleep(30)
 
     elif platform.system() == 'Darwin':
-        print(f'即将重启 mongodb 服务')
+        logger.info(f'即将重启 mongodb 服务')
         time.sleep(60)
         result = get_pid('mongod')  # 获取进程号
         if result:  # 有 pid, 重启服务
@@ -59,13 +61,13 @@ def restart_mongodb():
             time.sleep(10)
             command = f'mongod --config /usr/local/mongodb/mongod.conf'
             subprocess.call(command, shell=True)
-            # print('已重启服务')
+            # logger.info('已重启服务')
         else:  # 没有启动, 则启动服务
             command = f'mongod --config /usr/local/mongodb/mongod.conf'
             subprocess.call(command, shell=True)
 
     elif platform.system() == 'Linux':
-        print(f'即将重启 mongodb 服务')
+        logger.info(f'即将重启 mongodb 服务')
         time.sleep(60)
         command = f'service mongod restart'
         subprocess.call(command, shell=True)

mdbq/aggregation/query_data.py

@@ -20,6 +20,7 @@ import time
 import calendar
 import concurrent.futures
 import traceback
+import logging
 
 """
 
@@ -73,6 +74,7 @@ else:
         charset='utf8mb4'
     )
     targe_host = 'xigua_lx'
+logger = logging.getLogger(__name__)
 
 
 class MysqlDatasQuery:
@@ -96,13 +98,13 @@ class MysqlDatasQuery:
             try:
                 return func(*args, **kwargs)
             except Exception as e:
-                print(f'{func.__name__}, {e}')  # 将异常信息返回
+                logger.info(f'{func.__name__}, {e}')  # 将异常信息返回
                 with open(error_file, 'a') as f:
                     now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     f.write(f'\n{now}\n')
                     # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n')  # 发生异常所在的文件
                 traceback.print_exc(file=open(error_file, 'a'))  # 返回完整的堆栈信息
-                print(f'更多信息请查看日志文件: {error_file}')
+                logger.info(f'更多信息请查看日志文件: {error_file}')
 
         return wrapper
 
@@ -215,7 +217,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -276,7 +278,7 @@ class MysqlDatasQuery:
             '更新时间': 'timestamp',
         }
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) 属性设置3/商品索引表_主推排序调用')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) 属性设置3/商品索引表_主推排序调用')
         m_engine.df_to_mysql(
             df=df_pic,
             db_name='属性设置3',
@@ -400,7 +402,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -524,7 +526,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -634,7 +636,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -866,7 +868,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1015,7 +1017,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1153,7 +1155,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1279,7 +1281,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1349,7 +1351,7 @@ class MysqlDatasQuery:
         if not self.update_service:
             return
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1419,7 +1421,7 @@ class MysqlDatasQuery:
         if not self.update_service:
             return
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1481,7 +1483,7 @@ class MysqlDatasQuery:
         p = df.pop('上市年份')
         df.insert(loc=5, column='上市年份', value=p)
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name}')
         set_typ = {
             '日期': 'date',
             '店铺id': 'bigint',
@@ -1632,7 +1634,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1755,7 +1757,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1815,7 +1817,7 @@ class MysqlDatasQuery:
         min_date = pd.to_datetime(df['日期'].min()).strftime('%Y-%m-%d')
         max_date = pd.to_datetime(df['日期'].max()).strftime('%Y-%m-%d')
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1933,7 +1935,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -1980,7 +1982,7 @@ class MysqlDatasQuery:
         # min_date = df['日期'].min()
         # max_date = df['日期'].max()
         # now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        # print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/京东_京准通_按spu -> {min_date}~{max_date}')
+        # logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/京东_京准通_按spu -> {min_date}~{max_date}')
         # m_engine.df_to_mysql(
         #     df=df,
         #     db_name=db_name,
@@ -2061,7 +2063,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2175,7 +2177,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2267,7 +2269,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2351,7 +2353,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2452,7 +2454,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -2603,7 +2605,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         new_dict = {
             '日期': '',
             '店铺名称': '',
@@ -3005,7 +3007,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min().strftime('%Y-%m-%d')
         max_date = df['日期'].max().strftime('%Y-%m-%d')
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3108,7 +3110,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3173,7 +3175,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3298,7 +3300,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3675,7 +3677,7 @@ class MysqlDatasQuery:
                 '退货量_发货后': ('退货量_发货后', np.sum),
             }
         )
-        # print(df.info())
+        # logger.info(df.info())
 
         idbm['宝贝id'] = idbm['宝贝id'].astype('int64')
         # 1.  id 编码表合并图片表
@@ -3684,7 +3686,7 @@ class MysqlDatasQuery:
         # 2.  df 合并商品成本表
         df_cb = pd.merge(df_cb, cost, how='left', left_on='商家编码', right_on='款号')
         df_cb = df_cb[['宝贝id', '商家编码', '商品图片', '成本价']]
-        # print(df_cb.info())
+        # logger.info(df_cb.info())
         # 3.  合并 df
         df = pd.merge(df, df_cb, how='left', left_on='商品id', right_on='宝贝id')
         df.drop(labels='宝贝id', axis=1, inplace=True)
@@ -3723,7 +3725,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3846,7 +3848,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min()
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -3930,7 +3932,7 @@ class MysqlDatasQuery:
         min_date = df['日期'].min().strftime("%Y-%m-%d")
         max_date = df['日期'].max().strftime("%Y-%m-%d")
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,
@@ -4005,11 +4007,11 @@ def date_table():
     for name, group in grouped:
         if num > 52:
             num = 1
-        # print(f'Group: {name}')
+        # logger.info(f'Group: {name}')
         group['第n周_new'] = f'第{num}周'
         num += 1
         __res.append(group.copy())
-        # print(group)
+        # logger.info(group)
         # break
     df = pd.concat(__res, ignore_index=True)
     # df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x))
@@ -4072,7 +4074,7 @@ def date_table():
 
 def query1(months=1, less_dict=[]):
     if months == 0:
-        print(f'months 不建议为 0 ')
+        logger.info(f'months 不建议为 0 ')
         return
     sdq = MysqlDatasQuery()  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
@@ -4101,7 +4103,7 @@ def query1(months=1, less_dict=[]):
 
 def query2(months=1, less_dict=[]):
     if months == 0:
-        print(f'months 不建议为 0 ')
+        logger.info(f'months 不建议为 0 ')
         return
     sdq = MysqlDatasQuery()  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
@@ -4118,7 +4120,7 @@ def query2(months=1, less_dict=[]):
 
 def query3(months=1, less_dict=[]):
     if months == 0:
-        print(f'months 不建议为 0 ')
+        logger.info(f'months 不建议为 0 ')
         return
     sdq = MysqlDatasQuery()  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月