mdbq 3.2.6__py3-none-any.whl → 3.2.8__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1272,19 +1272,19 @@ def cut_as_year_month(as_month=False):
 
 
 if __name__ == '__main__':
-    cut_as_year_month(as_month=False)
+    # cut_as_year_month(as_month=False)
 
     # username = 'root'
     # password = ''
     # host = ''
     # port = ''
     #
-    # # 上传 1 个文件到数据库
-    # one_file_to_mysql(
-    #     file=r'/Users/xigua/Downloads/批量SKU导出-批量任务.xlsx',
-    #     db_name='属性设置3',
-    #     table_name='京东商品属性',
-    # )
+    # 上传 1 个文件到数据库
+    one_file_to_mysql(
+        file=r'/Users/xigua/Downloads/城市等级.csv',
+        db_name='属性设置3',
+        table_name='城市等级',
+    )
 
 
     # col = 1

mdbq/aggregation/query_data.py

@@ -6,6 +6,7 @@ from mdbq.mysql import s_query
 from mdbq.aggregation import optimize_data
 from mdbq.config import myconfig
 from mdbq.config import products
+from mdbq.config import set_support
 import datetime
 from dateutil.relativedelta import relativedelta
 import pandas as pd
@@ -18,10 +19,12 @@ import os
 import time
 import calendar
 import concurrent.futures
+import traceback
 
 """
 
 """
+error_file = os.path.join(set_support.SetSupport(dirname='support').dirname, 'error.log')
 username, password, host, port, service_database = None, None, None, None, None,
 if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
     conf = myconfig.main()
@@ -65,6 +68,12 @@ class MysqlDatasQuery:
                 return func(*args, **kwargs)
             except Exception as e:
                 print(f'{func.__name__}, {e}')  # 将异常信息返回
+                with open(error_file, 'a') as f:
+                    now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                    f.write(f'\n{now}\n')
+                    # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n')  # 发生异常所在的文件
+                traceback.print_exc(file=open(error_file, 'a'))  # 返回完整的堆栈信息
+                print(f'更多信息请查看日志文件: {error_file}')
 
         return wrapper
 
@@ -3227,10 +3236,10 @@ def main(days=150, months=3):
 
 
 if __name__ == '__main__':
-    main(
-        days=150,  # 清理聚合数据的日期长度
-        months=3  # 生成聚合数据的长度
-    )
+    # main(
+    #     days=150,  # 清理聚合数据的日期长度
+    #     months=3  # 生成聚合数据的长度
+    # )
 
     # query_list = [query1, query2]
     # # 使用 ThreadPoolExecutor 来并行运行
@@ -3244,10 +3253,10 @@ if __name__ == '__main__':
     #             ),
     #         }
 
-    # # 3. 清理聚合数据
-    # optimize_data.op_data(
-    #     db_name_lists=['聚合数据'],
-    #     days=180,  # 清理聚合数据的日期长度
-    #     is_mongo=False,
-    #     is_mysql=True,
-    # )
+    # 3. 清理聚合数据
+    optimize_data.op_data(
+        db_name_lists=['聚合数据'],
+        days=180,  # 清理聚合数据的日期长度
+        is_mongo=False,
+        is_mysql=True,
+    )

mdbq/mysql/mysql.py

@@ -12,7 +12,9 @@ import pandas as pd
 from sqlalchemy import create_engine
 import os
 import calendar
+from mdbq.config import set_support
 from mdbq.config import myconfig
+import traceback
 
 warnings.filterwarnings('ignore')
 """
@@ -25,6 +27,7 @@ warnings.filterwarnings('ignore')
 3. 小数必须使用 decimal, 禁止 float 和 double, 因为计算精度差异，后续需要聚合数据时会引发很多问题
 
 """
+error_file = os.path.join(set_support.SetSupport(dirname='support').dirname, 'error.log')
 
 
 def is_valid_date(date_string):
@@ -112,6 +115,12 @@ class MysqlUpload:
                 return func(*args, **kwargs)
             except Exception as e:
                 print(f'{func.__name__}, {e}')  # 将异常信息返回
+                with open(error_file, 'a') as f:
+                    now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                    f.write(f'\n{now}\n')
+                    # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n')  # 发生异常所在的文件
+                traceback.print_exc(file=open(error_file, 'a'))  # 返回完整的堆栈信息
+                print(f'更多信息请查看日志文件: {error_file}')
 
         return wrapper
 
@@ -196,6 +205,7 @@ class MysqlUpload:
                     else:
                         sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
                     # sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    # print(sql)
                     cursor.execute(sql)
                     print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
@@ -861,6 +871,12 @@ class OptimizeDatas:
                 return func(*args, **kwargs)
             except Exception as e:
                 print(f'{func.__name__}, {e}')  # 将异常信息返回
+                with open(error_file, 'a') as f:
+                    now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                    f.write(f'\n{now}\n')
+                    # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n')  # 发生异常所在的文件
+                traceback.print_exc(file=open(error_file, 'a'))  # 返回完整的堆栈信息
+                print(f'更多信息请查看日志文件: {error_file}')
 
         return wrapper
 
@@ -1162,9 +1178,3 @@ if __name__ == '__main__':
     data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     print(username, password, host, port)
-
-    df = pd.read_excel('/Users/xigua/Downloads/66563857.xlsx')
-    ss = MysqlUpload(username, password, host, port)
-    res, data = ss.convert_df_dtypes(df=df)
-    print(data)
-    print(res)

{mdbq-3.2.6.dist-info → mdbq-3.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.2.6
+Version: 3.2.8
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.2.6.dist-info → mdbq-3.2.8.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=ltYntRjxeN9YX1uTR2_zoYEik2PVPD70xAF98d4TcAo,74732
+mdbq/aggregation/aggregation.py,sha256=X4BETpYM_WxBjMZRW1nKZ3Ex2VoNtjRUj7A1MD074gA,74699
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
-mdbq/aggregation/query_data.py,sha256=Q7_brR_9BMc4HDluPmkvrApGFtUCnXReVYJnRQcSOLI,145414
+mdbq/aggregation/query_data.py,sha256=EGRiub-oowM7ImxeYrfpXouv_rlLJFnweYB03Bv1og8,146034
 mdbq/aggregation/query_data_bak.py,sha256=r1FU0C4zjXln7oVSrRkElh4Ehl-9mYhGcq57jLbViUA,104071
 mdbq/aggregation/query_data_bak20241124.py,sha256=oY95ZK3qt3Wx9pdZKZ5cvDh45Yi5yGj1kl8G6riumHA,144513
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=shsMl3B9dJqIyztekHt0L01XP7_nLEEY1jsEkCm9FMc,62855
+mdbq/mysql/mysql.py,sha256=apmrfk7Sx3WXFXvEsNtU_VQ5ybgXdZvQEP8QptOkces,63788
 mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
 mdbq/mysql/s_query.py,sha256=MbIprZ4yJDAZ9AahZPzl7hqS695Vs0P-AJNwAtA_EEc,9287
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -46,7 +46,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=48isoL6nEi_uniV-ja2HwYAI7O8D1i1goO4SzBwDGSU,19036
-mdbq-3.2.6.dist-info/METADATA,sha256=JJmWUxS4RYuDOrfy-nj0ufraDWHl2z7hGt_CWnpsIP8,243
-mdbq-3.2.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.2.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.2.6.dist-info/RECORD,,
+mdbq-3.2.8.dist-info/METADATA,sha256=_RMYkup1WgIb9RBJcVoL-Sv7KWrznuOnN4W1xmT-CCs,243
+mdbq-3.2.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.2.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.2.8.dist-info/RECORD,,