Functions-d 1.38__tar.gz → 1.40__tar.gz

{functions_d-1.38 → functions_d-1.40}/Functions_d/Functions_d.py

@@ -7,7 +7,7 @@ import shutil
 import time
 import urllib.request
 import WeComMsg
-import xlwings as xw
+# import xlwings as xw
 import yagmail
 import HiveClient
 from PIL import ImageGrab, Image
@@ -20,7 +20,7 @@ from selenium.webdriver.edge.service import Service
 from selenium.webdriver.edge.options import Options
 from webdriver_manager.microsoft import EdgeChromiumDriverManager
 import psutil
-import pythoncom
+# import pythoncom
 from tenacity import retry, stop_after_attempt, wait_fixed
 import requests
 import pandas as pd
@@ -676,10 +676,10 @@ class DataProcessingAndMessaging:
 
     # -------------------------- 写入Excel数据 --------------------------
     def writer_excel_data(self, path, filename, send_file, sheet_data, headers):
+        import xlwings as xw
         self.logger.info('开始处理Excel表格')
         print('开始处理Excel表格')
         filename = path + filename
-        send_file = send_file
         dfs = []
         sheet_names = []
         clear_ranges = []
@@ -755,6 +755,8 @@ class DataProcessingAndMessaging:
 
     # -------------------------- Excel截图（安全版） --------------------------
     def screen(self, filename, sheetname, screen_area, img_name):
+        import pythoncom
+        import xlwings as xw
         self.logger.info('开始截图')
         print('开始截图')
         pythoncom.CoInitialize()
@@ -1912,20 +1914,14 @@ class DataProcessingAndMessaging:
     # -------------------------- Hive数据导出Snappy压缩Parquet文件 --------------------------
     def export_parquet(self, df, file_no_suffix, save_dir=None):
         """
-        将Hive查询返回的DataFrame去除库表前缀列名，自动处理重复字段名，导出snappy压缩parquet文件
-        :param df: Hive读取的DataFrame
-        :param file_no_suffix: 不带后缀的文件名，方法内部自动拼接 .parquet
-        :param save_dir: 存储目录路径；不传则默认使用实例内置self.path
+        将Hive查询返回的DataFrame，自动去重列名，输出优化参数Snappy Parquet
         """
-        # 判定存储目录，为空则使用默认脚本路径self.path
         target_dir = save_dir if save_dir is not None else self.path
-        # 拼接完整文件路径
         file_name = f"{file_no_suffix}.parquet"
         save_path = os.path.join(target_dir, file_name)
         self.logger.info(f"开始处理Parquet导出，目标路径：{save_path}")
         print(f"开始处理Parquet导出，目标路径：{save_path}")
         try:
-            # 截取列名，去掉库表前缀，重复字段自动加后缀区分
             old_columns = list(df.columns)
             new_cols = []
             name_count = {}
@@ -1939,22 +1935,24 @@ class DataProcessingAndMessaging:
                     final_name = short_name
                 new_cols.append(final_name)
             df.columns = new_cols
-            self.logger.info(f"列名清洗完成，原列名：{old_columns} → 新列名：{list(df.columns)}")
+            self.logger.info(f"列名清洗完成，共{len(df.columns)}个字段")
             print(f"列名清洗完成，共{len(df.columns)}个字段")
 
-            # 导出snappy压缩parquet
+            # 增加优化参数，大幅提升PowerBI读取性能
             df.to_parquet(
                 save_path,
                 engine="pyarrow",
-                compression="snappy"
+                compression="snappy",
+                row_group_size=100000,
+                use_dictionary=True,
+                data_page_size=1048576
             )
-            # 获取文件大小日志
             file_size_kb = round(os.path.getsize(save_path) / 1024, 2)
-            success_msg = f"Parquet文件导出完成，文件大小：{file_size_kb} KB，数据行数：{len(df)}"
+            success_msg = f"Parquet导出完成，文件大小：{file_size_kb} KB，数据行数：{len(df)}"
             self.logger.info(success_msg)
             print(success_msg)
         except Exception as e:
-            err_msg = f"导出Snappy Parquet文件失败，保存路径：{save_path}，报错信息：{str(e)}"
+            err_msg = f"导出Parquet失败，路径：{save_path}，报错：{str(e)}"
             self.logger.error(err_msg, exc_info=True)
             print(err_msg)
             raise
@@ -1989,7 +1987,7 @@ class DataProcessingAndMessaging:
                     final_name = short_name
                 new_cols.append(final_name)
             df.columns = new_cols
-            self.logger.info(f"列名清洗完成，原列名：{old_columns} → 新列名：{list(df.columns)}")
+            self.logger.info(f"列名清洗完成，共{len(df.columns)}个字段")
             print(f"列名清洗完成，共{len(df.columns)}个字段")
 
             # 导出csv，utf-8-sig解决中文乱码，不输出索引

{functions_d-1.38 → functions_d-1.40}/Functions_d.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Functions_d
-Version: 1.38
+Version: 1.40
 Summary: 包含数据处理、Hive交互、企业微信消息发送、Excel操作等功能的工具类库
 Author: DongYang
 Author-email: 649898871@qq.com

{functions_d-1.38 → functions_d-1.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Functions_d
-Version: 1.38
+Version: 1.40
 Summary: 包含数据处理、Hive交互、企业微信消息发送、Excel操作等功能的工具类库
 Author: DongYang
 Author-email: 649898871@qq.com

{functions_d-1.38 → functions_d-1.40}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Functions_d
-version = 1.38
+version = 1.40
 author = DongYang
 author_email = 649898871@qq.com
 description = 包含数据处理、Hive交互、企业微信消息发送、Excel操作等功能的工具类库