Functions-d 1.34__tar.gz → 1.36__tar.gz

{functions_d-1.34 → functions_d-1.36}/Functions_d/Functions_d.py

@@ -9,10 +9,6 @@ import urllib.request
 import WeComMsg
 import xlwings as xw
 import yagmail
-import smtplib
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-from email.mime.application import MIMEApplication
 import HiveClient
 from PIL import ImageGrab, Image
 from bs4 import BeautifulSoup
@@ -29,9 +25,12 @@ from tenacity import retry, stop_after_attempt, wait_fixed
 import requests
 import pandas as pd
 import os
-from io import BytesIO
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
+import time
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.application import MIMEApplication
+from email.mime.image import MIMEImage
 
 
 class DataProcessingAndMessaging:
@@ -79,6 +78,15 @@ class DataProcessingAndMessaging:
         self.wechat_doc_access_token = None
         self._wechat_doc_logger = None  # 企微文档独立日志器（延迟初始化）
 
+        # -------------------------- 邮件SMTP全局配置 --------------------------
+        self.SMTP_SENDER = "cc_yingxiao@xin.com"
+        self.SMTP_PWD = "cw46pfeznNQx"
+        self.SMTP_HOST = "mail.xin.com"
+        self.SMTP_PORT = 587
+        self.SMTP_TIMEOUT = 300  # 超时延长至5分钟
+        self.MAIL_RETRY_TIMES = 3  # 失败重试3次
+        self.MAIL_RETRY_INTERVAL = 5  # 重试间隔5秒
+
 
     # -------------------------- 企微文档日志：延迟初始化独立日志器 --------------------------
     def _init_wechat_doc_logger(self):
@@ -130,6 +138,8 @@ class DataProcessingAndMessaging:
     # -------------------------- 脚本路径和文件名初始化 --------------------------
     def Start_Get_filepath_and_filename(self):
         self.start_time = time.time()
+        # ========== 新增：运行开始时追加10行空行分隔 ==========
+        self.logger.info('\n' * 5)
         caller_frame = inspect.stack()[1]
         self.current_script_name = caller_frame.filename
         self.log_filename = os.path.splitext(self.current_script_name)[0] + ".log"
@@ -148,7 +158,7 @@ class DataProcessingAndMessaging:
         elapsed_time = round(end_time - self.start_time, 0)
         print(f"运行时间：{elapsed_time} 秒")
         self.logger.info(f"运行时间：{elapsed_time} 秒")
-        self.logger.info('\n' * 10)
+        # self.logger.info('\n' * 10)
 
     # -------------------------- 企业微信消息发送 --------------------------
     # 仅展示修改后的 uxin_wx 方法，其余代码不变
@@ -815,16 +825,14 @@ class DataProcessingAndMessaging:
         self.screen(filename, sheet_name,
                     f"{start_range}:%s" % (self.column_label(len(data.columns)) + str(len(data) + 3)), image_path)
 
-    # -------------------------- 发送邮件（新版-优化增强版） --------------------------
+    # -------------------------- 发送邮件（新版-优化增强版+重试+防超时） --------------------------
     def send_email_new(self, recipient_emails, cc_emails=None, bcc_emails=None, subject="", html_body="",
                        attachments=None):
         # 1. 内部工具：格式化邮箱（和旧版完全对齐，转列表+过滤无效值）
         def format_email(emails):
             if not emails:
                 return []
-            # 单个邮箱转列表，列表直接使用
             email_list = [emails] if isinstance(emails, str) else emails
-            # 过滤：非空字符串 + 简单格式校验（含@和.）
             valid_emails = [
                 e.strip() for e in email_list
                 if isinstance(e, str) and e.strip() and '@' in e.strip() and '.' in e.strip()
@@ -853,17 +861,16 @@ class DataProcessingAndMessaging:
             }
             return mime_map.get(suffix, 'application/octet-stream')
 
-        # 3. 参数标准化处理（修复None报错隐患）
+        # 3. 参数标准化处理
         subject = subject.strip()
         to_emails = format_email(recipient_emails)
         cc_emails = format_email(cc_emails)
         bcc_emails = format_email(bcc_emails)
         attachments = attachments or []
-        # 附件标准化：单个字符串转列表
         if isinstance(attachments, str):
             attachments = [attachments.strip()]
 
-        # 4. 核心参数校验（提前报错，避免无效连接）
+        # 4. 核心参数校验
         if not to_emails:
             err_msg = "❌ 邮件发送失败：无有效收件人邮箱"
             self.logger.error(err_msg)
@@ -875,7 +882,7 @@ class DataProcessingAndMessaging:
             print(err_msg)
             raise ValueError(err_msg)
 
-        # 5. 控制台打印（和旧版完全对齐，实时展示发送进度）
+        # 5. 控制台打印
         print(f"📧 开始发送邮件 - 主题: {subject}")
         print(f"📥 收件人: {', '.join(to_emails)}")
         if cc_emails:
@@ -886,7 +893,7 @@ class DataProcessingAndMessaging:
             attach_names = [os.path.basename(att) for att in attachments]
             print(f"📎 附件: {', '.join(attach_names)}")
 
-        # 6. 日志记录（和原新版逻辑对齐）
+        # 6. 日志记录
         self.logger.info(f"开始发送邮件 - 主题: {subject}")
         self.logger.info(f"收件人: {', '.join(to_emails)}")
         if cc_emails:
@@ -897,106 +904,147 @@ class DataProcessingAndMessaging:
             attach_names = [os.path.basename(att) for att in attachments]
             self.logger.info(f"附件: {', '.join(attach_names)}")
 
-        # 7. SMTP固定配置
-        sender_email = 'cc_yingxiao@xin.com'
-        sender_password = 'cw46pfeznNQx'
-        smtp_host = 'mail.xin.com'
-        smtp_port = 587
+        # 所有收件人提前合并去重
+        all_recipients = list(set(to_emails + cc_emails + bcc_emails))
+        last_exception = None
 
-        try:
-            # 8. 构建邮件对象
-            msg = MIMEMultipart()
-            msg['From'] = sender_email
-            msg['To'] = ', '.join(to_emails)
-            if cc_emails:
-                msg['Cc'] = ', '.join(cc_emails)
-            if bcc_emails:
-                msg['Bcc'] = ', '.join(bcc_emails)
-            msg['Subject'] = subject
-
-            # 9. 邮件正文（修复中文乱码，指定utf-8编码）
-            body = MIMEText(html_body, 'html', 'utf-8')
-            msg.attach(body)
-
-            # 10. 附件处理（全格式支持+提前校验）
-            valid_attachments = []
-            for attachment in attachments:
-                attachment = attachment.strip()
-                if not os.path.exists(attachment):
-                    error_msg = f"❌ 邮件发送失败：附件不存在 -> {attachment}"
-                    self.logger.error(error_msg)
-                    print(error_msg)
-                    raise FileNotFoundError(error_msg)
-
-                # 自动识别MIME类型，适配不同文件
-                file_name = os.path.basename(attachment)
-                mime_type = get_mime_type(attachment)
-
-                with open(attachment, 'rb') as file:
-                    file_content = file.read()
-
-                # 按文件类型选择对应的MIME对象
-                if mime_type.startswith('image/'):
-                    part = MIMEImage(file_content, _subtype=mime_type.split('/')[1])
-                elif mime_type == 'text/plain' or mime_type == 'text/csv':
-                    part = MIMEText(file_content.decode('utf-8'), _subtype=mime_type.split('/')[1], _charset='utf-8')
-                else:
-                    part = MIMEApplication(file_content, _subtype=mime_type.split('/')[1])
-
-                part['Content-Disposition'] = f'attachment; filename="{file_name}"'
-                part['Content-Type'] = f'{mime_type}; name="{file_name}"'
-                msg.attach(part)
-                valid_attachments.append(file_name)
-                self.logger.debug(f"已添加附件: {file_name}")
-                print(f"✅ 已添加附件: {file_name}")
-
-            # 11. 连接SMTP服务器并发送（修复核心bug：587端口必须启用TLS）
-            print(f"🔌 正在连接SMTP服务器 {smtp_host}:{smtp_port}...")
-            with smtplib.SMTP(smtp_host, smtp_port, timeout=120) as smtp:
-                smtp.starttls()  # 587端口强制启用TLS，绝大多数服务器要求
-                smtp.login(sender_email, sender_password)
-                print(f"🔐 SMTP服务器登录成功")
-
-                # 所有收件人去重，避免重复发送
-                all_recipients = list(set(to_emails + cc_emails + bcc_emails))
-                smtp.sendmail(sender_email, all_recipients, msg.as_string())
-
-            # 12. 发送成功：控制台打印+日志（和旧版完全对齐）
-            success_msg = f"✅ 邮件发送成功 | 主题：{subject} | 收件人：{len(to_emails)}人"
-            if cc_emails:
-                success_msg += f" | 抄送：{len(cc_emails)}人"
-            if bcc_emails:
-                success_msg += f" | 密送：{len(bcc_emails)}人"
-            if valid_attachments:
-                success_msg += f" | 附件：{len(valid_attachments)}个"
+        # 7. 循环重试发送（解决网络超时/抖动）
+        for retry_idx in range(1, self.MAIL_RETRY_TIMES + 1):
+            try:
+                # 8. 构建邮件对象
+                msg = MIMEMultipart()
+                msg['From'] = self.SMTP_SENDER
+                msg['To'] = ', '.join(to_emails)
+                if cc_emails:
+                    msg['Cc'] = ', '.join(cc_emails)
+                if bcc_emails:
+                    msg['Bcc'] = ', '.join(bcc_emails)
+                msg['Subject'] = subject
+
+                # 9. 邮件正文
+                body = MIMEText(html_body, 'html', 'utf-8')
+                msg.attach(body)
+
+                # 10. 附件处理 + 安全解码 + 中文文件名修复
+                valid_attachments = []
+                for attachment in attachments:
+                    attachment = attachment.strip()
+                    if not os.path.exists(attachment):
+                        error_msg = f"❌ 邮件发送失败：附件不存在 -> {attachment}"
+                        self.logger.error(error_msg)
+                        print(error_msg)
+                        raise FileNotFoundError(error_msg)
+
+                    file_name = os.path.basename(attachment)
+                    mime_type = get_mime_type(attachment)
+
+                    # 二进制读取，避免大文件/编码报错
+                    try:
+                        with open(attachment, 'rb') as file:
+                            file_content = file.read()
+                    except Exception as e:
+                        err = f"❌ 读取附件失败 {attachment}：{str(e)}"
+                        self.logger.error(err, exc_info=True)
+                        print(err)
+                        raise RuntimeError(err)
+
+                    # 构造附件
+                    if mime_type.startswith('image/'):
+                        part = MIMEImage(file_content, _subtype=mime_type.split('/')[1])
+                    elif mime_type in ('text/plain', 'text/csv'):
+                        # 文本文件容错解码
+                        text = file_content.decode("utf-8", errors="replace")
+                        part = MIMEText(text, _subtype=mime_type.split('/')[1], _charset='utf-8')
+                    else:
+                        part = MIMEApplication(file_content, _subtype=mime_type.split('/')[1])
+
+                    # 修复中文附件名乱码（RFC2231标准）
+                    part.add_header('Content-Disposition', 'attachment', filename=file_name)
+                    part.set_param("filename*", f"utf-8''{file_name}", header="Content-Disposition")
+                    part['Content-Type'] = f'{mime_type}'
+                    msg.attach(part)
+
+                    valid_attachments.append(file_name)
+                    self.logger.debug(f"已添加附件: {file_name}")
+                    print(f"✅ 已添加附件: {file_name}")
+
+                # 11. 连接SMTP服务器（延长超时 + 双重ehlo提升兼容性）
+                print(f"🔌 第{retry_idx}次尝试连接 SMTP {self.SMTP_HOST}:{self.SMTP_PORT}")
+                self.logger.info(f"第{retry_idx}次连接 SMTP 服务器")
+
+                with smtplib.SMTP(self.SMTP_HOST, self.SMTP_PORT, timeout=self.SMTP_TIMEOUT) as smtp:
+                    smtp.ehlo()
+                    smtp.starttls()
+                    smtp.ehlo()
+                    smtp.login(self.SMTP_SENDER, self.SMTP_PWD)
+                    print(f"🔐 SMTP服务器登录成功")
+
+                    smtp.sendmail(self.SMTP_SENDER, all_recipients, msg.as_string())
+
+                # 发送成功
+                success_msg = f"✅ 邮件发送成功 | 主题：{subject} | 收件人：{len(to_emails)}人"
+                if cc_emails:
+                    success_msg += f" | 抄送：{len(cc_emails)}人"
+                if bcc_emails:
+                    success_msg += f" | 密送：{len(bcc_emails)}人"
+                if valid_attachments:
+                    success_msg += f" | 附件：{len(valid_attachments)}个"
+
+                self.logger.info(success_msg)
+                print(success_msg)
+                return True
 
-            self.logger.info(success_msg)
-            print(success_msg)
-            return True
+            # 可重试异常：超时、连接断开
+            except (TimeoutError, smtplib.SMTPServerDisconnected) as e:
+                last_exception = e
+                err = f"⚠️ 第{retry_idx}次发送超时/连接断开：{str(e)}"
+                self.logger.warning(err)
+                print(err)
+                if retry_idx < self.MAIL_RETRY_TIMES:
+                    print(f"⏳ {self.MAIL_RETRY_INTERVAL} 秒后进行重试...")
+                    time.sleep(self.MAIL_RETRY_INTERVAL)
+                continue
+
+            # 账号密码错误：致命异常，直接终止重试
+            except smtplib.SMTPAuthenticationError:
+                err_msg = "❌ 邮件发送失败：SMTP账号/密码错误，终止重试"
+                self.logger.error(err_msg, exc_info=True)
+                print(err_msg)
+                raise ValueError(err_msg)
 
-        # 13. 分类异常捕获（和旧版对齐，易排查问题）
-        except smtplib.SMTPAuthenticationError:
-            err_msg = "❌ 邮件发送失败：SMTP账号/密码/授权码错误"
-            self.logger.error(err_msg, exc_info=True)
-            print(err_msg)
-            raise ValueError(err_msg)
-        except smtplib.SMTPConnectError:
-            err_msg = "❌ 邮件发送失败：SMTP服务器连接失败（检查host/port/防火墙/网络）"
-            self.logger.error(err_msg, exc_info=True)
-            print(err_msg)
-            raise ConnectionError(err_msg)
-        except smtplib.SMTPException as e:
-            err_msg = f"❌ 邮件发送失败：SMTP协议错误 - {str(e)}"
-            self.logger.error(err_msg, exc_info=True)
-            print(err_msg)
-            raise RuntimeError(err_msg)
-        except FileNotFoundError:
-            raise  # 附件错误已提前处理，直接抛出
-        except Exception as e:
-            err_msg = f"❌ 邮件发送失败：{str(e)}"
-            self.logger.error(err_msg, exc_info=True)
-            print(err_msg)
-            raise RuntimeError(err_msg)
+            # 连接失败
+            except smtplib.SMTPConnectError:
+                err_msg = "❌ 邮件发送失败：SMTP服务器连接失败（检查网络/防火墙）"
+                self.logger.error(err_msg, exc_info=True)
+                print(err_msg)
+                raise ConnectionError(err_msg)
+
+            # 其他SMTP协议异常
+            except smtplib.SMTPException as e:
+                err_msg = f"❌ 邮件发送失败：SMTP协议错误 - {str(e)}"
+                self.logger.error(err_msg, exc_info=True)
+                print(err_msg)
+                raise RuntimeError(err_msg)
+
+            # 文件类异常，直接抛出
+            except (FileNotFoundError, RuntimeError):
+                raise
+
+            # 通用未知异常
+            except Exception as e:
+                last_exception = e
+                err = f"⚠️ 第{retry_idx}次发送未知异常：{str(e)}"
+                self.logger.error(err, exc_info=True)
+                print(err)
+                if retry_idx < self.MAIL_RETRY_TIMES:
+                    time.sleep(self.MAIL_RETRY_INTERVAL)
+                continue
+
+        # 所有重试全部失败
+        final_err = f"❌ 邮件{self.MAIL_RETRY_TIMES}次重试全部失败，最终异常：{str(last_exception)}"
+        self.logger.error(final_err, exc_info=True)
+        print(final_err)
+        raise RuntimeError(final_err)
 
     # -------------------------- Excel格式化 --------------------------
     def format_excel_worksheet(self, worksheet, df, workbook):
@@ -1687,28 +1735,38 @@ class DataProcessingAndMessaging:
             self.logger.warning(f"关闭Excel进程时发生异常: {str(e)}")
 
     # -------------------------- 上传文件/DF到内网文件服务器（最终完整版） --------------------------
-    def upload_to_internal_server(self, input_data, filename_in_server: str = "dataframe_result.xlsx"):
+    # -------------------------- 上传文件/DF到内网文件服务器（带可控自动下载最终版） --------------------------
+    def upload_to_internal_server(self, input_data, filename_in_server: str = "dataframe_result.xlsx",
+                                  auto_download: bool = False, download_save_path: str = None):
         """
-        上传数据到内网 http://fs-cc.xin.com（按要求定制：重试3次、读超时5分钟，无拆分无压缩）
-        :param input_data: 支持 3 种输入 →
-                           1) 单个 DataFrame
-                           2) 字典 {sheet名: df} → 自动生成多Sheet Excel
-                           3) 本地Excel文件路径(str)
+        上传数据到内网 http://fs-cc.xin.com（带可控自动下载功能）
+        :param input_data: 支持 3 种输入 → 1) 单个 DataFrame 2) 字典 {sheet名: df} 3) 本地Excel文件路径(str)
         :param filename_in_server: 服务器上显示的文件名
+        :param auto_download: 上传成功后是否自动下载，【默认False=关闭下载】，仅传入True时启动下载
+        :param download_save_path: 下载文件的本地保存路径（可选），默认与日志文件同目录
         :return: 下载链接（失败返回None）
         """
+        import requests
+        import pandas as pd
+        import os
+        import datetime
+        from io import BytesIO
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
+
         upload_url = "http://fs-cc.xin.com"
         self.logger.info(f"开始上传数据到内网服务器，服务器文件名：{filename_in_server}")
         print(f"📤 正在上传 → 服务器文件名：{filename_in_server}")
 
-        # ====================== 【按要求修改：重试固定3次】 ======================
-        def create_retry_session(retries=3):
+        # ====================== 【完全保留原有配置：重试3次、读超时5分钟】 ======================
+        def create_retry_session(retries=3, allowed_methods=None):
+            allowed_methods = allowed_methods or ["POST"]
             session = requests.Session()
             retry_strategy = Retry(
                 total=retries,
                 backoff_factor=2,
                 status_forcelist=[500, 502, 503, 504, 408],
-                allowed_methods=["POST"],
+                allowed_methods=allowed_methods,
                 raise_on_status=False
             )
             adapter = HTTPAdapter(max_retries=retry_strategy)
@@ -1718,9 +1776,48 @@ class DataProcessingAndMessaging:
 
         # ======================================================================
 
+        # ====================== 【新增：自动下载核心函数（与上传逻辑完全对齐）】 ======================
+        def _auto_download_file(download_url, save_path, file_name):
+            """内部自动下载函数，带重试保障、流式下载、日志同步"""
+            try:
+                # 自动创建保存目录，不存在则新建
+                os.makedirs(save_path, exist_ok=True)
+                full_save_path = os.path.join(save_path, file_name)
+
+                self.logger.info(f"📥 启动自动下载，文件保存路径：{full_save_path}")
+                print(f"📥 启动自动下载，文件保存路径：{full_save_path}")
+
+                # 流式下载（支持大文件，避免内存溢出），带3次重试、5分钟超时
+                session = create_retry_session(retries=3, allowed_methods=["GET"])
+                response = session.get(download_url, timeout=(10, 300), stream=True)
+
+                if response.status_code == 200:
+                    # 分块写入文件
+                    with open(full_save_path, 'wb') as f:
+                        for chunk in response.iter_content(chunk_size=8192):
+                            if chunk:
+                                f.write(chunk)
+
+                    self.logger.info(f"✅ 自动下载成功，文件已保存至：{full_save_path}")
+                    print(f"✅ 自动下载成功，文件已保存至：{full_save_path}")
+                    return full_save_path
+                else:
+                    err_msg = f"❌ 自动下载失败，服务器响应状态码：{response.status_code}"
+                    self.logger.error(err_msg)
+                    print(err_msg)
+                    return None
+
+            except Exception as e:
+                err_msg = f"❌ 自动下载异常：{str(e)}"
+                self.logger.error(err_msg, exc_info=True)
+                print(err_msg)
+                return None
+
+        # ======================================================================
+
         try:
             # ==============================================
-            # 【完全保留原逻辑：无Sheet拆分、无ZIP压缩】
+            # 【100%完全保留原有上传逻辑：无Sheet拆分、无ZIP压缩】
             # ==============================================
             if isinstance(input_data, dict):
                 # 字典 → 多Sheet Excel
@@ -1759,25 +1856,23 @@ class DataProcessingAndMessaging:
             else:
                 raise ValueError(f"不支持输入类型：{type(input_data)}")
 
-            # 上传
+            # 上传执行
             session = create_retry_session()
             files = {"file": (filename_in_server, file_obj, file_type)}
-            # ====================== 【按要求修改：读超时改为5分钟（300秒）】 ======================
             response = session.post(upload_url, files=files, timeout=(10, 600))
-            # ======================================================================
 
             # 关闭文件
             if isinstance(input_data, str):
                 file_obj.close()
 
-            # 结果处理
+            # 上传结果处理
             if response.status_code == 200:
                 download_url = response.text.strip()
-                self.logger.info(f"✅ 上传成功，链接：{download_url}")
+                self.logger.info(f"✅ 上传成功，下载链接：{download_url}")
                 print("✅ 上传成功！")
                 print("🔗 下载地址：", download_url)
 
-                # 生成同名链接文件（完全保留原逻辑）
+                # 生成同名链接文件（完全保留原有逻辑）
                 file_base_name = os.path.splitext(filename_in_server)[0]
                 link_file_name = f"{file_base_name}.log"
                 link_path = os.path.join(os.path.dirname(self.main_log_file), link_file_name)
@@ -1789,10 +1884,19 @@ class DataProcessingAndMessaging:
                     f.write(file_content)
 
                 self.logger.info(f"上传链接已保存到：{link_path}")
+
+                # ====================== 【核心：仅当auto_download=True时，才启动自动下载】 ======================
+                if auto_download:
+                    # 下载路径优先级：用户自定义路径 > 默认日志文件同目录
+                    final_save_path = download_save_path if download_save_path else os.path.dirname(self.main_log_file)
+                    # 执行自动下载
+                    _auto_download_file(download_url, final_save_path, filename_in_server)
+                # ======================================================================
+
                 return download_url
 
             else:
-                err = f"❌ 上传失败，状态码：{response.status_code}"
+                err = f"❌ 上传失败，服务器响应状态码：{response.status_code}"
                 self.logger.error(err)
                 print(err)
                 return None

{functions_d-1.34 → functions_d-1.36}/Functions_d.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Functions_d
-Version: 1.34
+Version: 1.36
 Summary: 包含数据处理、Hive交互、企业微信消息发送、Excel操作等功能的工具类库
 Author: DongYang
 Author-email: 649898871@qq.com

{functions_d-1.34 → functions_d-1.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Functions_d
-Version: 1.34
+Version: 1.36
 Summary: 包含数据处理、Hive交互、企业微信消息发送、Excel操作等功能的工具类库
 Author: DongYang
 Author-email: 649898871@qq.com

{functions_d-1.34 → functions_d-1.36}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Functions_d
-version = 1.34
+version = 1.36
 author = DongYang
 author_email = 649898871@qq.com
 description = 包含数据处理、Hive交互、企业微信消息发送、Excel操作等功能的工具类库