xinops 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xinops-1.0.1/PKG-INFO +155 -0
- xinops-1.0.1/README.md +109 -0
- xinops-1.0.1/pyproject.toml +4 -0
- xinops-1.0.1/setup.cfg +64 -0
- xinops-1.0.1/setup.py +2 -0
- xinops-1.0.1/xinops/__init__.py +27 -0
- xinops-1.0.1/xinops/_core.py +213 -0
- xinops-1.0.1/xinops/_email.py +338 -0
- xinops-1.0.1/xinops/_excel.py +289 -0
- xinops-1.0.1/xinops/_export.py +103 -0
- xinops-1.0.1/xinops/_feishu.py +305 -0
- xinops-1.0.1/xinops/_hive.py +260 -0
- xinops-1.0.1/xinops/_misc.py +294 -0
- xinops-1.0.1/xinops/_wecom.py +700 -0
- xinops-1.0.1/xinops.egg-info/PKG-INFO +155 -0
- xinops-1.0.1/xinops.egg-info/SOURCES.txt +18 -0
- xinops-1.0.1/xinops.egg-info/dependency_links.txt +1 -0
- xinops-1.0.1/xinops.egg-info/requires.txt +33 -0
- xinops-1.0.1/xinops.egg-info/top_level.txt +1 -0
xinops-1.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xinops
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: 营销数据运维工具库:Hive交互、企业微信/飞书消息、Excel/飞书表格、邮件推送等一体化工具集
|
|
5
|
+
Author: DongYang
|
|
6
|
+
Author-email: 649898871@qq.com
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Python: >=3.7
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
Requires-Dist: pandas>=1.3.0
|
|
18
|
+
Requires-Dist: numpy>=1.21.0
|
|
19
|
+
Requires-Dist: xlsxwriter>=3.0.0
|
|
20
|
+
Requires-Dist: requests>=2.26.0
|
|
21
|
+
Requires-Dist: pyhive>=0.6.5
|
|
22
|
+
Requires-Dist: thrift>=0.15.0
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.10.0
|
|
24
|
+
Requires-Dist: lxml>=4.6.3
|
|
25
|
+
Requires-Dist: pillow>=8.4.0
|
|
26
|
+
Requires-Dist: psutil>=5.8.0
|
|
27
|
+
Requires-Dist: tenacity>=8.0.0
|
|
28
|
+
Requires-Dist: openpyxl>=3.0.0
|
|
29
|
+
Requires-Dist: pyarrow>=10.0.0
|
|
30
|
+
Requires-Dist: yagmail>=0.15.293
|
|
31
|
+
Requires-Dist: PyYAML>=6.0
|
|
32
|
+
Requires-Dist: WeComMsg>=1.0.9
|
|
33
|
+
Requires-Dist: HiveClient>=1.0.4
|
|
34
|
+
Provides-Extra: excel-win
|
|
35
|
+
Requires-Dist: xlwings>=0.27.0; extra == "excel-win"
|
|
36
|
+
Provides-Extra: browser
|
|
37
|
+
Requires-Dist: selenium>=4.0.0; extra == "browser"
|
|
38
|
+
Requires-Dist: webdriver-manager>=3.8.0; extra == "browser"
|
|
39
|
+
Provides-Extra: pinyin
|
|
40
|
+
Requires-Dist: pypinyin>=0.48.0; extra == "pinyin"
|
|
41
|
+
Provides-Extra: all
|
|
42
|
+
Requires-Dist: xlwings>=0.27.0; extra == "all"
|
|
43
|
+
Requires-Dist: selenium>=4.0.0; extra == "all"
|
|
44
|
+
Requires-Dist: webdriver-manager>=3.8.0; extra == "all"
|
|
45
|
+
Requires-Dist: pypinyin>=0.48.0; extra == "all"
|
|
46
|
+
|
|
47
|
+
# xinops
|
|
48
|
+
|
|
49
|
+
营销数据运维一体化工具库(原 Functions_d + feishu_to_excel 合并重构版),**Windows / Mac 通用**。
|
|
50
|
+
|
|
51
|
+
## 与旧库的关系
|
|
52
|
+
|
|
53
|
+
- 原 `Functions_d`(1.46)与 `feishu_to_excel`(0.0.5)**保留为备份,不再更新**
|
|
54
|
+
- 本库为新库名下的持续维护版本,`DataProcessingAndMessaging` 类的方法名与 Functions_d 1.46 **完全一致**
|
|
55
|
+
- 业务脚本迁移只需改一行 import:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
# 旧写法
|
|
59
|
+
import Functions_d
|
|
60
|
+
ux = Functions_d.DataProcessingAndMessaging()
|
|
61
|
+
|
|
62
|
+
# 新写法
|
|
63
|
+
import xinops
|
|
64
|
+
ux = xinops.DataProcessingAndMessaging()
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## 安装
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install xinops # 核心功能(Windows/Mac 通用)
|
|
71
|
+
|
|
72
|
+
# 可选功能按需加装:
|
|
73
|
+
pip install "xinops[excel_win]" # writer_excel_data / screen(xlwings 操作本机 Excel,仅 Windows)
|
|
74
|
+
pip install "xinops[browser]" # init_edge_driver(Selenium Edge 自动化)
|
|
75
|
+
pip install "xinops[pinyin]" # name_to_pinyin / car_task_transfer
|
|
76
|
+
pip install "xinops[all]" # 全部装上(等同旧版 Functions_d 的安装效果)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
> Mac 用户注意:`writer_excel_data`(xlwings)与 `screen`(Excel 截图)依赖 Windows 上的本机 Excel,
|
|
80
|
+
> Mac 上请使用跨平台的 `writer_excel_data_new`(openpyxl 方案,无需安装 Excel)。
|
|
81
|
+
> xlwings/pythoncom 均为方法内延迟导入,Mac 上未安装不影响库的导入与其他功能。
|
|
82
|
+
|
|
83
|
+
## 密钥配置文件(必须)
|
|
84
|
+
|
|
85
|
+
本库不再在代码中硬编码任何密码。首次使用前,创建 `xinops_secrets.yaml`:
|
|
86
|
+
|
|
87
|
+
```yaml
|
|
88
|
+
# 企业微信消息
|
|
89
|
+
wecom_corpid: "wxd4e113eb4c0136b9"
|
|
90
|
+
wecom_corpsecret: "你的secret"
|
|
91
|
+
wecom_agentid: "1000026"
|
|
92
|
+
|
|
93
|
+
# 企业微信文档
|
|
94
|
+
wecom_doc_corpid: "wxd4e113eb4c0136b9"
|
|
95
|
+
wecom_doc_corpsecret: "你的secret"
|
|
96
|
+
|
|
97
|
+
# 邮件 SMTP
|
|
98
|
+
smtp_sender: "cc_yingxiao@xin.com"
|
|
99
|
+
smtp_pwd: "你的密码"
|
|
100
|
+
smtp_host: "smtp.exmail.qq.com"
|
|
101
|
+
smtp_port: 465
|
|
102
|
+
|
|
103
|
+
# Hive 连接
|
|
104
|
+
hive_host: "172.20.2.190"
|
|
105
|
+
hive_port: 10023
|
|
106
|
+
hive_user: "cc_yingxiao"
|
|
107
|
+
hive_pwd: "你的密码"
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**查找顺序**(找到即用):
|
|
111
|
+
|
|
112
|
+
1. 环境变量 `XINOPS_CONFIG`(或旧名 `PYPI_KU_CONFIG`)指定的路径
|
|
113
|
+
2. 业务脚本所在目录及其上层目录(最多向上 3 层)——推荐放在脚本目录的上层,多台机器迁移时随脚本目录一起拷贝
|
|
114
|
+
3. 用户主目录
|
|
115
|
+
|
|
116
|
+
所有配置项也支持同名环境变量覆盖(如 `WECOM_CORPSECRET`、`SMTP_PWD`、`HIVE_PWD`)。
|
|
117
|
+
**配置文件含敏感信息,切勿提交 git、切勿放进本仓库目录**(仓库 .gitignore 已加双保险)。
|
|
118
|
+
|
|
119
|
+
## 模块结构
|
|
120
|
+
|
|
121
|
+
| 模块 | 功能 |
|
|
122
|
+
|---|---|
|
|
123
|
+
| `_core.py` | 日志初始化、密钥配置加载、脚本生命周期、日期时间与通用工具 |
|
|
124
|
+
| `_wecom.py` | 企业微信消息 + 企微智能表格文档 |
|
|
125
|
+
| `_email.py` | 邮件发送(旧版 yagmail / 新版 smtplib 带重试) |
|
|
126
|
+
| `_hive.py` | Hive SQL 执行(dml/dmls/dql/dqls)与错误提取 |
|
|
127
|
+
| `_excel.py` | Excel 模板写入(xlwings/openpyxl 双方案)、截图、格式化导出 |
|
|
128
|
+
| `_export.py` | Parquet / CSV 导出 |
|
|
129
|
+
| `_misc.py` | Edge 驱动、数据量检查、拼音转换、车辆任务、内网文件服务器上传 |
|
|
130
|
+
| `_feishu.py` | 飞书表格上传器 FeishuSheetUploader(原 feishu_to_excel) |
|
|
131
|
+
|
|
132
|
+
## 使用示例
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
import xinops
|
|
136
|
+
|
|
137
|
+
# 主工具类(用法与 Functions_d 完全一致)
|
|
138
|
+
ux = xinops.DataProcessingAndMessaging()
|
|
139
|
+
ux.Start_Get_filepath_and_filename()
|
|
140
|
+
|
|
141
|
+
data = ux.run_sql(ux.path, "xxx.sql")
|
|
142
|
+
ux.writer_excel_data_new(path=ux.path, filename="muban.xlsx",
|
|
143
|
+
send_file=ux.path + "result.xlsx",
|
|
144
|
+
sheet_data=[{"data": data, "sheet_name": "明细",
|
|
145
|
+
"clear_range": "A2:U1000000", "date_range": "A1"}],
|
|
146
|
+
headers=True)
|
|
147
|
+
ux.uxin_wx("dongyang", "处理完成")
|
|
148
|
+
|
|
149
|
+
# 飞书表格上传(原 feishu_to_excel 功能)
|
|
150
|
+
uploader = xinops.FeishuSheetUploader(app_id="cli_xxx", app_secret="xxx")
|
|
151
|
+
token = uploader.create_spreadsheet(title="日报")
|
|
152
|
+
uploader.upload_to_sheet(token, "汇总", data, start_cell="A1")
|
|
153
|
+
|
|
154
|
+
ux.End_operation()
|
|
155
|
+
```
|
xinops-1.0.1/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# xinops
|
|
2
|
+
|
|
3
|
+
营销数据运维一体化工具库(原 Functions_d + feishu_to_excel 合并重构版),**Windows / Mac 通用**。
|
|
4
|
+
|
|
5
|
+
## 与旧库的关系
|
|
6
|
+
|
|
7
|
+
- 原 `Functions_d`(1.46)与 `feishu_to_excel`(0.0.5)**保留为备份,不再更新**
|
|
8
|
+
- 本库为新库名下的持续维护版本,`DataProcessingAndMessaging` 类的方法名与 Functions_d 1.46 **完全一致**
|
|
9
|
+
- 业务脚本迁移只需改一行 import:
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
# 旧写法
|
|
13
|
+
import Functions_d
|
|
14
|
+
ux = Functions_d.DataProcessingAndMessaging()
|
|
15
|
+
|
|
16
|
+
# 新写法
|
|
17
|
+
import xinops
|
|
18
|
+
ux = xinops.DataProcessingAndMessaging()
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## 安装
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install xinops # 核心功能(Windows/Mac 通用)
|
|
25
|
+
|
|
26
|
+
# 可选功能按需加装:
|
|
27
|
+
pip install "xinops[excel_win]" # writer_excel_data / screen(xlwings 操作本机 Excel,仅 Windows)
|
|
28
|
+
pip install "xinops[browser]" # init_edge_driver(Selenium Edge 自动化)
|
|
29
|
+
pip install "xinops[pinyin]" # name_to_pinyin / car_task_transfer
|
|
30
|
+
pip install "xinops[all]" # 全部装上(等同旧版 Functions_d 的安装效果)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
> Mac 用户注意:`writer_excel_data`(xlwings)与 `screen`(Excel 截图)依赖 Windows 上的本机 Excel,
|
|
34
|
+
> Mac 上请使用跨平台的 `writer_excel_data_new`(openpyxl 方案,无需安装 Excel)。
|
|
35
|
+
> xlwings/pythoncom 均为方法内延迟导入,Mac 上未安装不影响库的导入与其他功能。
|
|
36
|
+
|
|
37
|
+
## 密钥配置文件(必须)
|
|
38
|
+
|
|
39
|
+
本库不再在代码中硬编码任何密码。首次使用前,创建 `xinops_secrets.yaml`:
|
|
40
|
+
|
|
41
|
+
```yaml
|
|
42
|
+
# 企业微信消息
|
|
43
|
+
wecom_corpid: "wxd4e113eb4c0136b9"
|
|
44
|
+
wecom_corpsecret: "你的secret"
|
|
45
|
+
wecom_agentid: "1000026"
|
|
46
|
+
|
|
47
|
+
# 企业微信文档
|
|
48
|
+
wecom_doc_corpid: "wxd4e113eb4c0136b9"
|
|
49
|
+
wecom_doc_corpsecret: "你的secret"
|
|
50
|
+
|
|
51
|
+
# 邮件 SMTP
|
|
52
|
+
smtp_sender: "cc_yingxiao@xin.com"
|
|
53
|
+
smtp_pwd: "你的密码"
|
|
54
|
+
smtp_host: "smtp.exmail.qq.com"
|
|
55
|
+
smtp_port: 465
|
|
56
|
+
|
|
57
|
+
# Hive 连接
|
|
58
|
+
hive_host: "172.20.2.190"
|
|
59
|
+
hive_port: 10023
|
|
60
|
+
hive_user: "cc_yingxiao"
|
|
61
|
+
hive_pwd: "你的密码"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**查找顺序**(找到即用):
|
|
65
|
+
|
|
66
|
+
1. 环境变量 `XINOPS_CONFIG`(或旧名 `PYPI_KU_CONFIG`)指定的路径
|
|
67
|
+
2. 业务脚本所在目录及其上层目录(最多向上 3 层)——推荐放在脚本目录的上层,多台机器迁移时随脚本目录一起拷贝
|
|
68
|
+
3. 用户主目录
|
|
69
|
+
|
|
70
|
+
所有配置项也支持同名环境变量覆盖(如 `WECOM_CORPSECRET`、`SMTP_PWD`、`HIVE_PWD`)。
|
|
71
|
+
**配置文件含敏感信息,切勿提交 git、切勿放进本仓库目录**(仓库 .gitignore 已加双保险)。
|
|
72
|
+
|
|
73
|
+
## 模块结构
|
|
74
|
+
|
|
75
|
+
| 模块 | 功能 |
|
|
76
|
+
|---|---|
|
|
77
|
+
| `_core.py` | 日志初始化、密钥配置加载、脚本生命周期、日期时间与通用工具 |
|
|
78
|
+
| `_wecom.py` | 企业微信消息 + 企微智能表格文档 |
|
|
79
|
+
| `_email.py` | 邮件发送(旧版 yagmail / 新版 smtplib 带重试) |
|
|
80
|
+
| `_hive.py` | Hive SQL 执行(dml/dmls/dql/dqls)与错误提取 |
|
|
81
|
+
| `_excel.py` | Excel 模板写入(xlwings/openpyxl 双方案)、截图、格式化导出 |
|
|
82
|
+
| `_export.py` | Parquet / CSV 导出 |
|
|
83
|
+
| `_misc.py` | Edge 驱动、数据量检查、拼音转换、车辆任务、内网文件服务器上传 |
|
|
84
|
+
| `_feishu.py` | 飞书表格上传器 FeishuSheetUploader(原 feishu_to_excel) |
|
|
85
|
+
|
|
86
|
+
## 使用示例
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
import xinops
|
|
90
|
+
|
|
91
|
+
# 主工具类(用法与 Functions_d 完全一致)
|
|
92
|
+
ux = xinops.DataProcessingAndMessaging()
|
|
93
|
+
ux.Start_Get_filepath_and_filename()
|
|
94
|
+
|
|
95
|
+
data = ux.run_sql(ux.path, "xxx.sql")
|
|
96
|
+
ux.writer_excel_data_new(path=ux.path, filename="muban.xlsx",
|
|
97
|
+
send_file=ux.path + "result.xlsx",
|
|
98
|
+
sheet_data=[{"data": data, "sheet_name": "明细",
|
|
99
|
+
"clear_range": "A2:U1000000", "date_range": "A1"}],
|
|
100
|
+
headers=True)
|
|
101
|
+
ux.uxin_wx("dongyang", "处理完成")
|
|
102
|
+
|
|
103
|
+
# 飞书表格上传(原 feishu_to_excel 功能)
|
|
104
|
+
uploader = xinops.FeishuSheetUploader(app_id="cli_xxx", app_secret="xxx")
|
|
105
|
+
token = uploader.create_spreadsheet(title="日报")
|
|
106
|
+
uploader.upload_to_sheet(token, "汇总", data, start_cell="A1")
|
|
107
|
+
|
|
108
|
+
ux.End_operation()
|
|
109
|
+
```
|
xinops-1.0.1/setup.cfg
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
[metadata]
|
|
2
|
+
name = xinops
|
|
3
|
+
version = 1.0.1
|
|
4
|
+
author = DongYang
|
|
5
|
+
author_email = 649898871@qq.com
|
|
6
|
+
description = 营销数据运维工具库:Hive交互、企业微信/飞书消息、Excel/飞书表格、邮件推送等一体化工具集
|
|
7
|
+
long_description = file: README.md
|
|
8
|
+
long_description_content_type = text/markdown
|
|
9
|
+
classifiers =
|
|
10
|
+
Programming Language :: Python :: 3
|
|
11
|
+
Programming Language :: Python :: 3.7
|
|
12
|
+
Programming Language :: Python :: 3.8
|
|
13
|
+
Programming Language :: Python :: 3.9
|
|
14
|
+
Operating System :: OS Independent
|
|
15
|
+
Intended Audience :: Developers
|
|
16
|
+
Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
license = MIT
|
|
18
|
+
|
|
19
|
+
[options]
|
|
20
|
+
packages = find:
|
|
21
|
+
python_requires = >=3.7
|
|
22
|
+
install_requires =
|
|
23
|
+
pandas>=1.3.0
|
|
24
|
+
numpy>=1.21.0
|
|
25
|
+
xlsxwriter>=3.0.0
|
|
26
|
+
requests>=2.26.0
|
|
27
|
+
pyhive>=0.6.5
|
|
28
|
+
thrift>=0.15.0
|
|
29
|
+
beautifulsoup4>=4.10.0
|
|
30
|
+
lxml>=4.6.3
|
|
31
|
+
pillow>=8.4.0
|
|
32
|
+
psutil>=5.8.0
|
|
33
|
+
tenacity>=8.0.0
|
|
34
|
+
openpyxl>=3.0.0
|
|
35
|
+
pyarrow>=10.0.0
|
|
36
|
+
yagmail>=0.15.293
|
|
37
|
+
PyYAML>=6.0
|
|
38
|
+
WeComMsg>=1.0.9
|
|
39
|
+
HiveClient>=1.0.4
|
|
40
|
+
|
|
41
|
+
[options.extras_require]
|
|
42
|
+
excel_win =
|
|
43
|
+
xlwings>=0.27.0
|
|
44
|
+
browser =
|
|
45
|
+
selenium>=4.0.0
|
|
46
|
+
webdriver-manager>=3.8.0
|
|
47
|
+
pinyin =
|
|
48
|
+
pypinyin>=0.48.0
|
|
49
|
+
all =
|
|
50
|
+
xlwings>=0.27.0
|
|
51
|
+
selenium>=4.0.0
|
|
52
|
+
webdriver-manager>=3.8.0
|
|
53
|
+
pypinyin>=0.48.0
|
|
54
|
+
|
|
55
|
+
[options.packages.find]
|
|
56
|
+
where = .
|
|
57
|
+
exclude =
|
|
58
|
+
tests*
|
|
59
|
+
docs*
|
|
60
|
+
|
|
61
|
+
[egg_info]
|
|
62
|
+
tag_build =
|
|
63
|
+
tag_date = 0
|
|
64
|
+
|
xinops-1.0.1/setup.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Xinops/__init__.py
|
|
2
|
+
|
|
3
|
+
# 包版本号(与 setup.cfg 中的 version 保持一致)
|
|
4
|
+
__version__ = "1.0.0"
|
|
5
|
+
|
|
6
|
+
# 各功能模块 Mixin 导入
|
|
7
|
+
from ._core import CoreMixin
|
|
8
|
+
from ._wecom import WeComMixin
|
|
9
|
+
from ._email import EmailMixin
|
|
10
|
+
from ._hive import HiveMixin
|
|
11
|
+
from ._excel import ExcelMixin
|
|
12
|
+
from ._export import ExportMixin
|
|
13
|
+
from ._misc import MiscMixin
|
|
14
|
+
|
|
15
|
+
# 飞书表格上传器(独立类,不参与 Mixin 组装)
|
|
16
|
+
from ._feishu import FeishuSheetUploader
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# 组装为对外统一的类:方法名与 Functions_d 1.46 完全一致,业务脚本仅需改 import
|
|
20
|
+
class DataProcessingAndMessaging(CoreMixin, WeComMixin, EmailMixin, HiveMixin,
|
|
21
|
+
ExcelMixin, ExportMixin, MiscMixin):
|
|
22
|
+
"""数据处理与消息推送工具类(由多个功能 Mixin 组装而成)"""
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# 明确指定包对外暴露的成员(规范导入)
|
|
27
|
+
__all__ = ["DataProcessingAndMessaging", "FeishuSheetUploader", "__version__"]
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# -*- coding:utf-8 -*-
|
|
2
|
+
"""核心模块:日志初始化、脚本生命周期管理、日期时间与通用工具方法"""
|
|
3
|
+
import datetime
|
|
4
|
+
import inspect
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import time
|
|
8
|
+
import psutil
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CoreMixin:
|
|
13
|
+
"""基础设施 Mixin:日志器、运行状态属性、密钥配置加载、通用工具方法"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, enable_console_log=None):
|
|
16
|
+
# -------------------------- 1. 主类日志初始化 --------------------------
|
|
17
|
+
caller_frame = inspect.stack()[1]
|
|
18
|
+
caller_filename = caller_frame.filename
|
|
19
|
+
caller_filename = os.path.abspath(caller_filename)
|
|
20
|
+
script_dir = os.path.dirname(caller_filename)
|
|
21
|
+
log_filename = os.path.splitext(os.path.basename(caller_filename))[0] + ".log"
|
|
22
|
+
self.main_log_file = os.path.join(script_dir, log_filename)
|
|
23
|
+
|
|
24
|
+
self.logger = logging.getLogger("main_logger")
|
|
25
|
+
self.logger.setLevel(logging.INFO)
|
|
26
|
+
self.logger.propagate = False # 防止日志扩散
|
|
27
|
+
if not self.logger.handlers:
|
|
28
|
+
file_handler = logging.FileHandler(self.main_log_file, encoding='utf-8')
|
|
29
|
+
file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
|
30
|
+
self.logger.addHandler(file_handler)
|
|
31
|
+
if enable_console_log:
|
|
32
|
+
console_handler = logging.StreamHandler()
|
|
33
|
+
console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
|
34
|
+
self.logger.addHandler(console_handler)
|
|
35
|
+
|
|
36
|
+
self.logger.info("\n" * 1) # 1行空行,可自行修改数字
|
|
37
|
+
self.logger.info("初始化 DataProcessingAndMessaging 类")
|
|
38
|
+
|
|
39
|
+
# -------------------------- 2. 主类核心参数初始化 --------------------------
|
|
40
|
+
self.start_time = None
|
|
41
|
+
self.current_script_name = None
|
|
42
|
+
self.log_filename = None
|
|
43
|
+
self.current_script_names = None
|
|
44
|
+
self.current_path = None
|
|
45
|
+
self.path = None
|
|
46
|
+
|
|
47
|
+
# -------------------------- 2.1 密钥配置加载(外部yaml,代码不再硬编码) --------------------------
|
|
48
|
+
sec = self._load_secrets(script_dir)
|
|
49
|
+
|
|
50
|
+
# 企业微信消息发送参数(环境变量 > 配置文件)
|
|
51
|
+
self.corpid = os.getenv("WECOM_CORPID") or sec.get("wecom_corpid")
|
|
52
|
+
self.corpsecret = os.getenv("WECOM_CORPSECRET") or sec.get("wecom_corpsecret")
|
|
53
|
+
self.agentid = str(sec.get("wecom_agentid", "1000026"))
|
|
54
|
+
|
|
55
|
+
# -------------------------- 3. 企业微信文档功能初始化 --------------------------
|
|
56
|
+
self.WECHAT_DOC_CORP_ID = os.getenv("WECOM_DOC_CORPID") or sec.get("wecom_doc_corpid")
|
|
57
|
+
self.WECHAT_DOC_SECRET = os.getenv("WECOM_DOC_SECRET") or sec.get("wecom_doc_corpsecret")
|
|
58
|
+
self.WECHAT_DOC_SPACE_ID = None
|
|
59
|
+
self.WECHAT_DOC_LOG_FILE = os.path.join(script_dir, "docs_operation.log")
|
|
60
|
+
self.wechat_doc_access_token = None
|
|
61
|
+
self._wechat_doc_logger = None # 企微文档独立日志器(延迟初始化)
|
|
62
|
+
|
|
63
|
+
# -------------------------- 邮件SMTP全局配置 --------------------------
|
|
64
|
+
self.SMTP_SENDER = sec.get("smtp_sender")
|
|
65
|
+
self.SMTP_PWD = sec.get("smtp_pwd")
|
|
66
|
+
self.SMTP_HOST = sec.get("smtp_host", "smtp.exmail.qq.com")
|
|
67
|
+
self.SMTP_PORT = int(sec.get("smtp_port", 465))
|
|
68
|
+
self.SMTP_TIMEOUT = 300 # 超时延长至5分钟
|
|
69
|
+
self.MAIL_RETRY_TIMES = 3 # 失败重试3次
|
|
70
|
+
self.MAIL_RETRY_INTERVAL = 5 # 重试间隔5秒
|
|
71
|
+
|
|
72
|
+
# -------------------------- Hive连接配置(run_sql使用) --------------------------
|
|
73
|
+
self.hive_host = os.getenv("HIVE_HOST") or sec.get("hive_host")
|
|
74
|
+
self.hive_port = int(sec.get("hive_port", 10023))
|
|
75
|
+
self.hive_user = os.getenv("HIVE_USER") or sec.get("hive_user")
|
|
76
|
+
self.hive_pwd = os.getenv("HIVE_PWD") or sec.get("hive_pwd")
|
|
77
|
+
|
|
78
|
+
# -------------------------- 密钥配置文件加载 --------------------------
|
|
79
|
+
def _load_secrets(self, script_dir):
|
|
80
|
+
"""按优先级查找密钥配置文件(xinops_secrets.yaml 或 pypi_ku_secrets.yaml):
|
|
81
|
+
1. 环境变量 XINOPS_CONFIG / PYPI_KU_CONFIG 指定的路径
|
|
82
|
+
2. 调用方脚本所在目录及其各级上层目录(最多向上3层)
|
|
83
|
+
3. 用户主目录
|
|
84
|
+
配置文件放在代码仓库之外,切勿提交到 git 或打包进 PyPI。
|
|
85
|
+
"""
|
|
86
|
+
candidates = []
|
|
87
|
+
for env_key in ("XINOPS_CONFIG", "PYPI_KU_CONFIG"):
|
|
88
|
+
env_path = os.getenv(env_key)
|
|
89
|
+
if env_path:
|
|
90
|
+
candidates.append(env_path)
|
|
91
|
+
|
|
92
|
+
file_names = ("xinops_secrets.yaml", "pypi_ku_secrets.yaml")
|
|
93
|
+
# 从脚本所在目录开始向上查找(本目录 + 上3层)
|
|
94
|
+
d = script_dir
|
|
95
|
+
for _ in range(4):
|
|
96
|
+
for fn in file_names:
|
|
97
|
+
candidates.append(os.path.join(d, fn))
|
|
98
|
+
parent = os.path.dirname(d)
|
|
99
|
+
if parent == d: # 已到根目录
|
|
100
|
+
break
|
|
101
|
+
d = parent
|
|
102
|
+
|
|
103
|
+
# 用户主目录兜底
|
|
104
|
+
home = os.path.expanduser("~")
|
|
105
|
+
for fn in file_names:
|
|
106
|
+
candidates.append(os.path.join(home, fn))
|
|
107
|
+
|
|
108
|
+
for path in candidates:
|
|
109
|
+
if os.path.exists(path):
|
|
110
|
+
try:
|
|
111
|
+
with open(path, encoding="utf-8") as f:
|
|
112
|
+
return yaml.safe_load(f) or {}
|
|
113
|
+
except yaml.YAMLError as e:
|
|
114
|
+
raise ValueError(f"密钥配置文件格式错误({path}):{e}")
|
|
115
|
+
|
|
116
|
+
raise FileNotFoundError(
|
|
117
|
+
"未找到密钥配置文件 xinops_secrets.yaml(或 pypi_ku_secrets.yaml),已查找以下位置:\n"
|
|
118
|
+
+ "\n".join(candidates)
|
|
119
|
+
+ "\n请把配置文件放到业务脚本上层目录,或设置环境变量 XINOPS_CONFIG 指向实际路径"
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# -------------------------- 脚本路径和文件名初始化 --------------------------
|
|
123
|
+
def Start_Get_filepath_and_filename(self):
|
|
124
|
+
self.start_time = time.time()
|
|
125
|
+
# ========== 新增:运行开始时追加10行空行分隔 ==========
|
|
126
|
+
# self.logger.info('\n' * 5)
|
|
127
|
+
caller_frame = inspect.stack()[1]
|
|
128
|
+
self.current_script_name = caller_frame.filename
|
|
129
|
+
self.log_filename = os.path.splitext(self.current_script_name)[0] + ".log"
|
|
130
|
+
self.current_script_names = os.path.basename(self.current_script_name)
|
|
131
|
+
self.current_path = os.path.dirname(os.path.abspath(self.current_script_name))
|
|
132
|
+
self.path = self.current_path + os.sep
|
|
133
|
+
print(f"当前时间:{self.get_date_and_time('%Y-%m-%d %H:%M:%S', 0)}")
|
|
134
|
+
print(f"开始执行脚本:{self.current_script_names}")
|
|
135
|
+
self.logger.info(f"开始执行脚本:{self.current_script_names}")
|
|
136
|
+
|
|
137
|
+
# -------------------------- 脚本结束处理 --------------------------
|
|
138
|
+
def End_operation(self):
|
|
139
|
+
print(f"脚本:{self.current_script_names} 执行成功")
|
|
140
|
+
self.logger.info(f"脚本:{self.current_script_names} 执行成功")
|
|
141
|
+
end_time = time.time()
|
|
142
|
+
elapsed_time = round(end_time - self.start_time, 0)
|
|
143
|
+
print(f"运行时间:{elapsed_time} 秒")
|
|
144
|
+
self.logger.info(f"运行时间:{elapsed_time} 秒")
|
|
145
|
+
# self.logger.info('\n' * 2)
|
|
146
|
+
|
|
147
|
+
# -------------------------- 获取指定格式的日期时间 --------------------------
|
|
148
|
+
def get_date_and_time(self, format_type, days):
|
|
149
|
+
today = datetime.datetime.today()
|
|
150
|
+
target_date = today - datetime.timedelta(days=days)
|
|
151
|
+
result = target_date.strftime(format_type)
|
|
152
|
+
return result
|
|
153
|
+
|
|
154
|
+
# -------------------------- 获取文件大小(KB) --------------------------
|
|
155
|
+
def get_FileSize(self, img_name):
|
|
156
|
+
fsize = os.path.getsize(img_name)
|
|
157
|
+
fsize = fsize / float(1024)
|
|
158
|
+
return round(fsize, 2)
|
|
159
|
+
|
|
160
|
+
# -------------------------- 列标转换(A,B,C...) --------------------------
|
|
161
|
+
def column_label(self, n):
|
|
162
|
+
result = ""
|
|
163
|
+
while n > 0:
|
|
164
|
+
n, remainder = divmod(n - 1, 26)
|
|
165
|
+
result = chr(65 + remainder) + result
|
|
166
|
+
return result
|
|
167
|
+
|
|
168
|
+
# -------------------------- 辅助方法:检查文件是否被占用 --------------------------
|
|
169
|
+
def is_file_locked(self, file_path):
|
|
170
|
+
if not os.path.exists(file_path):
|
|
171
|
+
return False
|
|
172
|
+
try:
|
|
173
|
+
with open(file_path, 'a', encoding='utf-8') as f:
|
|
174
|
+
pass
|
|
175
|
+
return False
|
|
176
|
+
except IOError:
|
|
177
|
+
return True
|
|
178
|
+
|
|
179
|
+
# -------------------------- 辅助方法:安全关闭Excel进程 --------------------------
|
|
180
|
+
def _safe_quit_excel(self, app):
|
|
181
|
+
"""安全关闭Excel进程,优化时序检查逻辑"""
|
|
182
|
+
try:
|
|
183
|
+
pid = app.pid
|
|
184
|
+
# 先尝试正常退出
|
|
185
|
+
app.quit()
|
|
186
|
+
|
|
187
|
+
# 增加短暂延迟,等待进程自然退出(解决时序问题)
|
|
188
|
+
time.sleep(0.5)
|
|
189
|
+
|
|
190
|
+
# 检查进程是否还存在
|
|
191
|
+
if psutil.pid_exists(pid):
|
|
192
|
+
# 尝试优雅终止
|
|
193
|
+
proc = psutil.Process(pid)
|
|
194
|
+
proc.terminate() # 发送终止信号
|
|
195
|
+
|
|
196
|
+
# 等待进程退出,最多等待3秒
|
|
197
|
+
try:
|
|
198
|
+
proc.wait(timeout=3)
|
|
199
|
+
self.logger.debug(f"Excel进程(PID: {pid})已被强制终止")
|
|
200
|
+
except psutil.TimeoutExpired:
|
|
201
|
+
# 超时后强制杀死进程
|
|
202
|
+
proc.kill()
|
|
203
|
+
self.logger.debug(f"Excel进程(PID: {pid})超时未退出,已强制杀死")
|
|
204
|
+
else:
|
|
205
|
+
# 进程已正常退出,不记录警告
|
|
206
|
+
self.logger.debug(f"Excel进程(PID: {pid})已正常退出")
|
|
207
|
+
|
|
208
|
+
except psutil.NoSuchProcess:
|
|
209
|
+
# 进程在检查前已退出,属于正常情况,不记录警告
|
|
210
|
+
self.logger.debug(f"Excel进程已提前退出(PID: {pid})")
|
|
211
|
+
except Exception as e:
|
|
212
|
+
# 其他未知错误才记录警告
|
|
213
|
+
self.logger.warning(f"关闭Excel进程时发生异常: {str(e)}")
|