mdbq 3.3.5__py3-none-any.whl → 3.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/mongo/mongo.py +24 -22
- mdbq/mysql/recheck_mysql.py +1 -1
- mdbq/spider/aikucun.py +2 -24
- {mdbq-3.3.5.dist-info → mdbq-3.3.7.dist-info}/METADATA +1 -1
- {mdbq-3.3.5.dist-info → mdbq-3.3.7.dist-info}/RECORD +7 -18
- mdbq/aggregation/df_types.py +0 -188
- mdbq/aggregation/mysql_types.py +0 -240
- mdbq/clean/__init__.py +0 -4
- mdbq/clean/clean_upload.py +0 -1350
- mdbq/clean/data_clean.py +0 -1551
- mdbq/company/__init__.py +0 -4
- mdbq/company/copysh.py +0 -447
- mdbq/config/get_myconf.py +0 -131
- mdbq/config/update_conf.py +0 -102
- mdbq/req_post/__init__.py +0 -4
- mdbq/req_post/req_tb.py +0 -624
- {mdbq-3.3.5.dist-info → mdbq-3.3.7.dist-info}/WHEEL +0 -0
- {mdbq-3.3.5.dist-info → mdbq-3.3.7.dist-info}/top_level.txt +0 -0
mdbq/company/__init__.py
DELETED
mdbq/company/copysh.py
DELETED
@@ -1,447 +0,0 @@
|
|
1
|
-
# -*- coding: UTF-8 –*-
|
2
|
-
import os
|
3
|
-
import platform
|
4
|
-
import warnings
|
5
|
-
import getpass
|
6
|
-
import sys
|
7
|
-
import configparser
|
8
|
-
import datetime
|
9
|
-
import shutil
|
10
|
-
import time
|
11
|
-
import re
|
12
|
-
import socket
|
13
|
-
from mdbq.bdup import bdup
|
14
|
-
from mdbq.aggregation import query_data
|
15
|
-
from mdbq.aggregation import optimize_data
|
16
|
-
from mdbq.config import set_support
|
17
|
-
from mdbq.config import myconfig
|
18
|
-
from mdbq.mysql import mysql
|
19
|
-
from mdbq.clean import clean_upload
|
20
|
-
warnings.filterwarnings('ignore')
|
21
|
-
|
22
|
-
|
23
|
-
if platform.system() == 'Windows':
|
24
|
-
# windows版本
|
25
|
-
from mdbq.pbix import refresh_all
|
26
|
-
Data_Path = r'C:\同步空间\BaiduSyncdisk'
|
27
|
-
D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
|
28
|
-
Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表') # 共享文件根目录
|
29
|
-
elif platform.system() == 'Linux':
|
30
|
-
Data_Path = '数据中心'
|
31
|
-
D_PATH = 'Downloads'
|
32
|
-
if not os.path.exists(D_PATH):
|
33
|
-
os.makedirs(D_PATH)
|
34
|
-
Share_Path = '' # linux 通常是远程服务器,不需要访问共享
|
35
|
-
else:
|
36
|
-
Data_Path = f'/Users/{getpass.getuser()}/数据中心' # 使用Mac独立网络时
|
37
|
-
D_PATH = os.path.join(f'/Users/{getpass.getuser()}/Downloads')
|
38
|
-
Share_Path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表') # 共享文件根目录
|
39
|
-
|
40
|
-
upload_path = os.path.join(D_PATH, '数据上传中心') # 此目录位于下载文件夹,将统一上传百度云备份
|
41
|
-
source_path = os.path.join(Data_Path, '原始文件3') # 此目录保存下载并清洗过的文件,作为数据库备份
|
42
|
-
if not os.path.exists(upload_path): # 数据中心根目录
|
43
|
-
os.makedirs(upload_path)
|
44
|
-
if not os.path.exists(Data_Path): # 数据中心根目录
|
45
|
-
os.makedirs(Data_Path)
|
46
|
-
if not os.path.exists(source_path): # 原始文件
|
47
|
-
os.makedirs(source_path)
|
48
|
-
|
49
|
-
|
50
|
-
class TbFiles:
|
51
|
-
"""
|
52
|
-
用于在公司台式机中 定时同步pandas数据源文件到共享
|
53
|
-
"""
|
54
|
-
def __init__(self):
|
55
|
-
|
56
|
-
support_path = set_support.SetSupport(dirname='support').dirname
|
57
|
-
|
58
|
-
self.my_conf = os.path.join(support_path, '.copysh_conf')
|
59
|
-
self.path1 = os.path.join(support_path, 'tb_list.txt')
|
60
|
-
self.path2 = os.path.join(support_path, 'cp_list.txt')
|
61
|
-
self.d_path = None
|
62
|
-
self.data_path = None
|
63
|
-
self.share_path = None
|
64
|
-
self.before_max_time = []
|
65
|
-
self.sleep_minutes = 30
|
66
|
-
self.tomorrow = datetime.date.today()
|
67
|
-
|
68
|
-
def check_change(self):
|
69
|
-
""" 检查 source_path 的所有文件修改日期, 函数返回最新修改日期 """
|
70
|
-
source_path = os.path.join(self.data_path, 'pandas数据源')
|
71
|
-
if not os.path.exists(source_path):
|
72
|
-
return
|
73
|
-
results = []
|
74
|
-
for root, dirs, files in os.walk(source_path, topdown=False):
|
75
|
-
for name in files:
|
76
|
-
if '~$' in name or 'baiduyun' in name or name.startswith('.') or 'Icon' in name or 'xunlei' in name:
|
77
|
-
continue # 排除这些文件的变动
|
78
|
-
# stat_info = os.path.getmtime(os.path.join(root, name))
|
79
|
-
_c = os.stat(os.path.join(root, name)).st_mtime # 读取文件的元信息 >>>文件修改时间
|
80
|
-
c_time = datetime.datetime.fromtimestamp(_c) # 格式化修改时间
|
81
|
-
results.append(c_time)
|
82
|
-
return max(results).strftime('%Y%m%d%H%M%S')
|
83
|
-
|
84
|
-
def check_conf(self):
|
85
|
-
if not os.path.isfile(self.my_conf):
|
86
|
-
self.set_conf() # 添加配置文件
|
87
|
-
print('因缺少配置文件, 已自动初始化')
|
88
|
-
config = configparser.ConfigParser() # 初始化configparser类
|
89
|
-
try:
|
90
|
-
config.read(self.my_conf, 'UTF-8')
|
91
|
-
self.d_path = config.get('database', 'd_path')
|
92
|
-
self.data_path = config.get('database', 'data_path')
|
93
|
-
self.share_path = config.get('database', 'share_path')
|
94
|
-
if self.d_path is None or self.data_path is None or self.share_path is None:
|
95
|
-
self.set_conf()
|
96
|
-
print('配置文件部分值不完整, 已自动初始化')
|
97
|
-
if not os.path.exists(self.d_path) or not os.path.exists(self.data_path) or not os.path.exists(self.share_path):
|
98
|
-
self.set_conf()
|
99
|
-
print('配置文件异常(可能跨系统), 已自动初始化')
|
100
|
-
except Exception as e:
|
101
|
-
print(e)
|
102
|
-
print('配置文件部分值缺失, 已自动初始化')
|
103
|
-
self.set_conf()
|
104
|
-
sys.path.append(self.share_path)
|
105
|
-
|
106
|
-
def set_conf(self):
|
107
|
-
if platform.system() == 'Windows':
|
108
|
-
self.d_path = os.path.join('C:\\Users', getpass.getuser(), 'Downloads')
|
109
|
-
self.data_path = os.path.join('C:\\同步空间', 'BaiduSyncdisk')
|
110
|
-
self.share_path = os.path.join('\\\\192.168.1.198', '时尚事业部\\01.运营部\\天猫报表') # 共享文件根目录
|
111
|
-
elif platform.system() == 'Darwin':
|
112
|
-
self.d_path = os.path.join('/Users', getpass.getuser(), 'Downloads')
|
113
|
-
self.data_path = os.path.join('/Users', getpass.getuser(), '数据中心')
|
114
|
-
self.share_path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表') # 共享文件根目录
|
115
|
-
else:
|
116
|
-
self.d_path = 'Downloads'
|
117
|
-
self.data_path = os.path.join(getpass.getuser(), '数据中心')
|
118
|
-
self.share_path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表') # 共享文件根目录
|
119
|
-
|
120
|
-
if not os.path.exists(self.share_path):
|
121
|
-
self.share_path = re.sub('时尚事业部', '时尚事业部-1', self.share_path)
|
122
|
-
|
123
|
-
with open(self.my_conf, 'w+', encoding='utf-8') as f:
|
124
|
-
f.write('[database]\n')
|
125
|
-
f.write(f'# 配置文件\n')
|
126
|
-
f.write('# 下载目录\n')
|
127
|
-
f.write(f'd_path = {self.d_path}\n\n')
|
128
|
-
f.write('# 数据中心目录\n')
|
129
|
-
f.write(f'data_path = {self.data_path}\n\n')
|
130
|
-
f.write('# 共享目录\n')
|
131
|
-
f.write(f'share_path = {self.share_path}\n\n')
|
132
|
-
f.write('# 公司台式机中,用于触发下载百度云文件,更新至本机数据库\n')
|
133
|
-
f.write(f'ch_record = False\n\n')
|
134
|
-
print('目录初始化!')
|
135
|
-
|
136
|
-
def tb_file(self):
|
137
|
-
|
138
|
-
self.check_conf() # 检查配置文件
|
139
|
-
|
140
|
-
now_max_time = self.check_change()
|
141
|
-
if now_max_time in self.before_max_time:
|
142
|
-
return # 不更新
|
143
|
-
else:
|
144
|
-
self.before_max_time = [] # 重置变量,以免越来越占内存
|
145
|
-
self.before_max_time.append(now_max_time)
|
146
|
-
|
147
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
|
148
|
-
res = self.check_upload_mysql()
|
149
|
-
if not res:
|
150
|
-
print(f'检测到源文件修改, 但今日已经同步过, 不再同步')
|
151
|
-
return
|
152
|
-
print(f'{now}pandas数据源文件修改, 触发同步 ({self.sleep_minutes}分钟后开始)')
|
153
|
-
|
154
|
-
if not os.path.exists(self.data_path):
|
155
|
-
print(f'{self.data_path}: 本地目录不存在或配置文件异常, 无法同步此目录')
|
156
|
-
return None
|
157
|
-
if not os.path.exists(self.share_path):
|
158
|
-
print(f'{self.share_path}: 本机未连接共享或配置文件异常, 无法同步')
|
159
|
-
return None
|
160
|
-
|
161
|
-
time.sleep(self.sleep_minutes*60) # 开始同步前休眠时间
|
162
|
-
recent_time = 48 # 同步近N小时内更新过的文件,单位:小时
|
163
|
-
tb_list = []
|
164
|
-
pd_list = []
|
165
|
-
try:
|
166
|
-
with open(self.path1, 'r', encoding='utf-8') as f:
|
167
|
-
content = f.readlines()
|
168
|
-
content = [item.strip() for item in content if not item.strip().startswith('#')]
|
169
|
-
tb_list = [item for item in content if item]
|
170
|
-
|
171
|
-
with open(self.path2, 'r', encoding='utf-8') as f:
|
172
|
-
content = f.readlines()
|
173
|
-
content = [item.strip() for item in content if not item.strip().startswith('#')]
|
174
|
-
pd_list = [item for item in content if item]
|
175
|
-
except Exception as e:
|
176
|
-
print(e)
|
177
|
-
|
178
|
-
source_path = os.path.join(self.data_path, 'pandas数据源') # \BaiduSyncdisk\pandas数据源
|
179
|
-
target_path = os.path.join(self.share_path, 'pandas数据源') # \01.运营部\天猫报表\pandas数据源
|
180
|
-
|
181
|
-
if not os.path.exists(target_path): # 检查共享主目录,创建目录
|
182
|
-
os.makedirs(target_path, exist_ok=True)
|
183
|
-
|
184
|
-
# 删除共享的副本
|
185
|
-
file_list = os.listdir(self.share_path)
|
186
|
-
for file_1 in file_list:
|
187
|
-
if '副本_' in file_1 or 'con' in file_1: # or '.DS' in file_1
|
188
|
-
try:
|
189
|
-
os.remove(os.path.join(self.share_path, file_1))
|
190
|
-
print(f'移除: {os.path.join(self.share_path, file_1)}')
|
191
|
-
except Exception as e:
|
192
|
-
print(e)
|
193
|
-
print(f'移除失败:{os.path.join(self.share_path, file_1)}')
|
194
|
-
file_list2 = os.listdir(target_path) # 删除乱七八糟的临时文件
|
195
|
-
for file_1 in file_list2:
|
196
|
-
if '.DS' in file_1 or 'con' in file_1:
|
197
|
-
try:
|
198
|
-
os.remove(os.path.join(target_path, file_1))
|
199
|
-
print(f'移除: {os.path.join(target_path, file_1)}')
|
200
|
-
except Exception as e:
|
201
|
-
print(e)
|
202
|
-
|
203
|
-
# 删除 run_py的 副本
|
204
|
-
del_p = os.path.join(self.data_path, '自动0备份', 'py', '数据更新', 'run_py')
|
205
|
-
for file_1 in os.listdir(del_p):
|
206
|
-
if '副本_' in file_1:
|
207
|
-
try:
|
208
|
-
os.remove(os.path.join(del_p, file_1))
|
209
|
-
print(f'移除: {os.path.join(del_p, file_1)}')
|
210
|
-
except Exception as e:
|
211
|
-
print(e)
|
212
|
-
print(f'移除失败:{os.path.join(del_p, file_1)}')
|
213
|
-
|
214
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
215
|
-
print(f'{now} 正在同步文件...')
|
216
|
-
# 复制 run_py的文件到共享
|
217
|
-
for file_1 in tb_list:
|
218
|
-
s = os.path.join(del_p, file_1)
|
219
|
-
t = os.path.join(self.share_path, file_1)
|
220
|
-
try:
|
221
|
-
shutil.copy2(s, t)
|
222
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
|
223
|
-
print(f'{now}复制: {s}')
|
224
|
-
except Exception as e:
|
225
|
-
print(e)
|
226
|
-
s1 = os.path.join(del_p, f'副本_{file_1}')
|
227
|
-
t1 = os.path.join(self.share_path, f'副本_{file_1}')
|
228
|
-
shutil.copy2(s, s1) # 创建副本
|
229
|
-
shutil.copy2(s1, t1) # 复制副本到共享
|
230
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
|
231
|
-
print(f'{now}已创建副本 -->> {s1}')
|
232
|
-
|
233
|
-
# 同步 pandas 文件到共享
|
234
|
-
now_time = time.time()
|
235
|
-
for filenames in pd_list:
|
236
|
-
src = os.path.join(source_path, filenames) # 原位置,可能是文件或文件夹
|
237
|
-
dst = os.path.join(target_path, filenames) # 目标位置,可能是文件或文件夹
|
238
|
-
if os.path.isdir(src): # 如果是文件夹
|
239
|
-
for root, dirs, files in os.walk(src, topdown=False):
|
240
|
-
for name in files:
|
241
|
-
if '~$' in name or 'DS_Store' in name:
|
242
|
-
continue
|
243
|
-
if name.endswith('csv') or name.endswith('xlsx') or name.endswith('pbix') or name.endswith(
|
244
|
-
'xls'):
|
245
|
-
new_src = os.path.join(root, name)
|
246
|
-
# share_path = dst + '\\' + new_src.split(src)[1] # 拼接目标路径
|
247
|
-
share_path = os.path.join(f'{dst}{new_src.split(src)[1]}') # 拼接目标路径
|
248
|
-
ls_paths = os.path.dirname(os.path.abspath(share_path)) # 获取上级目录,用来创建
|
249
|
-
if not os.path.exists(ls_paths): # 目录不存在则创建
|
250
|
-
os.makedirs(ls_paths, exist_ok=True)
|
251
|
-
c_stat = os.stat(new_src).st_mtime # 读取文件的元信息 >>>文件修改时间
|
252
|
-
if now_time - c_stat < recent_time * 3600: # 仅同步近期更新的文件
|
253
|
-
# res_name = os.path.basename(new_src)
|
254
|
-
try:
|
255
|
-
shutil.copy2(new_src, share_path)
|
256
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
|
257
|
-
print(f'{now}复制文件: {new_src}')
|
258
|
-
except Exception as e:
|
259
|
-
print(e)
|
260
|
-
elif os.path.isfile(src) and 'DS_Store' not in src: # 如果是文件
|
261
|
-
if src.endswith('csv') or src.endswith('xlsx') or src.endswith('pbix') or src.endswith('xls'):
|
262
|
-
c_stat = os.stat(src).st_mtime # 读取文件的元信息 >>>文件修改时间
|
263
|
-
if now_time - c_stat < recent_time * 3600:
|
264
|
-
ls_paths = os.path.dirname(os.path.abspath(src)) # 获取上级目录,用来创建
|
265
|
-
if not os.path.exists(ls_paths): # 目录不存在则创建
|
266
|
-
os.makedirs(ls_paths, exist_ok=True)
|
267
|
-
# new_name = os.path.basename(src)
|
268
|
-
try:
|
269
|
-
shutil.copy2(src, dst)
|
270
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
|
271
|
-
print(f'{now}复制文件: {src}')
|
272
|
-
except Exception as e:
|
273
|
-
print(e)
|
274
|
-
else:
|
275
|
-
print(f'{src} 所需同步的文件不存在,请检查:pd_list参数')
|
276
|
-
|
277
|
-
# 刷新共享位置的指定文件/文件夹
|
278
|
-
if platform.system() == 'Windows':
|
279
|
-
excel_path = os.path.join(self.share_path, 'EXCEL报表')
|
280
|
-
files = os.listdir(excel_path)
|
281
|
-
files = [f'{excel_path}\\{item}' for item in files if item.endswith('.xlsx') or item.endswith('.xls')]
|
282
|
-
r = refresh_all.RefreshAll()
|
283
|
-
for file in files:
|
284
|
-
if '~' in file or 'DS_Store' in file or 'baidu' in file or 'xunlei' in file:
|
285
|
-
continue
|
286
|
-
if file.endswith('.xlsx') or file.endswith('.xls'):
|
287
|
-
r.refresh_excel(file=file)
|
288
|
-
time.sleep(5)
|
289
|
-
|
290
|
-
# 临时加的
|
291
|
-
# excel_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\0-WLM_运营周报-1012输出.xlsx'
|
292
|
-
dir_files = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新'
|
293
|
-
files = os.listdir(dir_files)
|
294
|
-
for file in files:
|
295
|
-
if file.endswith('.xlsx') and file.startswith('0-WLM_运营周报') and '~' not in file and 'baidu' not in file:
|
296
|
-
excel_file = os.path.join(dir_files, file)
|
297
|
-
r.refresh_excel(file=excel_file)
|
298
|
-
|
299
|
-
self.before_max_time = self.check_change() # 重置值, 避免重复同步
|
300
|
-
|
301
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
302
|
-
print(f'{now} 同步完成!')
|
303
|
-
|
304
|
-
def refresh_excel(self):
|
305
|
-
# 刷新共享位置的指定文件/文件夹
|
306
|
-
if platform.system() == 'Windows' and socket.gethostname() == 'company':
|
307
|
-
excel_path = os.path.join(Share_Path, 'EXCEL报表')
|
308
|
-
files = os.listdir(excel_path)
|
309
|
-
files = [f'{excel_path}\\{item}' for item in files if item.endswith('.xlsx') or item.endswith('.xls')]
|
310
|
-
r = refresh_all.RefreshAll()
|
311
|
-
for file in files:
|
312
|
-
if '~' in file or 'DS_Store' in file or 'baidu' in file or 'xunlei' in file:
|
313
|
-
continue
|
314
|
-
if file.endswith('.xlsx') or file.endswith('.xls'):
|
315
|
-
r.refresh_excel(file=file)
|
316
|
-
time.sleep(5)
|
317
|
-
|
318
|
-
# 临时加的
|
319
|
-
# excel_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\0-WLM_运营周报-1012输出.xlsx'
|
320
|
-
dir_files = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新'
|
321
|
-
files = os.listdir(dir_files)
|
322
|
-
for file in files:
|
323
|
-
if file.endswith('.xlsx') and file.startswith(
|
324
|
-
'0-WLM_运营周报') and '~' not in file and 'baidu' not in file:
|
325
|
-
excel_file = os.path.join(dir_files, file)
|
326
|
-
r.refresh_excel(file=excel_file)
|
327
|
-
|
328
|
-
def check_upload_mysql(self):
|
329
|
-
# 每天只更新一次
|
330
|
-
today = datetime.date.today()
|
331
|
-
if today == self.tomorrow:
|
332
|
-
self.tomorrow = today + datetime.timedelta(days=1)
|
333
|
-
return True
|
334
|
-
else:
|
335
|
-
return False
|
336
|
-
|
337
|
-
|
338
|
-
def op_data(days: int =100):
|
339
|
-
|
340
|
-
# 清理数据库, 除了 聚合数据
|
341
|
-
if socket.gethostname() == 'company': # 公司台式机自身运行
|
342
|
-
# 清理所有非聚合数据的库
|
343
|
-
optimize_data.op_data(
|
344
|
-
db_name_lists=[
|
345
|
-
'京东数据2',
|
346
|
-
'属性设置3',
|
347
|
-
'推广数据2',
|
348
|
-
'推广数据_淘宝店',
|
349
|
-
'爱库存2',
|
350
|
-
'生意参谋3',
|
351
|
-
'生意经3',
|
352
|
-
# '聚合数据',
|
353
|
-
'达摩盘3',
|
354
|
-
],
|
355
|
-
days=days,
|
356
|
-
)
|
357
|
-
|
358
|
-
# 数据聚合
|
359
|
-
query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=3,)
|
360
|
-
time.sleep(60)
|
361
|
-
|
362
|
-
# 清理聚合数据
|
363
|
-
optimize_data.op_data(db_name_lists=['聚合数据'], days=3650, )
|
364
|
-
|
365
|
-
|
366
|
-
def main():
|
367
|
-
# if platform.system() != 'Windows':
|
368
|
-
# print(f'只可以在 windows 运行')
|
369
|
-
# return
|
370
|
-
t = TbFiles()
|
371
|
-
while True:
|
372
|
-
system = platform.system() # 本机系统
|
373
|
-
host_name = socket.gethostname() # 本机名
|
374
|
-
conf = myconfig.main()
|
375
|
-
data = conf[system][host_name]
|
376
|
-
is_download = data['is_download'] # 读取配置, 如果是 Ture 则执行更新
|
377
|
-
|
378
|
-
if is_download:
|
379
|
-
bd_remoto_path = f'windows2/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
|
380
|
-
b = bdup.BaiDu()
|
381
|
-
# 1. 从百度云下载文件
|
382
|
-
b.download_dir(local_path=upload_path, remote_path=bd_remoto_path)
|
383
|
-
|
384
|
-
clean_upload.main(
|
385
|
-
is_mysql=True, # 调试时加,False: 不进行后续的聚合数据及清理
|
386
|
-
is_company=True, # 公司电脑不需要移动文件到原始文件
|
387
|
-
)
|
388
|
-
|
389
|
-
#
|
390
|
-
# # 3. 数据清理和聚合
|
391
|
-
# op_data(days=100)
|
392
|
-
|
393
|
-
if socket.gethostname() == 'company':
|
394
|
-
# 此处不可以使用 data 更新,要使用具体键值,否则数据有覆盖
|
395
|
-
conf['Windows']['company'].update(
|
396
|
-
{
|
397
|
-
'is_download': False # 更新完成,下次需 all-datas.py 修改或者手动修改
|
398
|
-
}
|
399
|
-
)
|
400
|
-
# print(conf)
|
401
|
-
myconfig.write_back(datas=conf) # 写回文件生效
|
402
|
-
elif socket.gethostname() == 'xigua1':
|
403
|
-
conf['Windows']['xigua1'].update(
|
404
|
-
{
|
405
|
-
'is_download': False # 更新完成,下次需 all-datas.py 修改或者手动修改
|
406
|
-
}
|
407
|
-
)
|
408
|
-
# print(conf)
|
409
|
-
myconfig.write_back(datas=conf) # 写回文件生效
|
410
|
-
elif socket.gethostname() == 'xigua_lx':
|
411
|
-
conf['Windows']['xigua_lx'].update(
|
412
|
-
{
|
413
|
-
'is_download': False # 更新完成,下次需 all-datas.py 修改或者手动修改
|
414
|
-
}
|
415
|
-
)
|
416
|
-
# print(conf)
|
417
|
-
myconfig.write_back(datas=conf) # 写回文件生效
|
418
|
-
elif socket.gethostname() == 'Mac2.local':
|
419
|
-
conf['Darwin']['Mac2.local'].update(
|
420
|
-
{
|
421
|
-
'is_download': False # 更新完成,下次需 all-datas.py 修改或者手动修改
|
422
|
-
}
|
423
|
-
)
|
424
|
-
# print(conf)
|
425
|
-
myconfig.write_back(datas=conf) # 写回文件生效
|
426
|
-
elif socket.gethostname() == 'MacBookPro':
|
427
|
-
conf['Darwin']['MacBookPro'].update(
|
428
|
-
{
|
429
|
-
'is_download': False # 更新完成,下次需 all-datas.py 修改或者手动修改
|
430
|
-
}
|
431
|
-
)
|
432
|
-
# print(conf)
|
433
|
-
myconfig.write_back(datas=conf) # 写回文件生效
|
434
|
-
|
435
|
-
if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
|
436
|
-
t.refresh_excel()
|
437
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
|
438
|
-
print(f'{now}数据完成!')
|
439
|
-
|
440
|
-
# t.sleep_minutes = 5 # 同步前休眠时间
|
441
|
-
# if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
|
442
|
-
# t.tb_file()
|
443
|
-
time.sleep(600) # 检测间隔
|
444
|
-
|
445
|
-
|
446
|
-
if __name__ == '__main__':
|
447
|
-
main()
|
mdbq/config/get_myconf.py
DELETED
@@ -1,131 +0,0 @@
|
|
1
|
-
# -*- coding: UTF-8 –*-
|
2
|
-
import platform
|
3
|
-
import getpass
|
4
|
-
import socket
|
5
|
-
import configparser
|
6
|
-
import os
|
7
|
-
import sys
|
8
|
-
from posixpath import dirname
|
9
|
-
from urllib import parse
|
10
|
-
from mdbq.config import set_support
|
11
|
-
|
12
|
-
|
13
|
-
class MyConf:
|
14
|
-
"""
|
15
|
-
读取配置文件信息
|
16
|
-
"""
|
17
|
-
def __init__(self, path='support'):
|
18
|
-
self.top_path = os.path.realpath(os.path.dirname(sys.argv[0])) # 程序运行目录, 打包时使用
|
19
|
-
self.conf_file = os.path.join(self.top_path, path, '.my_conf')
|
20
|
-
self.config = None
|
21
|
-
|
22
|
-
def get_myconf(self, options: list):
|
23
|
-
if not os.path.exists(self.conf_file):
|
24
|
-
print(f'尚未配置: 缺少 .my_conf 文件 {self.conf_file}')
|
25
|
-
return
|
26
|
-
if not options:
|
27
|
-
print(f'传入的参数为空: {options}')
|
28
|
-
return
|
29
|
-
self.config = configparser.ConfigParser()
|
30
|
-
self.config.read(self.conf_file, 'UTF-8')
|
31
|
-
results = []
|
32
|
-
for option in options:
|
33
|
-
try:
|
34
|
-
results.append(self.config.get('database', option))
|
35
|
-
except configparser.NoOptionError:
|
36
|
-
results.append('')
|
37
|
-
return results
|
38
|
-
|
39
|
-
|
40
|
-
def select_config_values(target_service, database, path=None):
|
41
|
-
"""
|
42
|
-
target_service: 指向: home_lx, aliyun
|
43
|
-
database: 指向: mongodb, mysql
|
44
|
-
"""
|
45
|
-
if not path:
|
46
|
-
path = set_support.SetSupport(dirname='support').dirname
|
47
|
-
|
48
|
-
m = MyConf(path=path)
|
49
|
-
options = []
|
50
|
-
if target_service == 'home_lx': # 1. 家里笔记本
|
51
|
-
if database == 'mongodb':
|
52
|
-
if socket.gethostname() == 'xigua_lx':
|
53
|
-
# 本机自身运行使用 127.0.0.1
|
54
|
-
options = ['username_db_lx_nw', 'password_db_lx_nw', 'host_bd', 'port_db_lx_nw',]
|
55
|
-
elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
|
56
|
-
# 内网地址:正在运行的是 家里笔记本或者台式机,或者 macbook pro
|
57
|
-
options = ['username_db_lx_nw', 'password_db_lx_nw', 'host_db_lx_nw', 'port_db_lx_nw',]
|
58
|
-
else:
|
59
|
-
options = ['username_db_lx', 'password_db_lx', 'host_db_lx', 'port_db_lx']
|
60
|
-
|
61
|
-
elif database == 'mysql':
|
62
|
-
if socket.gethostname() == 'xigua_lx':
|
63
|
-
# 本机自身运行使用 127.0.0.1
|
64
|
-
options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_bd', 'port_mysql_lx_nw',]
|
65
|
-
elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBookPro':
|
66
|
-
# 内网地址:正在运行的是 家里笔记本或者台式机,或者 macb ook pro
|
67
|
-
options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_mysql_lx_nw', 'port_mysql_lx_nw',]
|
68
|
-
else:
|
69
|
-
options = ['username_mysql_lx', 'password_mysql_lx', 'host_mysql_lx', 'port_mysql_lx']
|
70
|
-
|
71
|
-
elif target_service == 'home_xigua1':
|
72
|
-
if database == 'mongodb':
|
73
|
-
print('未配置')
|
74
|
-
elif database == 'mysql':
|
75
|
-
if socket.gethostname() == 'xigua_lx':
|
76
|
-
# 本机自身运行使用 127.0.0.1
|
77
|
-
options = ['username_mysql_xigua1_nw', 'password_mysql_xigua1_nw', 'host_mysql_xigua1_nw', 'port_mysql_xigua1_nw',]
|
78
|
-
elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'macbook pro':
|
79
|
-
# 内网地址:正在运行的是 家里笔记本或者台式机,或者 macb ook pro
|
80
|
-
options = ['username_mysql_xigua1_nw', 'password_mysql_xigua1_nw', 'host_bd', 'port_mysql_xigua1_nw',]
|
81
|
-
else:
|
82
|
-
print('未配置')
|
83
|
-
options = ['', '', '', '']
|
84
|
-
|
85
|
-
elif target_service == 'aliyun': # 2. 阿里云服务器
|
86
|
-
if database == 'mongodb':
|
87
|
-
if socket.gethostname() == 'xigua-cloud':
|
88
|
-
# 阿里云自身运行使用 127.0.0.1
|
89
|
-
options = ['username_db_aliyun', 'password_db_aliyun', 'host_bd', 'port_db_aliyun', ]
|
90
|
-
else:
|
91
|
-
options = ['username_db_aliyun', 'password_db_aliyun', 'host_db_aliyun', 'port_db_aliyun', ]
|
92
|
-
elif database == 'mysql':
|
93
|
-
if socket.gethostname() == 'xigua-cloud':
|
94
|
-
# 阿里云自身运行使用 127.0.0.1
|
95
|
-
options = ['username_mysql_aliyun', 'password_mysql_aliyun', 'host_bd', 'port_mysql_aliyun', ]
|
96
|
-
else:
|
97
|
-
options = ['username_mysql_aliyun', 'password_mysql_aliyun', 'host_mysql_aliyun', 'port_mysql_aliyun', ]
|
98
|
-
|
99
|
-
elif target_service == 'company': # 3. 公司台式机
|
100
|
-
if database == 'mongodb':
|
101
|
-
options = ['username_db_company_nw', 'password_db_company_nw', 'host_db_company_nw', 'port_db_company_nw', ]
|
102
|
-
elif database == 'mysql':
|
103
|
-
options = ['username_mysql_company_nw', 'password_mysql_company_nw', 'host_mysql_company_nw', 'port_mysql_company_nw', ]
|
104
|
-
|
105
|
-
elif target_service == 'nas': # 4. 群晖
|
106
|
-
if database == 'mysql':
|
107
|
-
options = ['username_mysql_nas_nw', 'password_mysql_nas_nw', 'host_mysql_nas_nw', 'port_mysql_nas_nw', ]
|
108
|
-
|
109
|
-
value = m.get_myconf(options=options)
|
110
|
-
if not value:
|
111
|
-
return '', '', '', 0
|
112
|
-
if database == 'mongodb': # mongodb 特殊字符要转码, mysql 不需要转
|
113
|
-
username = parse.quote_plus(str(value[0]).strip()) # 对可能存在的特殊字符进行编码
|
114
|
-
password = parse.quote_plus(str(value[1]).strip()) # 如果密码含有 @、/ 字符,一定要进行编码
|
115
|
-
else:
|
116
|
-
username = str(value[0]).strip()
|
117
|
-
password = str(value[1]).strip()
|
118
|
-
host = str(value[2]).strip()
|
119
|
-
port = int(value[3])
|
120
|
-
return username, password, host, port
|
121
|
-
|
122
|
-
|
123
|
-
def main():
|
124
|
-
pass
|
125
|
-
|
126
|
-
|
127
|
-
if __name__ == '__main__':
|
128
|
-
# main()
|
129
|
-
r, d, s, g = select_config_values(target_service='home_lx', database='mysql')
|
130
|
-
print(r, d, s, g, type(r), type(d), type(s), type(g))
|
131
|
-
print(f'本机: {platform.system()} // {socket.gethostname()}')
|