mdbq 0.0.9__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/__init__.py +4 -0
- mdbq/aggregation/aggregation.py +1003 -0
- mdbq/aggregation/query_data.py +354 -0
- mdbq/config/get_myconf.py +4 -0
- mdbq/dataframe/converter.py +9 -5
- mdbq/mongo/mongo.py +40 -5
- mdbq/mysql/mysql.py +60 -14
- mdbq/mysql/s_query.py +3 -0
- {mdbq-0.0.9.dist-info → mdbq-0.1.1.dist-info}/METADATA +1 -1
- {mdbq-0.0.9.dist-info → mdbq-0.1.1.dist-info}/RECORD +12 -9
- {mdbq-0.0.9.dist-info → mdbq-0.1.1.dist-info}/WHEEL +0 -0
- {mdbq-0.0.9.dist-info → mdbq-0.1.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,354 @@
|
|
1
|
+
# -*- coding: UTF-8 –*-
|
2
|
+
from mdbq.mongo import mongo
|
3
|
+
from mdbq.mysql import s_query
|
4
|
+
from mdbq.config import get_myconf
|
5
|
+
import datetime
|
6
|
+
from dateutil.relativedelta import relativedelta
|
7
|
+
import pandas as pd
|
8
|
+
import numpy as np
|
9
|
+
import platform
|
10
|
+
import getpass
|
11
|
+
import json
|
12
|
+
import os
|
13
|
+
"""
|
14
|
+
程序用于下载数据库(调用 s_query.py 下载并清洗), 并对数据进行聚合清洗, 不会更新数据库信息;
|
15
|
+
"""
|
16
|
+
|
17
|
+
|
18
|
+
class MongoDatasQuery:
|
19
|
+
"""
|
20
|
+
从 数据库 中下载数据
|
21
|
+
self.output: 数据库默认导出目录
|
22
|
+
self.is_maximize: 是否最大转化数据
|
23
|
+
"""
|
24
|
+
def __init__(self, target_service):
|
25
|
+
# target_service 从哪个服务器下载数据
|
26
|
+
self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
27
|
+
# 实例化一个下载类
|
28
|
+
username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
|
29
|
+
self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
|
30
|
+
|
31
|
+
def tg_wxt(self):
|
32
|
+
self.download.start_date, self.download.end_date = self.months_data(num=self.months)
|
33
|
+
projection = {
|
34
|
+
'日期': 1,
|
35
|
+
'场景名字': 1,
|
36
|
+
'主体id': 1,
|
37
|
+
'花费': 1,
|
38
|
+
'展现量': 1,
|
39
|
+
'点击量': 1,
|
40
|
+
'总购物车数': 1,
|
41
|
+
'总成交笔数': 1,
|
42
|
+
'总成交金额': 1,
|
43
|
+
'自然流量曝光量': 1,
|
44
|
+
'直接成交笔数': 1,
|
45
|
+
'直接成交金额': 1,
|
46
|
+
}
|
47
|
+
df = self.download.data_to_df(
|
48
|
+
db_name='天猫数据2',
|
49
|
+
collection_name='推广数据_宝贝主体报表',
|
50
|
+
projection=projection,
|
51
|
+
)
|
52
|
+
return df
|
53
|
+
|
54
|
+
@staticmethod
|
55
|
+
def days_data(days, end_date=None):
|
56
|
+
""" 读取近 days 天的数据 """
|
57
|
+
if not end_date:
|
58
|
+
end_date = datetime.datetime.now()
|
59
|
+
start_date = end_date - datetime.timedelta(days=days)
|
60
|
+
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
61
|
+
|
62
|
+
@staticmethod
|
63
|
+
def months_data(num=0, end_date=None):
|
64
|
+
""" 读取近 num 个月的数据, 0 表示读取当月的数据 """
|
65
|
+
if not end_date:
|
66
|
+
end_date = datetime.datetime.now()
|
67
|
+
start_date = end_date - relativedelta(months=num) # n 月以前的今天
|
68
|
+
start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
|
69
|
+
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
70
|
+
|
71
|
+
|
72
|
+
class MysqlDatasQuery:
|
73
|
+
"""
|
74
|
+
从数据库中下载数据
|
75
|
+
"""
|
76
|
+
def __init__(self, target_service):
|
77
|
+
# target_service 从哪个服务器下载数据
|
78
|
+
self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
79
|
+
# 实例化一个下载类
|
80
|
+
username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
|
81
|
+
self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
|
82
|
+
|
83
|
+
def tg_wxt(self):
|
84
|
+
start_date, end_date = self.months_data(num=self.months)
|
85
|
+
projection = {
|
86
|
+
'日期': 1,
|
87
|
+
'场景名字': 1,
|
88
|
+
'主体id': 1,
|
89
|
+
'花费': 1,
|
90
|
+
'展现量': 1,
|
91
|
+
'点击量': 1,
|
92
|
+
'总购物车数': 1,
|
93
|
+
'总成交笔数': 1,
|
94
|
+
'总成交金额': 1,
|
95
|
+
'自然流量曝光量': 1,
|
96
|
+
'直接成交笔数': 1,
|
97
|
+
'直接成交金额': 1,
|
98
|
+
}
|
99
|
+
df = self.download.data_to_df(
|
100
|
+
db_name='天猫数据2',
|
101
|
+
tabel_name='推广数据_宝贝主体报表',
|
102
|
+
start_date=start_date,
|
103
|
+
end_date=end_date,
|
104
|
+
projection=projection,
|
105
|
+
)
|
106
|
+
return df
|
107
|
+
|
108
|
+
def syj(self):
|
109
|
+
start_date, end_date = self.months_data(num=self.months)
|
110
|
+
projection = {
|
111
|
+
'日期': 1,
|
112
|
+
'宝贝id': 1,
|
113
|
+
'商家编码': 1,
|
114
|
+
'行业类目': 1,
|
115
|
+
'销售额': 1,
|
116
|
+
'销售量': 1,
|
117
|
+
'订单数': 1,
|
118
|
+
'退货量': 1,
|
119
|
+
'退款额': 1,
|
120
|
+
'退货量_发货后_': 1,
|
121
|
+
}
|
122
|
+
df = self.download.data_to_df(
|
123
|
+
db_name='生意经2',
|
124
|
+
tabel_name='宝贝指标',
|
125
|
+
start_date=start_date,
|
126
|
+
end_date=end_date,
|
127
|
+
projection=projection,
|
128
|
+
)
|
129
|
+
return df
|
130
|
+
|
131
|
+
def dplyd(self):
|
132
|
+
start_date, end_date = self.months_data(num=self.months)
|
133
|
+
projection = {
|
134
|
+
'日期': 1,
|
135
|
+
'一级来源': 1,
|
136
|
+
'二级来源': 1,
|
137
|
+
'三级来源': 1,
|
138
|
+
'访客数': 1,
|
139
|
+
'支付金额': 1,
|
140
|
+
'支付买家数': 1,
|
141
|
+
'支付转化率': 1,
|
142
|
+
'加购人数': 1,
|
143
|
+
}
|
144
|
+
df = self.download.data_to_df(
|
145
|
+
db_name='生意参谋数据2',
|
146
|
+
tabel_name='店铺来源_日数据',
|
147
|
+
start_date=start_date,
|
148
|
+
end_date=end_date,
|
149
|
+
projection=projection,
|
150
|
+
)
|
151
|
+
return df
|
152
|
+
|
153
|
+
@staticmethod
|
154
|
+
def months_data(num=0, end_date=None):
|
155
|
+
""" 读取近 num 个月的数据, 0 表示读取当月的数据 """
|
156
|
+
if not end_date:
|
157
|
+
end_date = datetime.datetime.now()
|
158
|
+
start_date = end_date - relativedelta(months=num) # n 月以前的今天
|
159
|
+
start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
|
160
|
+
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
161
|
+
|
162
|
+
|
163
|
+
class GroupBy:
|
164
|
+
"""
|
165
|
+
数据聚合和导出
|
166
|
+
"""
|
167
|
+
def __init__(self):
|
168
|
+
# self.output: 数据库默认导出目录
|
169
|
+
if platform.system() == 'Darwin':
|
170
|
+
self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
|
171
|
+
elif platform.system() == 'Windows':
|
172
|
+
self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
|
173
|
+
else:
|
174
|
+
self.output = os.path.join('数据中心/数据库导出')
|
175
|
+
|
176
|
+
def groupby(self, df, tabel_name, is_maximize=True):
|
177
|
+
"""
|
178
|
+
self.is_maximize: 是否最大转化数据
|
179
|
+
"""
|
180
|
+
|
181
|
+
if '宝贝主体报表' in tabel_name:
|
182
|
+
df.rename(columns={
|
183
|
+
'场景名字': '营销场景',
|
184
|
+
'主体id': '商品id',
|
185
|
+
'总购物车数': '加购量',
|
186
|
+
'总成交笔数': '成交笔数',
|
187
|
+
'总成交金额': '成交金额'
|
188
|
+
}, inplace=True)
|
189
|
+
df = df.astype({
|
190
|
+
'花费': float,
|
191
|
+
'展现量': int,
|
192
|
+
'点击量': int,
|
193
|
+
'加购量': int,
|
194
|
+
'成交笔数': int,
|
195
|
+
'成交金额': float,
|
196
|
+
'自然流量曝光量': int,
|
197
|
+
'直接成交笔数': int,
|
198
|
+
'直接成交金额': float,
|
199
|
+
}, errors='raise')
|
200
|
+
df.fillna(0, inplace=True)
|
201
|
+
if is_maximize:
|
202
|
+
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
203
|
+
**{'加购量': ('加购量', np.max),
|
204
|
+
'成交笔数': ('成交笔数', np.max),
|
205
|
+
'成交金额': ('成交金额', np.max),
|
206
|
+
'自然流量曝光量': ('自然流量曝光量', np.max),
|
207
|
+
'直接成交笔数': ('直接成交笔数', np.max),
|
208
|
+
'直接成交金额': ('直接成交金额', np.max)
|
209
|
+
}
|
210
|
+
)
|
211
|
+
else:
|
212
|
+
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
213
|
+
**{'加购量': ('加购量', np.min),
|
214
|
+
'成交笔数': ('成交笔数', np.min),
|
215
|
+
'成交金额': ('成交金额', np.min),
|
216
|
+
'自然流量曝光量': ('自然流量曝光量', np.min),
|
217
|
+
'直接成交笔数': ('直接成交笔数', np.max),
|
218
|
+
'直接成交金额': ('直接成交金额', np.max)
|
219
|
+
}
|
220
|
+
)
|
221
|
+
df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
|
222
|
+
return df
|
223
|
+
elif '宝贝指标' in tabel_name:
|
224
|
+
df.fillna(0, inplace=True)
|
225
|
+
df = df[(df['销售额'] != 0) | (df['退款额'] != 0)]
|
226
|
+
df = df.groupby(['日期', '宝贝id', '商家编码', '行业类目'], as_index=False).agg(
|
227
|
+
**{'销售额': ('销售额', np.min),
|
228
|
+
'销售量': ('销售量', np.min),
|
229
|
+
'订单数': ('订单数', np.min),
|
230
|
+
'退货量': ('退货量', np.max),
|
231
|
+
'退款额': ('退款额', np.max),
|
232
|
+
'退货量_发货后_': ('退货量_发货后_', np.max),
|
233
|
+
}
|
234
|
+
)
|
235
|
+
df['件均价'] = df.apply(lambda x: x['销售额'] / x['销售量'] if x['销售量'] > 0 else 0, axis=1).round(
|
236
|
+
0) # 两列运算, 避免除以0
|
237
|
+
df['价格带'] = df['件均价'].apply(
|
238
|
+
lambda x: '2000+' if x >= 2000
|
239
|
+
else '1000+' if x >= 1000
|
240
|
+
else '500+' if x >= 500
|
241
|
+
else '300+' if x >= 300
|
242
|
+
else '300以下'
|
243
|
+
)
|
244
|
+
return df
|
245
|
+
elif '店铺来源_日数据' in tabel_name:
|
246
|
+
return df
|
247
|
+
else:
|
248
|
+
print(f'<{tabel_name}>: Groupby 类尚未配置,数据为空')
|
249
|
+
return pd.DataFrame({})
|
250
|
+
|
251
|
+
def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
|
252
|
+
index=False, header=True, st_ascend=None, ascend=None, freq=None):
|
253
|
+
"""
|
254
|
+
path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹,可以不传,
|
255
|
+
st_ascend: 排序参数 ['column1', 'column2']
|
256
|
+
ascend: 升降序 [True, False]
|
257
|
+
freq: 将创建子文件夹并按月分类存储, freq='Y', 或 freq='M'
|
258
|
+
"""
|
259
|
+
if len(df) == 0:
|
260
|
+
return
|
261
|
+
if not path:
|
262
|
+
path = self.output
|
263
|
+
else:
|
264
|
+
path = os.path.join(self.output, path)
|
265
|
+
if not os.path.exists(path):
|
266
|
+
os.makedirs(path)
|
267
|
+
if st_ascend and ascend:
|
268
|
+
try:
|
269
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
270
|
+
except:
|
271
|
+
print(f'{filename}: sort_values排序参数错误!')
|
272
|
+
if freq:
|
273
|
+
if '日期' not in df.columns.tolist():
|
274
|
+
return print(f'{filename}: 数据缺少日期列,无法按日期分组')
|
275
|
+
groups = df.groupby(pd.Grouper(key='日期', freq=freq))
|
276
|
+
for name1, df in groups:
|
277
|
+
if freq == 'M':
|
278
|
+
sheet_name = name1.strftime('%Y-%m')
|
279
|
+
elif freq == 'Y':
|
280
|
+
sheet_name = name1.strftime('%Y年')
|
281
|
+
else:
|
282
|
+
sheet_name = '_未分类'
|
283
|
+
new_path = os.path.join(path, filename)
|
284
|
+
if not os.path.exists(new_path):
|
285
|
+
os.makedirs(new_path)
|
286
|
+
new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
|
287
|
+
if st_ascend and ascend: # 这里需要重新排序一次,原因未知
|
288
|
+
try:
|
289
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
290
|
+
except:
|
291
|
+
print(f'{filename}: sort_values排序参数错误!')
|
292
|
+
|
293
|
+
df.to_csv(new_path, encoding=encoding, index=index, header=header)
|
294
|
+
else:
|
295
|
+
df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
|
296
|
+
|
297
|
+
def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
|
298
|
+
if len(df) == 0:
|
299
|
+
return
|
300
|
+
if not path:
|
301
|
+
path = self.output
|
302
|
+
else:
|
303
|
+
path = os.path.join(self.output, path)
|
304
|
+
if not os.path.exists(path):
|
305
|
+
os.makedirs(path)
|
306
|
+
if st_ascend and ascend:
|
307
|
+
try:
|
308
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
309
|
+
except:
|
310
|
+
print(f'{filename}: sort_values排序参数错误!')
|
311
|
+
df.to_json(os.path.join(path, filename + '.json'),
|
312
|
+
orient=orient, force_ascii=force_ascii)
|
313
|
+
|
314
|
+
def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
|
315
|
+
freeze_panes=(1, 0), st_ascend=None, ascend=None):
|
316
|
+
if len(df) == 0:
|
317
|
+
return
|
318
|
+
if not path:
|
319
|
+
path = self.output
|
320
|
+
else:
|
321
|
+
path = os.path.join(self.output, path)
|
322
|
+
if not os.path.exists(path):
|
323
|
+
os.makedirs(path)
|
324
|
+
if st_ascend and ascend:
|
325
|
+
try:
|
326
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
327
|
+
except:
|
328
|
+
print(f'{filename}: sort_values排序参数错误!')
|
329
|
+
df.to_excel(os.path.join(path, filename + '.xlsx'),
|
330
|
+
index=index, header=header, engine=engine, freeze_panes=freeze_panes)
|
331
|
+
|
332
|
+
|
333
|
+
def data_output():
|
334
|
+
sdq = MysqlDatasQuery(target_service='home_lx')
|
335
|
+
sdq.months = 0
|
336
|
+
|
337
|
+
# df = sdq.tg_wxt() # 从数据库中获取数据并转为 df
|
338
|
+
# g = GroupBy() # 数据聚合
|
339
|
+
# df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
|
340
|
+
# g.as_csv(df=df, filename='推广数据_宝贝主体报表') # 数据导出
|
341
|
+
|
342
|
+
# df = sdq.syj()
|
343
|
+
# g = GroupBy()
|
344
|
+
# df = g.groupby(df=df, tabel_name='宝贝指标', is_maximize=True)
|
345
|
+
# g.as_csv(df=df, filename='宝贝指标')
|
346
|
+
|
347
|
+
df = sdq.dplyd()
|
348
|
+
g = GroupBy()
|
349
|
+
df = g.groupby(df=df, tabel_name='店铺来源_日数据', is_maximize=True)
|
350
|
+
g.as_csv(df=df, filename='店铺来源_日数据')
|
351
|
+
|
352
|
+
|
353
|
+
if __name__ == '__main__':
|
354
|
+
main()
|
mdbq/config/get_myconf.py
CHANGED
@@ -106,6 +106,10 @@ def select_config_values(target_service, database, path=None):
|
|
106
106
|
elif database == 'mysql':
|
107
107
|
options = ['username_mysql_company_nw', 'password_mysql_company_nw', 'host_mysql_company_nw', 'port_mysql_company_nw', ]
|
108
108
|
|
109
|
+
elif target_service == 'nas': # 4. 群晖
|
110
|
+
if database == 'mysql':
|
111
|
+
options = ['username_mysql_nas_nw', 'password_mysql_nas_nw', 'host_mysql_nas_nw', 'port_mysql_nas_nw', ]
|
112
|
+
|
109
113
|
value = m.get_myconf(options=options)
|
110
114
|
if not value:
|
111
115
|
return '', '', '', 0
|
mdbq/dataframe/converter.py
CHANGED
@@ -44,14 +44,18 @@ class DataFrameConverter(object):
|
|
44
44
|
pass
|
45
45
|
new_col = col.lower()
|
46
46
|
new_col = re.sub(r'[\',,()()/=<>+\-*^"’\[\]~#|&% .;]', '_', new_col)
|
47
|
+
new_col = re.sub(r'_+$', '', new_col)
|
47
48
|
df.rename(columns={col: new_col}, inplace=True)
|
48
49
|
df.fillna(0, inplace=True)
|
49
50
|
return df
|
50
51
|
|
51
52
|
|
52
53
|
if __name__ == '__main__':
|
53
|
-
df = pd.DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
|
54
|
-
converter = DataFrameConverter()
|
55
|
-
df = converter.convert_df_cols(df)
|
56
|
-
print(df['a'].dtype)
|
57
|
-
print(df)
|
54
|
+
# df = pd.DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
|
55
|
+
# converter = DataFrameConverter()
|
56
|
+
# df = converter.convert_df_cols(df)
|
57
|
+
# print(df['a'].dtype)
|
58
|
+
# print(df)
|
59
|
+
pattern = 'dfa_dfawr__'
|
60
|
+
pattern = re.sub(r'_+$', '', pattern)
|
61
|
+
print(pattern)
|
mdbq/mongo/mongo.py
CHANGED
@@ -195,6 +195,7 @@ class DownMongo:
|
|
195
195
|
|
196
196
|
def data_to_file(self, file_type, db_name, collection_name):
|
197
197
|
"""
|
198
|
+
用于 GUI 的函数
|
198
199
|
将 mongodb 数据保存本地
|
199
200
|
db_name: 数据库名
|
200
201
|
collections 集合名
|
@@ -206,9 +207,9 @@ class DownMongo:
|
|
206
207
|
_collection = self.client[self.db_name][self.collection_name] # 连接集合
|
207
208
|
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
208
209
|
if not self.start_date:
|
209
|
-
print(f'{now}
|
210
|
+
print(f'{now}正在下载 ({self.host}) {self.db_name}: {self.collection_name}, 区间: 近 {self.days} 天\n...')
|
210
211
|
else:
|
211
|
-
print(f'{now}
|
212
|
+
print(f'{now}正在下载 ({self.host}) {self.db_name}: {self.collection_name}, 区间: {self.start_date}~{self.end_date}')
|
212
213
|
|
213
214
|
if not self.start_date:
|
214
215
|
self.start_date = datetime.datetime.now() - datetime.timedelta(days=self.days)
|
@@ -221,11 +222,10 @@ class DownMongo:
|
|
221
222
|
{'$project': {'_id': 0}}, # 不保留 id 字段
|
222
223
|
]
|
223
224
|
results = _collection.aggregate(pipeline)
|
224
|
-
|
225
|
+
|
225
226
|
# 输出结果
|
226
227
|
datas = []
|
227
228
|
for doc in results:
|
228
|
-
# print(doc)
|
229
229
|
datas.append(doc)
|
230
230
|
_df = pd.DataFrame(datas)
|
231
231
|
if len(_df) == 0:
|
@@ -251,7 +251,7 @@ class DownMongo:
|
|
251
251
|
_df.to_excel(_path, index=False, header=True, engine='openpyxl', freeze_panes=(1, 0)) # freeze_ 冻结列索引
|
252
252
|
else:
|
253
253
|
print(f'{file_type}: 未支持的文件类型')
|
254
|
-
print(f'{self.collection_name}
|
254
|
+
print(f'<{self.collection_name}> 导出: {_path}, 数据完成!')
|
255
255
|
self.client.close()
|
256
256
|
|
257
257
|
|
@@ -655,6 +655,28 @@ class OptimizeDatas:
|
|
655
655
|
start_date += datetime.timedelta(days=1)
|
656
656
|
return date_list
|
657
657
|
|
658
|
+
def rename_column(self):
|
659
|
+
""" 批量修改数据库的列名 """
|
660
|
+
"""
|
661
|
+
# for db_name in ['京东数据2', '天猫数据2', '市场数据2', '生意参谋数据2', '生意经2', '属性设置2',]:
|
662
|
+
# s = OptimizeDatas(username=username, password=password, host=host, port=port)
|
663
|
+
# s.db_name = db_name
|
664
|
+
# s.rename_column()
|
665
|
+
# s.client.close()
|
666
|
+
"""
|
667
|
+
self.client = pymongo.MongoClient(self.link) # 连接数据库
|
668
|
+
database_names = self.client.list_database_names() # 所有数据库名称
|
669
|
+
collections = self.my_collection_names(db_name=self.db_name) # 所有集合名称
|
670
|
+
for collection_name in collections:
|
671
|
+
collection = self.client[self.db_name].get_collection(collection_name)
|
672
|
+
has_date_field = collection.find_one({})
|
673
|
+
for key, value in has_date_field.items():
|
674
|
+
if key.endswith('_'):
|
675
|
+
new_name = re.sub(r'_+$', '', key)
|
676
|
+
query = {key: {'$exists': True}}
|
677
|
+
update = {'$rename': {key: new_name}}
|
678
|
+
collection.update_many(query, update)
|
679
|
+
|
658
680
|
|
659
681
|
def upload_one_dir():
|
660
682
|
username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
|
@@ -690,3 +712,16 @@ if __name__ == '__main__':
|
|
690
712
|
# main()
|
691
713
|
username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
|
692
714
|
print(username, password, host, port)
|
715
|
+
|
716
|
+
# for db_name in [
|
717
|
+
# '京东数据2',
|
718
|
+
# '天猫数据2',
|
719
|
+
# '市场数据2',
|
720
|
+
# '生意参谋数据2',
|
721
|
+
# '生意经2',
|
722
|
+
# '属性设置2',
|
723
|
+
# ]:
|
724
|
+
# s = OptimizeDatas(username=username, password=password, host=host, port=port)
|
725
|
+
# s.db_name = db_name
|
726
|
+
# s.rename_column()
|
727
|
+
# s.client.close()
|
mdbq/mysql/mysql.py
CHANGED
@@ -81,8 +81,11 @@ class MysqlUpload:
|
|
81
81
|
if '8.138.27' in str(self.host) or platform.system() == "Linux": # 阿里云 mysql 低版本不支持 0900
|
82
82
|
cursor.execute(f"CREATE DATABASE {db_name} COLLATE utf8mb4_unicode_ci")
|
83
83
|
self.config.update({'charset': 'utf8mb4_unicode_ci'})
|
84
|
+
if '192.168.1.100' in str(self.host):
|
85
|
+
cursor.execute(f"CREATE DATABASE {db_name}")
|
84
86
|
else:
|
85
87
|
cursor.execute(f"CREATE DATABASE {db_name} COLLATE utf8mb4_0900_ai_ci")
|
88
|
+
# cursor.execute(f"CREATE DATABASE {db_name}")
|
86
89
|
connection.commit()
|
87
90
|
print(f"创建Database: {db_name}")
|
88
91
|
except Exception as e:
|
@@ -140,23 +143,36 @@ class MysqlUpload:
|
|
140
143
|
# connection.commit()
|
141
144
|
|
142
145
|
# 4. 更新插入数据
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
engine = create_engine(
|
147
|
-
f'mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}'
|
148
|
-
)
|
149
|
-
df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
|
150
|
-
except Exception as e: # 如果异常则回滚
|
146
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
147
|
+
print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{tabel_name}')
|
148
|
+
if str(self.host) == '192.168.1.100': # 群晖服务器
|
151
149
|
try:
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
150
|
+
datas = df.to_dict('records')
|
151
|
+
for data in datas:
|
152
|
+
cols = ', '.join(data.keys())
|
153
|
+
values = ', '.join([f'"{v}"' for v in data.values()])
|
154
|
+
sql = f"INSERT INTO {tabel_name} ({cols}) VALUES ({values})"
|
155
|
+
cursor.execute(sql)
|
156
|
+
connection.commit()
|
156
157
|
except Exception as e:
|
157
|
-
|
158
|
-
print(f'{now}{db_name}/{tabel_name}数据异常, 正在回滚: {e}')
|
158
|
+
print(e)
|
159
159
|
connection.rollback()
|
160
|
+
else: # 其他服务器
|
161
|
+
try:
|
162
|
+
engine = create_engine(
|
163
|
+
f'mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}'
|
164
|
+
)
|
165
|
+
df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
|
166
|
+
except Exception as e: # 如果异常则回滚
|
167
|
+
try:
|
168
|
+
connection.rollback()
|
169
|
+
print(f'{e}, 发生异常,正在重试...')
|
170
|
+
# df = df.replace([np.inf, -np.inf], 0)
|
171
|
+
df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
|
172
|
+
except Exception as e:
|
173
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
174
|
+
print(f'{now}{db_name}/{tabel_name}数据异常, 正在回滚: {e}')
|
175
|
+
connection.rollback()
|
160
176
|
finally:
|
161
177
|
connection.close()
|
162
178
|
|
@@ -545,6 +561,33 @@ class OptimizeDatas:
|
|
545
561
|
start_date += datetime.timedelta(days=1)
|
546
562
|
return date_list
|
547
563
|
|
564
|
+
def rename_column(self):
|
565
|
+
""" 批量修改数据库的列名 """
|
566
|
+
"""
|
567
|
+
# for db_name in ['京东数据2', '天猫数据2', '市场数据2', '生意参谋数据2', '生意经2', '属性设置2',]:
|
568
|
+
# s = OptimizeDatas(username=username, password=password, host=host, port=port)
|
569
|
+
# s.db_name = db_name
|
570
|
+
# s.rename_column()
|
571
|
+
"""
|
572
|
+
tables = self.table_list(db_name=self.db_name)
|
573
|
+
for table_dict in tables:
|
574
|
+
for key, table_name in table_dict.items():
|
575
|
+
self.config.update({'database': self.db_name}) # 添加更新 config 字段
|
576
|
+
self.connection = pymysql.connect(**self.config)
|
577
|
+
with self.connection.cursor() as cursor:
|
578
|
+
cursor.execute(f"SHOW FULL COLUMNS FROM {table_name}") # 查询数据表的列信息
|
579
|
+
columns = cursor.fetchall()
|
580
|
+
columns = [{column['Field']: column['Type']} for column in columns]
|
581
|
+
for column in columns:
|
582
|
+
for key, value in column.items():
|
583
|
+
if key.endswith('_'):
|
584
|
+
new_name = re.sub(r'_+$', '', key)
|
585
|
+
sql = f"ALTER TABLE {table_name} CHANGE COLUMN {key} {new_name} {value}"
|
586
|
+
cursor.execute(sql)
|
587
|
+
self.connection.commit()
|
588
|
+
if self.connection:
|
589
|
+
self.connection.close()
|
590
|
+
|
548
591
|
|
549
592
|
def year_month_day(start_date, end_date):
|
550
593
|
"""
|
@@ -594,3 +637,6 @@ def download_datas(tabel_name, save_path, start_date):
|
|
594
637
|
if __name__ == '__main__':
|
595
638
|
username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
|
596
639
|
print(username, password, host, port)
|
640
|
+
|
641
|
+
|
642
|
+
|
mdbq/mysql/s_query.py
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
2
2
|
mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
3
|
+
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
4
|
+
mdbq/aggregation/aggregation.py,sha256=n2MoKwN1ltJD3juu59zkhc7PGUMDTkn0zCcZGs8RnXI,56775
|
5
|
+
mdbq/aggregation/query_data.py,sha256=RaZ0NJwTeNhSgpyj9eUjEVc6Kp83etr6luyzSf9U5WE,14465
|
3
6
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
4
7
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
5
8
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
@@ -7,17 +10,17 @@ mdbq/clean/data_clean.py,sha256=33OmeQFl9AW21P5EOay52W_S8DF96H5oHwCg4fSuBxA,8535
|
|
7
10
|
mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
|
8
11
|
mdbq/company/copysh.py,sha256=0exynzeqf85gCBQXAgKycVxddMhr0TjkFcBP_NK0QTA,15609
|
9
12
|
mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
10
|
-
mdbq/config/get_myconf.py,sha256=
|
13
|
+
mdbq/config/get_myconf.py,sha256=9v3xebfcS1tptxpvk3_tGxfXjAehGVCveYe4iRUzLQQ,6372
|
11
14
|
mdbq/config/update_conf.py,sha256=YjGjjRchu5BcrmLJkoLjHEF2TbGOmsgCWX4LroXOYWQ,3455
|
12
15
|
mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
|
13
|
-
mdbq/dataframe/converter.py,sha256=
|
16
|
+
mdbq/dataframe/converter.py,sha256=5hrGx-lPVwYLuyZNOHf6K7O9_AAKZ7mCp0MHxlxSCnk,2816
|
14
17
|
mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
|
15
18
|
mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
|
16
19
|
mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
|
17
|
-
mdbq/mongo/mongo.py,sha256=
|
20
|
+
mdbq/mongo/mongo.py,sha256=q0B4wXDSTtXg_vMN7MPh6zdxl6tT68tM74LmdVNQQek,31892
|
18
21
|
mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
19
|
-
mdbq/mysql/mysql.py,sha256=
|
20
|
-
mdbq/mysql/s_query.py,sha256=
|
22
|
+
mdbq/mysql/mysql.py,sha256=Ul-QJHFKjSHu2uxk_CRswVvCV-oFtN-t-3wIOoKfLLg,30550
|
23
|
+
mdbq/mysql/s_query.py,sha256=pzxRGBRP3Ku4oVLqfpMd1VWWct3YlxxnVavtKa9kgSM,5302
|
21
24
|
mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
|
22
25
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
23
26
|
mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
|
@@ -27,7 +30,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
|
27
30
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
28
31
|
mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
|
29
32
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
30
|
-
mdbq-0.
|
31
|
-
mdbq-0.
|
32
|
-
mdbq-0.
|
33
|
-
mdbq-0.
|
33
|
+
mdbq-0.1.1.dist-info/METADATA,sha256=60TKH9pmImZH4nV2l_b0ge4Cu6K9YmWvsZzdt8fNNZA,245
|
34
|
+
mdbq-0.1.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
|
35
|
+
mdbq-0.1.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
36
|
+
mdbq-0.1.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|