mdbq 0.0.9__py3-none-any.whl → 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,354 @@
1
+ # -*- coding: UTF-8 –*-
2
+ from mdbq.mongo import mongo
3
+ from mdbq.mysql import s_query
4
+ from mdbq.config import get_myconf
5
+ import datetime
6
+ from dateutil.relativedelta import relativedelta
7
+ import pandas as pd
8
+ import numpy as np
9
+ import platform
10
+ import getpass
11
+ import json
12
+ import os
13
+ """
14
+ 程序用于下载数据库(调用 s_query.py 下载并清洗), 并对数据进行聚合清洗, 不会更新数据库信息;
15
+ """
16
+
17
+
18
+ class MongoDatasQuery:
19
+ """
20
+ 从 数据库 中下载数据
21
+ self.output: 数据库默认导出目录
22
+ self.is_maximize: 是否最大转化数据
23
+ """
24
+ def __init__(self, target_service):
25
+ # target_service 从哪个服务器下载数据
26
+ self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
27
+ # 实例化一个下载类
28
+ username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
29
+ self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
30
+
31
+ def tg_wxt(self):
32
+ self.download.start_date, self.download.end_date = self.months_data(num=self.months)
33
+ projection = {
34
+ '日期': 1,
35
+ '场景名字': 1,
36
+ '主体id': 1,
37
+ '花费': 1,
38
+ '展现量': 1,
39
+ '点击量': 1,
40
+ '总购物车数': 1,
41
+ '总成交笔数': 1,
42
+ '总成交金额': 1,
43
+ '自然流量曝光量': 1,
44
+ '直接成交笔数': 1,
45
+ '直接成交金额': 1,
46
+ }
47
+ df = self.download.data_to_df(
48
+ db_name='天猫数据2',
49
+ collection_name='推广数据_宝贝主体报表',
50
+ projection=projection,
51
+ )
52
+ return df
53
+
54
+ @staticmethod
55
+ def days_data(days, end_date=None):
56
+ """ 读取近 days 天的数据 """
57
+ if not end_date:
58
+ end_date = datetime.datetime.now()
59
+ start_date = end_date - datetime.timedelta(days=days)
60
+ return pd.to_datetime(start_date), pd.to_datetime(end_date)
61
+
62
+ @staticmethod
63
+ def months_data(num=0, end_date=None):
64
+ """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
65
+ if not end_date:
66
+ end_date = datetime.datetime.now()
67
+ start_date = end_date - relativedelta(months=num) # n 月以前的今天
68
+ start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
69
+ return pd.to_datetime(start_date), pd.to_datetime(end_date)
70
+
71
+
72
+ class MysqlDatasQuery:
73
+ """
74
+ 从数据库中下载数据
75
+ """
76
+ def __init__(self, target_service):
77
+ # target_service 从哪个服务器下载数据
78
+ self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
79
+ # 实例化一个下载类
80
+ username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
81
+ self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
82
+
83
+ def tg_wxt(self):
84
+ start_date, end_date = self.months_data(num=self.months)
85
+ projection = {
86
+ '日期': 1,
87
+ '场景名字': 1,
88
+ '主体id': 1,
89
+ '花费': 1,
90
+ '展现量': 1,
91
+ '点击量': 1,
92
+ '总购物车数': 1,
93
+ '总成交笔数': 1,
94
+ '总成交金额': 1,
95
+ '自然流量曝光量': 1,
96
+ '直接成交笔数': 1,
97
+ '直接成交金额': 1,
98
+ }
99
+ df = self.download.data_to_df(
100
+ db_name='天猫数据2',
101
+ tabel_name='推广数据_宝贝主体报表',
102
+ start_date=start_date,
103
+ end_date=end_date,
104
+ projection=projection,
105
+ )
106
+ return df
107
+
108
+ def syj(self):
109
+ start_date, end_date = self.months_data(num=self.months)
110
+ projection = {
111
+ '日期': 1,
112
+ '宝贝id': 1,
113
+ '商家编码': 1,
114
+ '行业类目': 1,
115
+ '销售额': 1,
116
+ '销售量': 1,
117
+ '订单数': 1,
118
+ '退货量': 1,
119
+ '退款额': 1,
120
+ '退货量_发货后_': 1,
121
+ }
122
+ df = self.download.data_to_df(
123
+ db_name='生意经2',
124
+ tabel_name='宝贝指标',
125
+ start_date=start_date,
126
+ end_date=end_date,
127
+ projection=projection,
128
+ )
129
+ return df
130
+
131
+ def dplyd(self):
132
+ start_date, end_date = self.months_data(num=self.months)
133
+ projection = {
134
+ '日期': 1,
135
+ '一级来源': 1,
136
+ '二级来源': 1,
137
+ '三级来源': 1,
138
+ '访客数': 1,
139
+ '支付金额': 1,
140
+ '支付买家数': 1,
141
+ '支付转化率': 1,
142
+ '加购人数': 1,
143
+ }
144
+ df = self.download.data_to_df(
145
+ db_name='生意参谋数据2',
146
+ tabel_name='店铺来源_日数据',
147
+ start_date=start_date,
148
+ end_date=end_date,
149
+ projection=projection,
150
+ )
151
+ return df
152
+
153
+ @staticmethod
154
+ def months_data(num=0, end_date=None):
155
+ """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
156
+ if not end_date:
157
+ end_date = datetime.datetime.now()
158
+ start_date = end_date - relativedelta(months=num) # n 月以前的今天
159
+ start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
160
+ return pd.to_datetime(start_date), pd.to_datetime(end_date)
161
+
162
+
163
+ class GroupBy:
164
+ """
165
+ 数据聚合和导出
166
+ """
167
+ def __init__(self):
168
+ # self.output: 数据库默认导出目录
169
+ if platform.system() == 'Darwin':
170
+ self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
171
+ elif platform.system() == 'Windows':
172
+ self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
173
+ else:
174
+ self.output = os.path.join('数据中心/数据库导出')
175
+
176
+ def groupby(self, df, tabel_name, is_maximize=True):
177
+ """
178
+ self.is_maximize: 是否最大转化数据
179
+ """
180
+
181
+ if '宝贝主体报表' in tabel_name:
182
+ df.rename(columns={
183
+ '场景名字': '营销场景',
184
+ '主体id': '商品id',
185
+ '总购物车数': '加购量',
186
+ '总成交笔数': '成交笔数',
187
+ '总成交金额': '成交金额'
188
+ }, inplace=True)
189
+ df = df.astype({
190
+ '花费': float,
191
+ '展现量': int,
192
+ '点击量': int,
193
+ '加购量': int,
194
+ '成交笔数': int,
195
+ '成交金额': float,
196
+ '自然流量曝光量': int,
197
+ '直接成交笔数': int,
198
+ '直接成交金额': float,
199
+ }, errors='raise')
200
+ df.fillna(0, inplace=True)
201
+ if is_maximize:
202
+ df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
203
+ **{'加购量': ('加购量', np.max),
204
+ '成交笔数': ('成交笔数', np.max),
205
+ '成交金额': ('成交金额', np.max),
206
+ '自然流量曝光量': ('自然流量曝光量', np.max),
207
+ '直接成交笔数': ('直接成交笔数', np.max),
208
+ '直接成交金额': ('直接成交金额', np.max)
209
+ }
210
+ )
211
+ else:
212
+ df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
213
+ **{'加购量': ('加购量', np.min),
214
+ '成交笔数': ('成交笔数', np.min),
215
+ '成交金额': ('成交金额', np.min),
216
+ '自然流量曝光量': ('自然流量曝光量', np.min),
217
+ '直接成交笔数': ('直接成交笔数', np.max),
218
+ '直接成交金额': ('直接成交金额', np.max)
219
+ }
220
+ )
221
+ df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
222
+ return df
223
+ elif '宝贝指标' in tabel_name:
224
+ df.fillna(0, inplace=True)
225
+ df = df[(df['销售额'] != 0) | (df['退款额'] != 0)]
226
+ df = df.groupby(['日期', '宝贝id', '商家编码', '行业类目'], as_index=False).agg(
227
+ **{'销售额': ('销售额', np.min),
228
+ '销售量': ('销售量', np.min),
229
+ '订单数': ('订单数', np.min),
230
+ '退货量': ('退货量', np.max),
231
+ '退款额': ('退款额', np.max),
232
+ '退货量_发货后_': ('退货量_发货后_', np.max),
233
+ }
234
+ )
235
+ df['件均价'] = df.apply(lambda x: x['销售额'] / x['销售量'] if x['销售量'] > 0 else 0, axis=1).round(
236
+ 0) # 两列运算, 避免除以0
237
+ df['价格带'] = df['件均价'].apply(
238
+ lambda x: '2000+' if x >= 2000
239
+ else '1000+' if x >= 1000
240
+ else '500+' if x >= 500
241
+ else '300+' if x >= 300
242
+ else '300以下'
243
+ )
244
+ return df
245
+ elif '店铺来源_日数据' in tabel_name:
246
+ return df
247
+ else:
248
+ print(f'<{tabel_name}>: Groupby 类尚未配置,数据为空')
249
+ return pd.DataFrame({})
250
+
251
+ def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
252
+ index=False, header=True, st_ascend=None, ascend=None, freq=None):
253
+ """
254
+ path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹,可以不传,
255
+ st_ascend: 排序参数 ['column1', 'column2']
256
+ ascend: 升降序 [True, False]
257
+ freq: 将创建子文件夹并按月分类存储, freq='Y', 或 freq='M'
258
+ """
259
+ if len(df) == 0:
260
+ return
261
+ if not path:
262
+ path = self.output
263
+ else:
264
+ path = os.path.join(self.output, path)
265
+ if not os.path.exists(path):
266
+ os.makedirs(path)
267
+ if st_ascend and ascend:
268
+ try:
269
+ df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
270
+ except:
271
+ print(f'{filename}: sort_values排序参数错误!')
272
+ if freq:
273
+ if '日期' not in df.columns.tolist():
274
+ return print(f'{filename}: 数据缺少日期列,无法按日期分组')
275
+ groups = df.groupby(pd.Grouper(key='日期', freq=freq))
276
+ for name1, df in groups:
277
+ if freq == 'M':
278
+ sheet_name = name1.strftime('%Y-%m')
279
+ elif freq == 'Y':
280
+ sheet_name = name1.strftime('%Y年')
281
+ else:
282
+ sheet_name = '_未分类'
283
+ new_path = os.path.join(path, filename)
284
+ if not os.path.exists(new_path):
285
+ os.makedirs(new_path)
286
+ new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
287
+ if st_ascend and ascend: # 这里需要重新排序一次,原因未知
288
+ try:
289
+ df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
290
+ except:
291
+ print(f'{filename}: sort_values排序参数错误!')
292
+
293
+ df.to_csv(new_path, encoding=encoding, index=index, header=header)
294
+ else:
295
+ df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
296
+
297
+ def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
298
+ if len(df) == 0:
299
+ return
300
+ if not path:
301
+ path = self.output
302
+ else:
303
+ path = os.path.join(self.output, path)
304
+ if not os.path.exists(path):
305
+ os.makedirs(path)
306
+ if st_ascend and ascend:
307
+ try:
308
+ df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
309
+ except:
310
+ print(f'{filename}: sort_values排序参数错误!')
311
+ df.to_json(os.path.join(path, filename + '.json'),
312
+ orient=orient, force_ascii=force_ascii)
313
+
314
+ def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
315
+ freeze_panes=(1, 0), st_ascend=None, ascend=None):
316
+ if len(df) == 0:
317
+ return
318
+ if not path:
319
+ path = self.output
320
+ else:
321
+ path = os.path.join(self.output, path)
322
+ if not os.path.exists(path):
323
+ os.makedirs(path)
324
+ if st_ascend and ascend:
325
+ try:
326
+ df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
327
+ except:
328
+ print(f'{filename}: sort_values排序参数错误!')
329
+ df.to_excel(os.path.join(path, filename + '.xlsx'),
330
+ index=index, header=header, engine=engine, freeze_panes=freeze_panes)
331
+
332
+
333
+ def data_output():
334
+ sdq = MysqlDatasQuery(target_service='home_lx')
335
+ sdq.months = 0
336
+
337
+ # df = sdq.tg_wxt() # 从数据库中获取数据并转为 df
338
+ # g = GroupBy() # 数据聚合
339
+ # df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
340
+ # g.as_csv(df=df, filename='推广数据_宝贝主体报表') # 数据导出
341
+
342
+ # df = sdq.syj()
343
+ # g = GroupBy()
344
+ # df = g.groupby(df=df, tabel_name='宝贝指标', is_maximize=True)
345
+ # g.as_csv(df=df, filename='宝贝指标')
346
+
347
+ df = sdq.dplyd()
348
+ g = GroupBy()
349
+ df = g.groupby(df=df, tabel_name='店铺来源_日数据', is_maximize=True)
350
+ g.as_csv(df=df, filename='店铺来源_日数据')
351
+
352
+
353
+ if __name__ == '__main__':
354
+ main()
mdbq/config/get_myconf.py CHANGED
@@ -106,6 +106,10 @@ def select_config_values(target_service, database, path=None):
106
106
  elif database == 'mysql':
107
107
  options = ['username_mysql_company_nw', 'password_mysql_company_nw', 'host_mysql_company_nw', 'port_mysql_company_nw', ]
108
108
 
109
+ elif target_service == 'nas': # 4. 群晖
110
+ if database == 'mysql':
111
+ options = ['username_mysql_nas_nw', 'password_mysql_nas_nw', 'host_mysql_nas_nw', 'port_mysql_nas_nw', ]
112
+
109
113
  value = m.get_myconf(options=options)
110
114
  if not value:
111
115
  return '', '', '', 0
@@ -44,14 +44,18 @@ class DataFrameConverter(object):
44
44
  pass
45
45
  new_col = col.lower()
46
46
  new_col = re.sub(r'[\',,()()/=<>+\-*^"’\[\]~#|&% .;]', '_', new_col)
47
+ new_col = re.sub(r'_+$', '', new_col)
47
48
  df.rename(columns={col: new_col}, inplace=True)
48
49
  df.fillna(0, inplace=True)
49
50
  return df
50
51
 
51
52
 
52
53
  if __name__ == '__main__':
53
- df = pd.DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
54
- converter = DataFrameConverter()
55
- df = converter.convert_df_cols(df)
56
- print(df['a'].dtype)
57
- print(df)
54
+ # df = pd.DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
55
+ # converter = DataFrameConverter()
56
+ # df = converter.convert_df_cols(df)
57
+ # print(df['a'].dtype)
58
+ # print(df)
59
+ pattern = 'dfa_dfawr__'
60
+ pattern = re.sub(r'_+$', '', pattern)
61
+ print(pattern)
mdbq/mongo/mongo.py CHANGED
@@ -195,6 +195,7 @@ class DownMongo:
195
195
 
196
196
  def data_to_file(self, file_type, db_name, collection_name):
197
197
  """
198
+ 用于 GUI 的函数
198
199
  将 mongodb 数据保存本地
199
200
  db_name: 数据库名
200
201
  collections 集合名
@@ -206,9 +207,9 @@ class DownMongo:
206
207
  _collection = self.client[self.db_name][self.collection_name] # 连接集合
207
208
  now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
208
209
  if not self.start_date:
209
- print(f'{now}正在下载数据 ({self.host}) {self.db_name}: {self.collection_name}, 数据区间: 近 {self.days} 天\n...')
210
+ print(f'{now}正在下载 ({self.host}) {self.db_name}: {self.collection_name}, 区间: 近 {self.days} 天\n...')
210
211
  else:
211
- print(f'{now}正在下载数据 ({self.host}) {self.db_name}: {self.collection_name}, 数据区间: {self.start_date} ~ {self.end_date}')
212
+ print(f'{now}正在下载 ({self.host}) {self.db_name}: {self.collection_name}, 区间: {self.start_date}~{self.end_date}')
212
213
 
213
214
  if not self.start_date:
214
215
  self.start_date = datetime.datetime.now() - datetime.timedelta(days=self.days)
@@ -221,11 +222,10 @@ class DownMongo:
221
222
  {'$project': {'_id': 0}}, # 不保留 id 字段
222
223
  ]
223
224
  results = _collection.aggregate(pipeline)
224
- # print(results)
225
+
225
226
  # 输出结果
226
227
  datas = []
227
228
  for doc in results:
228
- # print(doc)
229
229
  datas.append(doc)
230
230
  _df = pd.DataFrame(datas)
231
231
  if len(_df) == 0:
@@ -251,7 +251,7 @@ class DownMongo:
251
251
  _df.to_excel(_path, index=False, header=True, engine='openpyxl', freeze_panes=(1, 0)) # freeze_ 冻结列索引
252
252
  else:
253
253
  print(f'{file_type}: 未支持的文件类型')
254
- print(f'{self.collection_name} 保存路径: {_path}, 数据完成!\n-->> 注意数据库导出的数据没有排重!')
254
+ print(f'<{self.collection_name}> 导出: {_path}, 数据完成!')
255
255
  self.client.close()
256
256
 
257
257
 
@@ -655,6 +655,28 @@ class OptimizeDatas:
655
655
  start_date += datetime.timedelta(days=1)
656
656
  return date_list
657
657
 
658
+ def rename_column(self):
659
+ """ 批量修改数据库的列名 """
660
+ """
661
+ # for db_name in ['京东数据2', '天猫数据2', '市场数据2', '生意参谋数据2', '生意经2', '属性设置2',]:
662
+ # s = OptimizeDatas(username=username, password=password, host=host, port=port)
663
+ # s.db_name = db_name
664
+ # s.rename_column()
665
+ # s.client.close()
666
+ """
667
+ self.client = pymongo.MongoClient(self.link) # 连接数据库
668
+ database_names = self.client.list_database_names() # 所有数据库名称
669
+ collections = self.my_collection_names(db_name=self.db_name) # 所有集合名称
670
+ for collection_name in collections:
671
+ collection = self.client[self.db_name].get_collection(collection_name)
672
+ has_date_field = collection.find_one({})
673
+ for key, value in has_date_field.items():
674
+ if key.endswith('_'):
675
+ new_name = re.sub(r'_+$', '', key)
676
+ query = {key: {'$exists': True}}
677
+ update = {'$rename': {key: new_name}}
678
+ collection.update_many(query, update)
679
+
658
680
 
659
681
  def upload_one_dir():
660
682
  username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
@@ -690,3 +712,16 @@ if __name__ == '__main__':
690
712
  # main()
691
713
  username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
692
714
  print(username, password, host, port)
715
+
716
+ # for db_name in [
717
+ # '京东数据2',
718
+ # '天猫数据2',
719
+ # '市场数据2',
720
+ # '生意参谋数据2',
721
+ # '生意经2',
722
+ # '属性设置2',
723
+ # ]:
724
+ # s = OptimizeDatas(username=username, password=password, host=host, port=port)
725
+ # s.db_name = db_name
726
+ # s.rename_column()
727
+ # s.client.close()
mdbq/mysql/mysql.py CHANGED
@@ -81,8 +81,11 @@ class MysqlUpload:
81
81
  if '8.138.27' in str(self.host) or platform.system() == "Linux": # 阿里云 mysql 低版本不支持 0900
82
82
  cursor.execute(f"CREATE DATABASE {db_name} COLLATE utf8mb4_unicode_ci")
83
83
  self.config.update({'charset': 'utf8mb4_unicode_ci'})
84
+ if '192.168.1.100' in str(self.host):
85
+ cursor.execute(f"CREATE DATABASE {db_name}")
84
86
  else:
85
87
  cursor.execute(f"CREATE DATABASE {db_name} COLLATE utf8mb4_0900_ai_ci")
88
+ # cursor.execute(f"CREATE DATABASE {db_name}")
86
89
  connection.commit()
87
90
  print(f"创建Database: {db_name}")
88
91
  except Exception as e:
@@ -140,23 +143,36 @@ class MysqlUpload:
140
143
  # connection.commit()
141
144
 
142
145
  # 4. 更新插入数据
143
- try:
144
- now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
145
- print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{tabel_name}')
146
- engine = create_engine(
147
- f'mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}'
148
- )
149
- df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
150
- except Exception as e: # 如果异常则回滚
146
+ now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
147
+ print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{tabel_name}')
148
+ if str(self.host) == '192.168.1.100': # 群晖服务器
151
149
  try:
152
- connection.rollback()
153
- print(f'{e}, 发生异常,正在重试...')
154
- # df = df.replace([np.inf, -np.inf], 0)
155
- df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
150
+ datas = df.to_dict('records')
151
+ for data in datas:
152
+ cols = ', '.join(data.keys())
153
+ values = ', '.join([f'"{v}"' for v in data.values()])
154
+ sql = f"INSERT INTO {tabel_name} ({cols}) VALUES ({values})"
155
+ cursor.execute(sql)
156
+ connection.commit()
156
157
  except Exception as e:
157
- now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
158
- print(f'{now}{db_name}/{tabel_name}数据异常, 正在回滚: {e}')
158
+ print(e)
159
159
  connection.rollback()
160
+ else: # 其他服务器
161
+ try:
162
+ engine = create_engine(
163
+ f'mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}'
164
+ )
165
+ df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
166
+ except Exception as e: # 如果异常则回滚
167
+ try:
168
+ connection.rollback()
169
+ print(f'{e}, 发生异常,正在重试...')
170
+ # df = df.replace([np.inf, -np.inf], 0)
171
+ df.to_sql(tabel_name, con=engine, if_exists='append', index=False)
172
+ except Exception as e:
173
+ now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
174
+ print(f'{now}{db_name}/{tabel_name}数据异常, 正在回滚: {e}')
175
+ connection.rollback()
160
176
  finally:
161
177
  connection.close()
162
178
 
@@ -545,6 +561,33 @@ class OptimizeDatas:
545
561
  start_date += datetime.timedelta(days=1)
546
562
  return date_list
547
563
 
564
+ def rename_column(self):
565
+ """ 批量修改数据库的列名 """
566
+ """
567
+ # for db_name in ['京东数据2', '天猫数据2', '市场数据2', '生意参谋数据2', '生意经2', '属性设置2',]:
568
+ # s = OptimizeDatas(username=username, password=password, host=host, port=port)
569
+ # s.db_name = db_name
570
+ # s.rename_column()
571
+ """
572
+ tables = self.table_list(db_name=self.db_name)
573
+ for table_dict in tables:
574
+ for key, table_name in table_dict.items():
575
+ self.config.update({'database': self.db_name}) # 添加更新 config 字段
576
+ self.connection = pymysql.connect(**self.config)
577
+ with self.connection.cursor() as cursor:
578
+ cursor.execute(f"SHOW FULL COLUMNS FROM {table_name}") # 查询数据表的列信息
579
+ columns = cursor.fetchall()
580
+ columns = [{column['Field']: column['Type']} for column in columns]
581
+ for column in columns:
582
+ for key, value in column.items():
583
+ if key.endswith('_'):
584
+ new_name = re.sub(r'_+$', '', key)
585
+ sql = f"ALTER TABLE {table_name} CHANGE COLUMN {key} {new_name} {value}"
586
+ cursor.execute(sql)
587
+ self.connection.commit()
588
+ if self.connection:
589
+ self.connection.close()
590
+
548
591
 
549
592
  def year_month_day(start_date, end_date):
550
593
  """
@@ -594,3 +637,6 @@ def download_datas(tabel_name, save_path, start_date):
594
637
  if __name__ == '__main__':
595
638
  username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
596
639
  print(username, password, host, port)
640
+
641
+
642
+
mdbq/mysql/s_query.py CHANGED
@@ -14,6 +14,9 @@ import calendar
14
14
  from mdbq.config import get_myconf
15
15
 
16
16
  warnings.filterwarnings('ignore')
17
+ """
18
+ 程序专门用来下载数据库数据, 并返回 df, 不做清洗数据操作;
19
+ """
17
20
 
18
21
 
19
22
  class QueryDatas:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mdbq
3
- Version: 0.0.9
3
+ Version: 0.1.1
4
4
  Home-page: https://pypi.org/project/mdbsql
5
5
  Author: xigua,
6
6
  Author-email: 2587125111@qq.com
@@ -1,5 +1,8 @@
1
1
  mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
2
2
  mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
3
+ mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
4
+ mdbq/aggregation/aggregation.py,sha256=n2MoKwN1ltJD3juu59zkhc7PGUMDTkn0zCcZGs8RnXI,56775
5
+ mdbq/aggregation/query_data.py,sha256=RaZ0NJwTeNhSgpyj9eUjEVc6Kp83etr6luyzSf9U5WE,14465
3
6
  mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
4
7
  mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
5
8
  mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -7,17 +10,17 @@ mdbq/clean/data_clean.py,sha256=33OmeQFl9AW21P5EOay52W_S8DF96H5oHwCg4fSuBxA,8535
7
10
  mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
8
11
  mdbq/company/copysh.py,sha256=0exynzeqf85gCBQXAgKycVxddMhr0TjkFcBP_NK0QTA,15609
9
12
  mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
10
- mdbq/config/get_myconf.py,sha256=q6Pylsnh4-MsHP9JcX8IdlnGVWikz9hio1HI_qh6Wvs,6171
13
+ mdbq/config/get_myconf.py,sha256=9v3xebfcS1tptxpvk3_tGxfXjAehGVCveYe4iRUzLQQ,6372
11
14
  mdbq/config/update_conf.py,sha256=YjGjjRchu5BcrmLJkoLjHEF2TbGOmsgCWX4LroXOYWQ,3455
12
15
  mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
13
- mdbq/dataframe/converter.py,sha256=VLG6Y0Ca_OkgpVkHDQU6AHSxGqISx6GsGiyDGHkXq0g,2668
16
+ mdbq/dataframe/converter.py,sha256=5hrGx-lPVwYLuyZNOHf6K7O9_AAKZ7mCp0MHxlxSCnk,2816
14
17
  mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
15
18
  mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
16
19
  mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
17
- mdbq/mongo/mongo.py,sha256=AK028yQWPajf_A-PYY-NJTfWiGQJDpBwPY3aS6cOcHk,30431
20
+ mdbq/mongo/mongo.py,sha256=q0B4wXDSTtXg_vMN7MPh6zdxl6tT68tM74LmdVNQQek,31892
18
21
  mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
19
- mdbq/mysql/mysql.py,sha256=HX3keNO-Hbpra3XNJtWbBwsD7pLHiKfM4ZQaSBksAkk,28149
20
- mdbq/mysql/s_query.py,sha256=bRnW8Cpy4fSsbMhzGCvjiK2kin9uamVumJC3nLAyjMg,5213
22
+ mdbq/mysql/mysql.py,sha256=Ul-QJHFKjSHu2uxk_CRswVvCV-oFtN-t-3wIOoKfLLg,30550
23
+ mdbq/mysql/s_query.py,sha256=pzxRGBRP3Ku4oVLqfpMd1VWWct3YlxxnVavtKa9kgSM,5302
21
24
  mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
22
25
  mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
23
26
  mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
@@ -27,7 +30,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
27
30
  mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
28
31
  mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
29
32
  mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
30
- mdbq-0.0.9.dist-info/METADATA,sha256=NoHzL0QzffHfRgvnRG15APNN5GuHNiTU0U5iWIijFLU,245
31
- mdbq-0.0.9.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
32
- mdbq-0.0.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
33
- mdbq-0.0.9.dist-info/RECORD,,
33
+ mdbq-0.1.1.dist-info/METADATA,sha256=60TKH9pmImZH4nV2l_b0ge4Cu6K9YmWvsZzdt8fNNZA,245
34
+ mdbq-0.1.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
35
+ mdbq-0.1.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
36
+ mdbq-0.1.1.dist-info/RECORD,,
File without changes