mdbq 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/query_data.py +111 -152
- mdbq/mysql/s_query.py +58 -103
- mdbq/mysql/year_month_day.py +38 -0
- {mdbq-0.0.6.dist-info → mdbq-0.0.7.dist-info}/METADATA +1 -1
- {mdbq-0.0.6.dist-info → mdbq-0.0.7.dist-info}/RECORD +7 -6
- {mdbq-0.0.6.dist-info → mdbq-0.0.7.dist-info}/WHEEL +1 -1
- {mdbq-0.0.6.dist-info → mdbq-0.0.7.dist-info}/top_level.txt +0 -0
mdbq/aggregation/query_data.py
CHANGED
@@ -20,20 +20,13 @@ class MongoDatasQuery:
|
|
20
20
|
"""
|
21
21
|
def __init__(self, target_service):
|
22
22
|
# target_service 从哪个服务器下载数据
|
23
|
-
self.
|
24
|
-
if platform.system() == 'Darwin':
|
25
|
-
self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
|
26
|
-
elif platform.system() == 'Windows':
|
27
|
-
self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
|
28
|
-
else:
|
29
|
-
self.output = os.path.join('数据中心/数据库导出')
|
30
|
-
|
23
|
+
self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
31
24
|
# 实例化一个下载类
|
32
25
|
username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
|
33
26
|
self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
|
34
27
|
|
35
28
|
def tg_wxt(self):
|
36
|
-
self.download.start_date, self.download.end_date = self.months_data(num=
|
29
|
+
self.download.start_date, self.download.end_date = self.months_data(num=self.months)
|
37
30
|
projection = {
|
38
31
|
'日期': 1,
|
39
32
|
'场景名字': 1,
|
@@ -48,48 +41,11 @@ class MongoDatasQuery:
|
|
48
41
|
'直接成交笔数': 1,
|
49
42
|
'直接成交金额': 1,
|
50
43
|
}
|
51
|
-
df = self.download.data_to_df(
|
52
|
-
|
53
|
-
'
|
54
|
-
|
55
|
-
|
56
|
-
'总成交笔数': '成交笔数',
|
57
|
-
'总成交金额': '成交金额'
|
58
|
-
}, inplace=True)
|
59
|
-
df = df.astype({
|
60
|
-
'花费': float,
|
61
|
-
'展现量': int,
|
62
|
-
'点击量': int,
|
63
|
-
'加购量': int,
|
64
|
-
'成交笔数': int,
|
65
|
-
'成交金额': float,
|
66
|
-
'自然流量曝光量': int,
|
67
|
-
'直接成交笔数': int,
|
68
|
-
'直接成交金额': float,
|
69
|
-
}, errors='raise')
|
70
|
-
df.fillna(0, inplace=True)
|
71
|
-
if self.is_maximize:
|
72
|
-
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
73
|
-
**{'加购量': ('加购量', np.max),
|
74
|
-
'成交笔数': ('成交笔数', np.max),
|
75
|
-
'成交金额': ('成交金额', np.max),
|
76
|
-
'自然流量曝光量': ('自然流量曝光量', np.max),
|
77
|
-
'直接成交笔数': ('直接成交笔数', np.max),
|
78
|
-
'直接成交金额': ('直接成交金额', np.max)
|
79
|
-
}
|
80
|
-
)
|
81
|
-
else:
|
82
|
-
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
83
|
-
**{'加购量': ('加购量', np.min),
|
84
|
-
'成交笔数': ('成交笔数', np.min),
|
85
|
-
'成交金额': ('成交金额', np.min),
|
86
|
-
'自然流量曝光量': ('自然流量曝光量', np.min),
|
87
|
-
'直接成交笔数': ('直接成交笔数', np.max),
|
88
|
-
'直接成交金额': ('直接成交金额', np.max)
|
89
|
-
}
|
90
|
-
)
|
91
|
-
df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
|
92
|
-
# print(df)
|
44
|
+
df = self.download.data_to_df(
|
45
|
+
db_name='天猫数据2',
|
46
|
+
collection_name='推广数据_宝贝主体报表',
|
47
|
+
projection=projection,
|
48
|
+
)
|
93
49
|
return df
|
94
50
|
|
95
51
|
@staticmethod
|
@@ -109,112 +65,43 @@ class MongoDatasQuery:
|
|
109
65
|
start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
|
110
66
|
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
111
67
|
|
112
|
-
def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
|
113
|
-
index=False, header=True, st_ascend=None, ascend=None, freq=None):
|
114
|
-
"""
|
115
|
-
path: 子文件夹,可以不传,默认导出目录 self.output
|
116
|
-
st_ascend: 排序参数
|
117
|
-
ascend: 升降序
|
118
|
-
freq: 将创建子文件夹并按月分类存储, freq='Y',或 freq='M'
|
119
|
-
"""
|
120
|
-
if not path:
|
121
|
-
path = self.output
|
122
|
-
else:
|
123
|
-
path = os.path.join(self.output, path)
|
124
|
-
if not os.path.exists(path):
|
125
|
-
os.makedirs(path)
|
126
|
-
if st_ascend and ascend:
|
127
|
-
try:
|
128
|
-
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
129
|
-
except:
|
130
|
-
print(f'{filename}: sort_values排序参数错误!')
|
131
|
-
if freq:
|
132
|
-
if '日期' not in df.columns.tolist():
|
133
|
-
return print(f'{filename}: 数据缺少日期列,无法按日期分组')
|
134
|
-
groups = df.groupby(pd.Grouper(key='日期', freq=freq))
|
135
|
-
for name1, df in groups:
|
136
|
-
if freq == 'M':
|
137
|
-
sheet_name = name1.strftime('%Y-%m')
|
138
|
-
elif freq == 'Y':
|
139
|
-
sheet_name = name1.strftime('%Y年')
|
140
|
-
else:
|
141
|
-
sheet_name = '_未分类'
|
142
|
-
new_path = os.path.join(path, filename)
|
143
|
-
if not os.path.exists(new_path):
|
144
|
-
os.makedirs(new_path)
|
145
|
-
new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
|
146
|
-
if st_ascend and ascend: # 这里需要重新排序一次,原因未知
|
147
|
-
try:
|
148
|
-
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
149
|
-
except:
|
150
|
-
print(f'{filename}: sort_values排序参数错误!')
|
151
|
-
|
152
|
-
df.to_csv(new_path, encoding=encoding, index=index, header=header)
|
153
|
-
else:
|
154
|
-
df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
|
155
|
-
|
156
|
-
def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
|
157
|
-
if not path:
|
158
|
-
path = self.output
|
159
|
-
else:
|
160
|
-
path = os.path.join(self.output, path)
|
161
|
-
if st_ascend and ascend:
|
162
|
-
try:
|
163
|
-
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
164
|
-
except:
|
165
|
-
print(f'{filename}: sort_values排序参数错误!')
|
166
|
-
df.to_json(os.path.join(path, filename + '.json'),
|
167
|
-
orient=orient, force_ascii=force_ascii)
|
168
|
-
|
169
|
-
def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
|
170
|
-
freeze_panes=(1, 0), st_ascend=None, ascend=None):
|
171
|
-
if not path:
|
172
|
-
path = self.output
|
173
|
-
else:
|
174
|
-
path = os.path.join(self.output, path)
|
175
|
-
if st_ascend and ascend:
|
176
|
-
try:
|
177
|
-
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
178
|
-
except:
|
179
|
-
print(f'{filename}: sort_values排序参数错误!')
|
180
|
-
df.to_excel(os.path.join(path, filename + '.xlsx'),
|
181
|
-
index=index, header=header, engine=engine, freeze_panes=freeze_panes)
|
182
|
-
|
183
68
|
|
184
69
|
class MysqlDatasQuery:
|
185
70
|
"""
|
186
|
-
|
187
|
-
self.output: 数据库默认导出目录
|
188
|
-
self.is_maximize: 是否最大转化数据
|
71
|
+
从数据库中下载数据
|
189
72
|
"""
|
190
73
|
def __init__(self, target_service):
|
191
74
|
# target_service 从哪个服务器下载数据
|
192
|
-
self.
|
193
|
-
if platform.system() == 'Darwin':
|
194
|
-
self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
|
195
|
-
elif platform.system() == 'Windows':
|
196
|
-
self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
|
197
|
-
else:
|
198
|
-
self.output = os.path.join('数据中心/数据库导出')
|
199
|
-
self.months = 1 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
200
|
-
|
75
|
+
self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
201
76
|
# 实例化一个下载类
|
202
77
|
username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
|
203
78
|
self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
|
204
79
|
|
205
80
|
def tg_wxt(self):
|
206
81
|
start_date, end_date = self.months_data(num=self.months)
|
207
|
-
|
82
|
+
projection = {
|
83
|
+
'日期': 1,
|
84
|
+
'场景名字': 1,
|
85
|
+
'主体id': 1,
|
86
|
+
'花费': 1,
|
87
|
+
'展现量': 1,
|
88
|
+
'点击量': 1,
|
89
|
+
'总购物车数': 1,
|
90
|
+
'总成交笔数': 1,
|
91
|
+
'总成交金额': 1,
|
92
|
+
'自然流量曝光量': 1,
|
93
|
+
'直接成交笔数': 1,
|
94
|
+
'直接成交金额': 1,
|
95
|
+
}
|
96
|
+
df = self.download.data_to_df(
|
97
|
+
db_name='天猫数据2',
|
98
|
+
tabel_name='推广数据_宝贝主体报表',
|
99
|
+
start_date=start_date,
|
100
|
+
end_date=end_date,
|
101
|
+
projection=projection,
|
102
|
+
)
|
208
103
|
return df
|
209
104
|
|
210
|
-
@staticmethod
|
211
|
-
def days_data(days, end_date=None):
|
212
|
-
""" 读取近 days 天的数据 """
|
213
|
-
if not end_date:
|
214
|
-
end_date = datetime.datetime.now()
|
215
|
-
start_date = end_date - datetime.timedelta(days=days)
|
216
|
-
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
217
|
-
|
218
105
|
@staticmethod
|
219
106
|
def months_data(num=0, end_date=None):
|
220
107
|
""" 读取近 num 个月的数据, 0 表示读取当月的数据 """
|
@@ -224,14 +111,75 @@ class MysqlDatasQuery:
|
|
224
111
|
start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
|
225
112
|
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
226
113
|
|
114
|
+
|
115
|
+
class GroupBy:
|
116
|
+
""" 数据聚合和导出 """
|
117
|
+
def __init__(self):
|
118
|
+
# self.output: 数据库默认导出目录
|
119
|
+
if platform.system() == 'Darwin':
|
120
|
+
self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
|
121
|
+
elif platform.system() == 'Windows':
|
122
|
+
self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
|
123
|
+
else:
|
124
|
+
self.output = os.path.join('数据中心/数据库导出')
|
125
|
+
|
126
|
+
def groupby(self, df, tabel_name, is_maximize=True):
|
127
|
+
"""
|
128
|
+
self.is_maximize: 是否最大转化数据
|
129
|
+
"""
|
130
|
+
if '宝贝主体报表' in tabel_name:
|
131
|
+
df.rename(columns={
|
132
|
+
'场景名字': '营销场景',
|
133
|
+
'主体id': '商品id',
|
134
|
+
'总购物车数': '加购量',
|
135
|
+
'总成交笔数': '成交笔数',
|
136
|
+
'总成交金额': '成交金额'
|
137
|
+
}, inplace=True)
|
138
|
+
df = df.astype({
|
139
|
+
'花费': float,
|
140
|
+
'展现量': int,
|
141
|
+
'点击量': int,
|
142
|
+
'加购量': int,
|
143
|
+
'成交笔数': int,
|
144
|
+
'成交金额': float,
|
145
|
+
'自然流量曝光量': int,
|
146
|
+
'直接成交笔数': int,
|
147
|
+
'直接成交金额': float,
|
148
|
+
}, errors='raise')
|
149
|
+
df.fillna(0, inplace=True)
|
150
|
+
if is_maximize:
|
151
|
+
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
152
|
+
**{'加购量': ('加购量', np.max),
|
153
|
+
'成交笔数': ('成交笔数', np.max),
|
154
|
+
'成交金额': ('成交金额', np.max),
|
155
|
+
'自然流量曝光量': ('自然流量曝光量', np.max),
|
156
|
+
'直接成交笔数': ('直接成交笔数', np.max),
|
157
|
+
'直接成交金额': ('直接成交金额', np.max)
|
158
|
+
}
|
159
|
+
)
|
160
|
+
else:
|
161
|
+
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
162
|
+
**{'加购量': ('加购量', np.min),
|
163
|
+
'成交笔数': ('成交笔数', np.min),
|
164
|
+
'成交金额': ('成交金额', np.min),
|
165
|
+
'自然流量曝光量': ('自然流量曝光量', np.min),
|
166
|
+
'直接成交笔数': ('直接成交笔数', np.max),
|
167
|
+
'直接成交金额': ('直接成交金额', np.max)
|
168
|
+
}
|
169
|
+
)
|
170
|
+
df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
|
171
|
+
return df
|
172
|
+
|
227
173
|
def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
|
228
174
|
index=False, header=True, st_ascend=None, ascend=None, freq=None):
|
229
175
|
"""
|
230
|
-
path:
|
231
|
-
st_ascend: 排序参数
|
232
|
-
ascend: 升降序
|
233
|
-
freq: 将创建子文件夹并按月分类存储, freq='Y'
|
176
|
+
path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹,可以不传,
|
177
|
+
st_ascend: 排序参数 ['column1', 'column2']
|
178
|
+
ascend: 升降序 [True, False]
|
179
|
+
freq: 将创建子文件夹并按月分类存储, freq='Y', 或 freq='M'
|
234
180
|
"""
|
181
|
+
if len(df) == 0:
|
182
|
+
return
|
235
183
|
if not path:
|
236
184
|
path = self.output
|
237
185
|
else:
|
@@ -269,10 +217,14 @@ class MysqlDatasQuery:
|
|
269
217
|
df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
|
270
218
|
|
271
219
|
def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
|
220
|
+
if len(df) == 0:
|
221
|
+
return
|
272
222
|
if not path:
|
273
223
|
path = self.output
|
274
224
|
else:
|
275
225
|
path = os.path.join(self.output, path)
|
226
|
+
if not os.path.exists(path):
|
227
|
+
os.makedirs(path)
|
276
228
|
if st_ascend and ascend:
|
277
229
|
try:
|
278
230
|
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
@@ -283,10 +235,14 @@ class MysqlDatasQuery:
|
|
283
235
|
|
284
236
|
def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
|
285
237
|
freeze_panes=(1, 0), st_ascend=None, ascend=None):
|
238
|
+
if len(df) == 0:
|
239
|
+
return
|
286
240
|
if not path:
|
287
241
|
path = self.output
|
288
242
|
else:
|
289
243
|
path = os.path.join(self.output, path)
|
244
|
+
if not os.path.exists(path):
|
245
|
+
os.makedirs(path)
|
290
246
|
if st_ascend and ascend:
|
291
247
|
try:
|
292
248
|
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
@@ -294,13 +250,16 @@ class MysqlDatasQuery:
|
|
294
250
|
print(f'{filename}: sort_values排序参数错误!')
|
295
251
|
df.to_excel(os.path.join(path, filename + '.xlsx'),
|
296
252
|
index=index, header=header, engine=engine, freeze_panes=freeze_panes)
|
297
|
-
|
298
|
-
|
253
|
+
|
254
|
+
|
299
255
|
def main():
|
300
|
-
sdq = MysqlDatasQuery(target_service='
|
256
|
+
sdq = MysqlDatasQuery(target_service='home_lx')
|
301
257
|
sdq.months = 0
|
302
|
-
df = sdq.tg_wxt()
|
303
|
-
|
258
|
+
df = sdq.tg_wxt() # 从数据库中获取数据并转为 df
|
259
|
+
|
260
|
+
g = GroupBy() # 数据聚合
|
261
|
+
df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
|
262
|
+
g.as_csv(df=df, filename='test') # 数据导出
|
304
263
|
|
305
264
|
|
306
265
|
if __name__ == '__main__':
|
mdbq/mysql/s_query.py
CHANGED
@@ -31,135 +31,90 @@ class QueryDatas:
|
|
31
31
|
'cursorclass': pymysql.cursors.DictCursor,
|
32
32
|
}
|
33
33
|
|
34
|
-
def data_to_df(self, db_name, tabel_name, start_date, end_date, projection=[]):
|
34
|
+
def data_to_df(self, db_name, tabel_name, start_date, end_date, projection: dict=[]):
|
35
|
+
|
35
36
|
start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
|
36
37
|
end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
|
37
|
-
df = pd.DataFrame()
|
38
|
+
df = pd.DataFrame() # 初始化df
|
39
|
+
|
40
|
+
if self.check_infos(db_name, tabel_name) == False:
|
41
|
+
return df
|
38
42
|
|
43
|
+
self.config.update({'database': db_name})
|
44
|
+
connection = pymysql.connect(**self.config) # 重新连接数据库
|
45
|
+
try:
|
46
|
+
with connection.cursor() as cursor:
|
47
|
+
# 3. 获取数据表的所有列信息
|
48
|
+
sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
|
49
|
+
cursor.execute(sql, (db_name, {tabel_name}))
|
50
|
+
columns = cursor.fetchall()
|
51
|
+
cols_exist = [col['COLUMN_NAME'] for col in columns] # 数据表的所有列, 返回 list
|
52
|
+
|
53
|
+
# 4. 构建 SQL 查询语句
|
54
|
+
if projection: # 获取指定列
|
55
|
+
columns_in = []
|
56
|
+
for key, value in projection.items():
|
57
|
+
if value == 1 and key in cols_exist:
|
58
|
+
columns_in.append(key) # 提取值为 1 的键并清理不在数据表的键
|
59
|
+
columns_in = ', '.join(columns_in)
|
60
|
+
if '日期' in cols_exist: # 不论是否指定, 只要数据表有日期,则执行
|
61
|
+
sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
|
62
|
+
f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
|
63
|
+
else: # 数据表没有日期列时,返回指定列的所有数据
|
64
|
+
sql = f"SELECT {columns_in} FROM {db_name}.{tabel_name}"
|
65
|
+
else: # 没有指定获取列时
|
66
|
+
if '日期' in cols_exist: # 但数据表有日期,仍然执行
|
67
|
+
columns_in = ', '.join(cols_exist)
|
68
|
+
sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
|
69
|
+
f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
|
70
|
+
else: # 没有指定获取列,且数据表也没有日期列,则返回全部列的全部数据
|
71
|
+
sql = f"SELECT * FROM {db_name}.{tabel_name}"
|
72
|
+
cursor.execute(sql)
|
73
|
+
rows = cursor.fetchall() # 获取查询结果
|
74
|
+
columns = [desc[0] for desc in cursor.description]
|
75
|
+
df = pd.DataFrame(rows, columns=columns) # 转为 df
|
76
|
+
except Exception as e:
|
77
|
+
print(f'{e}')
|
78
|
+
return df
|
79
|
+
finally:
|
80
|
+
connection.close()
|
81
|
+
|
82
|
+
if len(df) == 0:
|
83
|
+
print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
|
84
|
+
return df
|
85
|
+
|
86
|
+
def check_infos(self, db_name, tabel_name) -> bool:
|
87
|
+
""" 检查数据库、数据表是否存在 """
|
39
88
|
connection = pymysql.connect(**self.config) # 连接数据库
|
40
89
|
try:
|
41
90
|
with connection.cursor() as cursor:
|
91
|
+
# 1. 检查数据库是否存在
|
42
92
|
cursor.execute(f"SHOW DATABASES LIKE '{db_name}'") # 检查数据库是否存在
|
43
93
|
database_exists = cursor.fetchone()
|
44
94
|
if not database_exists:
|
45
95
|
print(f"Database <{db_name}>: 数据库不存在")
|
96
|
+
return False
|
46
97
|
finally:
|
47
98
|
connection.close() # 这里要断开连接
|
48
|
-
time.sleep(0.2)
|
49
99
|
|
50
100
|
self.config.update({'database': db_name}) # 添加更新 config 字段
|
51
101
|
connection = pymysql.connect(**self.config) # 重新连接数据库
|
52
102
|
try:
|
53
103
|
with connection.cursor() as cursor:
|
54
|
-
#
|
104
|
+
# 2. 查询表是否存在
|
55
105
|
sql = f"SHOW TABLES LIKE '{tabel_name}'"
|
56
106
|
cursor.execute(sql)
|
57
107
|
if not cursor.fetchone():
|
58
108
|
print(f'{db_name} -> <{tabel_name}>: 表不存在')
|
59
|
-
return
|
60
|
-
|
61
|
-
# 查询列
|
62
|
-
for col in projection:
|
63
|
-
sql = ('SELECT 1 FROM information_schema.columns WHERE table_schema = %s AND table_name = %s AND '
|
64
|
-
'column_name = %s')
|
65
|
-
cursor.execute(sql, (db_name, {tabel_name}, col))
|
66
|
-
if cursor.fetchone() is None: # 移除不存在的列
|
67
|
-
projection.remove(col)
|
109
|
+
return False
|
110
|
+
return True
|
68
111
|
except Exception as e:
|
69
112
|
print(e)
|
70
|
-
return
|
113
|
+
return False
|
71
114
|
finally:
|
72
115
|
connection.close() # 断开连接
|
73
116
|
|
74
|
-
# before_time = time.time()
|
75
|
-
# 读取数据
|
76
|
-
self.config.update({'database': db_name})
|
77
|
-
connection = pymysql.connect(**self.config) # 重新连接数据库
|
78
|
-
try:
|
79
|
-
with connection.cursor() as cursor:
|
80
|
-
if not projection: # 如果未指定,则查询所有列,获取 cols_exist
|
81
|
-
sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
|
82
|
-
cursor.execute(sql, (db_name, {tabel_name}))
|
83
|
-
columns = cursor.fetchall()
|
84
|
-
cols_exist = [col['COLUMN_NAME'] for col in columns]
|
85
|
-
|
86
|
-
if '日期' in projection or '日期' in cols_exist: # 指定含日期的 projection 或者未指定 projection 但表中有日期列
|
87
|
-
sql = f"SELECT * FROM {db_name}.{tabel_name} WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
|
88
|
-
elif projection: # 指定未含日期的 projection
|
89
|
-
sql = f"SELECT '%s' FROM {db_name}.{tabel_name}" % (', '.join(projection))
|
90
|
-
else: # 未指定 projection 且表中无日期
|
91
|
-
sql = f"SELECT * FROM {db_name}.{tabel_name}"
|
92
|
-
cursor.execute(sql)
|
93
|
-
rows = cursor.fetchall() # 获取查询结果
|
94
|
-
columns = [desc[0] for desc in cursor.description]
|
95
|
-
df = pd.DataFrame(rows, columns=columns)
|
96
|
-
except Exception as e:
|
97
|
-
print(f'{e} {db_name} -> <{tabel_name}>: 表不存在')
|
98
|
-
return df
|
99
|
-
finally:
|
100
|
-
connection.close()
|
101
|
-
|
102
|
-
if len(df) == 0:
|
103
|
-
print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
|
104
|
-
# else:
|
105
|
-
# now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
106
|
-
# cost_time = int(time.time() - before_time)
|
107
|
-
# if cost_time < 1:
|
108
|
-
# cost_time = round(time.time() - before_time, 2)
|
109
|
-
# print(f'{now}mysql ({self.host}) 表: {tabel_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
|
110
|
-
return df
|
111
|
-
|
112
|
-
|
113
|
-
def year_month_day(start_date, end_date):
|
114
|
-
"""
|
115
|
-
使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
|
116
|
-
calendar.monthrange: 获取当月第一个工作日的星期值(0,6) 以及当月天数
|
117
|
-
"""
|
118
|
-
# 替换年月日中的日, 以便即使传入当月日期也有返回值
|
119
|
-
try:
|
120
|
-
start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
|
121
|
-
except Exception as e:
|
122
|
-
print(e)
|
123
|
-
return []
|
124
|
-
# 使用pandas的date_range创建一个日期范围,频率为'MS'代表每月开始
|
125
|
-
date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
|
126
|
-
# 转换格式
|
127
|
-
year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
|
128
|
-
|
129
|
-
results = []
|
130
|
-
for year_month in year_months:
|
131
|
-
year = re.findall(r'(\d{4})', year_month)[0]
|
132
|
-
month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
|
133
|
-
s, d = calendar.monthrange(int(year), int(month))
|
134
|
-
results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
|
135
|
-
|
136
|
-
return results # start_date至end_date之间的所有年月日
|
137
|
-
|
138
|
-
|
139
|
-
def download_datas(tabel_name, save_path, start_date):
|
140
|
-
username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
|
141
|
-
print(username, password, host, port)
|
142
|
-
m = MysqlUpload(username=username, password=password, host=host, port=port)
|
143
|
-
m.port = port
|
144
|
-
results = year_month_day(start_date=start_date, end_date='today')
|
145
|
-
# print(results)
|
146
|
-
for result in results:
|
147
|
-
start_date = result['起始日期']
|
148
|
-
end_date = result['结束日期']
|
149
|
-
# print(start_date, end_date)
|
150
|
-
df = m.data_to_df(db_name='市场数据2', tabel_name=tabel_name, start_date=start_date, end_date=end_date)
|
151
|
-
if len(df) == 0:
|
152
|
-
continue
|
153
|
-
path = os.path.join(save_path, f'{tabel_name}_{str(start_date)}_{str(end_date)}.csv')
|
154
|
-
df['日期'] = df['日期'].apply(lambda x: re.sub(' .*', '', str(x)))
|
155
|
-
df.to_csv(path, index=False, encoding='utf-8_sig', header=True)
|
156
|
-
|
157
117
|
|
158
118
|
if __name__ == '__main__':
|
159
|
-
# username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
|
160
|
-
# print(username, password, host, port)
|
161
|
-
|
162
119
|
username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
|
163
|
-
|
164
|
-
df = qd.data_to_df(db_name='市场数据2', tabel_name='市场排行_店铺', start_date='2024-08-13', end_date='2024-08-31')
|
165
|
-
print(df)
|
120
|
+
print(username, password, host, port)
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# -*- coding:utf-8 -*-
|
2
|
+
import warnings
|
3
|
+
import pandas as pd
|
4
|
+
import calendar
|
5
|
+
|
6
|
+
warnings.filterwarnings('ignore')
|
7
|
+
|
8
|
+
|
9
|
+
def year_month_day(start_date, end_date):
|
10
|
+
"""
|
11
|
+
使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
|
12
|
+
calendar.monthrange: 获取当月第一个工作日的星期值(0,6) 以及当月天数
|
13
|
+
返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
|
14
|
+
"""
|
15
|
+
# 替换年月日中的日, 以便即使传入当月日期也有返回值
|
16
|
+
try:
|
17
|
+
start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
|
18
|
+
except Exception as e:
|
19
|
+
print(e)
|
20
|
+
return []
|
21
|
+
# 使用pandas的date_range创建一个日期范围,频率为'MS'代表每月开始
|
22
|
+
date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
|
23
|
+
# 转换格式
|
24
|
+
year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
|
25
|
+
|
26
|
+
results = []
|
27
|
+
for year_month in year_months:
|
28
|
+
year = re.findall(r'(\d{4})', year_month)[0]
|
29
|
+
month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
|
30
|
+
s, d = calendar.monthrange(int(year), int(month))
|
31
|
+
results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
|
32
|
+
|
33
|
+
return results # start_date至end_date之间的所有年月日
|
34
|
+
|
35
|
+
|
36
|
+
if __name__ == '__main__':
|
37
|
+
results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
|
38
|
+
print(results)
|
@@ -2,7 +2,7 @@ mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
|
2
2
|
mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
3
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
4
4
|
mdbq/aggregation/aggregation.py,sha256=RHQa2rs4fimRvJzluujErg6I8fn7s9q1-kwC2bPZohE,59439
|
5
|
-
mdbq/aggregation/query_data.py,sha256=
|
5
|
+
mdbq/aggregation/query_data.py,sha256=Yb-gUPBm8r93oKFDiZ0-DiGhaqeL-jA7eAfZG0__DrA,11235
|
6
6
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
7
7
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
8
8
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
@@ -18,7 +18,8 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
|
|
18
18
|
mdbq/mongo/mongo.py,sha256=hF93-kP2lxK4WY1KCdBBszLQ_I7W0mQQxZ7t4qU2w3A,32930
|
19
19
|
mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
20
20
|
mdbq/mysql/mysql.py,sha256=H9onFYKSYRjdXghK_29Aj7vgvUgDHexJjIECrdxLbE0,29925
|
21
|
-
mdbq/mysql/s_query.py,sha256=
|
21
|
+
mdbq/mysql/s_query.py,sha256=bRnW8Cpy4fSsbMhzGCvjiK2kin9uamVumJC3nLAyjMg,5213
|
22
|
+
mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
|
22
23
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
23
24
|
mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
|
24
25
|
mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
|
@@ -27,7 +28,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
|
27
28
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
28
29
|
mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
|
29
30
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
30
|
-
mdbq-0.0.
|
31
|
-
mdbq-0.0.
|
32
|
-
mdbq-0.0.
|
33
|
-
mdbq-0.0.
|
31
|
+
mdbq-0.0.7.dist-info/METADATA,sha256=nlphrFIJBQKjQnQvqtyK5NDTCGNTIOXRvxZdDGmiX20,245
|
32
|
+
mdbq-0.0.7.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
|
33
|
+
mdbq-0.0.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
34
|
+
mdbq-0.0.7.dist-info/RECORD,,
|
File without changes
|