mdbq 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/__version__.py +3 -0
- mdbq/aggregation/aggregation.py +17 -26
- mdbq/aggregation/query_data.py +266 -0
- mdbq/mysql/s_query.py +120 -0
- mdbq/mysql/year_month_day.py +38 -0
- mdbq/other/{xigua_porxy.py → porxy.py} +0 -3
- {mdbq-0.0.5.dist-info → mdbq-0.0.7.dist-info}/METADATA +1 -1
- {mdbq-0.0.5.dist-info → mdbq-0.0.7.dist-info}/RECORD +10 -6
- {mdbq-0.0.5.dist-info → mdbq-0.0.7.dist-info}/WHEEL +1 -1
- {mdbq-0.0.5.dist-info → mdbq-0.0.7.dist-info}/top_level.txt +0 -0
mdbq/__version__.py
ADDED
mdbq/aggregation/aggregation.py
CHANGED
@@ -960,7 +960,22 @@ def main():
|
|
960
960
|
# print(d.datas)
|
961
961
|
|
962
962
|
|
963
|
+
def update_dtypte():
|
964
|
+
""" 更新一个文件的 dtype 信息到 json 文件 """
|
965
|
+
file = '/Users/xigua/数据中心/原始文件2/月数据/流量来源/【生意参谋平台】无线店铺流量来源-2023-04-01_2023-04-30.csv'
|
966
|
+
df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
|
967
|
+
d = DataTypes()
|
968
|
+
d.read_dtypes(
|
969
|
+
df=df,
|
970
|
+
db_name='生意参谋数据2',
|
971
|
+
collection_name='店铺来源_月数据',
|
972
|
+
is_file_dtype=False, # 关闭文件优先
|
973
|
+
)
|
974
|
+
d.dtypes_to_file()
|
975
|
+
|
976
|
+
|
963
977
|
def upload():
|
978
|
+
""" 上传一个文件夹到数据库 """
|
964
979
|
path = '/Users/xigua/数据中心/原始文件2/生意参谋/客户_客户概况_画像'
|
965
980
|
db_name = '生意参谋数据2'
|
966
981
|
collection_name = '客户_客户概况_画像'
|
@@ -995,7 +1010,7 @@ def upload():
|
|
995
1010
|
# print(dtypes)
|
996
1011
|
for root, dirs, files in os.walk(path, topdown=False):
|
997
1012
|
for name in files:
|
998
|
-
if '~$' in name or '.DS' in name or '.localized' in name or '
|
1013
|
+
if '~$' in name or '.DS' in name or '.localized' in name or 'baidu' in name:
|
999
1014
|
continue
|
1000
1015
|
if name.endswith('.csv'):
|
1001
1016
|
# print(name)
|
@@ -1005,7 +1020,7 @@ def upload():
|
|
1005
1020
|
continue
|
1006
1021
|
for col in df.columns.tolist():
|
1007
1022
|
df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
|
1008
|
-
df.replace(to_replace=['--'], value='', regex=False, inplace=True)
|
1023
|
+
# df.replace(to_replace=['--'], value='', regex=False, inplace=True)
|
1009
1024
|
df = dt.convert_df_cols(df=df)
|
1010
1025
|
try:
|
1011
1026
|
df = df.astype(dtypes)
|
@@ -1032,29 +1047,5 @@ if __name__ == '__main__':
|
|
1032
1047
|
username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
|
1033
1048
|
print(username, password, host, port)
|
1034
1049
|
|
1035
|
-
d = DatabaseUpdate(path='/Users/xigua/Downloads')
|
1036
|
-
d.upload_df(service_databases=[{'home_lx': 'mongodb'},])
|
1037
|
-
|
1038
1050
|
# main()
|
1039
1051
|
# upload()
|
1040
|
-
# path = '/Users/xigua/数据中心/原始文件2/月数据/流量来源-自助取数-月数据'
|
1041
|
-
# for root, dirs, files in os.walk(path, topdown=False):
|
1042
|
-
# for name in files:
|
1043
|
-
# if name.endswith('.csv') and 'baidu' not in name:
|
1044
|
-
# with open(os.path.join(root, name), 'rb') as f:
|
1045
|
-
# f1 = f.read()
|
1046
|
-
# encod = chardet.detect(f1).get('encoding')
|
1047
|
-
# print(name, encod)
|
1048
|
-
# # df = pd.read_csv(os.path.join(root, name), encoding=encod, header=0, na_filter=False)
|
1049
|
-
# # df.to_csv(os.path.join(root, name), index=False, encoding='utf-8_sig', header=True)
|
1050
|
-
|
1051
|
-
# file = '/Users/xigua/数据中心/原始文件2/月数据/流量来源/【生意参谋平台】无线店铺流量来源-2023-04-01_2023-04-30.csv'
|
1052
|
-
# df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
|
1053
|
-
# d = DataTypes()
|
1054
|
-
# d.read_dtypes(
|
1055
|
-
# df=df,
|
1056
|
-
# db_name='生意参谋数据2',
|
1057
|
-
# collection_name='店铺来源_月数据',
|
1058
|
-
# is_file_dtype=False, # 关闭文件优先
|
1059
|
-
# )
|
1060
|
-
# d.dtypes_to_file()
|
@@ -0,0 +1,266 @@
|
|
1
|
+
# -*- coding: UTF-8 –*-
|
2
|
+
from mdbq.mongo import mongo
|
3
|
+
from mdbq.mysql import s_query
|
4
|
+
from mdbq.config import get_myconf
|
5
|
+
import datetime
|
6
|
+
from dateutil.relativedelta import relativedelta
|
7
|
+
import pandas as pd
|
8
|
+
import numpy as np
|
9
|
+
import platform
|
10
|
+
import getpass
|
11
|
+
import json
|
12
|
+
import os
|
13
|
+
|
14
|
+
|
15
|
+
class MongoDatasQuery:
|
16
|
+
"""
|
17
|
+
从 数据库 中下载数据
|
18
|
+
self.output: 数据库默认导出目录
|
19
|
+
self.is_maximize: 是否最大转化数据
|
20
|
+
"""
|
21
|
+
def __init__(self, target_service):
|
22
|
+
# target_service 从哪个服务器下载数据
|
23
|
+
self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
24
|
+
# 实例化一个下载类
|
25
|
+
username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
|
26
|
+
self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
|
27
|
+
|
28
|
+
def tg_wxt(self):
|
29
|
+
self.download.start_date, self.download.end_date = self.months_data(num=self.months)
|
30
|
+
projection = {
|
31
|
+
'日期': 1,
|
32
|
+
'场景名字': 1,
|
33
|
+
'主体id': 1,
|
34
|
+
'花费': 1,
|
35
|
+
'展现量': 1,
|
36
|
+
'点击量': 1,
|
37
|
+
'总购物车数': 1,
|
38
|
+
'总成交笔数': 1,
|
39
|
+
'总成交金额': 1,
|
40
|
+
'自然流量曝光量': 1,
|
41
|
+
'直接成交笔数': 1,
|
42
|
+
'直接成交金额': 1,
|
43
|
+
}
|
44
|
+
df = self.download.data_to_df(
|
45
|
+
db_name='天猫数据2',
|
46
|
+
collection_name='推广数据_宝贝主体报表',
|
47
|
+
projection=projection,
|
48
|
+
)
|
49
|
+
return df
|
50
|
+
|
51
|
+
@staticmethod
|
52
|
+
def days_data(days, end_date=None):
|
53
|
+
""" 读取近 days 天的数据 """
|
54
|
+
if not end_date:
|
55
|
+
end_date = datetime.datetime.now()
|
56
|
+
start_date = end_date - datetime.timedelta(days=days)
|
57
|
+
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
58
|
+
|
59
|
+
@staticmethod
|
60
|
+
def months_data(num=0, end_date=None):
|
61
|
+
""" 读取近 num 个月的数据, 0 表示读取当月的数据 """
|
62
|
+
if not end_date:
|
63
|
+
end_date = datetime.datetime.now()
|
64
|
+
start_date = end_date - relativedelta(months=num) # n 月以前的今天
|
65
|
+
start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
|
66
|
+
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
67
|
+
|
68
|
+
|
69
|
+
class MysqlDatasQuery:
|
70
|
+
"""
|
71
|
+
从数据库中下载数据
|
72
|
+
"""
|
73
|
+
def __init__(self, target_service):
|
74
|
+
# target_service 从哪个服务器下载数据
|
75
|
+
self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
|
76
|
+
# 实例化一个下载类
|
77
|
+
username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
|
78
|
+
self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
|
79
|
+
|
80
|
+
def tg_wxt(self):
|
81
|
+
start_date, end_date = self.months_data(num=self.months)
|
82
|
+
projection = {
|
83
|
+
'日期': 1,
|
84
|
+
'场景名字': 1,
|
85
|
+
'主体id': 1,
|
86
|
+
'花费': 1,
|
87
|
+
'展现量': 1,
|
88
|
+
'点击量': 1,
|
89
|
+
'总购物车数': 1,
|
90
|
+
'总成交笔数': 1,
|
91
|
+
'总成交金额': 1,
|
92
|
+
'自然流量曝光量': 1,
|
93
|
+
'直接成交笔数': 1,
|
94
|
+
'直接成交金额': 1,
|
95
|
+
}
|
96
|
+
df = self.download.data_to_df(
|
97
|
+
db_name='天猫数据2',
|
98
|
+
tabel_name='推广数据_宝贝主体报表',
|
99
|
+
start_date=start_date,
|
100
|
+
end_date=end_date,
|
101
|
+
projection=projection,
|
102
|
+
)
|
103
|
+
return df
|
104
|
+
|
105
|
+
@staticmethod
|
106
|
+
def months_data(num=0, end_date=None):
|
107
|
+
""" 读取近 num 个月的数据, 0 表示读取当月的数据 """
|
108
|
+
if not end_date:
|
109
|
+
end_date = datetime.datetime.now()
|
110
|
+
start_date = end_date - relativedelta(months=num) # n 月以前的今天
|
111
|
+
start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
|
112
|
+
return pd.to_datetime(start_date), pd.to_datetime(end_date)
|
113
|
+
|
114
|
+
|
115
|
+
class GroupBy:
|
116
|
+
""" 数据聚合和导出 """
|
117
|
+
def __init__(self):
|
118
|
+
# self.output: 数据库默认导出目录
|
119
|
+
if platform.system() == 'Darwin':
|
120
|
+
self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
|
121
|
+
elif platform.system() == 'Windows':
|
122
|
+
self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
|
123
|
+
else:
|
124
|
+
self.output = os.path.join('数据中心/数据库导出')
|
125
|
+
|
126
|
+
def groupby(self, df, tabel_name, is_maximize=True):
|
127
|
+
"""
|
128
|
+
self.is_maximize: 是否最大转化数据
|
129
|
+
"""
|
130
|
+
if '宝贝主体报表' in tabel_name:
|
131
|
+
df.rename(columns={
|
132
|
+
'场景名字': '营销场景',
|
133
|
+
'主体id': '商品id',
|
134
|
+
'总购物车数': '加购量',
|
135
|
+
'总成交笔数': '成交笔数',
|
136
|
+
'总成交金额': '成交金额'
|
137
|
+
}, inplace=True)
|
138
|
+
df = df.astype({
|
139
|
+
'花费': float,
|
140
|
+
'展现量': int,
|
141
|
+
'点击量': int,
|
142
|
+
'加购量': int,
|
143
|
+
'成交笔数': int,
|
144
|
+
'成交金额': float,
|
145
|
+
'自然流量曝光量': int,
|
146
|
+
'直接成交笔数': int,
|
147
|
+
'直接成交金额': float,
|
148
|
+
}, errors='raise')
|
149
|
+
df.fillna(0, inplace=True)
|
150
|
+
if is_maximize:
|
151
|
+
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
152
|
+
**{'加购量': ('加购量', np.max),
|
153
|
+
'成交笔数': ('成交笔数', np.max),
|
154
|
+
'成交金额': ('成交金额', np.max),
|
155
|
+
'自然流量曝光量': ('自然流量曝光量', np.max),
|
156
|
+
'直接成交笔数': ('直接成交笔数', np.max),
|
157
|
+
'直接成交金额': ('直接成交金额', np.max)
|
158
|
+
}
|
159
|
+
)
|
160
|
+
else:
|
161
|
+
df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
|
162
|
+
**{'加购量': ('加购量', np.min),
|
163
|
+
'成交笔数': ('成交笔数', np.min),
|
164
|
+
'成交金额': ('成交金额', np.min),
|
165
|
+
'自然流量曝光量': ('自然流量曝光量', np.min),
|
166
|
+
'直接成交笔数': ('直接成交笔数', np.max),
|
167
|
+
'直接成交金额': ('直接成交金额', np.max)
|
168
|
+
}
|
169
|
+
)
|
170
|
+
df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
|
171
|
+
return df
|
172
|
+
|
173
|
+
def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
|
174
|
+
index=False, header=True, st_ascend=None, ascend=None, freq=None):
|
175
|
+
"""
|
176
|
+
path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹,可以不传,
|
177
|
+
st_ascend: 排序参数 ['column1', 'column2']
|
178
|
+
ascend: 升降序 [True, False]
|
179
|
+
freq: 将创建子文件夹并按月分类存储, freq='Y', 或 freq='M'
|
180
|
+
"""
|
181
|
+
if len(df) == 0:
|
182
|
+
return
|
183
|
+
if not path:
|
184
|
+
path = self.output
|
185
|
+
else:
|
186
|
+
path = os.path.join(self.output, path)
|
187
|
+
if not os.path.exists(path):
|
188
|
+
os.makedirs(path)
|
189
|
+
if st_ascend and ascend:
|
190
|
+
try:
|
191
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
192
|
+
except:
|
193
|
+
print(f'{filename}: sort_values排序参数错误!')
|
194
|
+
if freq:
|
195
|
+
if '日期' not in df.columns.tolist():
|
196
|
+
return print(f'{filename}: 数据缺少日期列,无法按日期分组')
|
197
|
+
groups = df.groupby(pd.Grouper(key='日期', freq=freq))
|
198
|
+
for name1, df in groups:
|
199
|
+
if freq == 'M':
|
200
|
+
sheet_name = name1.strftime('%Y-%m')
|
201
|
+
elif freq == 'Y':
|
202
|
+
sheet_name = name1.strftime('%Y年')
|
203
|
+
else:
|
204
|
+
sheet_name = '_未分类'
|
205
|
+
new_path = os.path.join(path, filename)
|
206
|
+
if not os.path.exists(new_path):
|
207
|
+
os.makedirs(new_path)
|
208
|
+
new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
|
209
|
+
if st_ascend and ascend: # 这里需要重新排序一次,原因未知
|
210
|
+
try:
|
211
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
212
|
+
except:
|
213
|
+
print(f'{filename}: sort_values排序参数错误!')
|
214
|
+
|
215
|
+
df.to_csv(new_path, encoding=encoding, index=index, header=header)
|
216
|
+
else:
|
217
|
+
df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
|
218
|
+
|
219
|
+
def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
|
220
|
+
if len(df) == 0:
|
221
|
+
return
|
222
|
+
if not path:
|
223
|
+
path = self.output
|
224
|
+
else:
|
225
|
+
path = os.path.join(self.output, path)
|
226
|
+
if not os.path.exists(path):
|
227
|
+
os.makedirs(path)
|
228
|
+
if st_ascend and ascend:
|
229
|
+
try:
|
230
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
231
|
+
except:
|
232
|
+
print(f'{filename}: sort_values排序参数错误!')
|
233
|
+
df.to_json(os.path.join(path, filename + '.json'),
|
234
|
+
orient=orient, force_ascii=force_ascii)
|
235
|
+
|
236
|
+
def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
|
237
|
+
freeze_panes=(1, 0), st_ascend=None, ascend=None):
|
238
|
+
if len(df) == 0:
|
239
|
+
return
|
240
|
+
if not path:
|
241
|
+
path = self.output
|
242
|
+
else:
|
243
|
+
path = os.path.join(self.output, path)
|
244
|
+
if not os.path.exists(path):
|
245
|
+
os.makedirs(path)
|
246
|
+
if st_ascend and ascend:
|
247
|
+
try:
|
248
|
+
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
249
|
+
except:
|
250
|
+
print(f'{filename}: sort_values排序参数错误!')
|
251
|
+
df.to_excel(os.path.join(path, filename + '.xlsx'),
|
252
|
+
index=index, header=header, engine=engine, freeze_panes=freeze_panes)
|
253
|
+
|
254
|
+
|
255
|
+
def main():
|
256
|
+
sdq = MysqlDatasQuery(target_service='home_lx')
|
257
|
+
sdq.months = 0
|
258
|
+
df = sdq.tg_wxt() # 从数据库中获取数据并转为 df
|
259
|
+
|
260
|
+
g = GroupBy() # 数据聚合
|
261
|
+
df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
|
262
|
+
g.as_csv(df=df, filename='test') # 数据导出
|
263
|
+
|
264
|
+
|
265
|
+
if __name__ == '__main__':
|
266
|
+
main()
|
mdbq/mysql/s_query.py
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
# -*- coding:utf-8 -*-
|
2
|
+
import datetime
|
3
|
+
import platform
|
4
|
+
import re
|
5
|
+
import time
|
6
|
+
from functools import wraps
|
7
|
+
import warnings
|
8
|
+
import pymysql
|
9
|
+
import numpy as np
|
10
|
+
import pandas as pd
|
11
|
+
from sqlalchemy import create_engine
|
12
|
+
import os
|
13
|
+
import calendar
|
14
|
+
from mdbq.config import get_myconf
|
15
|
+
|
16
|
+
warnings.filterwarnings('ignore')
|
17
|
+
|
18
|
+
|
19
|
+
class QueryDatas:
|
20
|
+
def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
|
21
|
+
self.username = username
|
22
|
+
self.password = password
|
23
|
+
self.host = host
|
24
|
+
self.port = port
|
25
|
+
self.config = {
|
26
|
+
'host': self.host,
|
27
|
+
'port': self.port,
|
28
|
+
'user': self.username,
|
29
|
+
'password': self.password,
|
30
|
+
'charset': charset, # utf8mb4 支持存储四字节的UTF-8字符集
|
31
|
+
'cursorclass': pymysql.cursors.DictCursor,
|
32
|
+
}
|
33
|
+
|
34
|
+
def data_to_df(self, db_name, tabel_name, start_date, end_date, projection: dict=[]):
|
35
|
+
|
36
|
+
start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
|
37
|
+
end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
|
38
|
+
df = pd.DataFrame() # 初始化df
|
39
|
+
|
40
|
+
if self.check_infos(db_name, tabel_name) == False:
|
41
|
+
return df
|
42
|
+
|
43
|
+
self.config.update({'database': db_name})
|
44
|
+
connection = pymysql.connect(**self.config) # 重新连接数据库
|
45
|
+
try:
|
46
|
+
with connection.cursor() as cursor:
|
47
|
+
# 3. 获取数据表的所有列信息
|
48
|
+
sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
|
49
|
+
cursor.execute(sql, (db_name, {tabel_name}))
|
50
|
+
columns = cursor.fetchall()
|
51
|
+
cols_exist = [col['COLUMN_NAME'] for col in columns] # 数据表的所有列, 返回 list
|
52
|
+
|
53
|
+
# 4. 构建 SQL 查询语句
|
54
|
+
if projection: # 获取指定列
|
55
|
+
columns_in = []
|
56
|
+
for key, value in projection.items():
|
57
|
+
if value == 1 and key in cols_exist:
|
58
|
+
columns_in.append(key) # 提取值为 1 的键并清理不在数据表的键
|
59
|
+
columns_in = ', '.join(columns_in)
|
60
|
+
if '日期' in cols_exist: # 不论是否指定, 只要数据表有日期,则执行
|
61
|
+
sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
|
62
|
+
f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
|
63
|
+
else: # 数据表没有日期列时,返回指定列的所有数据
|
64
|
+
sql = f"SELECT {columns_in} FROM {db_name}.{tabel_name}"
|
65
|
+
else: # 没有指定获取列时
|
66
|
+
if '日期' in cols_exist: # 但数据表有日期,仍然执行
|
67
|
+
columns_in = ', '.join(cols_exist)
|
68
|
+
sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
|
69
|
+
f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
|
70
|
+
else: # 没有指定获取列,且数据表也没有日期列,则返回全部列的全部数据
|
71
|
+
sql = f"SELECT * FROM {db_name}.{tabel_name}"
|
72
|
+
cursor.execute(sql)
|
73
|
+
rows = cursor.fetchall() # 获取查询结果
|
74
|
+
columns = [desc[0] for desc in cursor.description]
|
75
|
+
df = pd.DataFrame(rows, columns=columns) # 转为 df
|
76
|
+
except Exception as e:
|
77
|
+
print(f'{e}')
|
78
|
+
return df
|
79
|
+
finally:
|
80
|
+
connection.close()
|
81
|
+
|
82
|
+
if len(df) == 0:
|
83
|
+
print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
|
84
|
+
return df
|
85
|
+
|
86
|
+
def check_infos(self, db_name, tabel_name) -> bool:
|
87
|
+
""" 检查数据库、数据表是否存在 """
|
88
|
+
connection = pymysql.connect(**self.config) # 连接数据库
|
89
|
+
try:
|
90
|
+
with connection.cursor() as cursor:
|
91
|
+
# 1. 检查数据库是否存在
|
92
|
+
cursor.execute(f"SHOW DATABASES LIKE '{db_name}'") # 检查数据库是否存在
|
93
|
+
database_exists = cursor.fetchone()
|
94
|
+
if not database_exists:
|
95
|
+
print(f"Database <{db_name}>: 数据库不存在")
|
96
|
+
return False
|
97
|
+
finally:
|
98
|
+
connection.close() # 这里要断开连接
|
99
|
+
|
100
|
+
self.config.update({'database': db_name}) # 添加更新 config 字段
|
101
|
+
connection = pymysql.connect(**self.config) # 重新连接数据库
|
102
|
+
try:
|
103
|
+
with connection.cursor() as cursor:
|
104
|
+
# 2. 查询表是否存在
|
105
|
+
sql = f"SHOW TABLES LIKE '{tabel_name}'"
|
106
|
+
cursor.execute(sql)
|
107
|
+
if not cursor.fetchone():
|
108
|
+
print(f'{db_name} -> <{tabel_name}>: 表不存在')
|
109
|
+
return False
|
110
|
+
return True
|
111
|
+
except Exception as e:
|
112
|
+
print(e)
|
113
|
+
return False
|
114
|
+
finally:
|
115
|
+
connection.close() # 断开连接
|
116
|
+
|
117
|
+
|
118
|
+
if __name__ == '__main__':
|
119
|
+
username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
|
120
|
+
print(username, password, host, port)
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# -*- coding:utf-8 -*-
|
2
|
+
import warnings
|
3
|
+
import pandas as pd
|
4
|
+
import calendar
|
5
|
+
|
6
|
+
warnings.filterwarnings('ignore')
|
7
|
+
|
8
|
+
|
9
|
+
def year_month_day(start_date, end_date):
|
10
|
+
"""
|
11
|
+
使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
|
12
|
+
calendar.monthrange: 获取当月第一个工作日的星期值(0,6) 以及当月天数
|
13
|
+
返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
|
14
|
+
"""
|
15
|
+
# 替换年月日中的日, 以便即使传入当月日期也有返回值
|
16
|
+
try:
|
17
|
+
start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
|
18
|
+
except Exception as e:
|
19
|
+
print(e)
|
20
|
+
return []
|
21
|
+
# 使用pandas的date_range创建一个日期范围,频率为'MS'代表每月开始
|
22
|
+
date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
|
23
|
+
# 转换格式
|
24
|
+
year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
|
25
|
+
|
26
|
+
results = []
|
27
|
+
for year_month in year_months:
|
28
|
+
year = re.findall(r'(\d{4})', year_month)[0]
|
29
|
+
month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
|
30
|
+
s, d = calendar.monthrange(int(year), int(month))
|
31
|
+
results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
|
32
|
+
|
33
|
+
return results # start_date至end_date之间的所有年月日
|
34
|
+
|
35
|
+
|
36
|
+
if __name__ == '__main__':
|
37
|
+
results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
|
38
|
+
print(results)
|
@@ -1,6 +1,8 @@
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
2
|
+
mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
2
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
3
|
-
mdbq/aggregation/aggregation.py,sha256=
|
4
|
+
mdbq/aggregation/aggregation.py,sha256=RHQa2rs4fimRvJzluujErg6I8fn7s9q1-kwC2bPZohE,59439
|
5
|
+
mdbq/aggregation/query_data.py,sha256=Yb-gUPBm8r93oKFDiZ0-DiGhaqeL-jA7eAfZG0__DrA,11235
|
4
6
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
5
7
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
6
8
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
@@ -16,15 +18,17 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
|
|
16
18
|
mdbq/mongo/mongo.py,sha256=hF93-kP2lxK4WY1KCdBBszLQ_I7W0mQQxZ7t4qU2w3A,32930
|
17
19
|
mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
18
20
|
mdbq/mysql/mysql.py,sha256=H9onFYKSYRjdXghK_29Aj7vgvUgDHexJjIECrdxLbE0,29925
|
21
|
+
mdbq/mysql/s_query.py,sha256=bRnW8Cpy4fSsbMhzGCvjiK2kin9uamVumJC3nLAyjMg,5213
|
22
|
+
mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
|
19
23
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
24
|
+
mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
|
20
25
|
mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
|
21
26
|
mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
|
22
|
-
mdbq/other/xigua_porxy.py,sha256=zTOxsdkdDAyGfHWPUm_7WIztjrGExONAwvPzTaC7Rho,5007
|
23
27
|
mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
24
28
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
25
29
|
mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
|
26
30
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
27
|
-
mdbq-0.0.
|
28
|
-
mdbq-0.0.
|
29
|
-
mdbq-0.0.
|
30
|
-
mdbq-0.0.
|
31
|
+
mdbq-0.0.7.dist-info/METADATA,sha256=nlphrFIJBQKjQnQvqtyK5NDTCGNTIOXRvxZdDGmiX20,245
|
32
|
+
mdbq-0.0.7.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
|
33
|
+
mdbq-0.0.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
34
|
+
mdbq-0.0.7.dist-info/RECORD,,
|
File without changes
|