mdbq 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,20 +20,13 @@ class MongoDatasQuery:
20
20
  """
21
21
  def __init__(self, target_service):
22
22
  # target_service 从哪个服务器下载数据
23
- self.is_maximize = True
24
- if platform.system() == 'Darwin':
25
- self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
26
- elif platform.system() == 'Windows':
27
- self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
28
- else:
29
- self.output = os.path.join('数据中心/数据库导出')
30
-
23
+ self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
31
24
  # 实例化一个下载类
32
25
  username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
33
26
  self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
34
27
 
35
28
  def tg_wxt(self):
36
- self.download.start_date, self.download.end_date = self.months_data(num=1)
29
+ self.download.start_date, self.download.end_date = self.months_data(num=self.months)
37
30
  projection = {
38
31
  '日期': 1,
39
32
  '场景名字': 1,
@@ -48,48 +41,11 @@ class MongoDatasQuery:
48
41
  '直接成交笔数': 1,
49
42
  '直接成交金额': 1,
50
43
  }
51
- df = self.download.data_to_df(db_name='天猫数据2', collection_name='推广数据_宝贝主体报表', projection=projection)
52
- df.rename(columns={
53
- '场景名字': '营销场景',
54
- '主体id': '商品id',
55
- '总购物车数': '加购量',
56
- '总成交笔数': '成交笔数',
57
- '总成交金额': '成交金额'
58
- }, inplace=True)
59
- df = df.astype({
60
- '花费': float,
61
- '展现量': int,
62
- '点击量': int,
63
- '加购量': int,
64
- '成交笔数': int,
65
- '成交金额': float,
66
- '自然流量曝光量': int,
67
- '直接成交笔数': int,
68
- '直接成交金额': float,
69
- }, errors='raise')
70
- df.fillna(0, inplace=True)
71
- if self.is_maximize:
72
- df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
73
- **{'加购量': ('加购量', np.max),
74
- '成交笔数': ('成交笔数', np.max),
75
- '成交金额': ('成交金额', np.max),
76
- '自然流量曝光量': ('自然流量曝光量', np.max),
77
- '直接成交笔数': ('直接成交笔数', np.max),
78
- '直接成交金额': ('直接成交金额', np.max)
79
- }
80
- )
81
- else:
82
- df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
83
- **{'加购量': ('加购量', np.min),
84
- '成交笔数': ('成交笔数', np.min),
85
- '成交金额': ('成交金额', np.min),
86
- '自然流量曝光量': ('自然流量曝光量', np.min),
87
- '直接成交笔数': ('直接成交笔数', np.max),
88
- '直接成交金额': ('直接成交金额', np.max)
89
- }
90
- )
91
- df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
92
- # print(df)
44
+ df = self.download.data_to_df(
45
+ db_name='天猫数据2',
46
+ collection_name='推广数据_宝贝主体报表',
47
+ projection=projection,
48
+ )
93
49
  return df
94
50
 
95
51
  @staticmethod
@@ -109,112 +65,43 @@ class MongoDatasQuery:
109
65
  start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
110
66
  return pd.to_datetime(start_date), pd.to_datetime(end_date)
111
67
 
112
- def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
113
- index=False, header=True, st_ascend=None, ascend=None, freq=None):
114
- """
115
- path: 子文件夹,可以不传,默认导出目录 self.output
116
- st_ascend: 排序参数
117
- ascend: 升降序
118
- freq: 将创建子文件夹并按月分类存储, freq='Y',或 freq='M'
119
- """
120
- if not path:
121
- path = self.output
122
- else:
123
- path = os.path.join(self.output, path)
124
- if not os.path.exists(path):
125
- os.makedirs(path)
126
- if st_ascend and ascend:
127
- try:
128
- df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
129
- except:
130
- print(f'{filename}: sort_values排序参数错误!')
131
- if freq:
132
- if '日期' not in df.columns.tolist():
133
- return print(f'{filename}: 数据缺少日期列,无法按日期分组')
134
- groups = df.groupby(pd.Grouper(key='日期', freq=freq))
135
- for name1, df in groups:
136
- if freq == 'M':
137
- sheet_name = name1.strftime('%Y-%m')
138
- elif freq == 'Y':
139
- sheet_name = name1.strftime('%Y年')
140
- else:
141
- sheet_name = '_未分类'
142
- new_path = os.path.join(path, filename)
143
- if not os.path.exists(new_path):
144
- os.makedirs(new_path)
145
- new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
146
- if st_ascend and ascend: # 这里需要重新排序一次,原因未知
147
- try:
148
- df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
149
- except:
150
- print(f'{filename}: sort_values排序参数错误!')
151
-
152
- df.to_csv(new_path, encoding=encoding, index=index, header=header)
153
- else:
154
- df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
155
-
156
- def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
157
- if not path:
158
- path = self.output
159
- else:
160
- path = os.path.join(self.output, path)
161
- if st_ascend and ascend:
162
- try:
163
- df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
164
- except:
165
- print(f'{filename}: sort_values排序参数错误!')
166
- df.to_json(os.path.join(path, filename + '.json'),
167
- orient=orient, force_ascii=force_ascii)
168
-
169
- def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
170
- freeze_panes=(1, 0), st_ascend=None, ascend=None):
171
- if not path:
172
- path = self.output
173
- else:
174
- path = os.path.join(self.output, path)
175
- if st_ascend and ascend:
176
- try:
177
- df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
178
- except:
179
- print(f'{filename}: sort_values排序参数错误!')
180
- df.to_excel(os.path.join(path, filename + '.xlsx'),
181
- index=index, header=header, engine=engine, freeze_panes=freeze_panes)
182
-
183
68
 
184
69
  class MysqlDatasQuery:
185
70
  """
186
- 从 数据库 中下载数据
187
- self.output: 数据库默认导出目录
188
- self.is_maximize: 是否最大转化数据
71
+ 从数据库中下载数据
189
72
  """
190
73
  def __init__(self, target_service):
191
74
  # target_service 从哪个服务器下载数据
192
- self.is_maximize = True
193
- if platform.system() == 'Darwin':
194
- self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
195
- elif platform.system() == 'Windows':
196
- self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
197
- else:
198
- self.output = os.path.join('数据中心/数据库导出')
199
- self.months = 1 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
200
-
75
+ self.months = 0 # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
201
76
  # 实例化一个下载类
202
77
  username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
203
78
  self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
204
79
 
205
80
  def tg_wxt(self):
206
81
  start_date, end_date = self.months_data(num=self.months)
207
- df = self.download.data_to_df(db_name='天猫数据2', tabel_name='推广数据_宝贝主体报表', start_date=start_date, end_date=end_date)
82
+ projection = {
83
+ '日期': 1,
84
+ '场景名字': 1,
85
+ '主体id': 1,
86
+ '花费': 1,
87
+ '展现量': 1,
88
+ '点击量': 1,
89
+ '总购物车数': 1,
90
+ '总成交笔数': 1,
91
+ '总成交金额': 1,
92
+ '自然流量曝光量': 1,
93
+ '直接成交笔数': 1,
94
+ '直接成交金额': 1,
95
+ }
96
+ df = self.download.data_to_df(
97
+ db_name='天猫数据2',
98
+ tabel_name='推广数据_宝贝主体报表',
99
+ start_date=start_date,
100
+ end_date=end_date,
101
+ projection=projection,
102
+ )
208
103
  return df
209
104
 
210
- @staticmethod
211
- def days_data(days, end_date=None):
212
- """ 读取近 days 天的数据 """
213
- if not end_date:
214
- end_date = datetime.datetime.now()
215
- start_date = end_date - datetime.timedelta(days=days)
216
- return pd.to_datetime(start_date), pd.to_datetime(end_date)
217
-
218
105
  @staticmethod
219
106
  def months_data(num=0, end_date=None):
220
107
  """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
@@ -224,14 +111,75 @@ class MysqlDatasQuery:
224
111
  start_date = f'{start_date.year}-{start_date.month}-01' # 替换为 n 月以前的第一天
225
112
  return pd.to_datetime(start_date), pd.to_datetime(end_date)
226
113
 
114
+
115
+ class GroupBy:
116
+ """ 数据聚合和导出 """
117
+ def __init__(self):
118
+ # self.output: 数据库默认导出目录
119
+ if platform.system() == 'Darwin':
120
+ self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
121
+ elif platform.system() == 'Windows':
122
+ self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
123
+ else:
124
+ self.output = os.path.join('数据中心/数据库导出')
125
+
126
+ def groupby(self, df, tabel_name, is_maximize=True):
127
+ """
128
+ self.is_maximize: 是否最大转化数据
129
+ """
130
+ if '宝贝主体报表' in tabel_name:
131
+ df.rename(columns={
132
+ '场景名字': '营销场景',
133
+ '主体id': '商品id',
134
+ '总购物车数': '加购量',
135
+ '总成交笔数': '成交笔数',
136
+ '总成交金额': '成交金额'
137
+ }, inplace=True)
138
+ df = df.astype({
139
+ '花费': float,
140
+ '展现量': int,
141
+ '点击量': int,
142
+ '加购量': int,
143
+ '成交笔数': int,
144
+ '成交金额': float,
145
+ '自然流量曝光量': int,
146
+ '直接成交笔数': int,
147
+ '直接成交金额': float,
148
+ }, errors='raise')
149
+ df.fillna(0, inplace=True)
150
+ if is_maximize:
151
+ df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
152
+ **{'加购量': ('加购量', np.max),
153
+ '成交笔数': ('成交笔数', np.max),
154
+ '成交金额': ('成交金额', np.max),
155
+ '自然流量曝光量': ('自然流量曝光量', np.max),
156
+ '直接成交笔数': ('直接成交笔数', np.max),
157
+ '直接成交金额': ('直接成交金额', np.max)
158
+ }
159
+ )
160
+ else:
161
+ df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
162
+ **{'加购量': ('加购量', np.min),
163
+ '成交笔数': ('成交笔数', np.min),
164
+ '成交金额': ('成交金额', np.min),
165
+ '自然流量曝光量': ('自然流量曝光量', np.min),
166
+ '直接成交笔数': ('直接成交笔数', np.max),
167
+ '直接成交金额': ('直接成交金额', np.max)
168
+ }
169
+ )
170
+ df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
171
+ return df
172
+
227
173
  def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
228
174
  index=False, header=True, st_ascend=None, ascend=None, freq=None):
229
175
  """
230
- path: 子文件夹,可以不传,默认导出目录 self.output
231
- st_ascend: 排序参数
232
- ascend: 升降序
233
- freq: 将创建子文件夹并按月分类存储, freq='Y',或 freq='M'
176
+ path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹,可以不传,
177
+ st_ascend: 排序参数 ['column1', 'column2']
178
+ ascend: 升降序 [True, False]
179
+ freq: 将创建子文件夹并按月分类存储, freq='Y', freq='M'
234
180
  """
181
+ if len(df) == 0:
182
+ return
235
183
  if not path:
236
184
  path = self.output
237
185
  else:
@@ -269,10 +217,14 @@ class MysqlDatasQuery:
269
217
  df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
270
218
 
271
219
  def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
220
+ if len(df) == 0:
221
+ return
272
222
  if not path:
273
223
  path = self.output
274
224
  else:
275
225
  path = os.path.join(self.output, path)
226
+ if not os.path.exists(path):
227
+ os.makedirs(path)
276
228
  if st_ascend and ascend:
277
229
  try:
278
230
  df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
@@ -283,10 +235,14 @@ class MysqlDatasQuery:
283
235
 
284
236
  def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
285
237
  freeze_panes=(1, 0), st_ascend=None, ascend=None):
238
+ if len(df) == 0:
239
+ return
286
240
  if not path:
287
241
  path = self.output
288
242
  else:
289
243
  path = os.path.join(self.output, path)
244
+ if not os.path.exists(path):
245
+ os.makedirs(path)
290
246
  if st_ascend and ascend:
291
247
  try:
292
248
  df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
@@ -294,13 +250,16 @@ class MysqlDatasQuery:
294
250
  print(f'{filename}: sort_values排序参数错误!')
295
251
  df.to_excel(os.path.join(path, filename + '.xlsx'),
296
252
  index=index, header=header, engine=engine, freeze_panes=freeze_panes)
297
-
298
-
253
+
254
+
299
255
  def main():
300
- sdq = MysqlDatasQuery(target_service='company')
256
+ sdq = MysqlDatasQuery(target_service='home_lx')
301
257
  sdq.months = 0
302
- df = sdq.tg_wxt()
303
- print(df)
258
+ df = sdq.tg_wxt() # 从数据库中获取数据并转为 df
259
+
260
+ g = GroupBy() # 数据聚合
261
+ df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
262
+ g.as_csv(df=df, filename='test') # 数据导出
304
263
 
305
264
 
306
265
  if __name__ == '__main__':
mdbq/mysql/s_query.py CHANGED
@@ -31,135 +31,90 @@ class QueryDatas:
31
31
  'cursorclass': pymysql.cursors.DictCursor,
32
32
  }
33
33
 
34
- def data_to_df(self, db_name, tabel_name, start_date, end_date, projection=[]):
34
+ def data_to_df(self, db_name, tabel_name, start_date, end_date, projection: dict=[]):
35
+
35
36
  start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
36
37
  end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
37
- df = pd.DataFrame()
38
+ df = pd.DataFrame() # 初始化df
39
+
40
+ if self.check_infos(db_name, tabel_name) == False:
41
+ return df
38
42
 
43
+ self.config.update({'database': db_name})
44
+ connection = pymysql.connect(**self.config) # 重新连接数据库
45
+ try:
46
+ with connection.cursor() as cursor:
47
+ # 3. 获取数据表的所有列信息
48
+ sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
49
+ cursor.execute(sql, (db_name, {tabel_name}))
50
+ columns = cursor.fetchall()
51
+ cols_exist = [col['COLUMN_NAME'] for col in columns] # 数据表的所有列, 返回 list
52
+
53
+ # 4. 构建 SQL 查询语句
54
+ if projection: # 获取指定列
55
+ columns_in = []
56
+ for key, value in projection.items():
57
+ if value == 1 and key in cols_exist:
58
+ columns_in.append(key) # 提取值为 1 的键并清理不在数据表的键
59
+ columns_in = ', '.join(columns_in)
60
+ if '日期' in cols_exist: # 不论是否指定, 只要数据表有日期,则执行
61
+ sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
62
+ f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
63
+ else: # 数据表没有日期列时,返回指定列的所有数据
64
+ sql = f"SELECT {columns_in} FROM {db_name}.{tabel_name}"
65
+ else: # 没有指定获取列时
66
+ if '日期' in cols_exist: # 但数据表有日期,仍然执行
67
+ columns_in = ', '.join(cols_exist)
68
+ sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
69
+ f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
70
+ else: # 没有指定获取列,且数据表也没有日期列,则返回全部列的全部数据
71
+ sql = f"SELECT * FROM {db_name}.{tabel_name}"
72
+ cursor.execute(sql)
73
+ rows = cursor.fetchall() # 获取查询结果
74
+ columns = [desc[0] for desc in cursor.description]
75
+ df = pd.DataFrame(rows, columns=columns) # 转为 df
76
+ except Exception as e:
77
+ print(f'{e}')
78
+ return df
79
+ finally:
80
+ connection.close()
81
+
82
+ if len(df) == 0:
83
+ print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
84
+ return df
85
+
86
+ def check_infos(self, db_name, tabel_name) -> bool:
87
+ """ 检查数据库、数据表是否存在 """
39
88
  connection = pymysql.connect(**self.config) # 连接数据库
40
89
  try:
41
90
  with connection.cursor() as cursor:
91
+ # 1. 检查数据库是否存在
42
92
  cursor.execute(f"SHOW DATABASES LIKE '{db_name}'") # 检查数据库是否存在
43
93
  database_exists = cursor.fetchone()
44
94
  if not database_exists:
45
95
  print(f"Database <{db_name}>: 数据库不存在")
96
+ return False
46
97
  finally:
47
98
  connection.close() # 这里要断开连接
48
- time.sleep(0.2)
49
99
 
50
100
  self.config.update({'database': db_name}) # 添加更新 config 字段
51
101
  connection = pymysql.connect(**self.config) # 重新连接数据库
52
102
  try:
53
103
  with connection.cursor() as cursor:
54
- # 1. 查询表是否存在
104
+ # 2. 查询表是否存在
55
105
  sql = f"SHOW TABLES LIKE '{tabel_name}'"
56
106
  cursor.execute(sql)
57
107
  if not cursor.fetchone():
58
108
  print(f'{db_name} -> <{tabel_name}>: 表不存在')
59
- return df
60
-
61
- # 查询列
62
- for col in projection:
63
- sql = ('SELECT 1 FROM information_schema.columns WHERE table_schema = %s AND table_name = %s AND '
64
- 'column_name = %s')
65
- cursor.execute(sql, (db_name, {tabel_name}, col))
66
- if cursor.fetchone() is None: # 移除不存在的列
67
- projection.remove(col)
109
+ return False
110
+ return True
68
111
  except Exception as e:
69
112
  print(e)
70
- return df
113
+ return False
71
114
  finally:
72
115
  connection.close() # 断开连接
73
116
 
74
- # before_time = time.time()
75
- # 读取数据
76
- self.config.update({'database': db_name})
77
- connection = pymysql.connect(**self.config) # 重新连接数据库
78
- try:
79
- with connection.cursor() as cursor:
80
- if not projection: # 如果未指定,则查询所有列,获取 cols_exist
81
- sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
82
- cursor.execute(sql, (db_name, {tabel_name}))
83
- columns = cursor.fetchall()
84
- cols_exist = [col['COLUMN_NAME'] for col in columns]
85
-
86
- if '日期' in projection or '日期' in cols_exist: # 指定含日期的 projection 或者未指定 projection 但表中有日期列
87
- sql = f"SELECT * FROM {db_name}.{tabel_name} WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
88
- elif projection: # 指定未含日期的 projection
89
- sql = f"SELECT '%s' FROM {db_name}.{tabel_name}" % (', '.join(projection))
90
- else: # 未指定 projection 且表中无日期
91
- sql = f"SELECT * FROM {db_name}.{tabel_name}"
92
- cursor.execute(sql)
93
- rows = cursor.fetchall() # 获取查询结果
94
- columns = [desc[0] for desc in cursor.description]
95
- df = pd.DataFrame(rows, columns=columns)
96
- except Exception as e:
97
- print(f'{e} {db_name} -> <{tabel_name}>: 表不存在')
98
- return df
99
- finally:
100
- connection.close()
101
-
102
- if len(df) == 0:
103
- print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
104
- # else:
105
- # now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
106
- # cost_time = int(time.time() - before_time)
107
- # if cost_time < 1:
108
- # cost_time = round(time.time() - before_time, 2)
109
- # print(f'{now}mysql ({self.host}) 表: {tabel_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
110
- return df
111
-
112
-
113
- def year_month_day(start_date, end_date):
114
- """
115
- 使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
116
- calendar.monthrange: 获取当月第一个工作日的星期值(0,6) 以及当月天数
117
- """
118
- # 替换年月日中的日, 以便即使传入当月日期也有返回值
119
- try:
120
- start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
121
- except Exception as e:
122
- print(e)
123
- return []
124
- # 使用pandas的date_range创建一个日期范围,频率为'MS'代表每月开始
125
- date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
126
- # 转换格式
127
- year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
128
-
129
- results = []
130
- for year_month in year_months:
131
- year = re.findall(r'(\d{4})', year_month)[0]
132
- month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
133
- s, d = calendar.monthrange(int(year), int(month))
134
- results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
135
-
136
- return results # start_date至end_date之间的所有年月日
137
-
138
-
139
- def download_datas(tabel_name, save_path, start_date):
140
- username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
141
- print(username, password, host, port)
142
- m = MysqlUpload(username=username, password=password, host=host, port=port)
143
- m.port = port
144
- results = year_month_day(start_date=start_date, end_date='today')
145
- # print(results)
146
- for result in results:
147
- start_date = result['起始日期']
148
- end_date = result['结束日期']
149
- # print(start_date, end_date)
150
- df = m.data_to_df(db_name='市场数据2', tabel_name=tabel_name, start_date=start_date, end_date=end_date)
151
- if len(df) == 0:
152
- continue
153
- path = os.path.join(save_path, f'{tabel_name}_{str(start_date)}_{str(end_date)}.csv')
154
- df['日期'] = df['日期'].apply(lambda x: re.sub(' .*', '', str(x)))
155
- df.to_csv(path, index=False, encoding='utf-8_sig', header=True)
156
-
157
117
 
158
118
  if __name__ == '__main__':
159
- # username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
160
- # print(username, password, host, port)
161
-
162
119
  username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
163
- qd = QueryDatas(username=username, password=password, host=host, port=port)
164
- df = qd.data_to_df(db_name='市场数据2', tabel_name='市场排行_店铺', start_date='2024-08-13', end_date='2024-08-31')
165
- print(df)
120
+ print(username, password, host, port)
@@ -0,0 +1,38 @@
1
+ # -*- coding:utf-8 -*-
2
+ import warnings
3
+ import pandas as pd
4
+ import calendar
5
+
6
+ warnings.filterwarnings('ignore')
7
+
8
+
9
+ def year_month_day(start_date, end_date):
10
+ """
11
+ 使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
12
+ calendar.monthrange: 获取当月第一个工作日的星期值(0,6) 以及当月天数
13
+ 返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
14
+ """
15
+ # 替换年月日中的日, 以便即使传入当月日期也有返回值
16
+ try:
17
+ start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
18
+ except Exception as e:
19
+ print(e)
20
+ return []
21
+ # 使用pandas的date_range创建一个日期范围,频率为'MS'代表每月开始
22
+ date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
23
+ # 转换格式
24
+ year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
25
+
26
+ results = []
27
+ for year_month in year_months:
28
+ year = re.findall(r'(\d{4})', year_month)[0]
29
+ month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
30
+ s, d = calendar.monthrange(int(year), int(month))
31
+ results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
32
+
33
+ return results # start_date至end_date之间的所有年月日
34
+
35
+
36
+ if __name__ == '__main__':
37
+ results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
38
+ print(results)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mdbq
3
- Version: 0.0.6
3
+ Version: 0.0.7
4
4
  Home-page: https://pypi.org/project/mdbsql
5
5
  Author: xigua,
6
6
  Author-email: 2587125111@qq.com
@@ -2,7 +2,7 @@ mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
2
2
  mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
3
3
  mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
4
4
  mdbq/aggregation/aggregation.py,sha256=RHQa2rs4fimRvJzluujErg6I8fn7s9q1-kwC2bPZohE,59439
5
- mdbq/aggregation/query_data.py,sha256=avbMc36kCuyTjLOXMzIEIKSc0x227c1t-Ydf0vdkViM,13756
5
+ mdbq/aggregation/query_data.py,sha256=Yb-gUPBm8r93oKFDiZ0-DiGhaqeL-jA7eAfZG0__DrA,11235
6
6
  mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
7
7
  mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
8
8
  mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -18,7 +18,8 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
18
18
  mdbq/mongo/mongo.py,sha256=hF93-kP2lxK4WY1KCdBBszLQ_I7W0mQQxZ7t4qU2w3A,32930
19
19
  mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
20
20
  mdbq/mysql/mysql.py,sha256=H9onFYKSYRjdXghK_29Aj7vgvUgDHexJjIECrdxLbE0,29925
21
- mdbq/mysql/s_query.py,sha256=P0QNwJL3ytyN75c8Qny1xfxrOUI4ks-FuRghNsyMWic,7409
21
+ mdbq/mysql/s_query.py,sha256=bRnW8Cpy4fSsbMhzGCvjiK2kin9uamVumJC3nLAyjMg,5213
22
+ mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
22
23
  mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
23
24
  mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
24
25
  mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
@@ -27,7 +28,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
27
28
  mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
28
29
  mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
29
30
  mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
30
- mdbq-0.0.6.dist-info/METADATA,sha256=G_RkhGDJMmXpNdrECY4LBT0f6HF9sHpfU-LYxNUzWPk,245
31
- mdbq-0.0.6.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
32
- mdbq-0.0.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
33
- mdbq-0.0.6.dist-info/RECORD,,
31
+ mdbq-0.0.7.dist-info/METADATA,sha256=nlphrFIJBQKjQnQvqtyK5NDTCGNTIOXRvxZdDGmiX20,245
32
+ mdbq-0.0.7.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
33
+ mdbq-0.0.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
34
+ mdbq-0.0.7.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (70.2.0)
2
+ Generator: setuptools (70.1.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5