mdbq 2.0.1__py3-none-any.whl → 2.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. mdbq/aggregation/aggregation.py +25 -13
  2. mdbq/clean/data_clean.py +12 -0
  3. {mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/METADATA +1 -1
  4. {mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/RECORD +6 -6
  5. {mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/WHEEL +1 -1
  6. {mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/top_level.txt +0 -0
mdbq/aggregation/aggregation.py CHANGED
@@ -320,6 +320,14 @@ class DatabaseUpdate:
320
320
  )
321
321
  df['日期'] = df['数据周期'].apply(lambda x: re.findall('(.*) ~', x)[0])
322
322
  check_remove_file = True
323
+ elif name.endswith('.csv') and '分天数据-计划_活动类型-推广概览-数据汇总' in name:
324
+ df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
325
+ df['日期'].replace(to_replace=['\\t'], value='', regex=True, inplace=True)
326
+ df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
327
+ # min_clm = str(df['日期'].min()).split(' ')[0]
328
+ # max_clm = str(df['日期'].max()).split(' ')[0]
329
+ # new_name = f'淘宝联盟_分天数据_计划_活动类型_推广概览_数据汇总_{min_clm}_{max_clm}'
330
+ check_remove_file = True
323
331
  elif name.endswith('.csv') and 'baobei' in name:
324
332
  # 生意经宝贝指标日数据
325
333
  date = re.findall(r's-(\d{4})(\d{2})(\d{2})\.', str(name))
@@ -443,15 +451,19 @@ class DatabaseUpdate:
443
451
  elif name.endswith('.csv') and '人群洞察' in name:
444
452
  df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
445
453
  df.replace(to_replace=['--'], value='', regex=False, inplace=True)
446
- df = df[df['人群规模'] != '']
454
+ df = df[(df['人群规模'] != '') & (df['人群规模'] != '--')]
447
455
  if len(df) == 0:
448
- if is_move:
449
- try:
450
- os.remove(os.path.join(root, name)) # 是否移除原文件
451
- check_remove_file = True
452
- except Exception as e:
453
- print(f'{name}, {e}')
456
+ try:
457
+ os.remove(os.path.join(root, name)) # 移除原文件
458
+ except:
459
+ pass
454
460
  continue
461
+ if is_move:
462
+ try:
463
+ os.remove(os.path.join(root, name)) # 是否移除原文件
464
+ check_remove_file = True
465
+ except Exception as e:
466
+ print(f'{name}, {e}')
455
467
  elif name.endswith('.csv') and '客户_客户概况_画像' in name:
456
468
  df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
457
469
  check_remove_file = True
@@ -1240,7 +1252,7 @@ def test2():
1240
1252
  if __name__ == '__main__':
1241
1253
  username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
1242
1254
  print(username, password, host, port)
1243
- file_dir(one_file=False, target_service='home_lx')
1255
+ # file_dir(one_file=False, target_service='home_lx')
1244
1256
  # one_file_to_mysql(
1245
1257
  # file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
1246
1258
  # db_name='京东数据2',
@@ -1260,11 +1272,11 @@ if __name__ == '__main__':
1260
1272
  # )
1261
1273
 
1262
1274
 
1263
- # # 新版 数据分类
1264
- # dp = DatabaseUpdate(path='/Users/xigua/Downloads')
1265
- # dp.new_unzip(is_move=True)
1266
- # dp.cleaning(is_move=False) # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
1267
- # # 将 self.datas 更新至数据库
1275
+ # 新版 数据分类
1276
+ dp = DatabaseUpdate(path='/Users/xigua/Downloads')
1277
+ dp.new_unzip(is_move=True)
1278
+ dp.cleaning(is_move=False) # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
1279
+ # 将 self.datas 更新至数据库
1268
1280
  # dp.upload_df(service_databases=[
1269
1281
  # # {'home_lx': 'mongodb'},
1270
1282
  # {'company': 'mysql'},
mdbq/clean/data_clean.py CHANGED
@@ -474,6 +474,15 @@ class DataClean:
474
474
  new_name = re.sub(r' ?(\(\d+\))', '',new_name)
475
475
  self.save_to_csv(df, root, new_name) # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
476
476
  os.remove(os.path.join(root, name))
477
+ elif name.endswith('.csv') and '分天数据-计划_活动类型-推广概览-数据汇总' in name:
478
+ df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
479
+ df['日期'].replace(to_replace=['\\t'], value='', regex=True, inplace=True)
480
+ df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
481
+ min_clm = str(df['日期'].min()).split(' ')[0]
482
+ max_clm = str(df['日期'].max()).split(' ')[0]
483
+ new_name = f'淘宝联盟_分天数据_计划_活动类型_推广概览_数据汇总_{min_clm}_{max_clm}'
484
+ self.save_to_csv(df, root, new_name) # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
485
+ os.remove(os.path.join(root, name))
477
486
  elif name.endswith('.csv') and 'baobei' in name:
478
487
  # 生意经宝贝指标日数据
479
488
  # print(name)
@@ -1143,6 +1152,9 @@ class DataClean:
1143
1152
  elif name.endswith('.csv') and '参谋店铺流量来源（月）' in name:
1144
1153
  t_path = str(pathlib.Path(self.source_path, '月数据/流量来源-自助取数-月数据'))
1145
1154
  bib(t_path, _as_month=True)
1155
+ elif name.endswith('.csv') and '淘宝联盟_分天数据_计划_活动类型_推广概览_数据汇总' in name:
1156
+ t_path = str(pathlib.Path(self.source_path, '月数据/淘宝联盟'))
1157
+ bib(t_path, _as_month=False)
1146
1158
  elif name.endswith('.csv') and '竞店分析' in name and '来源分析-入店来源' in name:
1147
1159
  t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析/来源分析/入店来源'))
1148
1160
  bib(t_path, _as_month=False)
{mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mdbq
3
- Version: 2.0.1
3
+ Version: 2.0.3
4
4
  Home-page: https://pypi.org/project/mdbsql
5
5
  Author: xigua,
6
6
  Author-email: 2587125111@qq.com
{mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/RECORD RENAMED
@@ -1,7 +1,7 @@
1
1
  mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
2
2
  mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
3
3
  mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
4
- mdbq/aggregation/aggregation.py,sha256=0V7pnamxjvz5cpPYm6cE5-EFkktXVfB60fauC3SrZ9Y,73008
4
+ mdbq/aggregation/aggregation.py,sha256=ysHgpGoO2aXme0l1G9fqKMzWnvtTeqPwy07n4CNpdDA,73922
5
5
  mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
6
6
  mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
7
7
  mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
@@ -9,7 +9,7 @@ mdbq/aggregation/query_data.py,sha256=ixvfjEai-zIN5fUiVg9nSFrzjiBfJv6QC9_mLTvFfg
9
9
  mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
10
10
  mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
11
11
  mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
12
- mdbq/clean/data_clean.py,sha256=BgOZlJwuv5X4BL24DOHC6XtMFspFD-AdCBC_EJJ9rjA,101382
12
+ mdbq/clean/data_clean.py,sha256=kKPVvKq2WVUI9qLF7U-jW1IEJtBE9_a6rwJwpR9hfhI,102556
13
13
  mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
14
14
  mdbq/company/copysh.py,sha256=VUaaJPXPYPHWwnkdK77PWz_dAXZyEmYBA9Df1yROHAc,17764
15
15
  mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
36
36
  mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
37
37
  mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
38
38
  mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
39
- mdbq-2.0.1.dist-info/METADATA,sha256=oqzIwWYKBRXwGEedgZbZPsi8JPSHyOceKS9ArHbgMMU,245
40
- mdbq-2.0.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
41
- mdbq-2.0.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
42
- mdbq-2.0.1.dist-info/RECORD,,
39
+ mdbq-2.0.3.dist-info/METADATA,sha256=keQRDwBmHcrEjRS5EuqnKz9tfVGRUzntuD3Cmum4dy8,245
40
+ mdbq-2.0.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
41
+ mdbq-2.0.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
42
+ mdbq-2.0.3.dist-info/RECORD,,
{mdbq-2.0.1.dist-info → mdbq-2.0.3.dist-info}/WHEEL RENAMED
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (70.1.0)
2
+ Generator: bdist_wheel (0.44.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5