mdbq 2.0.2__py3-none-any.whl → 2.0.3__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -320,6 +320,14 @@ class DatabaseUpdate:
                     )
                     df['日期'] = df['数据周期'].apply(lambda x: re.findall('(.*) ~', x)[0])
                     check_remove_file = True
+                elif name.endswith('.csv') and '分天数据-计划_活动类型-推广概览-数据汇总' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    df['日期'].replace(to_replace=['\\t'], value='', regex=True, inplace=True)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    # min_clm = str(df['日期'].min()).split(' ')[0]
+                    # max_clm = str(df['日期'].max()).split(' ')[0]
+                    # new_name = f'淘宝联盟_分天数据_计划_活动类型_推广概览_数据汇总_{min_clm}_{max_clm}'
+                    check_remove_file = True
                 elif name.endswith('.csv') and 'baobei' in name:
                     # 生意经宝贝指标日数据
                     date = re.findall(r's-(\d{4})(\d{2})(\d{2})\.', str(name))
@@ -1244,7 +1252,7 @@ def test2():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
-    file_dir(one_file=False, target_service='home_lx')
+    # file_dir(one_file=False, target_service='home_lx')
     # one_file_to_mysql(
     #     file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
     #     db_name='京东数据2',
@@ -1264,11 +1272,11 @@ if __name__ == '__main__':
     # )
 
 
-    # # 新版 数据分类
-    # dp = DatabaseUpdate(path='/Users/xigua/Downloads')
-    # dp.new_unzip(is_move=True)
-    # dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # # 将 self.datas 更新至数据库
+    # 新版 数据分类
+    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
+    dp.new_unzip(is_move=True)
+    dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
+    # 将 self.datas 更新至数据库
     # dp.upload_df(service_databases=[
     #     # {'home_lx': 'mongodb'},
     #     {'company': 'mysql'},

mdbq/clean/data_clean.py

@@ -474,6 +474,15 @@ class DataClean:
                         new_name = re.sub(r' ?(\(\d+\))', '',new_name)
                         self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                         os.remove(os.path.join(root, name))
+                    elif name.endswith('.csv') and '分天数据-计划_活动类型-推广概览-数据汇总' in name:
+                        df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                        df['日期'].replace(to_replace=['\\t'], value='', regex=True, inplace=True)
+                        df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                        min_clm = str(df['日期'].min()).split(' ')[0]
+                        max_clm = str(df['日期'].max()).split(' ')[0]
+                        new_name = f'淘宝联盟_分天数据_计划_活动类型_推广概览_数据汇总_{min_clm}_{max_clm}'
+                        self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
+                        os.remove(os.path.join(root, name))
                     elif name.endswith('.csv') and 'baobei' in name:
                         # 生意经宝贝指标日数据
                         # print(name)
@@ -1143,6 +1152,9 @@ class DataClean:
                 elif name.endswith('.csv') and '参谋店铺流量来源（月）' in name:
                     t_path = str(pathlib.Path(self.source_path, '月数据/流量来源-自助取数-月数据'))
                     bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '淘宝联盟_分天数据_计划_活动类型_推广概览_数据汇总' in name:
+                    t_path = str(pathlib.Path(self.source_path, '月数据/淘宝联盟'))
+                    bib(t_path, _as_month=False)
                 elif name.endswith('.csv') and '竞店分析' in name and '来源分析-入店来源' in name:
                     t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析/来源分析/入店来源'))
                     bib(t_path, _as_month=False)

{mdbq-2.0.2.dist-info → mdbq-2.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.0.2
+Version: 2.0.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.0.2.dist-info → mdbq-2.0.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=uaclXkIaZVhTqyI6VipI4Re7sXQDtlvVCopA_ONpcKM,73194
+mdbq/aggregation/aggregation.py,sha256=ysHgpGoO2aXme0l1G9fqKMzWnvtTeqPwy07n4CNpdDA,73922
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
@@ -9,7 +9,7 @@ mdbq/aggregation/query_data.py,sha256=ixvfjEai-zIN5fUiVg9nSFrzjiBfJv6QC9_mLTvFfg
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=BgOZlJwuv5X4BL24DOHC6XtMFspFD-AdCBC_EJJ9rjA,101382
+mdbq/clean/data_clean.py,sha256=kKPVvKq2WVUI9qLF7U-jW1IEJtBE9_a6rwJwpR9hfhI,102556
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=VUaaJPXPYPHWwnkdK77PWz_dAXZyEmYBA9Df1yROHAc,17764
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-2.0.2.dist-info/METADATA,sha256=n4o0hi_JT0eYxpVVIGm-LZnSixtlu_Cc3BfLpdmoEL4,245
-mdbq-2.0.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.0.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.0.2.dist-info/RECORD,,
+mdbq-2.0.3.dist-info/METADATA,sha256=keQRDwBmHcrEjRS5EuqnKz9tfVGRUzntuD3Cmum4dy8,245
+mdbq-2.0.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.0.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.0.3.dist-info/RECORD,,