mdbq 2.0.3__py3-none-any.whl → 2.0.5__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -39,10 +39,11 @@ class DatabaseUpdate:
         self.datas: list = []  # 带更新进数据库的数据集合
         self.start_date = '2022-01-01'  # 日期表的起始日期
 
-    def cleaning(self, is_move=True):
+    def cleaning(self, is_move=True, is_except=[]):
         """
         数据清洗, 返回包含 数据库名, 集合名称, 和 df 主体
         修改 cleaning 时，要同步 support 下的 标题对照表.csv
+        is_except: 需要排除不做处理的文件或文件夹
         """
         if not os.path.exists(self.path):
             print(f'1.1.0 初始化时传入了不存在的目录: {self.path}')
@@ -62,6 +63,16 @@ class DatabaseUpdate:
                 check_remove_file = False  # 设置这个参数的目的: 避免误删其他文件， 不是本程序数据清洗覆盖的文件不做干预
                 if '~$' in name or '.DS' in name or '.localized' in name or '.ini' in name or '$RECYCLE.BIN' in name or 'Icon' in name:
                     continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
                 db_name = None  # 初始化/重置变量，避免进入下一个循环
                 collection_name = None
                 for data in datas:  # 根据标题对照表适配 db_name 和 collection_name
@@ -755,7 +766,15 @@ class DatabaseUpdate:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
                 db_name = None  # 初始化/重置变量，避免进入下一个循环
                 collection_name = None
                 for data in datas:  # 根据标题对照表适配 db_name 和 collection_name
@@ -1252,7 +1271,7 @@ def test2():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
-    # file_dir(one_file=False, target_service='home_lx')
+    file_dir(one_file=False, target_service='company')
     # one_file_to_mysql(
     #     file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
     #     db_name='京东数据2',
@@ -1272,13 +1291,13 @@ if __name__ == '__main__':
     # )
 
 
-    # 新版 数据分类
-    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
-    dp.new_unzip(is_move=True)
-    dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # 将 self.datas 更新至数据库
-    # dp.upload_df(service_databases=[
-    #     # {'home_lx': 'mongodb'},
-    #     {'company': 'mysql'},
-    #     # {'nas': 'mysql'},
-    # ])
+    # # 新版 数据分类
+    # dp = DatabaseUpdate(path='/Users/xigua/Downloads')
+    # dp.new_unzip(is_move=True)
+    # dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
+    # # 将 self.datas 更新至数据库
+    # # dp.upload_df(service_databases=[
+    # #     # {'home_lx': 'mongodb'},
+    # #     {'company': 'mysql'},
+    # #     # {'nas': 'mysql'},
+    # # ])

mdbq/aggregation/query_data.py

@@ -1535,7 +1535,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=24)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=7)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

mdbq/clean/data_clean.py

@@ -67,7 +67,7 @@ class DataClean:
         _df.to_csv(os.path.join(_save_paths, filenames), encoding=encoding, index=False, header=True)
 
     # @try_except
-    def change_and_sort(self, path=None):
+    def change_and_sort(self, path=None, is_except=[]):
         """数据转换"""
         if not path:
             path = self.path
@@ -87,6 +87,16 @@ class DataClean:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
 
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
                 try:
                     encoding = self.get_encoding(file_path=pathlib.Path(root, name))
                     # ----------------- 推广报表 分割线 -----------------
@@ -272,11 +282,14 @@ class DataClean:
                             df.insert(loc=0, column='数据周期', value=data_lis)
                         df.insert(loc=0, column='日期', value=date01[0])
                         # 2024-2-19 官方更新了推广渠道来源名称
+                        # df['三级来源'] = df['三级来源'].apply(
+                        #     lambda x: '精准人群推广' if x == '精准人群推广(原引力魔方)'
+                        #     else '关键词推广' if x == '关键词推广(原直通车)'
+                        #     else '智能场景' if x == '智能场景(原万相台)'
+                        #     else x
+                        # )
                         df['三级来源'] = df['三级来源'].apply(
-                            lambda x: '精准人群推广' if x == '精准人群推广(原引力魔方)'
-                            else '关键词推广' if x == '关键词推广(原直通车)'
-                            else '智能场景' if x == '智能场景(原万相台)'
-                            else x
+                            lambda x: re.sub('(.*)', '', str(x) if x else x)
                         )
                         # df = df[df['访客数'] != '0']
                         df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
@@ -1130,7 +1143,7 @@ class DataClean:
                         bib(t_path)
                     elif '_新版' in name:
                         t_path = str(pathlib.Path(self.source_path, '生意参谋/流量来源'))
-                        bib(t_path)
+                        bib(t_path, _as_month=True)
                     else:
                         t_path = str(pathlib.Path(self.source_path, '生意参谋/流量来源_旧版'))
                         bib(t_path, _as_month=True)

mdbq/config/products.py

@@ -20,6 +20,9 @@ class Products:
 
     def update_my_datas(self):
         my_datas = [
+            {
+                '平台': '天猫', '商品id': '840499705810', '上市年份': '2024年10月'
+            },
             {
                 '平台': '天猫', '商品id': '830789689032', '上市年份': '2024年9月'
             },

{mdbq-2.0.3.dist-info → mdbq-2.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.0.3
+Version: 2.0.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.0.3.dist-info → mdbq-2.0.5.dist-info}/RECORD

@@ -1,20 +1,20 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=ysHgpGoO2aXme0l1G9fqKMzWnvtTeqPwy07n4CNpdDA,73922
+mdbq/aggregation/aggregation.py,sha256=TiSMZHa9F_f6iMptzCVdukWhCzXzpcYIh3lN61P-i94,74825
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=ixvfjEai-zIN5fUiVg9nSFrzjiBfJv6QC9_mLTvFfg0,72380
+mdbq/aggregation/query_data.py,sha256=qBNjGTxaQl6rg2-_jlJKGz_sop9UVgoNj5z75XGl_iQ,72379
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=kKPVvKq2WVUI9qLF7U-jW1IEJtBE9_a6rwJwpR9hfhI,102556
+mdbq/clean/data_clean.py,sha256=hyhLsX5UEmj2ROVScQMRdR52vUuuLE5uSG5QJ60gtQU,103176
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=VUaaJPXPYPHWwnkdK77PWz_dAXZyEmYBA9Df1yROHAc,17764
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
-mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
+mdbq/config/products.py,sha256=L1uhzdbqTprQg_rekKt0ucgpeIuMvi3H2v48_GZWPuY,5803
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-2.0.3.dist-info/METADATA,sha256=keQRDwBmHcrEjRS5EuqnKz9tfVGRUzntuD3Cmum4dy8,245
-mdbq-2.0.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.0.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.0.3.dist-info/RECORD,,
+mdbq-2.0.5.dist-info/METADATA,sha256=q3s1z7iCeWS4qXY4yzg05F7K_JUDYhIp1H5Zlo-uYV4,245
+mdbq-2.0.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.0.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.0.5.dist-info/RECORD,,