mdbq 2.0.4__py3-none-any.whl → 2.0.6__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -39,10 +39,11 @@ class DatabaseUpdate:
         self.datas: list = []  # 带更新进数据库的数据集合
         self.start_date = '2022-01-01'  # 日期表的起始日期
 
-    def cleaning(self, is_move=True):
+    def cleaning(self, is_move=True, is_except=[]):
         """
         数据清洗, 返回包含 数据库名, 集合名称, 和 df 主体
         修改 cleaning 时，要同步 support 下的 标题对照表.csv
+        is_except: 需要排除不做处理的文件或文件夹
         """
         if not os.path.exists(self.path):
             print(f'1.1.0 初始化时传入了不存在的目录: {self.path}')
@@ -62,6 +63,16 @@ class DatabaseUpdate:
                 check_remove_file = False  # 设置这个参数的目的: 避免误删其他文件， 不是本程序数据清洗覆盖的文件不做干预
                 if '~$' in name or '.DS' in name or '.localized' in name or '.ini' in name or '$RECYCLE.BIN' in name or 'Icon' in name:
                     continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
                 db_name = None  # 初始化/重置变量，避免进入下一个循环
                 collection_name = None
                 for data in datas:  # 根据标题对照表适配 db_name 和 collection_name
@@ -755,7 +766,15 @@ class DatabaseUpdate:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
                 db_name = None  # 初始化/重置变量，避免进入下一个循环
                 collection_name = None
                 for data in datas:  # 根据标题对照表适配 db_name 和 collection_name

mdbq/aggregation/optimize_data.py

@@ -57,7 +57,7 @@ def restart_mongodb():
         subprocess.call(command, shell=True)
 
 
-def op_data(db_name_lists, service_databases=None, days: int = 63):
+def op_data(db_name_lists, service_databases=None, days: int = 63, is_mongo=True, is_mysql=True):
     """ service_databases 这个参数暂时没有用 """
     # for service_database in service_databases:
     #     for service_name, database in service_database.items():
@@ -71,42 +71,45 @@ def op_data(db_name_lists, service_databases=None, days: int = 63):
 
     if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
         # mongodb
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
-            database='mongodb',
-        )
-        m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
-        m.db_name_lists = db_name_lists
-        m.days = days
-        m.optimize_list()
-        if m.client:
-            m.client.close()
-            print(f'已关闭 mongodb 连接')
+        if is_mongo:
+            username, password, host, port = get_myconf.select_config_values(
+                target_service='home_lx',
+                database='mongodb',
+            )
+            m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
+            m.db_name_lists = db_name_lists
+            m.days = days
+            m.optimize_list()
+            if m.client:
+                m.client.close()
+                print(f'已关闭 mongodb 连接')
 
-        if socket.gethostname() == 'xigua_lx':
-            restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
+            if socket.gethostname() == 'xigua_lx':
+                restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
 
         # Mysql
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
-            database='mysql',
-        )
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = db_name_lists
-        s.days = days
-        s.optimize_list()
+        if is_mysql:
+            username, password, host, port = get_myconf.select_config_values(
+                target_service='home_lx',
+                database='mysql',
+            )
+            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+            s.db_name_lists = db_name_lists
+            s.days = days
+            s.optimize_list()
 
     elif socket.gethostname() == 'company':
         # Mysql
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='company',
-            database='mysql',
-        )
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = db_name_lists
-        s.days = days
-        s.optimize_list()
+        if is_mysql:
+            username, password, host, port = get_myconf.select_config_values(
+                target_service='company',
+                database='mysql',
+            )
+            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+            s.db_name_lists = db_name_lists
+            s.days = days
+            s.optimize_list()
 
 
 if __name__ == '__main__':
-    op_data(db_name_lists=['聚合数据'], service_databases=[{'company': 'mysql'}], days=3650)
+    op_data(db_name_lists=['聚合数据'], service_databases=[{'company': 'mysql'}], days=3650, is_mongo=True, is_mysql=True)

mdbq/clean/data_clean.py

@@ -67,7 +67,7 @@ class DataClean:
         _df.to_csv(os.path.join(_save_paths, filenames), encoding=encoding, index=False, header=True)
 
     # @try_except
-    def change_and_sort(self, path=None):
+    def change_and_sort(self, path=None, is_except=[]):
         """数据转换"""
         if not path:
             path = self.path
@@ -87,6 +87,16 @@ class DataClean:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
 
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
                 try:
                     encoding = self.get_encoding(file_path=pathlib.Path(root, name))
                     # ----------------- 推广报表 分割线 -----------------

{mdbq-2.0.4.dist-info → mdbq-2.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.0.4
+Version: 2.0.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.0.4.dist-info → mdbq-2.0.6.dist-info}/RECORD

@@ -1,15 +1,15 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=DAwLaTJCqolPz3QNoQnmWb_DTLc0HtvjprhkO8vhVpE,73940
+mdbq/aggregation/aggregation.py,sha256=TiSMZHa9F_f6iMptzCVdukWhCzXzpcYIh3lN61P-i94,74825
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
-mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
+mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
 mdbq/aggregation/query_data.py,sha256=qBNjGTxaQl6rg2-_jlJKGz_sop9UVgoNj5z75XGl_iQ,72379
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=CTnybgn5mMGzQ5dKJ0Hh-iwV1qehbaBHRFtttX9AO1M,102758
+mdbq/clean/data_clean.py,sha256=hyhLsX5UEmj2ROVScQMRdR52vUuuLE5uSG5QJ60gtQU,103176
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=VUaaJPXPYPHWwnkdK77PWz_dAXZyEmYBA9Df1yROHAc,17764
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-2.0.4.dist-info/METADATA,sha256=JFnnucukEykE5vpzO7TcoA9fqXt3brI1pDJARIKn9wU,245
-mdbq-2.0.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.0.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.0.4.dist-info/RECORD,,
+mdbq-2.0.6.dist-info/METADATA,sha256=hqNt7JUo-6t9oYpg0FXAl-SV6wbluV8bD9mR7hzVJoM,245
+mdbq-2.0.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.0.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.0.6.dist-info/RECORD,,