mdbq 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

mdbq/aggregation/__init__.py

@@ -0,0 +1,4 @@
+
+
+
+# 数据清洗, 数据聚合, 入库

mdbq/{pipeline/database.py → aggregation/aggregation.py}

@@ -156,9 +156,6 @@ class DataTypes:
 class DatabaseUpdate:
     def __init__(self, path):
         self.path = path  # 数据所在目录, 即: 下载文件夹
-        self.up_to_mogo: bool = False  # 不设置则不上传 mongodb
-        self.up_to_mysql: bool = False  # 不设置则不上传 mysql
-        self.up_to_company_mysql: bool = False  # 公司台式机更新
         self.datas: list = []  # 带更新进数据库的数据集合
 
     def cleaning(self, is_move=True):
@@ -764,53 +761,43 @@ class DatabaseUpdate:
                         os.remove(os.path.join(root, name))
         json_data.dtypes_to_file()  # 写入 json 文件, 包含数据的 dtypes 信息
 
-    def upload_df(self):
+    def upload_df(self, service_databases=[{}]):
         """
         将清洗后的 df 上传数据库
         """
-        if self.up_to_mogo:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mongodb',
-            )
-            d = mongo.UploadMongo(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-                drop_duplicates=False,
-            )
-        if self.up_to_mysql:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mysql',
-            )
-            m = mysql.MysqlUpload(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-            )
-        if self.up_to_company_mysql:  # 公司台式机更新
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='company',
-                database='mysql',
-            )
-            cmp = mysql.MysqlUpload(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-            )
+        for service_database in service_databases:
+            for service_name, database in service_database.items():
+                # print(service_name, database)
+                if database == 'mongodb':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    d = mongo.UploadMongo(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                        drop_duplicates=False,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
 
-        for data in self.datas:
-            df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
-            if self.up_to_mogo:
-                d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-            if self.up_to_mysql:
-                m.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
-            if self.up_to_company_mysql:
-                cmp.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
+                elif database == 'mysql':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    m = mysql.MysqlUpload(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        m.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
 
     def convert_df_cols(self, df):
         """"
@@ -967,11 +954,9 @@ class DatabaseUpdate:
 
 def main():
     d = DatabaseUpdate(path='/Users/xigua/Downloads')
-    d.up_to_mogo = True
-    d.up_to_mysql = True
     d.new_unzip(is_move=True)
     d.cleaning(is_move=False)
-    d.upload_df()
+    d.upload_df(service_databases=[{'home_lx': 'mongodb'}, {'home_lx': 'mysql'}])
     # print(d.datas)
 
 
@@ -1046,6 +1031,10 @@ def upload():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
     print(username, password, host, port)
+
+    d = DatabaseUpdate(path='/Users/xigua/Downloads')
+    d.upload_df(service_databases=[{'home_lx': 'mongodb'},])
+
     # main()
     # upload()
     # path = '/Users/xigua/数据中心/原始文件2/月数据/流量来源-自助取数-月数据'

mdbq/company/copysh.py

@@ -297,12 +297,9 @@ def main():
             b.download_dir(local_path=d_path, remote_path=upload_path)
 
             dp = database.DatabaseUpdate(path=d_path)
-            dp.up_to_mogo = False  # 公司台式机不需要
-            dp.up_to_mysql = False  # 公司台式机不需要
-            dp.up_to_company_mysql = True
             dp.new_unzip(is_move=True)
             dp.cleaning(is_move=True)  # 公司台式机需要移除
-            dp.upload_df()
+            d.upload_df(service_databases=[{'company': 'mysql'}])
 
             # 此操作修改 .copysh_conf 文件，更新完成之后将 ch_record 改为 false
             w = update_conf.UpdateConf(filename='.copysh_conf')

{mdbq-0.0.3.dist-info → mdbq-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.3
+Version: 0.0.4
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.0.3.dist-info → mdbq-0.0.4.dist-info}/RECORD

@@ -1,10 +1,12 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
+mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
+mdbq/aggregation/aggregation.py,sha256=HtaUlq6abeyT4V-C3jREbjJGj5L8cTKcDfDm4c4UNpI,60133
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bd.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=33OmeQFl9AW21P5EOay52W_S8DF96H5oHwCg4fSuBxA,85359
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=MzLsvv--9CPDa8qZnTBk34_HlPfRPN5b7VsjWAl8owU,15733
+mdbq/company/copysh.py,sha256=U_m6C6dBRGPhuw2prgiFgfnTIiTXxyFJV0K6OE_PY9g,15602
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=q6Pylsnh4-MsHP9JcX8IdlnGVWikz9hio1HI_qh6Wvs,6171
 mdbq/config/update_conf.py,sha256=YjGjjRchu5BcrmLJkoLjHEF2TbGOmsgCWX4LroXOYWQ,3455
@@ -21,10 +23,8 @@ mdbq/other/xigua_porxy.py,sha256=zTOxsdkdDAyGfHWPUm_7WIztjrGExONAwvPzTaC7Rho,500
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
-mdbq/pipeline/__init__.py,sha256=DJRrxEiZZvGfB-qZOfn1H9J_7RnLDXUnUIX7HJCyYPw,24
-mdbq/pipeline/database.py,sha256=aJty3f7sux1gr2NkHjrE1cXVsyEUhQvzK2051KyR_dc,60213
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-0.0.3.dist-info/METADATA,sha256=hhSbUt-hY9zLOcCSYTOWDP5yDQV0LE-HQq2slSc2RXI,245
-mdbq-0.0.3.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-mdbq-0.0.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-0.0.3.dist-info/RECORD,,
+mdbq-0.0.4.dist-info/METADATA,sha256=5CO9nsAMDCP49m1fNFh_UQ-tIyPIgBFdUl-rQd8HlOc,245
+mdbq-0.0.4.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+mdbq-0.0.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-0.0.4.dist-info/RECORD,,

mdbq/pipeline/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# 数据清洗任务