mdbq 0.0.2__tar.gz → 0.0.4__tar.gz

{mdbq-0.0.2 → mdbq-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.2
+Version: 0.0.4
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-0.0.4/mdbq/aggregation/__init__.py

@@ -0,0 +1,4 @@
+
+
+
+# 数据清洗, 数据聚合, 入库

mdbq-0.0.2/mdbq/pipeline/database.py → mdbq-0.0.4/mdbq/aggregation/aggregation.py

@@ -156,9 +156,6 @@ class DataTypes:
 class DatabaseUpdate:
     def __init__(self, path):
         self.path = path  # 数据所在目录, 即: 下载文件夹
-        self.up_to_mogo: bool = False  # 不设置则不上传 mongodb
-        self.up_to_mysql: bool = False  # 不设置则不上传 mysql
-        self.up_to_company_mysql: bool = False  # 公司台式机更新
         self.datas: list = []  # 带更新进数据库的数据集合
 
     def cleaning(self, is_move=True):
@@ -764,53 +761,43 @@ class DatabaseUpdate:
                         os.remove(os.path.join(root, name))
         json_data.dtypes_to_file()  # 写入 json 文件, 包含数据的 dtypes 信息
 
-    def upload_df(self):
+    def upload_df(self, service_databases=[{}]):
         """
         将清洗后的 df 上传数据库
         """
-        if self.up_to_mogo:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mongodb',
-            )
-            d = mongo.UploadMongo(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-                drop_duplicates=False,
-            )
-        if self.up_to_mysql:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mysql',
-            )
-            m = mysql.MysqlUpload(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-            )
-        if self.up_to_company_mysql:  # 公司台式机更新
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='company',
-                database='mysql',
-            )
-            cmp = mysql.MysqlUpload(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-            )
+        for service_database in service_databases:
+            for service_name, database in service_database.items():
+                # print(service_name, database)
+                if database == 'mongodb':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    d = mongo.UploadMongo(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                        drop_duplicates=False,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
 
-        for data in self.datas:
-            df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
-            if self.up_to_mogo:
-                d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-            if self.up_to_mysql:
-                m.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
-            if self.up_to_company_mysql:
-                cmp.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
+                elif database == 'mysql':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    m = mysql.MysqlUpload(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        m.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
 
     def convert_df_cols(self, df):
         """"
@@ -967,11 +954,9 @@ class DatabaseUpdate:
 
 def main():
     d = DatabaseUpdate(path='/Users/xigua/Downloads')
-    d.up_to_mogo = True
-    d.up_to_mysql = True
     d.new_unzip(is_move=True)
     d.cleaning(is_move=False)
-    d.upload_df()
+    d.upload_df(service_databases=[{'home_lx': 'mongodb'}, {'home_lx': 'mysql'}])
     # print(d.datas)
 
 
@@ -1046,6 +1031,10 @@ def upload():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
     print(username, password, host, port)
+
+    d = DatabaseUpdate(path='/Users/xigua/Downloads')
+    d.upload_df(service_databases=[{'home_lx': 'mongodb'},])
+
     # main()
     # upload()
     # path = '/Users/xigua/数据中心/原始文件2/月数据/流量来源-自助取数-月数据'

mdbq-0.0.4/mdbq/clean/__init__.py

@@ -0,0 +1,4 @@
+
+
+
+# 数据清洗, 此版本不再维护