mdbq 0.0.3__tar.gz → 0.0.5__tar.gz

{mdbq-0.0.3 → mdbq-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.3
+Version: 0.0.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-0.0.5/mdbq/aggregation/__init__.py

@@ -0,0 +1,4 @@
+
+
+
+# 数据清洗, 数据聚合, 入库

mdbq-0.0.3/mdbq/pipeline/database.py → mdbq-0.0.5/mdbq/aggregation/aggregation.py

@@ -156,9 +156,6 @@ class DataTypes:
 class DatabaseUpdate:
     def __init__(self, path):
         self.path = path  # 数据所在目录, 即: 下载文件夹
-        self.up_to_mogo: bool = False  # 不设置则不上传 mongodb
-        self.up_to_mysql: bool = False  # 不设置则不上传 mysql
-        self.up_to_company_mysql: bool = False  # 公司台式机更新
         self.datas: list = []  # 带更新进数据库的数据集合
 
     def cleaning(self, is_move=True):
@@ -764,53 +761,43 @@ class DatabaseUpdate:
                         os.remove(os.path.join(root, name))
         json_data.dtypes_to_file()  # 写入 json 文件, 包含数据的 dtypes 信息
 
-    def upload_df(self):
+    def upload_df(self, service_databases=[{}]):
         """
         将清洗后的 df 上传数据库
         """
-        if self.up_to_mogo:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mongodb',
-            )
-            d = mongo.UploadMongo(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-                drop_duplicates=False,
-            )
-        if self.up_to_mysql:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mysql',
-            )
-            m = mysql.MysqlUpload(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-            )
-        if self.up_to_company_mysql:  # 公司台式机更新
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='company',
-                database='mysql',
-            )
-            cmp = mysql.MysqlUpload(
-                username=username,
-                password=password,
-                host=host,
-                port=port,
-            )
+        for service_database in service_databases:
+            for service_name, database in service_database.items():
+                # print(service_name, database)
+                if database == 'mongodb':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    d = mongo.UploadMongo(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                        drop_duplicates=False,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
 
-        for data in self.datas:
-            df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
-            if self.up_to_mogo:
-                d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-            if self.up_to_mysql:
-                m.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
-            if self.up_to_company_mysql:
-                cmp.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
+                elif database == 'mysql':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    m = mysql.MysqlUpload(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        m.df_to_mysql(df=df, db_name=db_name, tabel_name=collection_name)
 
     def convert_df_cols(self, df):
         """"
@@ -967,11 +954,9 @@ class DatabaseUpdate:
 
 def main():
     d = DatabaseUpdate(path='/Users/xigua/Downloads')
-    d.up_to_mogo = True
-    d.up_to_mysql = True
     d.new_unzip(is_move=True)
     d.cleaning(is_move=False)
-    d.upload_df()
+    d.upload_df(service_databases=[{'home_lx': 'mongodb'}, {'home_lx': 'mysql'}])
     # print(d.datas)
 
 
@@ -1046,6 +1031,10 @@ def upload():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
     print(username, password, host, port)
+
+    d = DatabaseUpdate(path='/Users/xigua/Downloads')
+    d.upload_df(service_databases=[{'home_lx': 'mongodb'},])
+
     # main()
     # upload()
     # path = '/Users/xigua/数据中心/原始文件2/月数据/流量来源-自助取数-月数据'

{mdbq-0.0.3 → mdbq-0.0.5}/mdbq/company/copysh.py

@@ -9,8 +9,8 @@ import datetime
 import shutil
 import time
 import re
-from mdbq.bdup import bd
-from mdbq.pipeline import database
+from mdbq.bdup import bdup
+from mdbq.aggregation import database
 from mdbq.config import update_conf
 warnings.filterwarnings('ignore')
 
@@ -293,16 +293,13 @@ def main():
         res, d_path = u.check_date()  # 文件中的 ch_record 值，决定是否执行更新
         if res:
             upload_path = f'windows/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
-            b = bdup2.BaiDu()
+            b = bdup.BaiDu()
             b.download_dir(local_path=d_path, remote_path=upload_path)
 
-            dp = database.DatabaseUpdate(path=d_path)
-            dp.up_to_mogo = False  # 公司台式机不需要
-            dp.up_to_mysql = False  # 公司台式机不需要
-            dp.up_to_company_mysql = True
+            dp = aggregation.DatabaseUpdate(path=d_path)
             dp.new_unzip(is_move=True)
             dp.cleaning(is_move=True)  # 公司台式机需要移除
-            dp.upload_df()
+            d.upload_df(service_databases=[{'company': 'mysql'}])
 
             # 此操作修改 .copysh_conf 文件，更新完成之后将 ch_record 改为 false
             w = update_conf.UpdateConf(filename='.copysh_conf')

{mdbq-0.0.3 → mdbq-0.0.5}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.3
+Version: 0.0.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.0.3 → mdbq-0.0.5}/mdbq.egg-info/SOURCES.txt

@@ -5,8 +5,10 @@ mdbq.egg-info/PKG-INFO
 mdbq.egg-info/SOURCES.txt
 mdbq.egg-info/dependency_links.txt
 mdbq.egg-info/top_level.txt
+mdbq/aggregation/__init__.py
+mdbq/aggregation/aggregation.py
 mdbq/bdup/__init__.py
-mdbq/bdup/bd.py
+mdbq/bdup/bdup.py
 mdbq/clean/__init__.py
 mdbq/clean/data_clean.py
 mdbq/company/__init__.py
@@ -27,6 +29,4 @@ mdbq/other/xigua_porxy.py
 mdbq/pbix/__init__.py
 mdbq/pbix/pbix_refresh.py
 mdbq/pbix/refresh_all.py
-mdbq/pipeline/__init__.py
-mdbq/pipeline/database.py
 mdbq/spider/__init__.py

{mdbq-0.0.3 → mdbq-0.0.5}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='0.0.3',
+      version='0.0.5',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',

mdbq-0.0.3/mdbq/pipeline/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# 数据清洗任务