mdbq 1.1.4__tar.gz → 1.1.6__tar.gz

{mdbq-1.1.4 → mdbq-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.4
+Version: 1.1.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.4 → mdbq-1.1.6}/mdbq/aggregation/aggregation.py

@@ -849,8 +849,11 @@ class DatabaseUpdate:
         return df
 
 
-def upload(path, db_name, collection_name, one_file=False):
+def upload(path, db_name, collection_name):
     """ 上传一个文件夹到数据库 """
+    if not os.path.isdir(path):
+        print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload: 函数只接受文件夹路径，不是一个文件夹: {path}')
+        return
     username, password, host, port = get_myconf.select_config_values(
         target_service='home_lx',
         database='mongodb',
@@ -888,7 +891,7 @@ def upload(path, db_name, collection_name, one_file=False):
         db_name=db_name,
         collection_name=collection_name,
     )
-    # print(dtypes)
+
     count = 0
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
@@ -920,12 +923,18 @@ def upload(path, db_name, collection_name, one_file=False):
                 except Exception as e:
                     print(name, e)
             i += 1
-            if one_file:  # 给 file_dir 函数调用
-                break  # 每个文件夹只上传一个文件
     if d.client:
         d.client.close()  # 必须手动关闭数据库连接
 
 
+def one_file_to_mysql(file, db_name, table_name, target_service, database):
+    username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
+    filename = os.path.basename(file)
+    df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False, float_precision='high')
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename)
+
+
 def file_dir():
     """
     按照文件记录对照表
@@ -941,16 +950,26 @@ def file_dir():
     for data in datas:
         # print(data)
         if data['入库进度'] == 0:
-            sub_path, db_name, collection_name = data['子文件夹'], data['数据库名'], data['数据表']
-            # print(os.path.join(path, sub_path), db_name, collection_name)
-            upload(
-                path=os.path.join(path, sub_path),
-                db_name=db_name,
-                collection_name=collection_name,
-                one_file=True
-            )
+            sub_path, db_name, table_name = data['子文件夹'], data['数据库名'], data['数据表']
+            # print(os.path.join(path, sub_path), db_name, table_name)
+            # 从每个文件夹中取出一个文件
+            real_path_list = []
+            for root, dirs, files in os.walk(os.path.join(path, sub_path), topdown=False):
+                for name in files:
+                    if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
+                        real_path_list.append(os.path.join(root, name))
+                        break
+            for real_path in real_path_list:
+                one_file_to_mysql(
+                    file=real_path,
+                    db_name=db_name,
+                    table_name=table_name,
+                    target_service='home_lx',
+                    database='mysql'
+                )
             data.update({'入库进度': 1})  # 更新进度
     df = pd.DataFrame.from_dict(datas, orient='columns')
+    # print(df)
     df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
 
 

{mdbq-1.1.4 → mdbq-1.1.6}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.4
+Version: 1.1.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.4 → mdbq-1.1.6}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.1.4',
+      version='1.1.6',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',