mdbq 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -849,8 +849,11 @@ class DatabaseUpdate:
         return df
 
 
-def upload(path, db_name, collection_name, one_file=False):
+def upload(path, db_name, collection_name):
     """ 上传一个文件夹到数据库 """
+    if not os.path.isdir(path):
+        print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload: 函数只接受文件夹路径，不是一个文件夹: {path}')
+        return
     username, password, host, port = get_myconf.select_config_values(
         target_service='home_lx',
         database='mongodb',
@@ -888,7 +891,7 @@ def upload(path, db_name, collection_name, one_file=False):
         db_name=db_name,
         collection_name=collection_name,
     )
-    # print(dtypes)
+
     count = 0
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
@@ -920,12 +923,18 @@ def upload(path, db_name, collection_name, one_file=False):
                 except Exception as e:
                     print(name, e)
             i += 1
-            if one_file:  # 给 file_dir 函数调用
-                break  # 每个文件夹只上传一个文件
     if d.client:
         d.client.close()  # 必须手动关闭数据库连接
 
 
+def one_file_to_mysql(file, db_name, table_name, target_service, database):
+    username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
+    filename = os.path.basename(file)
+    df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False, float_precision='high')
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename)
+
+
 def file_dir():
     """
     按照文件记录对照表
@@ -941,16 +950,26 @@ def file_dir():
     for data in datas:
         # print(data)
         if data['入库进度'] == 0:
-            sub_path, db_name, collection_name = data['子文件夹'], data['数据库名'], data['数据表']
-            # print(os.path.join(path, sub_path), db_name, collection_name)
-            upload(
-                path=os.path.join(path, sub_path),
-                db_name=db_name,
-                collection_name=collection_name,
-                one_file=True
-            )
+            sub_path, db_name, table_name = data['子文件夹'], data['数据库名'], data['数据表']
+            # print(os.path.join(path, sub_path), db_name, table_name)
+            # 从每个文件夹中取出一个文件
+            real_path_list = []
+            for root, dirs, files in os.walk(os.path.join(path, sub_path), topdown=False):
+                for name in files:
+                    if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
+                        real_path_list.append(os.path.join(root, name))
+                        break
+            for real_path in real_path_list:
+                one_file_to_mysql(
+                    file=real_path,
+                    db_name=db_name,
+                    table_name=table_name,
+                    target_service='home_lx',
+                    database='mysql'
+                )
             data.update({'入库进度': 1})  # 更新进度
     df = pd.DataFrame.from_dict(datas, orient='columns')
+    # print(df)
     df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
 
 

{mdbq-1.1.4.dist-info → mdbq-1.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.4
+Version: 1.1.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.4.dist-info → mdbq-1.1.6.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=EgC_gN3glWtK69zgWzj3hfRK1LZyJXXhlFQ1XCTKVUo,55991
+mdbq/aggregation/aggregation.py,sha256=3dCahUs562As08tK9kCyOQVQquYv5aNQNzwXd7l3GDI,57041
 mdbq/aggregation/df_types.py,sha256=rHLIgv82PJSFmDvXkZyOJAffXkFyyMyFO23w9tUt8EQ,7525
 mdbq/aggregation/mysql_types.py,sha256=kzUAGM4FYp77tA_dvHjZNGyPoUAOU1WY5QD0uanh9I4,10418
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.1.4.dist-info/METADATA,sha256=YQd3jzKVU3WLWIyaBS4iLlx80Lh-eGFfS5STLnd6Jr0,245
-mdbq-1.1.4.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.1.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.1.4.dist-info/RECORD,,
+mdbq-1.1.6.dist-info/METADATA,sha256=V1FyrzllnS0bwYxliNCbDaqc8_MaGJNSuGqvdeNA6AE,245
+mdbq-1.1.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.1.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.1.6.dist-info/RECORD,,