mdbq 1.9.9__py3-none-any.whl → 2.0.1__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1055,7 +1055,7 @@ class DatabaseUpdate:
                 )
 
 
-def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None):
+def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, target_service='company'):
     """ 上传一个文件夹到 mysql 或者 mongodb 数据库 """
     if not os.path.isdir(path):
         print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 函数只接受文件夹路径，不是一个文件夹: {path}')
@@ -1063,7 +1063,7 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
 
     if dbs['mongodb']:
         username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
+            target_service=target_service,
             database='mongodb',
         )
         d = mongo.UploadMongo(
@@ -1076,7 +1076,7 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
 
     if dbs['mysql']:
         username, password, host, port = get_myconf.select_config_values(
-            target_service='company',
+            target_service=target_service,
             database='mysql',
         )
         m = mysql.MysqlUpload(
@@ -1107,10 +1107,11 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             count += 1
-    i = 0  # 用来统计当前处理文件进度
+    i = 1  # 用来统计当前处理文件进度
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~$' in name or '.DS' in name or '.localized' in name or 'baidu' in name:
+                i += 1
                 continue
             if name.endswith('.csv'):
                 try:
@@ -1161,7 +1162,7 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
     m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename, move_insert=False,  df_sql=True, drop_duplicates=False,)
 
 
-def file_dir(one_file=True):
+def file_dir(one_file=True, target_service='company'):
     """
     按照文件记录对照表上传数据
     批量上传数据库
@@ -1195,7 +1196,7 @@ def file_dir(one_file=True):
                         file=real_path,
                         db_name=db_name,
                         table_name=table_name,
-                        target_service='home_lx',
+                        target_service=target_service,
                         database='mysql'
                     )
             else:  # 上传全部文件夹
@@ -1203,7 +1204,8 @@ def file_dir(one_file=True):
                     path=os.path.join(path, sub_path),
                     db_name = db_name,
                     collection_name = table_name,
-                    dbs={'mysql': True, 'mongodb': True},
+                    dbs={'mysql': True, 'mongodb': False},
+                    target_service=target_service,
                 )
             data.update({'入库进度': 1})  # 更新进度为已上传
     # 将进度信息写回文件
@@ -1238,7 +1240,7 @@ def test2():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
-    # file_dir(one_file=False)
+    file_dir(one_file=False, target_service='home_lx')
     # one_file_to_mysql(
     #     file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
     #     db_name='京东数据2',
@@ -1247,23 +1249,24 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
 
-    # db_name = '生意经2'
-    # table_name = '省份城市分析'
+    # db_name = '推广数据2'
+    # table_name = '权益报表'
     # upload_dir(
-    #     path='/Users/xigua/数据中心/原始文件2/生意经/地域分布',
+    #     path='/Users/xigua/数据中心/原始文件2/推广报表/权益报表12313',
     #     db_name=db_name,
     #     collection_name=table_name,
     #     dbs={'mysql': True, 'mongodb': False},
+    #     target_service='home_lx',
     # )
-    #
 
-    # 新版 数据分类
-    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
-    dp.new_unzip(is_move=True)
-    dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # 将 self.datas 更新至数据库
-    dp.upload_df(service_databases=[
-        # {'home_lx': 'mongodb'},
-        {'company': 'mysql'},
-        # {'nas': 'mysql'},
-    ])
+
+    # # 新版 数据分类
+    # dp = DatabaseUpdate(path='/Users/xigua/Downloads')
+    # dp.new_unzip(is_move=True)
+    # dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
+    # # 将 self.datas 更新至数据库
+    # dp.upload_df(service_databases=[
+    #     # {'home_lx': 'mongodb'},
+    #     {'company': 'mysql'},
+    #     # {'nas': 'mysql'},
+    # ])

mdbq/aggregation/query_data.py

@@ -1535,7 +1535,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=24)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

mdbq/config/get_myconf.py

@@ -62,7 +62,7 @@ def select_config_values(target_service, database, path=None):
             if socket.gethostname() == 'xigua_lx':
                 # 本机自身运行使用 127.0.0.1
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_bd',  'port_mysql_lx_nw',]
-            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBookPro':
                 # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macb    ook pro
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_mysql_lx_nw',  'port_mysql_lx_nw',]
             else:

{mdbq-1.9.9.dist-info → mdbq-2.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.9.9
+Version: 2.0.1
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.9.9.dist-info → mdbq-2.0.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=ZDMYHZXuVx9MIiTYDjGUHLNBvj98h6-q6x2BrlBdaNw,72786
+mdbq/aggregation/aggregation.py,sha256=0V7pnamxjvz5cpPYm6cE5-EFkktXVfB60fauC3SrZ9Y,73008
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=32NjVVYLnfFkzD8TflmNVhpdQTLRRUrb9toMGApSOC8,72379
+mdbq/aggregation/query_data.py,sha256=ixvfjEai-zIN5fUiVg9nSFrzjiBfJv6QC9_mLTvFfg0,72380
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -13,7 +13,7 @@ mdbq/clean/data_clean.py,sha256=BgOZlJwuv5X4BL24DOHC6XtMFspFD-AdCBC_EJJ9rjA,1013
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=VUaaJPXPYPHWwnkdK77PWz_dAXZyEmYBA9Df1yROHAc,17764
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/get_myconf.py,sha256=-CFEW0dQh4OIwVgwK-cL0eVp1LN3PjJgN89d4P5TB9I,6011
+mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.9.9.dist-info/METADATA,sha256=zYBl4BSGNMyJDZRf5BKpMZKIxlJv6tPosWHyZlc7w04,245
-mdbq-1.9.9.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.9.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.9.9.dist-info/RECORD,,
+mdbq-2.0.1.dist-info/METADATA,sha256=oqzIwWYKBRXwGEedgZbZPsi8JPSHyOceKS9ArHbgMMU,245
+mdbq-2.0.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.0.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.0.1.dist-info/RECORD,,