mdbq 1.9.8__tar.gz → 2.0.0__tar.gz

{mdbq-1.9.8 → mdbq-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.9.8
+Version: 2.0.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.9.8 → mdbq-2.0.0}/mdbq/aggregation/aggregation.py

@@ -1055,7 +1055,7 @@ class DatabaseUpdate:
                 )
 
 
-def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None):
+def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, target_service='company'):
     """ 上传一个文件夹到 mysql 或者 mongodb 数据库 """
     if not os.path.isdir(path):
         print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 函数只接受文件夹路径，不是一个文件夹: {path}')
@@ -1063,7 +1063,7 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
 
     if dbs['mongodb']:
         username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
+            target_service=target_service,
             database='mongodb',
         )
         d = mongo.UploadMongo(
@@ -1076,7 +1076,7 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
 
     if dbs['mysql']:
         username, password, host, port = get_myconf.select_config_values(
-            target_service='company',
+            target_service=target_service,
             database='mysql',
         )
         m = mysql.MysqlUpload(
@@ -1107,10 +1107,11 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             count += 1
-    i = 0  # 用来统计当前处理文件进度
+    i = 1  # 用来统计当前处理文件进度
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~$' in name or '.DS' in name or '.localized' in name or 'baidu' in name:
+                i += 1
                 continue
             if name.endswith('.csv'):
                 try:
@@ -1161,7 +1162,7 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
     m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename, move_insert=False,  df_sql=True, drop_duplicates=False,)
 
 
-def file_dir(one_file=True):
+def file_dir(one_file=True, target_service='company'):
     """
     按照文件记录对照表上传数据
     批量上传数据库
@@ -1195,7 +1196,7 @@ def file_dir(one_file=True):
                         file=real_path,
                         db_name=db_name,
                         table_name=table_name,
-                        target_service='home_lx',
+                        target_service=target_service,
                         database='mysql'
                     )
             else:  # 上传全部文件夹
@@ -1203,7 +1204,8 @@ def file_dir(one_file=True):
                     path=os.path.join(path, sub_path),
                     db_name = db_name,
                     collection_name = table_name,
-                    dbs={'mysql': True, 'mongodb': True},
+                    dbs={'mysql': True, 'mongodb': False},
+                    target_service=target_service,
                 )
             data.update({'入库进度': 1})  # 更新进度为已上传
     # 将进度信息写回文件
@@ -1238,7 +1240,7 @@ def test2():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
-    # file_dir(one_file=False)
+    # file_dir(one_file=False, target_service='home_lx')
     # one_file_to_mysql(
     #     file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
     #     db_name='京东数据2',
@@ -1247,23 +1249,24 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
 
-    # db_name = '生意经2'
-    # table_name = '省份城市分析'
-    # upload_dir(
-    #     path='/Users/xigua/数据中心/原始文件2/生意经/地域分布',
-    #     db_name=db_name,
-    #     collection_name=table_name,
-    #     dbs={'mysql': True, 'mongodb': False},
-    # )
-    #
+    db_name = '推广数据2'
+    table_name = '权益报表'
+    upload_dir(
+        path='/Users/xigua/数据中心/原始文件2/推广报表/权益报表12313',
+        db_name=db_name,
+        collection_name=table_name,
+        dbs={'mysql': True, 'mongodb': False},
+        target_service='home_lx',
+    )
 
-    # 新版 数据分类
-    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
-    dp.new_unzip(is_move=True)
-    dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # 将 self.datas 更新至数据库
-    dp.upload_df(service_databases=[
-        # {'home_lx': 'mongodb'},
-        {'company': 'mysql'},
-        # {'nas': 'mysql'},
-    ])
+
+    # # 新版 数据分类
+    # dp = DatabaseUpdate(path='/Users/xigua/Downloads')
+    # dp.new_unzip(is_move=True)
+    # dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
+    # # 将 self.datas 更新至数据库
+    # dp.upload_df(service_databases=[
+    #     # {'home_lx': 'mongodb'},
+    #     {'company': 'mysql'},
+    #     # {'nas': 'mysql'},
+    # ])

{mdbq-1.9.8 → mdbq-2.0.0}/mdbq/clean/data_clean.py

@@ -116,7 +116,15 @@ class DataClean:
 
                             df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
                             df.fillna(0, inplace=True)
-                            col_ids = ['场景ID', '计划ID', '单元ID', '主体ID', '宝贝ID', '词ID/词包ID', '创意ID']
+                            col_ids = [
+                                # '场景ID',  # 2024.10.5 改为不加 =""
+                                '计划ID',
+                                '单元ID',
+                                '主体ID',
+                                '宝贝ID',
+                                '词ID/词包ID',
+                                '创意ID',
+                            ]
                             sb = df.columns.tolist()
                             if '日期' not in sb:
                                 print(f'{name} 注意：该报表不包含分日数据，数据不会保存，请重新下载！')
@@ -162,7 +170,10 @@ class DataClean:
                             shop_name = shop_name[0]
                         else:
                             shop_name = ''
-                        cols = ['场景ID', '计划ID']
+                        cols = [
+                            # '场景ID',  # 2024.10.5 改为不加 =""
+                            '计划ID',
+                        ]
                         for col in cols:
                             df[col] = df[col].apply(lambda x: f'="{x}"' if x and '=' not in str(x) else x)
                         df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符

{mdbq-1.9.8 → mdbq-2.0.0}/mdbq/config/get_myconf.py

@@ -62,7 +62,7 @@ def select_config_values(target_service, database, path=None):
             if socket.gethostname() == 'xigua_lx':
                 # 本机自身运行使用 127.0.0.1
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_bd',  'port_mysql_lx_nw',]
-            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBookPro':
                 # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macb    ook pro
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_mysql_lx_nw',  'port_mysql_lx_nw',]
             else:

{mdbq-1.9.8 → mdbq-2.0.0}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.9.8
+Version: 2.0.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.9.8 → mdbq-2.0.0}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.9.8',
+      version='2.0.0',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',