mdbq 2.6.4__tar.gz → 2.6.5__tar.gz

{mdbq-2.6.4 → mdbq-2.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.6.4
+Version: 2.6.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.6.4 → mdbq-2.6.5}/mdbq/aggregation/aggregation.py

@@ -1174,36 +1174,40 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
                 i += 1
                 continue
             if name.endswith('.csv'):
-                try:
-                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    if len(df) == 0:
-                        continue
-                    # if '新版' not in name:
-                    #     continue
-                    cv = converter.DataFrameConverter()
-                    df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
-
-                    try:
-                        df = df.astype(dtypes)  # 按本地文件更新 df 的数据类型, 可能因为字段不同产生异常
-                    except Exception as e:
-                        print(name, e)
-                        # 如果发生异常，这将 df 的数据和 json 中的数据取交集
-                        old_dt = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
-                        intersection_keys = dtypes.keys() & old_dt.keys()  # 获取两个字典键的交集
-                        dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
-                        df = df.astype(dtypes)  # 再次更新 df 的数据类型
+                df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+            if name.endswith('.xlsx'):
+                df = pd.read_excel(os.path.join(root, name), sheet_name=0, header=0, engine='openpyxl')
+            try:
+                if len(df) == 0:
+                    continue
+                # if '新版' not in name:
+                #     continue
+                cv = converter.DataFrameConverter()
+                df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
 
-                    if dbs['mongodb']:
-                        d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                    if dbs['mysql']:  # drop_duplicates: 值为 True 时检查重复数据再插入
-                        m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
-                                      move_insert=False,  # 先删除，再插入
-                                      df_sql = True,
-                                      drop_duplicates=False,
-                                      filename=name, count=f'{i}/{count}')
-                        # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
+                try:
+                    df = df.astype(dtypes)  # 按本地文件更新 df 的数据类型, 可能因为字段不同产生异常
                 except Exception as e:
                     print(name, e)
+                    # 如果发生异常，这将 df 的数据和 json 中的数据取交集
+                    old_dt = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
+                    intersection_keys = dtypes.keys() & old_dt.keys()  # 获取两个字典键的交集
+                    dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
+                    df = df.astype(dtypes)  # 再次更新 df 的数据类型
+
+                if dbs['mongodb']:
+                    d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
+                if dbs['mysql']:  # drop_duplicates: 值为 True 时检查重复数据再插入
+                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
+                                  move_insert=False,  # 先删除，再插入
+                                  df_sql = True,
+                                  drop_duplicates=False,
+                                  filename=name, count=f'{i}/{count}',
+                                  service_database={target_service: 'mysql'},   # 这个参数是用来设置更新哪台服务器的 types 信息到本地 json 文件
+                                  )
+                    # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
+            except Exception as e:
+                print(name, e)
             i += 1
     if dbs['mongodb']:
         if d.client:
@@ -1220,7 +1224,16 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False, float_precision='high')
     # df.replace(to_replace=[','], value='', regex=True, inplace=True)  # 替换掉特殊字符
     m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
-    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename, move_insert=False,  df_sql=True, drop_duplicates=False,)
+    m.df_to_mysql(
+        df=df,
+        db_name=db_name,
+        table_name=table_name,
+        filename=filename,
+        move_insert=False,
+        df_sql=True,
+        drop_duplicates=False,
+        service_database={target_service: database},
+    )
 
 
 def file_dir(one_file=True, target_service='company'):
@@ -1303,26 +1316,26 @@ if __name__ == '__main__':
     print(username, password, host, port)
     # file_dir(one_file=False, target_service='company')
 
-    # 上传 1 个文件到数据库
-    one_file_to_mysql(
-        file='/Users/xigua/Downloads/万里马箱包推广1_营销概况_qwqw全站营销_2024-08-18_2024-09-01.csv',
-        db_name='京东数据3',
-        table_name='推广数据_全站营销',
-        target_service='company',
-        database='mysql'
-    )
-
-    # # 上传一个目录到指定数据库
-    # db_name = '天猫_推广数据3'
-    # table_name = '主体报表'
-    # upload_dir(
-    #     path='/Users/xigua/数据中心/原始文件3/天猫推广报表/主体报表',
-    #     db_name=db_name,
-    #     collection_name=table_name,
-    #     dbs={'mysql': True, 'mongodb': False},
+    # # 上传 1 个文件到数据库
+    # one_file_to_mysql(
+    #     file='/Users/xigua/Downloads/万里马箱包推广1_营销概况_qwqw全站营销_2024-08-18_2024-09-01.csv',
+    #     db_name='京东数据3',
+    #     table_name='推广数据_全站营销',
     #     target_service='company',
+    #     database='mysql'
     # )
 
+    # 上传一个目录到指定数据库
+    db_name = '京东数据3'
+    table_name = '京东商智_spu_商品明细'
+    upload_dir(
+        path='/Users/xigua/数据中心/原始文件3/京东报表/spu_商品明细',
+        db_name=db_name,
+        collection_name=table_name,
+        dbs={'mysql': True, 'mongodb': False},
+        target_service='company',
+    )
+
 
     # # 新版 数据分类
     # dp = DatabaseUpdate(path='/Users/xigua/Downloads')

{mdbq-2.6.4 → mdbq-2.6.5}/mdbq/clean/clean_upload.py

@@ -103,6 +103,11 @@ class DataClean:
                 '数据库名': '爱库存2',
                 '集合名称': '商品spu榜单',
             },
+            {
+                '文件简称': '手淘搜索_本店引流词_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
+                '集合名称': '手淘搜索_本店引流词',
+            },
         ]
         for root, dirs, files in os.walk(path, topdown=False):
             for name in files:
@@ -134,7 +139,7 @@ class DataClean:
                 if name.endswith('.xls') and '商品排行_' in name:
                     df = pd.read_excel(os.path.join(root, name), header=4)
                     if len(df) == 0:
-                        print(f'{name} 报表数据为空')
+                        print(f'{name} 报表数据不能为空')
                         continue
                     df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
                     df.replace(to_replace=[','], value='', regex=True, inplace=True)
@@ -145,6 +150,25 @@ class DataClean:
                     new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
+                elif name.endswith('.xls') and '手淘搜索_本店引流词_' in name:
+                    df = pd.read_excel(os.path.join(root, name), header=5)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据不能为空')
+                        continue
+                    df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
+                    df.replace(to_replace=[','], value='', regex=True, inplace=True)
+                    df.rename(columns={'统计日期': '日期'}, inplace=True)
+                    shop_name = re.findall(r'本店.*_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
+                    kw_type = re.findall('手淘搜索_本店引流词_([\u4e00-\u9fff]+)_', name)[0]
+                    df.insert(loc=2, column='词类型', value=kw_type)
+                    if '店铺名称' in df.columns.tolist():
+                        df['店铺名称'] = shop_name
+                    else:
+                        df.insert(loc=1, column='店铺名称', value=shop_name)
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+
                 elif name.endswith('.csv') and '_来源构成_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
@@ -938,6 +962,9 @@ class DataClean:
                 elif name.endswith('.csv') and '爱库存_商品榜单_' in name:
                     t_path = os.path.join(self.source_path, '爱库存', 'spu商品榜单')
                     bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '手淘搜索_本店引流词_' in name:
+                    t_path = os.path.join(self.source_path, '生意参谋', '手淘搜索_本店引流词')
+                    bib(t_path, _as_month=True)
 
     def move_dmp(self, path=None, is_except=[]):
         """ 达摩盘 """
@@ -1517,15 +1544,15 @@ def main(service_databases=None, is_mysql=False):
 
 
 if __name__ == '__main__':
-    # main(
-    #     service_databases = [
-    #         {'company': 'mysql'},
-    #         # {'home_lx': 'mysql'},
-    #         # {'home_lx': 'mongodb'},
-    #         # {'nas': 'mysql'},
-    #     ],
-    #     is_mysql = False,  # 清理聚合数据
-    # )
+    main(
+        service_databases = [
+            {'company': 'mysql'},
+            # {'home_lx': 'mysql'},
+            # {'home_lx': 'mongodb'},
+            # {'nas': 'mysql'},
+        ],
+        is_mysql = False,  # 清理聚合数据
+    )
 
     # c = DataClean(
     #     path=upload_path,  # 源文件目录，下载文件夹
@@ -1536,4 +1563,4 @@ if __name__ == '__main__':
     # c.move_tg_tm(is_except=['临时文件', ])  # 天猫，移到文件到原始文件夹
 
 
-    test()
+    # test()

{mdbq-2.6.4 → mdbq-2.6.5}/mdbq/mysql/mysql.py

@@ -82,7 +82,7 @@ class MysqlUpload:
         icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用，设置此参数时需将 drop_duplicates 改为 False
                 使用增量更新: 必须确保 icm_update 传进来的列必须是数据表中唯一主键，值不会发生变化，不会重复，否则可能产生错乱覆盖情况
         filename: 用来追踪处理进度，传这个参数是方便定位产生错误的文件
-        service_databases: 这个参数是用来设置更新哪台服务器的 types 信息到本地 json 文件
+        service_database: 这个参数是用来设置更新哪台服务器的 types 信息到本地 json 文件
         json_path: 这个参数同样也是是用来设置更新 json 文件
         """
         self.filename = filename
@@ -168,18 +168,18 @@ class MysqlUpload:
                     chunksize=1000
                 )
                 # print(f'重置自增')
-                # 6. 重置自增列
-                try:
-                    cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                    result = cursor.fetchone()
-                    if result:
-                        cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                    cursor.execute(
-                        f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                    cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                except Exception as e:
-                    print(f'{e}')
-                    connection.rollback()
+                # # 6. 重置自增列
+                # try:
+                #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                #     result = cursor.fetchone()
+                #     if result:
+                #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                #     cursor.execute(
+                #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                # except Exception as e:
+                #     print(f'{e}')
+                #     connection.rollback()
                 connection.close()
                 return
 

{mdbq-2.6.4 → mdbq-2.6.5}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.6.4
+Version: 2.6.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.6.4 → mdbq-2.6.5}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.6.4',
+      version='2.6.5',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',