mdbq 2.9.6__py3-none-any.whl → 2.9.7__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1150,51 +1150,39 @@ def upload_dir(path, db_name, collection_name, json_path=None):
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
             if name.endswith('.xlsx'):
                 df = pd.read_excel(os.path.join(root, name), sheet_name=0, header=0, engine='openpyxl')
-            try:
-                if len(df) == 0:
-                    continue
-                # if '新版' not in name:
-                #     continue
-                cv = converter.DataFrameConverter()
-                df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
-
-                try:
-                    df = df.astype(dtypes)  # 按本地文件更新 df 的数据类型, 可能因为字段不同产生异常
-                except Exception as e:
-                    print(name, e)
-                    # 如果发生异常，这将 df 的数据和 json 中的数据取交集
-                    old_dt = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
-                    intersection_keys = dtypes.keys() & old_dt.keys()  # 获取两个字典键的交集
-                    dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
-                    df = df.astype(dtypes)  # 再次更新 df 的数据类型
-                df.fillna(0, inplace=True)
-
-                # for col in df.columns.tolist():
-                #     df[col] = df[col].apply(lambda x: 0 if str(x) == '' else x)
-                # print(f'{i}/{count}')
-                # sql_engine = create_engine(
-                #     f"mysql+pymysql://{username}:{password}@{host}:{port}/{db_name}")  # 创建数据库引擎
-                # df.to_sql(
-                #     name=collection_name,
-                #     con=sql_engine,
-                #     if_exists='append',
-                #     index=False,
-                #     chunksize=1000
-                # )
+            # try:
+            if len(df) == 0:
+                continue
+            # if '新版' not in name:
+            #     continue
+            # cv = converter.DataFrameConverter()
+            # df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
 
+            # try:
+            #     df = df.astype(dtypes)  # 按本地文件更新 df 的数据类型, 可能因为字段不同产生异常
+            # except Exception as e:
+            #     print(name, e)
+            #     # 如果发生异常，这将 df 的数据和 json 中的数据取交集
+            #     old_dt = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
+            #     intersection_keys = dtypes.keys() & old_dt.keys()  # 获取两个字典键的交集
+            #     dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
+            #     df = df.astype(dtypes)  # 再次更新 df 的数据类型
+            df.fillna(0, inplace=True)
+            for col in df.columns.tolist():
+                df[col] = df[col].apply(lambda x: 0 if str(x) == '' else x)
 
+            # if '更新时间' not in df.columns.tolist():
+            #     df['更新时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            #
+            m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
+                          move_insert=False,  # 先删除，再插入
+                          df_sql = True,
+                          drop_duplicates=False,
+                          count=f'{i}/{count}',
+                          filename=name,
+                          )
+            # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
 
-                #
-                m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
-                              move_insert=False,  # 先删除，再插入
-                              df_sql = True,
-                              drop_duplicates=False,
-                              count=f'{i}/{count}',
-                              filename=name,
-                              )
-                # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
-            except Exception as e:
-                print(name, e)
             i += 1
 
 
@@ -1282,13 +1270,13 @@ if __name__ == '__main__':
     # )
 
     # test()
-    col = 0
+    col = 1
     if col:
         # 上传一个目录到指定数据库
-        db_name = '生意参谋3'
-        table_name = '店铺流量来源构成'
+        db_name = '京东数据3'
+        table_name = '推广数据_全站营销'
         upload_dir(
-            path=r'/Users/xigua/数据中心/原始文件3/生意参谋/店铺流量来源',
+            path=r'/Users/xigua/数据中心/原始文件3/京东报表/京准通_全站营销',
             db_name=db_name,
             collection_name=table_name,
         )

mdbq/aggregation/query_data.py

@@ -2409,11 +2409,15 @@ def main(days=100, months=3):
 
 
 if __name__ == '__main__':
-    main(days=100, months=3)
+    # main(days=100, months=3)
 
     # data_aggregation(
     #     months=3,
     #     is_juhe=True,  # 生成聚合表
     #     # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
     # )
-
+    data_aggregation(
+        months=1,
+        is_juhe=True,  # 生成聚合表
+        # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
+    )

mdbq/mysql/mysql.py

@@ -376,17 +376,17 @@ class MysqlUpload:
         cols = df.columns.tolist()
         for col in cols:
             df[col] = df[col].apply(lambda x: float(re.sub(r'%$', '', str(x))) / 100 if (
-                    str(x) != '' and str(x).endswith('%')) else '0.0' if str(x) == '0%' else x)
+                    str(x) != '' and str(x).endswith('%')) and not re.findall('[\\u4e00-\\u9fa5]', str(x)) else '0.0' if str(x) == '0%' else x)
             try:
                 df[col] = df[col].apply(
                     lambda x: int(x) if '_' not in str(x) and '.' not in str(x) else x)  # 不含小数点尝试转整数
             except:
                 pass
-            if df[col].dtype == 'object':
-                try:
+            try:
+                if df[col].dtype == 'object':  # 有些列没有被 pandas 识别数据类型，会没有 dtype 属性
                     df[col] = df[col].apply(lambda x: float(x) if '.' in str(x) and '_' not in str(x) else x)
-                except:
-                    pass
+            except:
+                pass
             new_col = col.lower()
             new_col = re.sub(r'[()\-，,&~^、 （）\"\'“”=·/。》《><！!`]', '_', new_col, re.IGNORECASE)
             new_col = new_col.replace('）', '')
@@ -432,7 +432,7 @@ class MysqlUpload:
                 __res_dict.update({k: 'varchar(255)'})
         return __res_dict, df
 
-    @try_except
+    # @try_except
     def df_to_mysql(self, df, db_name, table_name, set_typ=None, icm_update=[], move_insert=False, df_sql=False, drop_duplicates=False,
                     filename=None, count=None, reset_id=False):
         """
@@ -532,18 +532,19 @@ class MysqlUpload:
                     chunksize=1000
                 )
                 if reset_id:
-                    # 6. 重置自增列
-                    try:
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                        result = cursor.fetchone()
-                        if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                        cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    except Exception as e:
-                        print(f'{e}')
-                        connection.rollback()
+                    pass
+                    # # 6. 重置自增列
+                    # try:
+                    #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    #     result = cursor.fetchone()
+                    #     if result:
+                    #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    #     cursor.execute(
+                    #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    # except Exception as e:
+                    #     print(f'{e}')
+                    #     connection.rollback()
                 connection.commit()  # 提交事务
                 connection.close()
                 return
@@ -570,17 +571,18 @@ class MysqlUpload:
                 )
                 # 6. 重置自增列
                 if reset_id:
-                    try:
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                        result = cursor.fetchone()
-                        if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                        cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    except Exception as e:
-                        print(f'{e}')
-                        connection.rollback()
+                    pass
+                    # try:
+                    #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    #     result = cursor.fetchone()
+                    #     if result:
+                    #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    #     cursor.execute(
+                    #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    # except Exception as e:
+                    #     print(f'{e}')
+                    #     connection.rollback()
                 connection.close()
                 return
 
@@ -589,6 +591,9 @@ class MysqlUpload:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
                 try:
+                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
                     condition = []
                     for k, v in data.items():
                         condition += [f'`{k}` = "{v}"']
@@ -670,18 +675,18 @@ class MysqlUpload:
                     print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
                     # breakpoint()
 
-            # 6. 重置自增列
-            try:
-                cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                result = cursor.fetchone()
-                if result:
-                    cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                cursor.execute(
-                    f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-            except Exception as e:
-                print(f'{e}')
-                connection.rollback()
+            # # 6. 重置自增列
+            # try:
+            #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+            #     result = cursor.fetchone()
+            #     if result:
+            #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+            #     cursor.execute(
+            #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+            #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+            # except Exception as e:
+            #     print(f'{e}')
+            #     connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 
@@ -911,18 +916,18 @@ class OptimizeDatas:
                     else:  # 不存在日期列的情况
                         self.delete_duplicate2(table_name=table_name)
 
-                    # 5. 重置自增列 (id 列)
-                    try:
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                        result = cursor.fetchone()
-                        if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                        cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    except Exception as e:
-                        print(f'{e}')
-                        self.connection.rollback()
+                    # # 5. 重置自增列 (id 列)
+                    # try:
+                    #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    #     result = cursor.fetchone()
+                    #     if result:
+                    #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    #     cursor.execute(
+                    #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    # except Exception as e:
+                    #     print(f'{e}')
+                    #     self.connection.rollback()
                 self.connection.close()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
         print(f'{now}mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')

mdbq/spider/aikucun.py

@@ -20,6 +20,9 @@ from selenium.webdriver.common.keys import Keys
 from mdbq.aggregation import aggregation
 from mdbq.clean import data_clean
 from mdbq.other import ua_sj
+from mdbq.mysql import mysql
+from mdbq.config import myconfig
+import socket
 
 warnings.filterwarnings('ignore')
 
@@ -44,6 +47,23 @@ else:
     Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
 upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+m_engine = mysql.MysqlUpload(
+    username=username,
+    password=password,
+    host=host,
+    port=port,
+    charset='utf8mb4'
+)
 
 
 def get_cookie_aikucun():
@@ -262,6 +282,12 @@ class AikuCun:
         _driver.quit()
 
     def clean_data(self, date):
+        set_typ = {
+            '店铺名称': 'varchar(100)',
+            'spu_id': 'varchar(100)',
+            '图片': 'varchar(255)',
+            '数据更新时间': 'timestamp',
+        }
         for root, dirs, files in os.walk(upload_path, topdown=False):
             for name in files:
                 if '~$' in name or 'DS_Store' in name:
@@ -285,6 +311,21 @@ class AikuCun:
                     # df['数据更新时间'] = df['数据更新时间'].apply(lambda x: re.sub('  ', ' ', str(x)) if x else x)
                     # print(df['数据更新时间'])
                     # breakpoint()
+
+                    m_engine.df_to_mysql(
+                        df=df,
+                        db_name='爱库存2',
+                        table_name='商品spu榜单',
+                        icm_update=[],  # 增量更新, 在聚合数据中使用，其他不要用
+                        move_insert=False,  # 先删除，再插入
+                        df_sql=True,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                        drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                        count=None,
+                        filename=None,  # 用来追踪处理进度
+                        reset_id=False,  # 是否重置自增列
+                        set_typ=set_typ,
+                    )
+
                     new_name = f'爱库存_商品榜单_spu_{date}_{date}.csv'
                     df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False)
                     os.remove(os.path.join(root, name))

{mdbq-2.9.6.dist-info → mdbq-2.9.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.9.6
+Version: 2.9.7
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.9.6.dist-info → mdbq-2.9.7.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=3d_sx-cFrW-c03D5Ry9jf144Ph3d0znIl3IHmnInsYA,73902
+mdbq/aggregation/aggregation.py,sha256=2KCVXZygQt4xVxGbFcDMBpL3PukY4yQF_uI-qLSTWaU,73460
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=0kiJQv7xLeH7kXxPmMiUUPYIlt5gcEyzSETmJTV372U,103891
+mdbq/aggregation/query_data.py,sha256=r1FU0C4zjXln7oVSrRkElh4Ehl-9mYhGcq57jLbViUA,104071
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -26,7 +26,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=sfETcLkxeOjDn-B6-ShHRPrq8BUJA9Qmtj4zyIIvi3w,59499
+mdbq/mysql/mysql.py,sha256=O5pmD_RB8TRKuatqL9Runrlr5E4XrLpDsPsi0vW_e8c,60075
 mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -42,8 +42,8 @@ mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,
 mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.9.6.dist-info/METADATA,sha256=rYMAeGz0dY4meetEVC4l49GVUVRf3E9kzMqoLAn0GW4,243
-mdbq-2.9.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.9.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.9.6.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=01qJo_Di5Kmi2lG5_HKb0OI283b1-Pgqh-nnA0pX4TY,19038
+mdbq-2.9.7.dist-info/METADATA,sha256=ekYpjMjTHFv6a4rLxf9LiRjwZC49l8XpNQdkXD2shQM,243
+mdbq-2.9.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.9.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.7.dist-info/RECORD,,