mdbq 2.9.6__py3-none-any.whl → 2.9.8__py3-none-any.whl

mdbq/dataframe/converter.py

@@ -71,8 +71,6 @@ class DataFrameConverter(object):
                         pass
             if df[col].dtype == 'float' or df[col].dtype == 'float64':  # 对于小数类型, 保留 6 位小数
                 df[col] = df[col].fillna(0.0).apply(lambda x: round(x, 6))
-                # df[col] = df[col].fillna(0.0).apply(lambda x: "{:.6f}".format(x))
-                # df[col] = df[col].apply('float64')
 
             # 转换日期样式的列为日期类型
             value = df.loc[0, col]
@@ -84,7 +82,9 @@ class DataFrameConverter(object):
                         df[col] = df[col].apply(lambda x: pd.to_datetime(x))
                     except:
                         pass
-            new_col = re.sub(r'[（）()-,，$&~^、* ]', '_', col.lower())
+            new_col = col.lower()
+            new_col = re.sub(r'[()\-，,&~^、 （）\"\'“”=·/。》《><！!`]', '_', new_col, re.IGNORECASE)
+            new_col = new_col.replace('）', '')
             new_col = re.sub(r'_{2,}', '_', new_col)
             new_col = re.sub(r'_+$', '', new_col)
             df.rename(columns={col: new_col}, inplace=True)

mdbq/mysql/mysql.py

@@ -376,17 +376,17 @@ class MysqlUpload:
         cols = df.columns.tolist()
         for col in cols:
             df[col] = df[col].apply(lambda x: float(re.sub(r'%$', '', str(x))) / 100 if (
-                    str(x) != '' and str(x).endswith('%')) else '0.0' if str(x) == '0%' else x)
+                    str(x) != '' and str(x).endswith('%')) and not re.findall('[\\u4e00-\\u9fa5]', str(x)) else '0.0' if str(x) == '0%' else x)
             try:
                 df[col] = df[col].apply(
                     lambda x: int(x) if '_' not in str(x) and '.' not in str(x) else x)  # 不含小数点尝试转整数
             except:
                 pass
-            if df[col].dtype == 'object':
-                try:
+            try:
+                if df[col].dtype == 'object':  # 有些列没有被 pandas 识别数据类型，会没有 dtype 属性
                     df[col] = df[col].apply(lambda x: float(x) if '.' in str(x) and '_' not in str(x) else x)
-                except:
-                    pass
+            except:
+                pass
             new_col = col.lower()
             new_col = re.sub(r'[()\-，,&~^、 （）\"\'“”=·/。》《><！!`]', '_', new_col, re.IGNORECASE)
             new_col = new_col.replace('）', '')
@@ -432,7 +432,7 @@ class MysqlUpload:
                 __res_dict.update({k: 'varchar(255)'})
         return __res_dict, df
 
-    @try_except
+    # @try_except
     def df_to_mysql(self, df, db_name, table_name, set_typ=None, icm_update=[], move_insert=False, df_sql=False, drop_duplicates=False,
                     filename=None, count=None, reset_id=False):
         """
@@ -532,18 +532,19 @@ class MysqlUpload:
                     chunksize=1000
                 )
                 if reset_id:
-                    # 6. 重置自增列
-                    try:
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                        result = cursor.fetchone()
-                        if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                        cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    except Exception as e:
-                        print(f'{e}')
-                        connection.rollback()
+                    pass
+                    # # 6. 重置自增列
+                    # try:
+                    #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    #     result = cursor.fetchone()
+                    #     if result:
+                    #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    #     cursor.execute(
+                    #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    # except Exception as e:
+                    #     print(f'{e}')
+                    #     connection.rollback()
                 connection.commit()  # 提交事务
                 connection.close()
                 return
@@ -570,17 +571,18 @@ class MysqlUpload:
                 )
                 # 6. 重置自增列
                 if reset_id:
-                    try:
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                        result = cursor.fetchone()
-                        if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                        cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    except Exception as e:
-                        print(f'{e}')
-                        connection.rollback()
+                    pass
+                    # try:
+                    #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    #     result = cursor.fetchone()
+                    #     if result:
+                    #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    #     cursor.execute(
+                    #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    # except Exception as e:
+                    #     print(f'{e}')
+                    #     connection.rollback()
                 connection.close()
                 return
 
@@ -589,6 +591,9 @@ class MysqlUpload:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
                 try:
+                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
                     condition = []
                     for k, v in data.items():
                         condition += [f'`{k}` = "{v}"']
@@ -665,23 +670,24 @@ class MysqlUpload:
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
                         cursor.execute(sql)
                 except Exception as e:
+                    pass
                     # print(data)
                     # print(values)
-                    print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
+                    # print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
                     # breakpoint()
 
-            # 6. 重置自增列
-            try:
-                cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                result = cursor.fetchone()
-                if result:
-                    cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                cursor.execute(
-                    f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-            except Exception as e:
-                print(f'{e}')
-                connection.rollback()
+            # # 6. 重置自增列
+            # try:
+            #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+            #     result = cursor.fetchone()
+            #     if result:
+            #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+            #     cursor.execute(
+            #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+            #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+            # except Exception as e:
+            #     print(f'{table_name}, -> {e}')
+            #     connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 
@@ -911,18 +917,18 @@ class OptimizeDatas:
                     else:  # 不存在日期列的情况
                         self.delete_duplicate2(table_name=table_name)
 
-                    # 5. 重置自增列 (id 列)
-                    try:
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                        result = cursor.fetchone()
-                        if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                        cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    except Exception as e:
-                        print(f'{e}')
-                        self.connection.rollback()
+                    # # 5. 重置自增列 (id 列)
+                    # try:
+                    #     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    #     result = cursor.fetchone()
+                    #     if result:
+                    #         cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    #     cursor.execute(
+                    #         f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #     cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    # except Exception as e:
+                    #     print(f'{e}')
+                    #     self.connection.rollback()
                 self.connection.close()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
         print(f'{now}mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')

mdbq/mysql/s_query.py

@@ -92,6 +92,7 @@ class QueryDatas:
 
         if len(df) == 0:
             print(f'database: {db_name}, table: {table_name} 查询的数据为空')
+            return pd.DataFrame()
         cv = converter.DataFrameConverter()
         df = cv.convert_df_cols(df)
         return df

mdbq/spider/aikucun.py

@@ -20,6 +20,9 @@ from selenium.webdriver.common.keys import Keys
 from mdbq.aggregation import aggregation
 from mdbq.clean import data_clean
 from mdbq.other import ua_sj
+from mdbq.mysql import mysql
+from mdbq.config import myconfig
+import socket
 
 warnings.filterwarnings('ignore')
 
@@ -44,6 +47,23 @@ else:
     Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
 upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+m_engine = mysql.MysqlUpload(
+    username=username,
+    password=password,
+    host=host,
+    port=port,
+    charset='utf8mb4'
+)
 
 
 def get_cookie_aikucun():
@@ -262,6 +282,12 @@ class AikuCun:
         _driver.quit()
 
     def clean_data(self, date):
+        set_typ = {
+            '店铺名称': 'varchar(100)',
+            'spu_id': 'varchar(100)',
+            '图片': 'varchar(255)',
+            '数据更新时间': 'timestamp',
+        }
         for root, dirs, files in os.walk(upload_path, topdown=False):
             for name in files:
                 if '~$' in name or 'DS_Store' in name:
@@ -285,6 +311,21 @@ class AikuCun:
                     # df['数据更新时间'] = df['数据更新时间'].apply(lambda x: re.sub('  ', ' ', str(x)) if x else x)
                     # print(df['数据更新时间'])
                     # breakpoint()
+
+                    m_engine.df_to_mysql(
+                        df=df,
+                        db_name='爱库存2',
+                        table_name='商品spu榜单',
+                        icm_update=[],  # 增量更新, 在聚合数据中使用，其他不要用
+                        move_insert=False,  # 先删除，再插入
+                        df_sql=True,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                        drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                        count=None,
+                        filename=None,  # 用来追踪处理进度
+                        reset_id=False,  # 是否重置自增列
+                        set_typ=set_typ,
+                    )
+
                     new_name = f'爱库存_商品榜单_spu_{date}_{date}.csv'
                     df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False)
                     os.remove(os.path.join(root, name))

{mdbq-2.9.6.dist-info → mdbq-2.9.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.9.6
+Version: 2.9.8
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.9.6.dist-info → mdbq-2.9.8.dist-info}/RECORD

@@ -1,11 +1,12 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=3d_sx-cFrW-c03D5Ry9jf144Ph3d0znIl3IHmnInsYA,73902
+mdbq/aggregation/aggregation.py,sha256=2KCVXZygQt4xVxGbFcDMBpL3PukY4yQF_uI-qLSTWaU,73460
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=0kiJQv7xLeH7kXxPmMiUUPYIlt5gcEyzSETmJTV372U,103891
+mdbq/aggregation/query_data.py,sha256=e6vb4hPYZL5KWE6O-MtDoY13GWhx5YMDvTyD3rdgy3c,111441
+mdbq/aggregation/query_data_bak.py,sha256=r1FU0C4zjXln7oVSrRkElh4Ehl-9mYhGcq57jLbViUA,104071
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -20,15 +21,15 @@ mdbq/config/products.py,sha256=ykvoQiA4OvFEYQ35wmCkREECdz0xIJzIs-Xix9mFpYI,6295
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=s-5EzHU9HlcSHlKgRd3autGODaKn523av1N0gh_56YY,4614
+mdbq/dataframe/converter.py,sha256=doWRcFMqf0_RQz7w5BxRNB8JeLetFSsNld43GWlhXXc,4600
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=sfETcLkxeOjDn-B6-ShHRPrq8BUJA9Qmtj4zyIIvi3w,59499
+mdbq/mysql/mysql.py,sha256=_geeu7LP-Ur76kr1ka7_jdifnwrnJJjWnUBzlPXOQOQ,60119
 mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
-mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
+mdbq/mysql/s_query.py,sha256=fnXncwSmA7CB0ELn1a-YxYZDrYkC2Bcgnj2J4dcQ8X8,8481
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
@@ -42,8 +43,8 @@ mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,
 mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.9.6.dist-info/METADATA,sha256=rYMAeGz0dY4meetEVC4l49GVUVRf3E9kzMqoLAn0GW4,243
-mdbq-2.9.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.9.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.9.6.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=01qJo_Di5Kmi2lG5_HKb0OI283b1-Pgqh-nnA0pX4TY,19038
+mdbq-2.9.8.dist-info/METADATA,sha256=-YqbseryNUHGqmaRj1Brh9FejzA62uUxgav_hmn14CQ,243
+mdbq-2.9.8.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.9.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.8.dist-info/RECORD,,

{mdbq-2.9.6.dist-info → mdbq-2.9.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any