mdbq 3.8.11__py3-none-any.whl → 3.8.13__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.8.11'
+VERSION = '3.8.13'

mdbq/aggregation/query_data.py

@@ -147,12 +147,12 @@ class MysqlDatasQuery:
             '直接成交笔数': 'int',
             '直接成交金额': 'decimal(12,2)',
         }
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['日期', '店铺名称', '商品id', '花费', '成交金额', '直接成交金额']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['日期', '店铺名称', '商品id', '花费', '成交金额', '直接成交金额']]
+        #     }
+        # )  # 制作其他聚合表
         self.pf_datas.append(
             {
                 '集合名称': '天猫汇总表调用',
@@ -633,12 +633,12 @@ class MysqlDatasQuery:
             '件均价': 'mediumint',
             '价格带': 'varchar(100)',
         }
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['日期', '店铺名称', '宝贝id', '销售额', '销售量', '退款额_发货后', '退货量_发货后']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['日期', '店铺名称', '宝贝id', '销售额', '销售量', '退款额_发货后', '退货量_发货后']]
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         min_date = df['日期'].min()
@@ -1333,12 +1333,12 @@ class MysqlDatasQuery:
             '二级类目': 'varchar(100)',
             '三级类目': 'varchar(100)',
         }
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['宝贝id', '商家编码']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['宝贝id', '商家编码']]
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1390,12 +1390,12 @@ class MysqlDatasQuery:
             '三级类目': 'varchar(100)',
             '更新时间': 'timestamp'
         }
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['宝贝id', '商家编码']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['宝贝id', '商家编码']]
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1447,12 +1447,12 @@ class MysqlDatasQuery:
             '日期': 'date',
             '商品链接': 'varchar(255)',
         }
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['商品id', '商品图片']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['商品id', '商品图片']]
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1505,13 +1505,13 @@ class MysqlDatasQuery:
             'sku地址': 'varchar(255)',
             '更新时间': 'timestamp'
         }
-        # 制作其他聚合表
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['商品id', '商品图片']]
-            }
-        )
+        # # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['商品id', '商品图片']]
+        #     }
+        # )
         if not self.update_service:  # 调试加，是否继续执行下面的入库操作
             return
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1852,12 +1852,12 @@ class MysqlDatasQuery:
             '天猫页面价': 'decimal(10,2)',
             '天猫中促价': 'decimal(10,2)',
         }
-        self.pf_datas.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['款号', '成本价']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['款号', '成本价']]
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         min_date = pd.to_datetime(df['日期'].min()).strftime('%Y-%m-%d')
@@ -1951,12 +1951,12 @@ class MysqlDatasQuery:
         p = df.pop('spu_id')
         df.insert(loc=3, column='spu_id', value=p)
 
-        self.pf_datas_jd.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df[['日期', '产品线', '触发sku_id', '跟单sku_id', '花费']]
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas_jd.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df[['日期', '产品线', '触发sku_id', '跟单sku_id', '花费']]
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         set_typ = {
@@ -2204,12 +2204,12 @@ class MysqlDatasQuery:
         idx = df.groupby(['日期', '店铺名称', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数'])['更新时间'].idxmax()
         df = df.loc[idx]
         df = df[['日期', '店铺名称', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数', '成交单量', '成交金额']]
-        self.pf_datas_jd.append(
-            {
-                '集合名称': table_name,
-                '数据主体': df
-            }
-        )  # 制作其他聚合表
+        # self.pf_datas_jd.append(
+        #     {
+        #         '集合名称': table_name,
+        #         '数据主体': df
+        #     }
+        # )  # 制作其他聚合表
         if not self.update_service:
             return
         set_typ = {
@@ -3758,15 +3758,6 @@ class MysqlDatasQuery:
             '直接成交金额': 'float64',
             '自然流量曝光量': 'int64',
         }, errors='raise')
-        # tg = tg.groupby(['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-        #     **{'加购量': ('加购量', np.max),
-        #        '成交笔数': ('成交笔数', np.max),
-        #        '成交金额': ('成交金额', np.max),
-        #        '自然流量曝光量': ('自然流量曝光量', np.max),
-        #        '直接成交笔数': ('直接成交笔数', np.max),
-        #        '直接成交金额': ('直接成交金额', np.max)
-        #        }
-        # )
 
         df = pd.concat([tg, zb, pxb], axis=0, ignore_index=True)
         df.fillna(0, inplace=True)  # concat 之后要填充空值
@@ -3994,25 +3985,29 @@ def query1(months=1, less_dict=[]):
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
     sdq.update_service = True  # 调试时加，true: 将数据写入 mysql 服务器
 
-    sdq._ald_wxt(db_name='聚合数据', table_name='奥莱店_主体报表')
-    sdq._sj_wxt(db_name='聚合数据', table_name='圣积天猫店_主体报表')
-    sdq._tb_wxt(db_name='聚合数据', table_name='淘宝_主体报表')
+    # 依赖表  -- >>
     sdq.tg_wxt(db_name='聚合数据', table_name='天猫_主体报表')
+    sdq.tg_cjzb(db_name='聚合数据', table_name='天猫_超级直播')
+    sdq.pxb_zh(db_name='聚合数据', table_name='天猫_品销宝账户报表')
+    # 依赖表  << --
+
     sdq.syj(db_name='聚合数据', table_name='生意经_宝贝指标')
     sdq.idbm(db_name='聚合数据', table_name='商品id编码表')
     sdq.sp_picture(db_name='聚合数据', table_name='商品id图片对照表')
     sdq.sp_cost(db_name='聚合数据', table_name='商品成本')
     sdq.jdjzt(db_name='聚合数据', table_name='京东_京准通')
-    sdq.jdqzyx(db_name='聚合数据', table_name='京东_京准通_全站营销')
     sdq.sku_sales(db_name='聚合数据', table_name='京东_sku_商品明细')
+
+    sdq._ald_wxt(db_name='聚合数据', table_name='奥莱店_主体报表')
+    sdq._sj_wxt(db_name='聚合数据', table_name='圣积天猫店_主体报表')
+    sdq._tb_wxt(db_name='聚合数据', table_name='淘宝_主体报表')
+    sdq.jdqzyx(db_name='聚合数据', table_name='京东_京准通_全站营销')
     sdq.spu_sales(db_name='聚合数据', table_name='京东_spu_商品明细')
-    sdq.tg_cjzb(db_name='聚合数据', table_name='天猫_超级直播')
-    sdq.pxb_zh(db_name='聚合数据', table_name='天猫_品销宝账户报表')
     sdq.zb_ccfx(db_name='聚合数据', table_name='生意参谋_直播场次分析')
     sdq.tg_by_day(db_name='聚合数据', table_name='多店推广场景_按日聚合')
-    sdq.performance(bb_tg=True, db_name='聚合数据', table_name='_全店商品销售')  # _全店商品销售
-    sdq.performance(bb_tg=False, db_name='聚合数据', table_name='_推广商品销售')  # _推广商品销售
-    sdq.performance_jd(jd_tg=False, db_name='聚合数据', table_name='_京东_推广商品销售')  # _推广商品销售
+    # sdq.performance(bb_tg=True, db_name='聚合数据', table_name='_全店商品销售')  # _全店商品销售
+    # sdq.performance(bb_tg=False, db_name='聚合数据', table_name='_推广商品销售')  # _推广商品销售
+    # sdq.performance_jd(jd_tg=False, db_name='聚合数据', table_name='_京东_推广商品销售')  # _推广商品销售
     sdq.performance_concat(bb_tg=False, db_name='聚合数据', table_name='天猫_推广汇总')  # _推广商品销售
 
 

mdbq/mysql/mysql.py

@@ -246,32 +246,6 @@ class MysqlUpload:
 
             if reset_id:
                 pass
-            #     # 6. 重置自增列
-            #     try:
-            #         # 查询所有复合主键
-            #         sql = (
-            #             f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-            #             f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-            #         )
-            #         cursor.execute(sql)
-            #         result = cursor.fetchall()  # 复合主键数
-            #         if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-            #             column_name = 'id'
-            #             sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-            #                    f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-            #             cursor.execute(sql, (db_name, table_name, column_name))
-            #             result = cursor.fetchone()
-            #             if result:
-            #                 sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-            #                 cursor.execute(sql)
-            #             cursor.execute(
-            #                 f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-            #             cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-            #         else:
-            #             logger.info(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-            #     except Exception as e:
-            #         logger.error(f'333 {table_name} {e}')
-            #         connection.rollback()
             connection.commit()
 
     @try_except
@@ -380,23 +354,6 @@ class MysqlUpload:
                         else:
                             sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`)"
                         cursor.execute(sql)
-                    # if col in main_key or col in unique_main_key:
-                    #     sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
-                    #     cursor.execute(sql, (col))
-                    #     result = cursor.fetchone()  # 检查索引是否存在
-                    #     if not result:
-                    #         if col in main_key:
-                    #             sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
-                    #             logger.info(f"设置为索引: {col}({dtypes[col]})")
-                    #             cursor.execute(sql)
-                    #         elif col in unique_main_key:
-                    #             if dtypes[col] == 'mediumtext':
-                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`({index_length}));"
-                    #             else:
-                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`);"
-                    #             logger.info(f"设置唯一索引: {col}({dtypes[col]})")
-                    #             logger.info(sql)
-                    #             cursor.execute(sql)
             connection.commit()  # 提交事务
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
@@ -455,11 +412,23 @@ class MysqlUpload:
                                 # logger.info(sql)
                                 cursor.execute(sql)
                     else:  # 没有数据返回，则直接插入数据
-                        cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
-                        # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                        values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                        cursor.execute(sql)
+                        # cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
+                        # # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                        # values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
+                        # sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                        # cursor.execute(sql)
+
+                        # 清理和验证列名
+                        safe_columns = [f"`{escape_string(str(col))}`" for col in dict_data.keys()]
+                        cols = ", ".join(safe_columns)
+                        # 使用参数化查询
+                        placeholders = ", ".join(["%s"] * len(dict_data))
+                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({placeholders})"
+                        # 转义值并作为参数传递
+                        escaped_values = [escape_string(str(v)) if isinstance(v, str) else v for v in
+                                          dict_data.values()]
+                        cursor.execute(sql, tuple(escaped_values))
+
                     connection.commit()  # 提交数据库
                     continue
 
@@ -585,23 +554,6 @@ class MysqlUpload:
                         else:
                             sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`)"
                         cursor.execute(sql)
-                    # if col in main_key or col in unique_main_key:
-                    #     sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
-                    #     cursor.execute(sql, (col))
-                    #     result = cursor.fetchone()  # 检查索引是否存在
-                    #     if not result:
-                    #         if col in main_key:
-                    #             sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
-                    #             logger.info(f"设置为索引: {col}({dtypes[col]})")
-                    #             cursor.execute(sql)
-                    #         elif col in unique_main_key:
-                    #             if dtypes[col] == 'mediumtext':
-                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`({index_length}));"
-                    #             else:
-                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`);"
-                    #             logger.info(f"设置唯一索引: {col}({dtypes[col]})")
-                    #             logger.info(sql)
-                    #             cursor.execute(sql)
             connection.commit()  # 提交事务
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
@@ -657,11 +609,22 @@ class MysqlUpload:
                             # logger.info(sql)
                             cursor.execute(sql)
                 else:  # 没有数据返回，则直接插入数据
-                    cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
-                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
-                    sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                    cursor.execute(sql)
+                    # cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
+                    # # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    # values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
+                    # sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                    # cursor.execute(sql)
+
+                    # 构建安全的INSERT语句
+                    safe_keys = [f"`{escape_string(str(k))}`" for k in dict_data.keys()]
+                    keys_data = ", ".join(safe_keys)
+                    placeholders = ", ".join(["%s"] * len(dict_data))
+
+                    # 使用参数化查询
+                    sql = f"INSERT INTO `{table_name}` ({keys_data}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_datas}"
+                    escaped_values = [escape_string(str(v)) if isinstance(v, str) else v for v in dict_data.values()]
+                    cursor.execute(sql, tuple(escaped_values))
+
                 connection.commit()  # 提交数据库
                 connection.close()
                 return
@@ -703,9 +666,8 @@ class MysqlUpload:
             if str(v) == '':
                 v = 0
             v = str(v)
-            # v = re.sub('^-$|^--$|^nan$|^null$', '0', v, re.I)
-            # v = re.sub(',|="|"', '', v, re.I)
             v = re.sub('^="|"$', '', v, re.I)
+            v = re.sub(r'[\x00-\x1F\x7F-\x9F]', '', str(v))  # 移除控制字符
             if re.findall(r'^[-+]?\d+\.?\d*%$', v):
                 v = str(float(v.rstrip("%")) / 100)
 
@@ -919,35 +881,6 @@ class MysqlUpload:
                 )
                 if reset_id:
                     pass
-                    # # 6. 重置自增列
-                    # try:
-                    #     # 查询所有复合主键
-                    #     sql = (
-                    #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                    #         f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                    #     )
-                    #     cursor.execute(sql)
-                    #     result = cursor.fetchall()  # 复合主键数
-                    #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                    #         column_name = 'id'
-                    #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                    #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                    #         # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
-                    #         cursor.execute(sql, (db_name, table_name, column_name))
-                    #         result = cursor.fetchone()
-                    #         if result:
-                    #             # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
-                    #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                    #             cursor.execute(sql)
-                    #         cursor.execute(
-                    #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                    #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    #         # logger.info(f'重置自增id')
-                    #     else:
-                    #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                    # except Exception as e:
-                    #     logger.error(f'333 {table_name} {e}')
-                    #     connection.rollback()
                 connection.commit()  # 提交事务
                 connection.close()
                 return
@@ -975,36 +908,6 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
-                # # 6. 重置自增列
-                # try:
-                #     # 查询所有复合主键
-                #     sql = (
-                #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                #         f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                #     )
-                #     cursor.execute(sql)
-                #     result = cursor.fetchall()  # 复合主键数
-                #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                #         column_name = 'id'
-                #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                #         # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
-                #         cursor.execute(sql, (db_name, table_name, column_name))
-                #         result = cursor.fetchone()
-                #         if result:
-                #             # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
-                #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                #             cursor.execute(sql)
-                #         cursor.execute(
-                #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                #         # logger.info(f'重置自增id')
-                #     else:
-                #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                # except Exception as e:
-                #     logger.error(f'333 {table_name} {e}')
-                #     connection.rollback()
-                # connection.close()
                 return
 
             datas = df.to_dict(orient='records')
@@ -1095,35 +998,6 @@ class MysqlUpload:
 
             if reset_id:
                 pass
-                # # 6. 重置自增列
-                # try:
-                #     # 查询所有复合主键
-                #     sql = (
-                #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                #         f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                #     )
-                #     cursor.execute(sql)
-                #     result = cursor.fetchall()  # 复合主键数
-                #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                #         column_name = 'id'
-                #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                #         # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
-                #         cursor.execute(sql, (db_name, table_name, column_name))
-                #         result = cursor.fetchone()
-                #         if result:
-                #             # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
-                #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                #             cursor.execute(sql)
-                #         cursor.execute(
-                #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                #         # logger.info(f'重置自增id')
-                #     else:
-                #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                # except Exception as e:
-                #     logger.error(f'333 {table_name} {e}')
-                #     connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 
@@ -1405,33 +1279,6 @@ class OptimizeDatas:
                         self.end_date = end_date_before
                     else:  # 不存在日期列的情况
                         self.delete_duplicate2(table_name=table_name, except_key=except_key)
-
-                    # # 6. 重置自增列
-                    # try:
-                    #     # 查询所有复合主键
-                    #     sql = (
-                    #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                    #         f"WHERE `TABLE_SCHEMA` = '{self.db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                    #     )
-                    #     cursor.execute(sql)
-                    #     result = cursor.fetchall()  # 复合主键数
-                    #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                    #         column_name = 'id'
-                    #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                    #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                    #         cursor.execute(sql, (self.db_name, table_name, column_name))
-                    #         result = cursor.fetchone()
-                    #         if result:
-                    #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                    #             cursor.execute(sql)
-                    #         cursor.execute(
-                    #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                    #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    #     else:
-                    #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                    # except Exception as e:
-                    #     logger.error(f'333 {table_name} {e}')
-                    #     self.connection.rollback()
                 self.connection.close()
         logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
 

mdbq/spider/aikucun.py

@@ -262,7 +262,7 @@ class AikuCun:
             print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
             # print(res.json())
             if not res.json()['success']:
-                print('没有获取到数据, requests 请求不成功, success 返回值应为 True')
+                print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
@@ -484,6 +484,6 @@ def main(start_date, end_date, item_type=['spu']):
 if __name__ == '__main__':
     main(
         start_date='2025-03-01',
-        end_date='2025-03-12',
+        # end_date='2025-03-26',  # 不传则默认到今天
         item_type=['spu', 'sku']
     )

{mdbq-3.8.11.dist-info → mdbq-3.8.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.11
+Version: 3.8.13
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.11.dist-info → mdbq-3.8.13.dist-info}/RECORD

@@ -1,13 +1,13 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=7-v8Hf_oxS-viT6kvnNbH682QvNI_Gm_tpfZZWxVbI0,18
+mdbq/__version__.py,sha256=cmqMpHGJCaMGsHUAVyC67-oQIJBBnfayuPIXO2haNMo,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=-4HWC1HZmgqUAuvcRiHZU4FLtI70nRq_Hp3eXVZTyH8,185843
+mdbq/aggregation/query_data.py,sha256=9doVoOBEaLyHlT0fVEXpWtwLLV4NhpQs17kHQQ0p3ys,185443
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=2cPuqX4zq2b6ghFWxTylr52DPZGE2WNrCdFV0RcF6LY,89048
+mdbq/mysql/mysql.py,sha256=xCWY-np41FE1yrQqVWs0AzKZK9DHDFlZ_gbnog5LaNY,77860
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -20,8 +20,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=YLRTDgOKPGDyNB-z5dPOJhBoTzM6Rmbjy1Qng_KyJQc,19906
-mdbq-3.8.11.dist-info/METADATA,sha256=ZAydGB5l32CZnEjgk21kkCz1gKhGrzxXgtoXNBscLc4,364
-mdbq-3.8.11.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.11.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=m7ZIvrc9pqoGCYEH3FtgKTwqhX7QB6qzgc2twDzhX4w,19962
+mdbq-3.8.13.dist-info/METADATA,sha256=HEw9yfBRwMt6PWbT5Mr1O6pwyYaZVKCT0Yf6qt8hrzU,364
+mdbq-3.8.13.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.8.13.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.8.13.dist-info/RECORD,,