mdbq 3.8.1__py3-none-any.whl → 3.8.3__py3-none-any.whl

mdbq/__version__.py

@@ -1,3 +1 @@
-VERSION = (0, 3, 6)
-
-__version__ = '.'.join(map(str, VERSION))
+VERSION = '3.8.3'

mdbq/aggregation/query_data.py

@@ -468,7 +468,7 @@ class MysqlDatasQuery:
             '店铺名称': 1,
         }
         __res = []
-        for year in range(2024, datetime.datetime.today().year+1):
+        for year in range(2025, datetime.datetime.today().year+1):
             df = self.download.data_to_df(
                 db_name='推广数据_圣积天猫店',
                 table_name=f'主体报表_{year}',
@@ -2459,6 +2459,7 @@ class MysqlDatasQuery:
         df_tm_living = pd.DataFrame()  # 天猫超级直播
         df_jd = pd.DataFrame()  # 京东推广
         df_jd_qzyx = pd.DataFrame()  # 京东全站推广
+        df_jd_ziying = pd.DataFrame()  # 京东推广
 
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -2525,7 +2526,7 @@ class MysqlDatasQuery:
             )
         # sj圣积
         __res = []
-        for year in range(2024, datetime.datetime.today().year + 1):
+        for year in range(2025, datetime.datetime.today().year + 1):
             df_sj = self.download.data_to_df(
                 db_name='推广数据_圣积天猫店',
                 table_name=f'营销场景报表_{year}',
@@ -2783,7 +2784,52 @@ class MysqlDatasQuery:
             df_jd_qzyx = df_jd_qzyx[['日期', '店铺名称', '营销场景', '花费', '展现量', '点击量', '成交笔数', '成交金额']]
             df_jd_qzyx = df_jd_qzyx[df_jd_qzyx['花费'] > 0]
 
-        _datas = [item for item in  [df_tm, df_tb, df_tb_qzt, df_al, df_sj, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
+        projection = {
+            '日期': 1,
+            '产品线': 1,
+            '触发sku_id': 1,
+            '跟单sku_id': 1,
+            '花费': 1,
+            '展现数': 1,
+            '点击数': 1,
+            '直接订单行': 1,
+            '直接订单金额': 1,
+            '总订单行': 1,
+            '总订单金额': 1,
+            '直接加购数': 1,
+            '总加购数': 1,
+            'spu_id': 1,
+            '店铺名称': 1,
+        }
+        __res = []
+        for year in range(2025, datetime.datetime.today().year + 1):
+            df_jd_ziying = self.download.data_to_df(
+                db_name='京东数据3',
+                table_name=f'推广数据_京准通_自营店_{year}',
+                start_date=start_date,
+                end_date=end_date,
+                projection=projection,
+            )
+            __res.append(df_jd_ziying)
+        df_jd_ziying = pd.concat(__res, ignore_index=True)
+        if len(df_jd_ziying) > 0:
+            df_jd_ziying = df_jd_ziying.groupby(
+                ['日期', '店铺名称', '产品线', '触发sku_id', '跟单sku_id', 'spu_id', '花费', '展现数', '点击数'],
+                as_index=False).agg(
+                **{
+                    '直接订单行': ('直接订单行', np.max),
+                    '直接订单金额': ('直接订单金额', np.max),
+                    '成交笔数': ('总订单行', np.max),
+                    '成交金额': ('总订单金额', np.max),
+                    '直接加购数': ('直接加购数', np.max),
+                    '加购量': ('总加购数', np.max),
+                }
+            )
+            df_jd_ziying = df_jd_ziying[['日期', '店铺名称', '产品线', '花费', '展现数', '点击数', '加购量', '成交笔数', '成交金额']]
+            df_jd_ziying.rename(columns={'产品线': '营销场景', '展现数': '展现量', '点击数': '点击量'}, inplace=True)
+            df_jd_ziying = df_jd_ziying[df_jd_ziying['花费'] > 0]
+
+        _datas = [item for item in  [df_tm, df_tb, df_tb_qzt, df_al, df_sj, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx, df_jd_ziying] if len(item) > 0]  # 阻止空的 dataframe
         df = pd.concat(_datas, axis=0, ignore_index=True)
         # 超级直播全站推广不包含在营销场景报表中，所以单独添加  2025-03-05
         projection = {

mdbq/mysql/mysql.py

@@ -290,32 +290,33 @@ class MysqlUpload:
                 cursor.execute(sql, dict_data['数据主体'])
 
             if reset_id:
-                # 6. 重置自增列
-                try:
-                    # 查询所有复合主键
-                    sql = (
-                        f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                        f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                    )
-                    cursor.execute(sql)
-                    result = cursor.fetchall()  # 复合主键数
-                    if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                        column_name = 'id'
-                        sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                               f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                        cursor.execute(sql, (db_name, table_name, column_name))
-                        result = cursor.fetchone()
-                        if result:
-                            sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                            cursor.execute(sql)
-                        cursor.execute(
-                            f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                    else:
-                        logger.info(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                except Exception as e:
-                    logger.error(f'333 {table_name} {e}')
-                    connection.rollback()
+                pass
+            #     # 6. 重置自增列
+            #     try:
+            #         # 查询所有复合主键
+            #         sql = (
+            #             f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
+            #             f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
+            #         )
+            #         cursor.execute(sql)
+            #         result = cursor.fetchall()  # 复合主键数
+            #         if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
+            #             column_name = 'id'
+            #             sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+            #                    f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+            #             cursor.execute(sql, (db_name, table_name, column_name))
+            #             result = cursor.fetchone()
+            #             if result:
+            #                 sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+            #                 cursor.execute(sql)
+            #             cursor.execute(
+            #                 f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+            #             cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+            #         else:
+            #             logger.info(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
+            #     except Exception as e:
+            #         logger.error(f'333 {table_name} {e}')
+            #         connection.rollback()
             connection.commit()
 
     @try_except
@@ -1014,35 +1015,36 @@ class MysqlUpload:
                     chunksize=1000
                 )
                 if reset_id:
-                    # 6. 重置自增列
-                    try:
-                        # 查询所有复合主键
-                        sql = (
-                            f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                            f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                        )
-                        cursor.execute(sql)
-                        result = cursor.fetchall()  # 复合主键数
-                        if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                            column_name = 'id'
-                            sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                                   f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                            # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
-                            cursor.execute(sql, (db_name, table_name, column_name))
-                            result = cursor.fetchone()
-                            if result:
-                                # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
-                                sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                                cursor.execute(sql)
-                            cursor.execute(
-                                f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                            cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                            # logger.info(f'重置自增id')
-                        else:
-                            logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                    except Exception as e:
-                        logger.error(f'333 {table_name} {e}')
-                        connection.rollback()
+                    pass
+                    # # 6. 重置自增列
+                    # try:
+                    #     # 查询所有复合主键
+                    #     sql = (
+                    #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
+                    #         f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
+                    #     )
+                    #     cursor.execute(sql)
+                    #     result = cursor.fetchall()  # 复合主键数
+                    #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
+                    #         column_name = 'id'
+                    #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                    #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                    #         # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                    #         cursor.execute(sql, (db_name, table_name, column_name))
+                    #         result = cursor.fetchone()
+                    #         if result:
+                    #             # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                    #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                    #             cursor.execute(sql)
+                    #         cursor.execute(
+                    #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    #         # logger.info(f'重置自增id')
+                    #     else:
+                    #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
+                    # except Exception as e:
+                    #     logger.error(f'333 {table_name} {e}')
+                    #     connection.rollback()
                 connection.commit()  # 提交事务
                 connection.close()
                 return
@@ -1070,36 +1072,36 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
-                # 6. 重置自增列
-                try:
-                    # 查询所有复合主键
-                    sql = (
-                        f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                        f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                    )
-                    cursor.execute(sql)
-                    result = cursor.fetchall()  # 复合主键数
-                    if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                        column_name = 'id'
-                        sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                               f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                        # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
-                        cursor.execute(sql, (db_name, table_name, column_name))
-                        result = cursor.fetchone()
-                        if result:
-                            # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
-                            sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                            cursor.execute(sql)
-                        cursor.execute(
-                            f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                        # logger.info(f'重置自增id')
-                    else:
-                        logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                except Exception as e:
-                    logger.error(f'333 {table_name} {e}')
-                    connection.rollback()
-                connection.close()
+                # # 6. 重置自增列
+                # try:
+                #     # 查询所有复合主键
+                #     sql = (
+                #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
+                #         f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
+                #     )
+                #     cursor.execute(sql)
+                #     result = cursor.fetchall()  # 复合主键数
+                #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
+                #         column_name = 'id'
+                #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                #         # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                #         cursor.execute(sql, (db_name, table_name, column_name))
+                #         result = cursor.fetchone()
+                #         if result:
+                #             # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                #             cursor.execute(sql)
+                #         cursor.execute(
+                #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                #         # logger.info(f'重置自增id')
+                #     else:
+                #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
+                # except Exception as e:
+                #     logger.error(f'333 {table_name} {e}')
+                #     connection.rollback()
+                # connection.close()
                 return
 
             datas = df.to_dict(orient='records')
@@ -1189,35 +1191,36 @@ class MysqlUpload:
                     pass
 
             if reset_id:
-                # 6. 重置自增列
-                try:
-                    # 查询所有复合主键
-                    sql = (
-                        f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                        f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                    )
-                    cursor.execute(sql)
-                    result = cursor.fetchall()  # 复合主键数
-                    if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                        column_name = 'id'
-                        sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                               f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                        # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
-                        cursor.execute(sql, (db_name, table_name, column_name))
-                        result = cursor.fetchone()
-                        if result:
-                            # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
-                            sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                            cursor.execute(sql)
-                        cursor.execute(
-                            f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                        # logger.info(f'重置自增id')
-                    else:
-                        logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                except Exception as e:
-                    logger.error(f'333 {table_name} {e}')
-                    connection.rollback()
+                pass
+                # # 6. 重置自增列
+                # try:
+                #     # 查询所有复合主键
+                #     sql = (
+                #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
+                #         f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
+                #     )
+                #     cursor.execute(sql)
+                #     result = cursor.fetchall()  # 复合主键数
+                #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
+                #         column_name = 'id'
+                #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                #         # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                #         cursor.execute(sql, (db_name, table_name, column_name))
+                #         result = cursor.fetchone()
+                #         if result:
+                #             # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                #             cursor.execute(sql)
+                #         cursor.execute(
+                #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                #         # logger.info(f'重置自增id')
+                #     else:
+                #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
+                # except Exception as e:
+                #     logger.error(f'333 {table_name} {e}')
+                #     connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 
@@ -1500,32 +1503,32 @@ class OptimizeDatas:
                     else:  # 不存在日期列的情况
                         self.delete_duplicate2(table_name=table_name, except_key=except_key)
 
-                    # 6. 重置自增列
-                    try:
-                        # 查询所有复合主键
-                        sql = (
-                            f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                            f"WHERE `TABLE_SCHEMA` = '{self.db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                        )
-                        cursor.execute(sql)
-                        result = cursor.fetchall()  # 复合主键数
-                        if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                            column_name = 'id'
-                            sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
-                                   f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
-                            cursor.execute(sql, (self.db_name, table_name, column_name))
-                            result = cursor.fetchone()
-                            if result:
-                                sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
-                                cursor.execute(sql)
-                            cursor.execute(
-                                f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                            cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                        else:
-                            logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
-                    except Exception as e:
-                        logger.error(f'333 {table_name} {e}')
-                        self.connection.rollback()
+                    # # 6. 重置自增列
+                    # try:
+                    #     # 查询所有复合主键
+                    #     sql = (
+                    #         f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
+                    #         f"WHERE `TABLE_SCHEMA` = '{self.db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
+                    #     )
+                    #     cursor.execute(sql)
+                    #     result = cursor.fetchall()  # 复合主键数
+                    #     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
+                    #         column_name = 'id'
+                    #         sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                    #                f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                    #         cursor.execute(sql, (self.db_name, table_name, column_name))
+                    #         result = cursor.fetchone()
+                    #         if result:
+                    #             sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                    #             cursor.execute(sql)
+                    #         cursor.execute(
+                    #             f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    #         cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    #     else:
+                    #         logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
+                    # except Exception as e:
+                    #     logger.error(f'333 {table_name} {e}')
+                    #     self.connection.rollback()
                 self.connection.close()
         logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')