mdbq 3.7.2__py3-none-any.whl → 3.7.4__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -456,6 +456,117 @@ class MysqlDatasQuery:
         )
         return True
 
+    def _sj_wxt(self, db_name='聚合数据', table_name='圣积天猫店_主体报表', is_maximize=True):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '场景名字': 1,
+            '主体id': 1,
+            '花费': 1,
+            '展现量': 1,
+            '点击量': 1,
+            '总购物车数': 1,
+            '总成交笔数': 1,
+            '总成交金额': 1,
+            '自然流量曝光量': 1,
+            '直接成交笔数': 1,
+            '直接成交金额': 1,
+            '店铺名称': 1,
+        }
+        __res = []
+        for year in range(2024, datetime.datetime.today().year+1):
+            df = self.download.data_to_df(
+                db_name='推广数据_圣积天猫店',
+                table_name=f'主体报表_{year}',
+                start_date=start_date,
+                end_date=end_date,
+                projection=projection,
+            )
+            __res.append(df)
+        df = pd.concat(__res, ignore_index=True)
+        df.rename(columns={
+            '场景名字': '营销场景',
+            '主体id': '商品id',
+            '总购物车数': '加购量',
+            '总成交笔数': '成交笔数',
+            '总成交金额': '成交金额'
+        }, inplace=True)
+        df = df.astype({
+            '商品id': str,
+            '花费': 'float64',
+            '展现量': 'int64',
+            '点击量': 'int64',
+            '加购量': 'int64',
+            '成交笔数': 'int64',
+            '成交金额': 'float64',
+            '自然流量曝光量': 'int64',
+            '直接成交笔数': 'int64',
+            '直接成交金额': 'float64',
+        }, errors='raise')
+        df = df[df['花费'] > 0]
+        if is_maximize:
+            df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '花费', '点击量'], as_index=False).agg(
+                **{
+                    '展现量': ('展现量', np.max),
+                    '加购量': ('加购量', np.max),
+                   '成交笔数': ('成交笔数', np.max),
+                   '成交金额': ('成交金额', np.max),
+                   '自然流量曝光量': ('自然流量曝光量', np.max),
+                   '直接成交笔数': ('直接成交笔数', np.max),
+                   '直接成交金额': ('直接成交金额', np.max)
+                   }
+            )
+        else:
+            df = df.groupby(['日期', '店铺名称', '营销场景', '商品id', '花费', '点击量'], as_index=False).agg(
+                **{
+                    '展现量': ('展现量', np.min),
+                    '加购量': ('加购量', np.min),
+                    '成交笔数': ('成交笔数', np.min),
+                    '成交金额': ('成交金额', np.min),
+                    '自然流量曝光量': ('自然流量曝光量', np.min),
+                    '直接成交笔数': ('直接成交笔数', np.max),
+                    '直接成交金额': ('直接成交金额', np.max)
+                }
+            )
+        df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
+        set_typ = {
+            '日期': 'date',
+            '推广渠道': 'varchar(100)',
+            '店铺名称': 'varchar(100)',
+            '营销场景': 'varchar(100)',
+            '商品id': 'bigint',
+            '花费': 'decimal(12,2)',
+            '展现量': 'int',
+            '点击量': 'int',
+            '加购量': 'int',
+            '成交笔数': 'int',
+            '成交金额': 'decimal(12,2)',
+            '自然流量曝光量': 'int',
+            '直接成交笔数': 'int',
+            '直接成交金额': 'decimal(12,2)',
+        }
+
+        if not self.update_service:
+            return
+        min_date = df['日期'].min()
+        max_date = df['日期'].max()
+        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        logger.info(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        m_engine.df_to_mysql(
+            df=df,
+            db_name=db_name,
+            table_name=table_name,
+            # icm_update=['日期', '推广渠道', '营销场景', '商品id', '花费'],  # 增量更新, 在聚合数据中使用，其他不要用
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+            drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+            count=None,
+            filename=None,  # 用来追踪处理进度
+            reset_id=True,  # 是否重置自增列
+            set_typ=set_typ,
+        )
+        return True
+
     @try_except
     def syj(self, db_name='聚合数据', table_name='生意经_宝贝指标'):
         start_date, end_date = self.months_data(num=self.months)
@@ -2366,6 +2477,32 @@ class MysqlDatasQuery:
                     '成交金额': ('总成交金额', np.max)
                 }
             )
+        # sj圣积
+        __res = []
+        for year in range(2024, datetime.datetime.today().year + 1):
+            df_sj = self.download.data_to_df(
+                db_name='推广数据_圣积天猫店',
+                table_name=f'营销场景报表_{year}',
+                start_date=start_date,
+                end_date=end_date,
+                projection=projection,
+            )
+            __res.append(df_sj)
+        df_sj = pd.concat(__res, ignore_index=True)
+        if len(df_sj) > 0:
+            df_sj.rename(columns={'场景名字': '营销场景'}, inplace=True)
+            df_sj['店铺名称'] = df_sj['店铺名称'].apply(lambda x: 'saintJack旗舰店' if x == 'SAINTJACK旗舰店' else x)
+            df_sj = df_sj.groupby(
+                ['日期', '店铺名称', '场景id', '营销场景', '花费', '展现量'],
+                as_index=False).agg(
+                **{
+                    # '展现量': ('展现量', np.max),
+                    '点击量': ('点击量', np.max),
+                    '加购量': ('总购物车数', np.max),
+                    '成交笔数': ('总成交笔数', np.max),
+                    '成交金额': ('总成交金额', np.max)
+                }
+            )
         # 淘宝店
         __res = []
         for year in range(2024, datetime.datetime.today().year + 1):
@@ -2600,7 +2737,7 @@ class MysqlDatasQuery:
             df_jd_qzyx = df_jd_qzyx[['日期', '店铺名称', '营销场景', '花费', '展现量', '点击量', '成交笔数', '成交金额']]
             df_jd_qzyx = df_jd_qzyx[df_jd_qzyx['花费'] > 0]
 
-        _datas = [item for item in  [df_tm, df_tb, df_tb_qzt, df_al, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
+        _datas = [item for item in  [df_tm, df_tb, df_tb_qzt, df_al, df_sj, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
         df = pd.concat(_datas, axis=0, ignore_index=True)
         df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
         df = df.groupby(
@@ -3604,6 +3741,7 @@ def query1(months=1, less_dict=[]):
     sdq.update_service = True  # 调试时加，true: 将数据写入 mysql 服务器
 
     sdq._ald_wxt(db_name='聚合数据', table_name='奥莱店_主体报表')
+    sdq._sj_wxt(db_name='聚合数据', table_name='圣积天猫店_主体报表')
     sdq._tb_wxt(db_name='聚合数据', table_name='淘宝_主体报表')
     sdq.tg_wxt(db_name='聚合数据', table_name='天猫_主体报表')
     sdq.syj(db_name='聚合数据', table_name='生意经_宝贝指标')

mdbq/log/spider_logging.py

@@ -5,7 +5,10 @@ import os
 import getpass
 
 
-def setup_logging():
+def setup_logging(reMoveOldHandler=True):
+    """
+    reMoveOldHandler: 替换根日志记录器的所有现有处理器
+    """
     if platform.system() == 'Windows':
         from mdbq.pbix import refresh_all
         D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
@@ -38,6 +41,10 @@ def setup_logging():
 
     # 获取根日志记录器并添加Handler
     logger = logging.getLogger()
+    if reMoveOldHandler:
+        # 移除根日志记录器的所有现有处理器
+        for handler in logger.handlers[:]:  # 使用[:]来创建handlers列表的一个副本，因为我们在迭代时修改列表
+            logger.removeHandler(handler)
     logger.addHandler(file_handler)
     logger.addHandler(stream_handler)
     logger.setLevel(logging.INFO)  # 设置根日志级别

mdbq/other/download_sku_picture.py

@@ -979,7 +979,7 @@ if __name__ == '__main__':
         password=password,
         host=host,
         port=port,
-        heads=30,
+        heads=100,
         col_name='白底图',
         save_path=os.path.join(upload_path, '商品id_商家编码_图片'),
     )

mdbq/spider/aikucun.py

@@ -19,7 +19,8 @@ from mdbq.config import set_support
 from selenium.webdriver.common.keys import Keys
 from mdbq.other import ua_sj
 from mdbq.mysql import mysql
-from mdbq.config import myconfig
+from mdbq.mysql import s_query
+from mdbq.config import default
 import socket
 
 warnings.filterwarnings('ignore')
@@ -37,6 +38,9 @@ else:
 upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+print(username, password, host, port)
+# 实例化一个数据查询类，用来获取 cookies 表数据
+download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 
 
 def get_cookie_aikucun():
@@ -118,6 +122,40 @@ def get_cookie_aikucun():
             continue
         else:
             new_cookies_list.append(cookie)
+
+    ######### 新增 写入 mysql #########
+    set_typ = {
+        '日期': 'date',
+        'domain': 'varchar(100)',
+        'expiry': 'int',
+        'httpOnly': 'varchar(20)',
+        'name': 'varchar(50)',
+        'path': 'varchar(50)',
+        'sameSite': 'varchar(50)',
+        'secure': 'varchar(50)',
+        'value': 'text',
+        '更新时间': 'timestamp'
+    }
+    _cookies_list = []
+    for item in cookies_list:
+        new_dict = {'日期': datetime.datetime.today().strftime('%Y-%m-%d'), }
+        for k, v in item.items():
+            if v is None:
+                v = 'None'
+            new_dict.update({k: v})
+        if 'expiry' not in new_dict:
+            new_dict.update({'expiry': 0})
+        new_dict.update({'更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
+        _cookies_list.append(new_dict)
+    m_engine.insert_many_dict(
+        db_name='cookie文件',
+        table_name='main_aikucun',
+        dict_data_list=_cookies_list,
+        set_typ=set_typ,
+        allow_not_null=True,  # 允许插入空值
+    )
+    #############################################
+
     json_file = os.path.join(cookie_path, filename_aikucun)
     with open(json_file, 'w', encoding='utf-8') as f:
         json.dump(new_cookies_list, f, ensure_ascii=False, sort_keys=True, indent=4)
@@ -219,6 +257,38 @@ class AikuCun:
                     cookies_list = json.load(f)  # 使用json读取cookies 注意读取的是文件 所以用load而不是loads
                     for cookie in cookies_list:
                         _driver.add_cookie(cookie)  # 添加cookies信息
+                        # print(cookie)
+        db_name = 'cookie文件'
+        table_name = f'main_{shop_name}'
+        df = download.data_to_df(
+            db_name=db_name,
+            table_name=table_name,
+            start_date='2025-01-01',
+            end_date='2030-12-11',
+            projection={
+                'domain': 1,
+                'expiry': 1,
+                'httpOnly': 1,
+                'name': 1,
+                'path': 1,
+                'sameSite': 1,
+                'secure': 1,
+                'value': 1,
+                '更新时间': 1
+            },
+        )
+        # 仅保留最新日期的数据
+        idx = df.groupby('name')['更新时间'].idxmax()
+        df = df.loc[idx]
+        df.pop('更新时间')
+        for item in df.to_dict('records'):
+            new_dict = {}
+            for k, v in item.items():
+                if v == 'False':
+                    v = False
+                new_dict.update({k: v})
+            # _driver.add_cookie(new_dict)  # 添加cookies信息
+
         _driver.refresh()
         time.sleep(3)
         return _driver
@@ -417,8 +487,8 @@ class AikuCunNew:
 
 
 if __name__ == '__main__':
-    get_cookie_aikucun()  # 登录并获取 cookies
-    akucun(date_num=10, headless=True)  # 下载数据
+    # get_cookie_aikucun()  # 登录并获取 cookies
+    akucun(date_num=30, headless=True)  # 下载数据
 
     # a = AikuCunNew(shop_name='aikucun')
     # a.akc()

{mdbq-3.7.2.dist-info → mdbq-3.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.7.2
+Version: 3.7.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.7.2.dist-info → mdbq-3.7.4.dist-info}/RECORD

@@ -4,7 +4,7 @@ mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,
 mdbq/aggregation/aggregation_bak.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2kzY,80264
 mdbq/aggregation/datashow_bak.py,sha256=1AYSIDkdUx-4as1Ax2rPj0cExM9d-qFMrFYLAaPHNuk,54962
 mdbq/aggregation/optimize_data.py,sha256=foZGLDGJRhM2qOr2mTvB3InDFId7r4KBXrJfB3-xq1k,2639
-mdbq/aggregation/query_data.py,sha256=AiG0W9Rum_hKCTPNB-nuA5ehpnVW12byLA7uwzOmO6Q,168639
+mdbq/aggregation/query_data.py,sha256=wYsBTRx6vmZ-dKQRdl_pnOPcI2djyIIb92MOUy097E0,174930
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -16,7 +16,7 @@ mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
-mdbq/log/spider_logging.py,sha256=EBy3_49RWs1qLO8QM6jB3rk9n35VvpJn65AoET46Ro0,1601
+mdbq/log/spider_logging.py,sha256=CzQIflfbMSBSBjHSO3fEhcG9Oiyj6cR-YU0tObMMyxM,1975
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
@@ -24,7 +24,7 @@ mdbq/mysql/mysql.py,sha256=YgmSLkwjIUpjiGH3S-bTiaJCKe8As0WvHDOS6_ePyYs,98591
 mdbq/mysql/s_query.py,sha256=pj5ioJfUT81Su9S-km9G49gF5F2MmXXfw_oAIUzhN28,8794
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=-ia7KIu7Ij3huWF7tXVrWEE7N4QwJfWDbBIP4pVzcQ4,44655
+mdbq/other/download_sku_picture.py,sha256=B5WA2-CQds2l36w4gLs1cgqzWqC1mbQTozqwb3vUQh0,44656
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/sku_picture_bak.py,sha256=JwSXYlzamVqcKCD2tRH2VqYVZNr8fM6f--kcGlTVRnM,50026
@@ -36,8 +36,8 @@ mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=oyFwE-8c6uErSGYNIO0z2ng93mH0zstRLD86MWqF6M8,25636
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=1gAEwCUmhCSpOSRPD2EEcbH3bFGn4sUQnQUAsJb5-qM,19391
-mdbq-3.7.2.dist-info/METADATA,sha256=8R-WnzR5mVYftpJ8JekalK3kHRDLqom2RM22uV8QMTI,243
-mdbq-3.7.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.7.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.7.2.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=qMU29gb72OK8GLBD-zENo1EmxuojRnuPyEdk5S2KDKw,21767
+mdbq-3.7.4.dist-info/METADATA,sha256=oviqiqBpDqxScLXtRS6FhvAg-WudJQX_--ykPyP9zlM,243
+mdbq-3.7.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.7.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.7.4.dist-info/RECORD,,