mdbq 3.8.3__py3-none-any.whl → 3.8.5__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.8.3'
+VERSION = '3.8.5'

mdbq/aggregation/query_data.py

@@ -22,7 +22,9 @@ import sys
 
 """
 
-content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 
@@ -2919,9 +2921,11 @@ class MysqlDatasQuery:
         start_date, end_date = self.months_data(num=self.months)
         projection = {
             '日期': 1,
-            'spi_id': 1,
+            '平台': 1,
+            '店铺名称': 1,
+            'spuid': 1,
             '商品名称': 1,
-            '品牌名称': 1,
+            '品牌名': 1,
             '商品款号': 1,
             '一级类目名称': 1,
             '二级类目名称': 1,
@@ -2934,7 +2938,7 @@ class MysqlDatasQuery:
             '成交gmv': 1,
             '供货额': 1,
             '供货价': 1,
-            '销售爱豆人数_成交': 1,
+            '销售爱豆人数': 1,
             '支付人数_交易': 1,
             '支付人数_成交': 1,
             '销售量_成交': 1,
@@ -2947,34 +2951,44 @@ class MysqlDatasQuery:
             '售罄率': 1,
             '在架sku数': 1,
             '可售sku数': 1,
-            'sku数_交易': 1,
-            'sku数_成交': 1,
-            '营销后供货额': 1,
-            '营销后供货价': 1,
-            '店铺名称': 1,
+            '下单sku': 1,
+            '成交sku': 1,
+            '图片': 1,
+            '更新时间': 1,
         }
         projection = {}
         df = self.download.data_to_df(
             db_name='爱库存2',
-            table_name='商品spu榜单',
+            table_name='spu榜单',
             start_date=start_date,
             end_date=end_date,
             projection=projection,
         )
-        df.drop_duplicates(
-            subset=[
-                '日期',
-                '店铺名称',
-                'spu_id',
-                '访客量',
-                '浏览量',
-                '下单gmv',
-                '成交gmv',
-            ], keep='last', inplace=True, ignore_index=True)
+        idx = df.groupby(['日期', '店铺名称', 'spuid'])['更新时间'].idxmax()
+        df = df.loc[idx]
+        # df.drop_duplicates(
+        #     subset=[
+        #         '日期',
+        #         '店铺名称',
+        #         'spuid',
+        #         '访客量',
+        #         '浏览量',
+        #         '下单gmv',
+        #         '成交gmv',
+        #     ], keep='last', inplace=True, ignore_index=True)
+
+        # 调整列顺序, 定义需要前置的列
+        cols_to_move = ['日期','平台','店铺名称','品牌名','商品名称', '商品款号','spuid', '一级类目名称', '二级类目名称', '三级类目名称']
+        # 生成新的列顺序：前置列 + 剩余列（保持原顺序）
+        new_columns = cols_to_move + [col for col in df.columns if col not in cols_to_move]
+        # 调整DataFrame列顺序
+        df = df[new_columns]
+        df['更新时间'] = df.pop('更新时间')
+        df = df.astype({'日期': 'datetime64[ns]'}, errors='ignore')
         set_typ = {
             '日期': 'date',
             '店铺名称': 'varchar(100)',
-            'spu_id': 'varchar(100)',
+            'spuid': 'varchar(100)',
             '图片': 'varchar(255)',
             '序号': 'smallint',
             '商品名称': 'varchar(255)',

mdbq/log/spider_logging.py

@@ -10,12 +10,11 @@ def setup_logging(reMoveOldHandler=True):
     """
     reMoveOldHandler: 替换根日志记录器的所有现有处理器
     """
-    D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])))
+    dir_path = os.path.expanduser("~")
+    if not os.path.isdir(os.path.join(dir_path, 'logfile')):
+        os.makedirs(os.path.join(dir_path, 'logfile'))
 
-    if not os.path.isdir(os.path.join(D_PATH, 'logfile')):
-        os.makedirs(os.path.join(D_PATH, 'logfile'))
-
-    log_file = os.path.join(D_PATH, 'logfile', 'spider_tg.log')
+    log_file = os.path.join(dir_path, 'logfile', 'spider_tg.log')
     file_handler = RotatingFileHandler(
         filename=log_file,
         maxBytes=3*1024*1024,  # 3MB

mdbq/other/download_sku_picture.py

@@ -47,7 +47,9 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 if not os.path.exists(upload_path):  # 数据中心根目录
     os.makedirs(upload_path)
 
-content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 

mdbq/redis/getredis.py

@@ -17,7 +17,8 @@ import orjson
 logger = logging.getLogger(__name__)
 
 # 创建一个文件处理器，用于将日志写入文件
-dir_path = os.path.realpath(os.path.dirname(sys.argv[0]))
+dir_path = os.path.expanduser("~")
+# config_file = os.path.join(dir_path, 'spd.txt')
 log_file = os.path.join(dir_path, 'logfile', 'redis.log')
 if not os.path.isdir(os.path.join(dir_path, 'logfile')):
     os.mkdir(os.path.join(dir_path, 'logfile'))

mdbq/spider/aikucun.py

@@ -22,8 +22,9 @@ from mdbq.mysql import s_query
 from mdbq.config import config
 from mdbq.other import ua_sj
 
-
-content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
@@ -68,6 +69,7 @@ class AikuCun:
         self.today = datetime.date.today()
         self.start_date = (self.today - datetime.timedelta(days=7)).strftime('%Y-%m-%d')
         self.end_date = (self.today - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+        self.error_count = 0
 
     def logining(self, shop_name='aikucun', headless=False):
         option = webdriver.ChromeOptions()
@@ -195,6 +197,9 @@ class AikuCun:
         )
 
     def get_data_from_bbx(self, start_date=None, end_date=None, item_type='spu', page_num=1, page_size=300):
+        """
+        这里获取的数据等同于"查询"按钮的数据, 没有"营销后供货额/供货价" 这2个字段, 如果通过下载按钮的报表则有两个字段
+        """
         if start_date:
             self.start_date = start_date
         if end_date:
@@ -238,6 +243,9 @@ class AikuCun:
         num = 1
         results = []
         for date in date_list:
+            if self.error_count > 5:
+                print('已退出请求 -> self.error_count > 5')
+                break
             req_date = re.sub('-', '', date)
             data = {
                 'beginDate': req_date,
@@ -265,11 +273,15 @@ class AikuCun:
             print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
             # print(res.json())
             if not res.json()['success']:
-                print('requests 请求不成功, success 返回值应为 True')
+                print('没有获取到数据, requests 请求不成功, success 返回值应为 True')
+                num += 1
+                self.error_count += 1
                 time.sleep(1)
                 continue
             if not res.json()['data']['rows']:
-                print("请求获取的数据 ['data']['rows'] 不能为空")
+                print("返回的数据字典异常, ['data']['rows'] 不能为空")
+                num += 1
+                self.error_count += 1
                 time.sleep(1)
                 continue
             results += [(date, res.json()['data']['rows'])]
@@ -389,7 +401,7 @@ class AikuCun:
             '图片': 'varchar(255)',
             '浏览量': 'INT',
             '序号': 'INT',
-            'spuId': 'varchar(50)',
+            'spuid': 'varchar(50)',
             '商品名称': 'varchar(50)',
             '供货额': 'decimal(10,2)',
             '供货价': 'decimal(10,2)',
@@ -468,6 +480,7 @@ def main(start_date, end_date, item_type=['spu']):
             if not data_list:
                 ak.logining()
                 ak.save_token()
+                ak.error_count = 0  # 重置错误计数器
             else:
                 break
 
@@ -481,7 +494,7 @@ def main(start_date, end_date, item_type=['spu']):
 
 if __name__ == '__main__':
     main(
-        start_date='2025-03-06',
-        end_date='2025-03-06',
+        start_date='2025-03-01',
+        end_date='2025-03-12',
         item_type=['spu', 'sku']
     )

{mdbq-3.8.3.dist-info → mdbq-3.8.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.3
+Version: 3.8.5
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.3.dist-info → mdbq-3.8.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=N2YV132Ke9nxG8e_N4DlVVVafsd63H4Sbusd36SCowU,17
+mdbq/__version__.py,sha256=FbrHE5dCkeXrcGtVfIXS2SdQe0n1QBEKhU3bP4mfMtM,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=dmLJ0qma2L9HoJuEZ1ha7_mCIWH0gN6hR2HVEgQn0Uw,180343
+mdbq/aggregation/query_data.py,sha256=nCdo2Qr6159-yuRGYYO7kcIStV2tv5UEQC4dfBl9kic,181075
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -11,26 +11,25 @@ mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
-mdbq/log/spider_logging.py,sha256=59xe4Ckb7m-sBt3GYk8DC_hQg7-jnjBRq1o718r1Ry8,1676
+mdbq/log/spider_logging.py,sha256=KX9TTUn9naZNBACCEFhyTktnWhr5JaSNQLppLGyrm9Y,1645
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=tR6l4Zzn9j6zKaFcy0Ktw2oL8OoX3QB6jDoDp1l2fiM,95474
 mdbq/mysql/s_query.py,sha256=09Dp7DrVXui6dAI6zFDfrsUOdjPblF_oYUpgqbZMhXg,8757
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=xX3jU2Nu2R9CoveM7xevRstg1P40hvR1KL8wqmwt3O0,44809
+mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
-mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=1pTga2iINx0NV2ffl0D-aspZhrZMDQR8SpohAv1acoo,24076
+mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=GBZsCjsN3i1ZW9pAxeMAmb-y3yW3p3HJpjfrnnna5gg,19702
-mdbq-3.8.3.dist-info/METADATA,sha256=B7KoQYy-rPaD3MHQb58QxTYVZZUNAoOaKlxH4IidSE4,363
-mdbq-3.8.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.3.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=bUjjPjNoW3EL6H89nnBdFEwnWgGuEB2CENuBxcvx0Kw,20284
+mdbq-3.8.5.dist-info/METADATA,sha256=kBSQW0qeIfkw7V8GWiRRRlb_QMfz5NZFYq6xpwrHhy0,363
+mdbq-3.8.5.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.8.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.8.5.dist-info/RECORD,,

mdbq/pbix/refresh_all_old.py

@@ -1,177 +0,0 @@
-import os
-import sys
-import win32com.client
-import time
-import multiprocessing
-import psutil
-from pywinauto.application import Application
-import warnings
-from mdbq.config import set_support
-
-warnings.filterwarnings('ignore')
-top_path = os.path.realpath(os.path.dirname(sys.argv[0]))  # 程序运行目录, 打包时使用
-sys.path.append(top_path)
-
-
-class RefreshAll:
-    def __init__(self):
-        self.my_conf = os.path.join(set_support.SetSupport(dirname='support').dirname, '.my_conf')
-        self.pbix_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'ref_list.txt')
-        self.excel_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'ref_list.txt')
-        self.run_py_path = 'run_py'
-        self.procname = 'PBIDesktop.exe'
-
-    def refresh_pbix(self):
-        # 刷新 PBI
-        if not os.path.exists(self.pbix_path):
-            print(f'{self.pbix_path}: PBIxx file not found. ')
-            return
-        with open(self.pbix_path, 'r', encoding='utf-8') as f:
-            content = f.readlines()
-            content = [item.strip() for item in content if not item.strip().startswith('#') and not item.strip().startswith('[')]
-            pbix_list = [item for item in content if item]
-
-        if not pbix_list:
-            return
-        for filename in pbix_list:
-            if filename.endswith('.pbix'):
-                newfile = os.path.join(self.run_py_path, filename)
-                """刷新程序"""
-                for es in range(4):
-                    try:
-                        print(f'正在刷新 >>>{filename}')
-                        self.pbi(path=newfile)
-                        print('文件刷新 >>>' + filename)
-                        break
-                    except Exception as e:
-                        print(e)
-                        print('报错的文件 >>>' + filename + ' >> ' + str(es + 1))
-
-    def refresh_excel(self):
-        # 刷新 excel
-        if not os.path.exists(self.excel_path):
-            print(f'{self.excel_path}: Excel file not found. ')
-            return
-        with open(self.excel_path, 'r', encoding='utf-8') as f:
-            content = f.readlines()
-            content = [item.strip() for item in content if not item.strip().startswith('#')]
-            excel_list = [item for item in content if item]
-
-        if not excel_list:
-            return
-        for filename in excel_list:
-            if filename.endswith('.xlsx'):
-                try:
-                    print(f'正在刷新 >>>{filename}')
-                    path = os.path.join(top_path, self.run_py_path, filename)  # 拼接文件路径
-                    xlapp = win32com.client.Dispatch('Excel.Application')  # 创建Excel程序App
-                    xlapp.Visible = False  # 窗口是否可见
-                    xlapp.DisplayAlerts = False  # 是否显示警告信息
-                    wb = xlapp.Workbooks.Open(path)
-                    conjuncts = wb.Connections.Count  # 统计工作簿含有多少外部链接
-                    if conjuncts == 0:
-                        wb.Close(SaveChanges=False)
-                        xlapp.Quit()
-                    else:
-                        time.sleep(2)
-                        wb.RefreshAll()
-                        xlapp.CalculateUntilAsyncQueriesDone()
-                        time.sleep(2)
-                        wb.Save()
-                        wb.Close(SaveChanges=True)
-                        xlapp.Quit()
-                    print('文件刷新 >>>' + filename)
-                except Exception as e:
-                    print(e)
-
-    def refresh_excel2(self, excel_file):
-        # 刷新 excel
-        if excel_file.endswith('.xlsx'):
-            try:
-                print(f'正在刷新 >>>{excel_file}')
-                xlapp = win32com.client.Dispatch('Excel.Application')  # 创建Excel程序App
-                xlapp.Visible = False  # 窗口是否可见
-                xlapp.DisplayAlerts = False  # 是否显示警告信息
-                wb = xlapp.Workbooks.Open(excel_file)
-                conjuncts = wb.Connections.Count  # 统计工作簿含有多少外部链接
-                if conjuncts == 0:
-                    wb.Close(SaveChanges=False)
-                    xlapp.Quit()
-                else:
-                    time.sleep(2)
-                    wb.RefreshAll()
-                    xlapp.CalculateUntilAsyncQueriesDone()
-                    time.sleep(2)
-                    wb.Save()
-                    wb.Close(SaveChanges=True)
-                    xlapp.Quit()
-                print('文件刷新 >>>' + excel_file)
-            except Exception as e:
-                print(e)
-
-    def pbi(self, path, _timeout=300):
-        """
-        这是原本属于独立的库模块:  pbix_refresh
-        path: pbi文件路径
-        _timeout: 刷新等待时间
-        如果连接失败，请将 power bi 文件名的特殊字符(空格等)去掉或改为英文文件名试试
-        """
-        # 关闭已打开的 power bi进程
-        for proc in psutil.process_iter():
-            if proc.name() == self.procname:
-                proc.kill()
-        time.sleep(1)
-
-        # 启动 Power bi
-        os.system('start "" "' + path + '"')
-        time.sleep(5)
-
-        # 通过 connect 方法连接进程
-        app = Application(backend='uia').connect(path=self.procname)
-        # title_re可以通过正则表达式连接：".*Power BI Desktop", 仅限 2023年9月之前的 Power bi版本
-        # 2023年10月之后的版本没有此窗口后缀, 因此需用文件名找窗口
-        # 如果同时打开了其他同名的文件（非 power bi文件时）, 可能会引发错误
-        _filename = os.path.splitext(os.path.basename(path))[0]  # 文件名不含后缀
-        win = app.window(title_re=f'{_filename}.*?')  # 连接 pbi 窗口
-        time.sleep(10)
-        # win.print_control_identifiers()  # 打印窗口全部信息, 推荐使用 inspect 开发工具查询窗口句柄信息
-        num = 0
-        while True:
-            try:
-                win['刷新'].wait("enabled", timeout=_timeout)
-                time.sleep(3)
-                win['刷新'].click()
-                break
-            except:
-                print(f'{path}, 未识别窗口句柄, 连接超时')
-                num += 1
-                if num > 1:
-                    break
-
-        num = 0
-        while True:
-            try:
-                win['保存'].wait("enabled", timeout=_timeout)  # timeout 通过"保存"按键状态, 等待刷新窗口关闭, 时间不要设置太短
-                time.sleep(3)
-                win['保存'].click()
-                break
-            except:
-                print(f'{path}, 未识别窗口句柄, 文件未保存')
-                num += 1
-                if num > 1:
-                    break
-        # win.type_keys("^s")
-        win.wait("enabled", timeout=15)
-        win.close()
-
-        # 关闭进程
-        for proc in psutil.process_iter():
-            if proc.name() == self.procname:
-                proc.kill()
-
-
-if __name__ == '__main__':
-    # r = RefreshAll()
-    # # r.refresh_pbix()
-    # r.refresh_excel()
-    pass