mdbq 3.8.1__py3-none-any.whl → 3.8.3__py3-none-any.whl

mdbq/spider/aikucun.py

@@ -1,190 +1,75 @@
 # -*- coding:utf-8 -*-
 import datetime
-import getpass
+import requests
 import json
 import os
 import sys
-import pathlib
-import platform
 import re
 import time
 import warnings
-import requests
-import pandas as pd
+import platform
+import getpass
 from selenium import webdriver
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
-from mdbq.config import set_support
-from selenium.webdriver.common.keys import Keys
-from mdbq.other import ua_sj
+import pymysql
+import pandas as pd
+from mdbq.log import spider_logging
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
-from mdbq.config import default
-
-warnings.filterwarnings('ignore')
+from mdbq.config import config
+from mdbq.other import ua_sj
 
 
-if platform.system() == 'Windows':
-    # windows版本
-    D_PATH = str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\Downloads'))
-elif platform.system() == 'Linux':
-    D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'Downloads')
-    if not os.path.exists(D_PATH):
-        os.makedirs(D_PATH)
-else:
-    D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
-upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
+content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
-targe_host, hostname, local =  default.return_default_host()
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
-print(username, password, host, port)
+m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 # 实例化一个数据查询类，用来获取 cookies 表数据
 download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+logger = spider_logging.setup_logging()
 
 
-def get_cookie_aikucun():
-    """
-    """
-    _url = 'https://gray-merc.aikucun.com/index.html'
-    cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
-    filename_aikucun = 'cookie_aikucun.json'
-    print(_url)
-
-    option = webdriver.ChromeOptions()  # 浏览器启动选项
-    option.headless = True  # False指定为无界面模式
-    # 调整chrome启动配置
-    option.add_argument("--disable-gpu")
-    option.add_argument("--no-sandbox")
-    option.add_argument("--disable-dev-shm-usage")
-    option.add_experimental_option("excludeSwitches", ["enable-automation"])
-    option.add_experimental_option("useAutomationExtension", False)
-    # if platform.system() == 'Windows':
-    #     service = Service(os.path.join(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe'))
-    # else:
-    #     service = Service('/usr/local/bin/chromedriver')
-    if platform.system() == 'Windows':
-        # 设置Chrome的路径
-        chrome_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chrome_win64\\chrome.exe')
-        chromedriver_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chromedriver.exe')
-        # os.environ["webdriver.chrome.driver"] = chrome_path
-        option.binary_location = chrome_path  # windows 设置此参数有效
-        service = Service(chromedriver_path)
-        # service = Service(str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe')))  # 旧路径
-    else:
-        # 设置Chrome的路径
-        chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
-        chromedriver_path = '/usr/local/chrome/chromedriver'
-        os.environ["webdriver.chrome.driver"] = chrome_path
-
-        service = Service(chromedriver_path)
-    _driver = webdriver.Chrome(service=service, options=option)  # 创建Chrome驱动程序实例
-
-    # 登录
-    _driver.get(_url)
-    time.sleep(0.1)
-    _driver.maximize_window()  # 窗口最大化 方便后续加载数据
-    print(f'请登录并切换到百宝箱，再保存 cookies: \n https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604')
-    wait = WebDriverWait(_driver, timeout=15)
-    input_box = wait.until(
-        EC.element_to_be_clickable(
-            (By.XPATH, '//input[@placeholder="请输入用户名"]')))  #
-    input_box.send_keys('广东万里马实业股份有限公司')
-    input_box = wait.until(
-        EC.element_to_be_clickable(
-            (By.XPATH, '//input[@placeholder="请输入密码"]')))  #
-    input_box.send_keys('wlm123$$$')
-    time.sleep(0.1)
-    elements = _driver.find_elements(
-        By.XPATH, '//button[@class="merchant_login_btn" and contains(text(), "登录")]')
-    _driver.execute_script("arguments[0].click();", elements[0])
-    for i in range(100):
+def keep_connect(_db_name, _config, max_try: int=10):
+    attempts = 1
+    while attempts <= max_try:
         try:
-            wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, '//div[@class="user-info nav-user-slider"]')))
-            _driver.get(' https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604')
-            time.sleep(3)
-            break
-        except:
-            time.sleep(5)
-
-    d_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-    print(f'{d_time} 登录成功，正在获取cookie...')
-    time.sleep(0.1)
-
-    # 将cookies保存为json格式
-    cookies_list = _driver.get_cookies()
-    new_cookies_list = []
-    for cookie in cookies_list:
-        # 该字段有问题所以删除就可以
-        if 'HWWAFSESTIME' in cookie:
-            continue
-        else:
-            new_cookies_list.append(cookie)
-
-    ######### 新增 写入 mysql #########
-    set_typ = {
-        '日期': 'date',
-        'domain': 'varchar(100)',
-        'expiry': 'int',
-        'httpOnly': 'varchar(20)',
-        'name': 'varchar(50)',
-        'path': 'varchar(50)',
-        'sameSite': 'varchar(50)',
-        'secure': 'varchar(50)',
-        'value': 'text',
-        '更新时间': 'timestamp'
-    }
-    _cookies_list = []
-    for item in cookies_list:
-        new_dict = {'日期': datetime.datetime.today().strftime('%Y-%m-%d'), }
-        for k, v in item.items():
-            if v is None:
-                v = 'None'
-            new_dict.update({k: v})
-        if 'expiry' not in new_dict:
-            new_dict.update({'expiry': 0})
-        new_dict.update({'更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
-        _cookies_list.append(new_dict)
-    m_engine.insert_many_dict(
-        db_name='cookie文件',
-        table_name='main_aikucun',
-        dict_data_list=_cookies_list,
-        set_typ=set_typ,
-        allow_not_null=True,  # 允许插入空值
-    )
-    #############################################
-
-    json_file = os.path.join(cookie_path, filename_aikucun)
-    with open(json_file, 'w', encoding='utf-8') as f:
-        json.dump(new_cookies_list, f, ensure_ascii=False, sort_keys=True, indent=4)
-    print(f'cookie已保存: {json_file}')
-
-    # _file = os.path.join(cookie_path, filename_aikucun)
-    # with open(_file, 'w') as f:
-    #     # 将cookies保存为json格式
-    #     cookies_list = _driver.get_cookies()
-    #     # for cookie in cookies_list:
-    #     #     # 该字段有问题所以删除就可以
-    #     #     if 'expiry' in cookie:
-    #     #         del cookie['expiry']
-    #     #     # if 'domain' in cookie:
-    #     #     #     cookie['domain'] = '.taobao.com'
-    #     cookies_list = json.dumps(cookies_list)
-    #     f.write(cookies_list)
-    #     print(f'cookie已保存: {_file}')
-    _driver.quit()
+            connection = pymysql.connect(**_config)  # 连接数据库
+            return connection
+        except Exception as e:
+            logger.error(f'{_db_name}: 连接失败，正在重试: {host}:{port}  {attempts}/{max_try} {e}')
+            attempts += 1
+            time.sleep(30)
+    logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+    return None
+
+
+def dates_between(start_date, end_date) -> list:
+    """ 获取两个日期之间的所有日期， 返回 list   """
+    start_date = pd.to_datetime(start_date)
+    end_date = pd.to_datetime(end_date)
+    dates = []
+    current_date = start_date
+    while current_date <= end_date:
+        dates.append(current_date.strftime('%Y-%m-%d'))
+        current_date += datetime.timedelta(days=1)
+    return dates
 
 
 class AikuCun:
     def __init__(self):
-        # self.url = 'https://gray-merc.aikucun.com/index.html'
-        self.sp_url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
-        self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+        self.url = 'https://gray-merc.aikucun.com/index.html'
+        self.db_name = 'cookie文件'
+        self.table_name = 'main_aikucun'
+        self.shop_name = '万里马爱库存'
+        self.token = None
+        self.today = datetime.date.today()
+        self.start_date = (self.today - datetime.timedelta(days=7)).strftime('%Y-%m-%d')
+        self.end_date = (self.today - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
 
-    def login(self, shop_name='aikucun', headless=False):
+    def logining(self, shop_name='aikucun', headless=False):
         option = webdriver.ChromeOptions()
         if headless:
             option.add_argument("--headless")  # 设置无界面模式
@@ -219,8 +104,8 @@ class AikuCun:
         option.add_experimental_option('prefs', prefs)
         option.add_experimental_option('excludeSwitches', ['enable-automation'])  # 实验性参数, 左上角小字
 
-        # 修改默认下载文件夹路径
-        option.add_experimental_option("prefs", {"download.default_directory": f'{upload_path}'})
+        # # 修改默认下载文件夹路径
+        # option.add_experimental_option("prefs", {"download.default_directory": f'{upload_path}'})
 
         # # 通过excludeSwitches参数禁用默认的启动路径
         # option.add_experimental_option('excludeSwitches', ['enable-automation'])
@@ -249,247 +134,354 @@ class AikuCun:
         _driver.maximize_window()  # 窗口最大化 方便后续加载数据
 
         # 登录
-        _driver.get(self.sp_url)
-        _driver.delete_all_cookies()  # 首先清除浏览器打开已有的cookies
-        name_lists = os.listdir(self.cookie_path)  # cookie 放在主目录下的 cookies 文件夹
-        for name in name_lists:
-            if shop_name in name and name.endswith('.json') and '~' not in name and '.DS' not in name:
-                with open(os.path.join(self.cookie_path, name), 'r') as f:
-                    cookies_list = json.load(f)  # 使用json读取cookies 注意读取的是文件 所以用load而不是loads
-                    for cookie in cookies_list:
-                        _driver.add_cookie(cookie)  # 添加cookies信息
-                        # print(cookie)
-        db_name = 'cookie文件'
-        table_name = f'main_{shop_name}'
+        _driver.get(url='https://gray-merc.aikucun.com/index.html')  # self.url 可能被修改，这里使用固定页面获取 sign
+        time.sleep(0.1)
+        _driver.maximize_window()  # 窗口最大化 方便后续加载数据
+        wait = WebDriverWait(_driver, timeout=15)
+        input_box = wait.until(
+            EC.element_to_be_clickable(
+                (By.XPATH, '//input[@placeholder="请输入用户名"]')))  #
+        input_box.send_keys('广东万里马实业股份有限公司')
+        input_box = wait.until(
+            EC.element_to_be_clickable(
+                (By.XPATH, '//input[@placeholder="请输入密码"]')))  #
+        input_box.send_keys('wlm123$$$')
+        time.sleep(0.1)
+        elements = _driver.find_elements(
+            By.XPATH, '//button[@class="merchant_login_btn" and contains(text(), "登录")]')
+        _driver.execute_script("arguments[0].click();", elements[0])
+        for i in range(100):
+            try:
+                wait.until(
+                    EC.element_to_be_clickable(
+                        (By.XPATH, '//div[@class="user-info nav-user-slider"]')))
+                break
+            except:
+                time.sleep(5)
+        local_storage = _driver.execute_script("return window.localStorage;")
+        if 'token' in local_storage.keys():
+            self.token = {
+                '日期': datetime.datetime.today().strftime('%Y-%m-%d'),
+                '平台': '爱库存',
+                '店铺名称': self.shop_name,
+                'token': local_storage['token'],
+                '来源位置': 'localstorage',
+                '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+            }
+        time.sleep(5)
+        _driver.quit()
+
+    def save_token(self):
+        if not self.token:
+            print('self.token 不能为空')
+            return
+        set_typ = {
+            '日期': 'DATE',
+            '平台': 'varchar(50)',
+            '店铺名称': 'varchar(50)',
+            'token': 'varchar(255)',
+            '来源位置': 'varchar(50)',
+            '更新时间': 'timestamp'
+        }
+        # 更新至数据库记录
+        m_engine.dict_to_mysql(
+            db_name=self.db_name,
+            table_name=self.table_name,
+            dict_data=self.token,
+            unique_main_key=None,
+            icm_update=[],  # 唯一组合键
+            main_key=None,  # 指定索引列, 通常用日期列，默认会设置日期为索引
+            set_typ={},  # 指定数据类型
+        )
+
+    def get_data_from_bbx(self, start_date=None, end_date=None, item_type='spu', page_num=1, page_size=300):
+        if start_date:
+            self.start_date = start_date
+        if end_date:
+            self.end_date = end_date
+        date_list = dates_between(start_date=self.start_date, end_date=self.end_date)
+
         df = download.data_to_df(
-            db_name=db_name,
-            table_name=table_name,
-            start_date='2025-01-01',
-            end_date='2030-12-11',
+            db_name=self.db_name,
+            table_name=self.table_name,
+            start_date='2025-03-07',
+            end_date='2039-12-31',
             projection={
-                'domain': 1,
-                'expiry': 1,
-                'httpOnly': 1,
-                'name': 1,
-                'path': 1,
-                'sameSite': 1,
-                'secure': 1,
-                'value': 1,
+                '日期': 1,
+                '平台': 1,
+                '店铺名称': 1,
+                'token': 1,
                 '更新时间': 1
             },
         )
-        # 仅保留最新日期的数据
-        idx = df.groupby('name')['更新时间'].idxmax()
-        df = df.loc[idx]
-        df.pop('更新时间')
-        for item in df.to_dict('records'):
-            new_dict = {}
-            for k, v in item.items():
-                if v == 'False':
-                    v = False
-                new_dict.update({k: v})
-            # _driver.add_cookie(new_dict)  # 添加cookies信息
-
-        _driver.refresh()
-        time.sleep(3)
-        return _driver
-
-    def get_data(self, shop_name='aikucun', date_num=1, headless=True):
-        """
-        date_num: 获取最近 N 天数据，0表示今天
-        所有数据都是逐日下载
-        """
-
-        _driver = self.login(shop_name=shop_name, headless=headless)
-
-        _driver.get(self.sp_url)
-        time.sleep(3)
-        # breakpoint()
-
-        today = datetime.date.today()
-        for date_s in range(date_num):
-            new_date = today - datetime.timedelta(days=date_s)  # 会用作文件名
-            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-            print(f'{now} 正在下载爱库存文件 {date_s+1}/{date_num}: {new_date}')
-            str_date = str(new_date)[2:]
-            wait = WebDriverWait(_driver, timeout=15)  #
-            elements = _driver.find_elements(
-                By.XPATH, '//input[@placeholder="开始日期"]')
-            # _driver.execute_script("arguments[0].click();", elements[0])  # 点击
-
-            input_box = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, '//input[@placeholder="开始日期"]')))  #
-
-            # from selenium.webdriver.common.keys import Keys
-            for i in range(8):
-                input_box.send_keys(Keys.BACKSPACE)
-            input_box.send_keys(str_date)
-            time.sleep(1)
-            input_box = wait.until(
-                EC.element_to_be_clickable(
-                    (By.XPATH, '//input[@placeholder="结束日期"]')))  # 文件名输入框
-
-            for i in range(8):
-                input_box.send_keys(Keys.BACKSPACE)
-            input_box.send_keys(str_date)
-            time.sleep(2)
-            input_box.send_keys(Keys.ENTER)
-            time.sleep(2)
-            wait.until(EC.presence_of_element_located((By.XPATH, '//button/span[contains(text(), "查询")]')))
-            elements = _driver.find_elements(
-                By.XPATH, '//button/span[contains(text(), "查询")]')
-            _driver.execute_script("arguments[0].click();", elements[0])  # 点击
-            time.sleep(5)
-            wait.until(EC.presence_of_element_located(
-                (By.XPATH,
-                 '//button[@class="el-button el-button--primary el-button--small is-plain"]/span[contains(text(), "下载数据")]')))
-
-            elements = _driver.find_elements(
-                By.XPATH,
-                '//div[@class="ak-page-list__table-empty" and contains(text(), "暂无数据")]')
-            if elements:
-                print(f'cookies 可能已过期，无法下载')
-                _driver.quit()
+        if len(df) == 0:
+            self.logining()
+            self.save_token()
+        else:
+            # 仅保留最新日期的数据
+            idx = df.groupby(['平台', '店铺名称'])['更新时间'].idxmax()
+            df = df.loc[idx][['token']]
+            if len(df) == 0:
+                print(f'从数据库获取的 token 不能为空')
                 return
-
-            elements = _driver.find_elements(
-                By.XPATH,
-                '//button[@class="el-button el-button--primary el-button--small is-plain"]/span[contains(text(), "下载数据")]')
-            _driver.execute_script("arguments[0].click();", elements[0])  # 点击
-            time.sleep(5)
-            self.clean_data(date=new_date)  # 每下载一个文件，需要立即清洗数据
-        _driver.quit()
-
-    def clean_data(self, date):
+            self.token = df.iloc[0, 0]
+
+        self.url = f'https://treasurebox.aikucun.com/api/web/merchant/treasure/commodity/{item_type}/list'
+        headers = {
+            'headers': ua_sj.get_ua(),
+            'referer': 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant',
+            'content-type': 'application/json;charset=UTF-8',
+            'origin': 'https://treasurebox.aikucun.com',
+            'system': 'merchant',
+            'token': self.token,  # 从浏览器本地存储空间获取
+        }
+        num = 1
+        results = []
+        for date in date_list:
+            req_date = re.sub('-', '', date)
+            data = {
+                'beginDate': req_date,
+                'brandIds': [],
+                'cropId': '',
+                'cropName': '',
+                'ctgryOneIds': [],
+                'ctgryThreeIds': [],
+                'ctgryTwoIds': [],
+                'dimValue': '',
+                'endDate': req_date,
+                'merchantShopCode': '',
+                'orderByName': 'dealGmv',
+                'orderType': 'desc',
+                'pageNum': page_num,
+                'pageSize': page_size
+            }
+
+            res = requests.post(
+                url=self.url,
+                headers=headers,
+                # cookies=cookies,
+                data=json.dumps(data)
+            )
+            print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
+            # print(res.json())
+            if not res.json()['success']:
+                print('requests 请求不成功, success 返回值应为 True')
+                time.sleep(1)
+                continue
+            if not res.json()['data']['rows']:
+                print("请求获取的数据 ['data']['rows'] 不能为空")
+                time.sleep(1)
+                continue
+            results += [(date, res.json()['data']['rows'])]
+            num += 1
+            time.sleep(1)
+            if num % 32 == 0:
+                print("避免频繁请求, 正在休眠...")
+                # time.sleep(60)
+
+        return results
+
+    def insert_datas(self, data_list, db_name, table_name):
+        """数据清洗"""
+        if not data_list:
+            return
+        chanel_name = {
+            'availableNum': '可售库存数',
+            'availableSkuCnt': '在架sku数',
+            'brandName': '品牌名',
+            'ctgryOneName': '一级类目名称',
+            'ctgryThreeName': '三级类目名称',
+            'ctgryTwoName': '二级类目名称',
+            'dealBuyerCnt': '支付人数_成交',
+            'dealBuyerCntRate': '成交率_成交',
+            'dealGmv': '成交gmv',
+            'dealIdolCnt': '销售爱豆人数',
+            'dealProductCnt': '销售量_成交',
+            'dealProductCntRate': '售罄率',
+            'dealSkuCnt': '成交sku数',
+            'dealTwoCnt': '订单数_成交',
+            'downSkuCnt': '可售sku数',
+            'etlInsertTime': '数据更新时间',
+            'forwardConfirmCnt': '转发爱豆人数',
+            'forwardConfirmNum': '转发次数',
+            'merStyleNo': '商品款号',  # spu 榜单
+            'styleNo': '商品货号',  # sku 榜单
+            'orderBuyerCnt': '支付人数_交易',
+            'orderBuyerCntRate': '成交率_交易',
+            'orderGmv': '下单gmv',
+            'orderProductCnt': '销售量_交易',
+            'orderSkuCnt': '下单sku数',
+            'orderTwoCnt': '订单数_交易',
+            'pictureUrl': '图片',
+            'pvNum': '浏览量',
+            'rn': '序号',
+            'spuId': 'spuid',
+            'spuName': '商品名称',
+            'supplyAmount': '供货额',
+            'supplyPerAmount': '供货价',
+            'uvNum': '访客量',
+            'colorName': '颜色',
+            'sizeName': '尺码',
+            'barCode': '条码',  # sku榜单   款号 + 颜色编码
+        }
+        # 移除未翻译的列名
+        res_col = [item for item in chanel_name.keys() if chanel_name[item] == '']
+        for item in res_col:
+            del chanel_name[item]
+
+        _results = []
+        for item_ in data_list:
+            end_date, d_list = item_
+            for main_data_dict in d_list:
+                dict_data_before = {}
+                # 添加数据
+                dict_data_before.update({k: v for k, v in main_data_dict.items()})
+                # 初始化 dict_data
+                dict_data = {
+                    '日期': end_date,
+                    '平台': '爱库存',
+                    '店铺名称': self.shop_name
+                }
+                for k, v in dict_data_before.items():
+                    # 翻译键名
+                    [dict_data.update({name_v: v}) for name_k, name_v in chanel_name.items() if k == name_k]
+                    # 没有翻译的键值也要保留
+                    not_in_rename = [item for item in dict_data_before.keys() if item not in chanel_name.keys()]
+                    [dict_data.update({item: dict_data_before[item]}) for item in not_in_rename]
+                dict_data.update(
+                    {
+                        '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                    }
+                )
+                new_dict_data = {}
+                for k, v in dict_data.items():
+                    if v and str(v).lower() != 'none' and str(v) != 'null':
+                        new_dict_data.update({k: v})
+                    else:
+                        new_dict_data.update({k: 0})
+                _results.append(new_dict_data)
         set_typ = {
-            '日期': 'date',
-            '店铺名称': 'varchar(100)',
-            'spu_id': 'varchar(100)',
+            '可售库存数': 'INT',
+            '在架sku数': 'INT',
+            '品牌名': 'varchar(50)',
+            '一级类目名称': 'varchar(50)',
+            '三级类目名称': 'varchar(50)',
+            '二级类目名称': 'varchar(50)',
+            '支付人数_成交': 'INT',
+            '成交率_成交': 'decimal(10,4)',
+            '成交gmv': 'decimal(10,2)',
+            '销售爱豆人数': 'INT',
+            '销售量_成交': 'INT',
+            '售罄率': 'decimal(10,4)',
+            '成交sku数': 'INT',
+            '订单数_成交': 'INT',
+            '可售sku数': 'INT',
+            '数据更新时间': 'DATETIME',
+            '转发爱豆人数': 'INT',
+            '转发次数': 'INT',
+            '商品款号': 'varchar(50)',
+            '支付人数_交易': 'INT',
+            '成交率_交易': 'decimal(10,4)',
+            '下单gmv': 'decimal(10,2)',
+            '销售量_交易': 'INT',
+            '下单sku数': 'INT',
+            '订单数_交易': 'INT',
             '图片': 'varchar(255)',
-            '序号': 'smallint',
-            '商品名称': 'varchar(255)',
-            '商品款号': 'varchar(255)',
-            '一级类目名称': 'varchar(255)',
-            '二级类目名称': 'varchar(255)',
-            '三级类目名称': 'varchar(255)',
-            '数据更新时间': 'timestamp',
-            '更新时间': 'timestamp',
+            '浏览量': 'INT',
+            '序号': 'INT',
+            'spuId': 'varchar(50)',
+            '商品名称': 'varchar(50)',
+            '供货额': 'decimal(10,2)',
+            '供货价': 'decimal(10,2)',
+            '访客量': 'INT',
+            '颜色': 'varchar(50)',
+            '尺码': 'varchar(50)',
+            '货号': 'varchar(50)',  # 款号 + 颜色编码
         }
-        for root, dirs, files in os.walk(upload_path, topdown=False):
-            for name in files:
-                if '~$' in name or 'DS_Store' in name:
-                    continue
-                if name.endswith('csv'):
-                    pattern = re.findall('[\u4e00-\u9fff]+', name)
-                    if pattern:
-                        continue
-                    pattern = re.findall('^[0-9a-zA-Z_]{5,}-[0-9a-zA-Z_]+-[0-9a-zA-Z_]+-[0-9a-zA-Z_]+', name)
-                    if not pattern:
-                        continue
-                    df = pd.read_csv(os.path.join(root, name), encoding='gb2312', header=0, na_filter=False)
-                    if len(df) == 0:
-                        print(f'数据长度为 0 : {name}')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    df.insert(loc=0, column='日期', value=date)  # df中插入新列
-                    df.insert(loc=1, column='店铺名称', value='爱库存平台')  # df中插入新列
-                    df.rename(columns={'spuId': 'spu_id'}, inplace=True)
-                    # df['数据更新时间'] = pd.to_datetime(df['数据更新时间'], format='%Y-%m-%d %H:%M:%S', errors='ignore')
-                    # df['数据更新时间'] = df['数据更新时间'].apply(lambda x: re.sub('  ', ' ', str(x)) if x else x)
-                    # print(df['数据更新时间'])
-                    # breakpoint()
-                    new_dict = {
-                        '日期': '',
-                        '店铺名称': '',
-                        '序号': '',
-                        '商品名称': '',
-                        'spu_id': '',
-                        '商品款号': '',
-                        '一级类目名称': '',
-                        '二级类目名称': '',
-                        '三级类目名称': '',
-                        '访客量': '',
-                        '浏览量': '',
-                        '下单gmv': '',
-                        '成交gmv': '',
-                        '支付人数_成交': '',
-                    }
-                    _results = []
-                    for dict_data in df.to_dict(orient='records'):
-                        new_dict.update(dict_data)
-                        new_dict.update({'更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
-                        _results.append(new_dict)
-                    if _results:
-                        m_engine.insert_many_dict(
-                            db_name='爱库存2',
-                            table_name='商品spu榜单',
-                            dict_data_list=_results,
-                            icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
-                            unique_main_key=None,
-                            set_typ=set_typ,
-                        )
-
-                    new_name = f'爱库存_商品榜单_spu_{date}_{date}.csv'
-                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False)
-                    os.remove(os.path.join(root, name))
-
-
-def akucun(headless=True, date_num=10):
-    akc = AikuCun()
-    akc.get_data(shop_name='aikucun', date_num=date_num, headless=headless)  # 获取最近 N 天数据，0表示今天
-
-
-class AikuCunNew:
-
-    def __init__(self, shop_name,):
-        self.shop_name = shop_name
-        self.today = datetime.date.today()
-        self.headers = {'User-Agent': ua_sj.get_ua()}
-        self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
-        self.cookies = {}
-        self.get_cookies()  # 更新 self.cookies 的值
-        self.support_path = set_support.SetSupport(dirname='support').dirname
-        self.start_date = (self.today - datetime.timedelta(days=15)).strftime('%Y-%m-%d')
-        self.end_date = (self.today - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+        print(f'{self.shop_name} 正在更新数据库 {db_name} -> {table_name}...')
+        if 'spu' in table_name:
+            drop_dup = ['日期', '平台', '店铺名称', '商品款号', '访客量']
+        else:
+            drop_dup = ['日期', '平台', '店铺名称', '条码']
+        m_engine.insert_many_dict(
+            db_name=db_name,
+            table_name=table_name,
+            dict_data_list=_results,
+            icm_update=drop_dup,  # 唯一组合键
+            # unique_main_key=['人群id'],
+            set_typ=set_typ,
+            allow_not_null=False,  # 创建允许插入空值的列
+        )
 
-    def akc(self):
-        """
-        """
-        start_date = self.start_date
-        end_date = self.end_date
-        url = 'https://treasurebox.aikucun.com/api/web/merchant/treasure/commodity/list/down?'
-        self.headers.update({'Referer': 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'})
-        now = datetime.datetime.now()
-        timestamp_ms = round(time.mktime(now.timetuple()) * 1000 + now.microsecond / 1000)
+    def get_sign(self):
+        sign = 'bbcf5b9cf3d3b8ba9c22550dcba8a3ce97be766f'
+        current_timestamp_ms = '1741396070777'
+        # current_timestamp_ms = int(round(time.time() * 1000))
+        self.url = f'https://treasurebox.aikucun.com/api/web/merchant/treasure/commodity/sku/list?time={current_timestamp_ms}&sign={sign}'
+        headers = {
+            'headers': ua_sj.get_ua(),
+            'referer': 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant',
+            'content-type': 'application/json;charset=UTF-8',
+            'origin': 'https://treasurebox.aikucun.com',
+            # 'system': 'merchant',
+            # 'token': self.token,  # 从浏览器本地存储空间获取
+        }
         data = {
-            'time': timestamp_ms,
-            'sign': '2DA6A7580C859B374AE830CAD78BB84B'
+            'beginDate': '20250307',
+            'brandIds': [],
+            'cropId': '',
+            'cropName': '',
+            'ctgryOneIds': [],
+            'ctgryThreeIds': [],
+            'ctgryTwoIds': [],
+            'dimValue': '',
+            'endDate': '20250307',
+            'merchantShopCode': '',
+            'orderByName': 'dealGmv',
+            'orderType': 'desc',
+            'pageNum': 1,
+            'pageSize': 10
         }
         res = requests.post(
-            url,
-            headers=self.headers,
-            cookies=self.cookies,
-            params=data
+            url=self.url,
+            headers=headers,
+            data=json.dumps(data)
         )
-        print(res.text)
-
+        print(res.json())
 
 
-    def get_cookies(self):
-        files = os.listdir(self.cookie_path)
-        for file in files:
-            if self.shop_name in file and '~' not in file:
-                with open(os.path.join(self.cookie_path, file), 'r') as f:
-                    cookies_data = json.load(f)
+def main(start_date, end_date, item_type=['spu']):
+    ak = AikuCun()
+    # ak.get_sign()
+    for type_ in item_type:
+        if type_ not in ['spu', 'sku']:
+            print(f'{item_type} 非法参数: {type_}')
+            continue
+        for i in range(2):
+            data_list = ak.get_data_from_bbx(
+                start_date=start_date,
+                end_date=end_date,
+                item_type=type_,
+                page_num=1,
+                page_size=300
+            )
+            if not data_list:
+                ak.logining()
+                ak.save_token()
+            else:
                 break
-        for data in cookies_data:
-            self.cookies.update({data['name']: data['value']})
+
+        ak.insert_datas(
+            data_list=data_list,
+            db_name='爱库存2',
+            table_name=f'{type_}榜单'
+        )
 
 
-if __name__ == '__main__':
-    # get_cookie_aikucun()  # 登录并获取 cookies
-    akucun(date_num=30, headless=True)  # 下载数据
 
-    # a = AikuCunNew(shop_name='aikucun')
-    # a.akc()
+if __name__ == '__main__':
+    main(
+        start_date='2025-03-06',
+        end_date='2025-03-06',
+        item_type=['spu', 'sku']
+    )