mdbq 2.2.9__tar.gz → 2.3.1__tar.gz

{mdbq-2.2.9 → mdbq-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.2.9
+Version: 2.3.1
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.2.9 → mdbq-2.3.1}/mdbq/aggregation/aggregation.py

@@ -1295,14 +1295,14 @@ def test2():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
-    file_dir(one_file=False, target_service='company')
-    # one_file_to_mysql(
-    #     file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
-    #     db_name='京东数据2',
-    #     table_name='推广数据_全站营销',
-    #     target_service='company',
-    #     database='mysql'
-    # )
+    # file_dir(one_file=False, target_service='company')
+    one_file_to_mysql(
+        file='/Users/xigua/Downloads/爱库存_商品榜单_spu_2024-10-17_2024-10-17.csv',
+        db_name='爱库存2',
+        table_name='商品spu榜单',
+        target_service='company',
+        database='mysql'
+    )
 
     # db_name = '推广数据2'
     # table_name = '权益报表'

mdbq-2.3.1/mdbq/req_post/__init__.py

@@ -0,0 +1,4 @@
+
+
+
+# 配置文件

mdbq-2.3.1/mdbq/req_post/req_tb.py

@@ -0,0 +1,330 @@
+# -*- coding: UTF-8 –*-
+import os
+import time
+import datetime
+import pandas as pd
+import warnings
+import requests
+from mdbq.other import ua_sj
+from mdbq.config import get_myconf
+from mdbq.mysql import mysql
+import json
+import socket
+import platform
+import random
+
+warnings.filterwarnings('ignore')
+
+
+class RequestData:
+    def __init__(self):
+        self.date = datetime.date.today().strftime('%Y%m%d')
+        self.url = None
+        self.headers = None
+        self.cookies = None
+        self.datas = []
+        self.path = None
+        self.filename = None
+        self.is_json_file = False
+
+    def request_data(self, date, url, headers, cookies, path, filename):
+        """ 活动预售页面 流量来源 """
+        # date = datetime.date.today().strftime('%Y%m%d')
+        # url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
+        #        f'dateRange={date}%7C{date}'
+        #        f'&dateType=today'
+        #        f'&pageSize=10'
+        #        f'&page=1'
+        #        f'&order=desc'
+        #        f'&orderBy=frontPreheatUv'  # 必传参数
+        #        f'&activityId=94040472'  # 关键，必传参数
+        #        # f'&activityStatus=3'
+        #        # f'&device=2'
+        #        # f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
+        #        # f'&_=1729079731795'
+        #        # f'&token=7e94ba030'
+        #        )
+        # headers = {
+        #     "referer": "https://dmp.taobao.com/index_new.html",
+        #     'User-Agent': ua_sj.get_ua(),
+        # }
+        # cookies = {
+        #     'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
+
+        self.date = date
+        self.url = url
+        self.headers = headers
+        self.cookies = cookies
+        self.path = path
+        self.filename = filename
+        result = requests.get(
+            self.url,
+            headers=self.headers,
+            cookies=self.cookies,
+        )
+        m_data = json.loads(result.text)
+        update_time = m_data['data']['updateTime']
+        # pt_data = data['data']['data'][0]  # 平台流量
+        # gg_data = data['data']['data'][1]  # 广告流量
+        for all_data in m_data['data']['data']:
+            self.datas.append({
+                'frontPayByrCnt': all_data['frontPayByrCnt']['value'],
+                '一级标识id': all_data['pageId']['value'],
+                '二级标识id': '',
+                '三级标识id': '',
+                '一级来源': all_data['pageName']['value'],
+                '二级来源': '',
+                '三级来源': '',
+                '活动商品访客数（定金期）': all_data['frontPreheatUv']['value'],
+                '定金支付买家数': all_data['frontPayByrCnt']['value'],
+                '定金支付转化率': all_data['frontPayRate']['value'],
+                '日期': all_data['statDateStr']['value'],
+                '更新时间': update_time,
+                '促销活动': '2024双11预售',
+            })
+            if 'children' not in all_data.keys():  # 这一句有点多余，因为一级来源必定细分有二级来源
+                continue
+            for children_data in all_data['children']:
+                one_source_id = children_data['pPageId']['value']
+                one_source_name = children_data['pPageName']['value']
+                self.datas.append(
+                    {
+                        'frontPayByrCnt': children_data['frontPayByrCnt']['value'],
+                        '一级标识id': children_data['pPageId']['value'],
+                        '二级标识id': children_data['pageId']['value'],
+                        '三级标识id': '',
+                        '一级来源': children_data['pPageName']['value'],
+                        '二级来源': children_data['pageName']['value'],
+                        '三级来源': '',
+                        '活动商品访客数（定金期）': children_data['frontPreheatUv']['value'],
+                        '定金支付买家数': children_data['frontPayByrCnt']['value'],
+                        '定金支付转化率': children_data['frontPayRate']['value'],
+                        '日期': children_data['statDateStr']['value'],
+                        '更新时间': update_time,
+                        '促销活动': '2024双11预售',
+                    }
+                )
+                # print(children_data['children'])
+                # print(children_data)
+                if 'children' not in children_data.keys():  # 部分二级来源没有细分的三级来源，因为需要跳过 children 字段
+                    continue
+                for children_children_data in children_data['children']:
+                    # print(children_children_data)
+                    # print(one_source_name)
+                    self.datas.append(
+                        {
+                            'frontPayByrCnt': children_children_data['frontPayByrCnt']['value'],
+                            '一级标识id': one_source_id,
+                            '二级标识id': children_children_data['pPageId']['value'],
+                            '三级标识id': children_children_data['pageId']['value'],
+                            '一级来源': one_source_name,
+                            '二级来源': children_children_data['pPageName']['value'],
+                            '三级来源': children_children_data['pageName']['value'],
+                            '活动商品访客数（定金期）': children_children_data['frontPreheatUv']['value'],
+                            '定金支付买家数': children_children_data['frontPayByrCnt']['value'],
+                            '定金支付转化率': children_children_data['frontPayRate']['value'],
+                            '日期': children_children_data['statDateStr']['value'],
+                            '更新时间': update_time,
+                            '促销活动': '2024双11预售',
+                        }
+                    )
+        for item in self.datas:
+            if item['日期'] != '':
+                item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
+        if self.is_json_file:
+            with open(os.path.join(self.path, f'{self.filename}.json'), 'w') as f:
+                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
+
+    def hd_sp(self, date, url, headers, cookies, path, filename, pages=5):
+        """ 活动预售页面 分商品效果 """
+
+        self.date = date
+        self.url = url
+        self.headers = headers
+        self.cookies = cookies
+        self.path = path
+        self.filename = filename
+        for page in range(1, pages + 1):
+            self.url = f'{self.url}&page={page}'
+            result = requests.get(
+                self.url,
+                headers=self.headers,
+                cookies=self.cookies,
+            )
+            m_data = json.loads(result.text)
+            # print(m_data)
+            # with open(os.path.join(self.path, f'{self.filename}.json'), 'w') as f:
+            #     json.dump(m_data, f, ensure_ascii=False, sort_keys=True, indent=4)
+            update_time = m_data['data']['updateTime']
+            time_stamp = m_data['data']['timestamp']
+            # pt_data = data['data']['data'][0]  # 平台流量
+            # gg_data = data['data']['data'][1]  # 广告流量
+            for all_data in m_data['data']['data']['data']:
+                self.datas.append({
+                    'activityItemDepUv': all_data['activityItemDepUv']['value'],
+                    '商品链接': all_data['item']['detailUrl'],
+                    '商品id': all_data['item']['itemId'],
+                    '商品图片': all_data['item']['pictUrl'],
+                    'startDate': all_data['item']['startDate'],
+                    '商品标题': all_data['item']['title'],
+                    '预售订单金额': all_data['presaleOrdAmt']['value'],
+                    '定金支付件数': all_data['presalePayItemCnt']['value'],
+                    '预售访客人数': all_data['presaleUv']['value'],
+                    '定金支付金额': all_data['sumPayDepositAmt']['value'],
+                    '定金支付买家数': all_data['sumPayDepositByrCnt']['value'],
+                    '支付转化率': all_data['uvPayRate']['value'],
+                    '日期': date,
+                    '时间戳': time_stamp,
+                    '更新时间': update_time,
+                    '促销活动': '2024双11预售',
+                    '类型': '分商品效果',
+                })
+            time.sleep(random.randint(5, 10))
+        for item in self.datas:
+            if item['日期'] != '':
+                item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
+        if self.is_json_file:
+            with open(os.path.join(self.path, f'{self.filename}.json'), 'w') as f:
+                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
+
+    def request_jd(self, date, url, headers, cookies, path, filename):
+        """ 京东 """
+        self.date = date
+        self.url = url
+        self.headers = headers
+        self.cookies = cookies
+        self.path = path
+        self.filename = filename
+        result = requests.post(
+            url,
+            headers=headers,
+            cookies=cookies,
+        )
+        print(result.text)
+
+
+def tb_data(service_databases=[], db_name=None, table_name=None):
+    """ 2024双11预售实时流量分析 """
+    date = datetime.date.today().strftime('%Y%m%d')
+    url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
+           f'dateRange={date}%7C{date}'
+           f'&dateType=today'
+           f'&pageSize=10'
+           f'&page=1'
+           f'&order=desc'
+           f'&orderBy=frontPreheatUv'  # 必传参数
+           f'&activityId=94040472'  # 关键，必传参数
+           # f'&activityStatus=3'
+           # f'&device=2'
+           # f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
+           # f'&_=1729079731795'
+           # f'&token=7e94ba030'
+           )
+    headers = {
+        "referer": "https://dmp.taobao.com/index_new.html",
+        'User-Agent': ua_sj.get_ua(),
+    }
+    cookies = {
+        'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
+    path = '/Users/xigua/Downloads'
+    filename = 'test'
+    r = RequestData()
+    r.is_json_file = False
+    r.request_data(
+        date=date,
+        url=url,
+        headers=headers,
+        cookies=cookies,
+        path=path,
+        filename=filename,
+    )
+    # print(r.datas)
+    df = pd.DataFrame(r.datas)
+    # df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
+
+    if not service_databases:
+        return
+    if not db_name or not table_name:
+        print(f'尚未指定 db_name/table_name 参数')
+        return
+    for dt in service_databases:
+        for service_name, database in dt.items():
+            username, password, host, port = get_myconf.select_config_values(
+                target_service=service_name,
+                database=database,
+            )
+            m = mysql.MysqlUpload(
+                username=username,
+                password=password,
+                host=host,
+                port=port,
+            )
+            m.df_to_mysql(
+                df=df,
+                db_name=db_name,
+                table_name=table_name,
+                move_insert=False,  # 先删除，再插入
+                df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                filename=None,  # 用来追踪处理进度
+                service_database=dt,  # 字典
+            )
+
+
+def company_run():
+    if platform.system() == 'Windows' and socket.gethostname() == 'company':
+        while True:
+            tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2',
+                    table_name='2024双11预售实时流量分析')
+            time.sleep(random.randint(1500, 2000))
+
+
+def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
+    """ 2024双11预售 分商品效果 """
+    date = datetime.date.today().strftime('%Y%m%d')
+    url = (
+        f'https://sycm.taobao.com/datawar/v7/presaleActivity/itemCoreIndex/getItemListLive.json?'
+        f'activityId=94040472'
+        f'&itemType=0'  # 必传， 查看全部商品 0， 活动商品 1 ， 跨店满减商品 2 ，官方立减 3（无数据）
+        f'&device=1'
+        f'&dateRange={date}%7C{date}'
+        f'&dateType=today'
+        f'&pageSize=10'  # 必传
+        # f'&page=1'   # 必传
+        # f'&order=desc'
+        # f'&orderBy=presaleOrdAmt'
+        # f'&indexCode=presaleOrdAmt%2CsumPayDepositByrCnt%2CpresalePayItemCnt'
+        # f'&_=1729133575797'
+           )
+    headers = {
+        # "referer": "https://dmp.taobao.com/index_new.html",
+        'User-Agent': ua_sj.get_ua(),
+    }
+    cookies = {
+        'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
+    path = '/Users/xigua/Downloads'
+    filename = 'test'
+    r = RequestData()
+    r.is_json_file = False
+    r.hd_sp(
+        date=date,
+        url=url,
+        headers=headers,
+        cookies=cookies,
+        path=path,
+        filename=filename,
+        pages = pages,
+    )
+    # print(r.datas)
+    df = pd.DataFrame(r.datas)
+    df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
+
+
+if __name__ == '__main__':
+    company_run()
+    # tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2', table_name='2024双11预售实时流量分析')
+    hd_sp_data(
+        service_databases=[{'company': 'mysql'}],
+        # db_name='生意参谋2',
+        # table_name='2024双11预售实时流量分析',
+    )

mdbq-2.3.1/mdbq/spider/aikucun.py

@@ -0,0 +1,293 @@
+# -*- coding:utf-8 -*-
+import datetime
+import getpass
+import json
+import os
+import pathlib
+import platform
+import re
+import time
+import warnings
+import pandas as pd
+from selenium import webdriver
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.service import Service
+from mdbq.config import set_support
+from selenium.webdriver.common.keys import Keys
+from mdbq.aggregation import aggregation
+from mdbq.clean import data_clean
+
+warnings.filterwarnings('ignore')
+
+
+if platform.system() == 'Windows':
+    # windows版本
+    Data_Path = r'C:\同步空间\BaiduSyncdisk'
+    D_PATH = str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\Downloads'))
+    Share_Path = str(pathlib.Path(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表'))  # 共享文件根目录
+elif platform.system() == 'Linux':
+    Data_Path = '数据中心'
+    D_PATH = 'Downloads'
+    if not os.path.exists(D_PATH):
+        os.makedirs(D_PATH)
+    Share_Path = ''  # linux 通常是远程服务器，不需要访问共享
+else:
+    Data_Path = f'/Users/{getpass.getuser()}/数据中心'  # 使用Mac独立网络时
+    # Data_Path = '/Volumes'  # 直接使用共享连接台式机时的配置， 后面接 + 自动0备份/***
+    D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
+    Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
+
+
+def test():
+    """
+    """
+    _url = 'https://gray-merc.aikucun.com/index.html'
+    cookie_path = '/Users/xigua/Downloads'
+    print(_url)
+
+    option = webdriver.ChromeOptions()  # 浏览器启动选项
+    option.headless = True  # False指定为无界面模式
+    # 调整chrome启动配置
+    option.add_argument("--disable-gpu")
+    option.add_argument("--no-sandbox")
+    option.add_argument("--disable-dev-shm-usage")
+    option.add_experimental_option("excludeSwitches", ["enable-automation"])
+    option.add_experimental_option("useAutomationExtension", False)
+    # if platform.system() == 'Windows':
+    #     service = Service(os.path.join(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe'))
+    # else:
+    #     service = Service('/usr/local/bin/chromedriver')
+    if platform.system() == 'Windows':
+        # 设置Chrome的路径
+        chrome_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chrome_win64\\chrome.exe')
+        chromedriver_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chromedriver.exe')
+        # os.environ["webdriver.chrome.driver"] = chrome_path
+        option.binary_location = chrome_path  # windows 设置此参数有效
+        service = Service(chromedriver_path)
+        # service = Service(str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe')))  # 旧路径
+    else:
+        # 设置Chrome的路径
+        chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+        chromedriver_path = '/usr/local/chrome/chromedriver'
+        os.environ["webdriver.chrome.driver"] = chrome_path
+
+        service = Service(chromedriver_path)
+    _driver = webdriver.Chrome(service=service, options=option)  # 创建Chrome驱动程序实例
+
+    print('yes')
+    # 登录
+    _driver.get(_url)
+    time.sleep(0.1)
+    _driver.maximize_window()  # 窗口最大化 方便后续加载数据
+    breakpoint()
+
+    d_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    print(f'{d_time} 登录成功，正在获取cookie...')
+    time.sleep(0.1)
+
+
+    _file = os.path.join(cookie_path, f'cookie_.txt')
+    with open(_file, 'w') as f:
+        # 将cookies保存为json格式
+        cookies_list = _driver.get_cookies()
+        for cookie in cookies_list:
+            # 该字段有问题所以删除就可以
+            if 'expiry' in cookie:
+                del cookie['expiry']
+            # if 'domain' in cookie:
+            #     cookie['domain'] = '.taobao.com'
+        cookies_list = json.dumps(cookies_list)
+        f.write(cookies_list)
+        print(f'cookie已保存: {_file}')
+    _driver.quit()
+
+
+class AikuCun:
+    def __init__(self):
+        self.url = 'https://gray-merc.aikucun.com/index.html'
+        self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+
+    def login(self, shop_name='aikucun'):
+        option = webdriver.ChromeOptions()
+        # option.add_argument("--headless")  # 设置无界面模式
+        # 调整chrome启动配置
+        option.add_argument("--disable-gpu")
+        option.add_argument("--no-sandbox")
+        option.add_argument("--disable-dev-shm-usage")
+        option.add_experimental_option("excludeSwitches", ["enable-automation"])
+        option.add_experimental_option('excludeSwitches', ['enable-logging'])  # 禁止日志输出，减少控制台干扰
+        option.add_experimental_option("useAutomationExtension", False)
+        option.add_argument('--ignore-ssl-error')  # 忽略ssl错误
+        prefs = {
+            'profile.default_content_settings.popups': 0,  # 禁止弹出所有窗口
+            "browser.download.manager. showAlertOnComplete": False,  # 下载完成后不显示下载完成提示框
+            "profile.default_content_setting_values.automatic_downloads": 1,  # 允许自动下载多个文件
+        }
+
+        option.add_experimental_option('perfLoggingPrefs', {
+            'enableNetwork': True,
+            'enablePage': False,
+        })
+        option.set_capability("goog:loggingPrefs", {
+            'browser': 'ALL',
+            'performance': 'ALL',
+        })
+        option.set_capability("goog:perfLoggingPrefs", {
+            'enableNetwork': True,
+            'enablePage': False,
+            'enableTimeline': False
+        })
+
+        option.add_experimental_option('prefs', prefs)
+        option.add_experimental_option('excludeSwitches', ['enable-automation'])  # 实验性参数, 左上角小字
+
+        # # 通过excludeSwitches参数禁用默认的启动路径
+        # option.add_experimental_option('excludeSwitches', ['enable-automation'])
+
+        if platform.system() == 'Windows':
+            # 设置 chrome 和 chromedriver 启动路径
+            chrome_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chrome_win64\\chrome.exe')
+            chromedriver_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chromedriver.exe')
+            # os.environ["webdriver.chrome.driver"] = chrome_path
+            option.binary_location = chrome_path  # windows 设置此参数有效
+            service = Service(chromedriver_path)
+            # service = Service(str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe')))  # 旧路径
+        elif platform.system() == 'Darwin':
+            chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+            chromedriver_path = '/usr/local/chrome/chromedriver'
+            os.environ["webdriver.chrome.driver"] = chrome_path
+            # option.binary_location = chrome_path  # Macos 设置此参数报错
+            service = Service(chromedriver_path)
+        else:
+            chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+            chromedriver_path = '/usr/local/chrome/chromedriver'
+            os.environ["webdriver.chrome.driver"] = chrome_path
+            # option.binary_location = chrome_path  # macos 设置此参数报错
+            service = Service(chromedriver_path)
+        _driver = webdriver.Chrome(options=option, service=service)  # 创建Chrome驱动程序实例
+        _driver.maximize_window()  # 窗口最大化 方便后续加载数据
+
+        # 登录
+        _driver.get(self.url)
+        _driver.delete_all_cookies()  # 首先清除浏览器打开已有的cookies
+        name_lists = os.listdir(self.cookie_path)  # cookie 放在主目录下的 cookies 文件夹
+        for name in name_lists:
+            if shop_name in name and name.endswith('.txt') and '~' not in name and '.DS' not in name:
+                with open(os.path.join(self.cookie_path, name), 'r') as f:
+                    cookies_list = json.load(f)  # 使用json读取cookies 注意读取的是文件 所以用load而不是loads
+                    for cookie in cookies_list:
+                        _driver.add_cookie(cookie)  # 添加cookies信息
+        _driver.refresh()
+        time.sleep(3)
+        return _driver
+
+    def get_data(self, shop_name='aikucun', date_num=1):
+        """
+        date_num: 获取最近 N 天数据，0表示今天
+        所有数据都是逐日下载
+        """
+
+        _driver = self.login(shop_name=shop_name)
+        _url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
+        _driver.get(_url)
+        time.sleep(3)
+
+        today = datetime.date.today()
+        for date_s in range(date_num):
+            new_date = today - datetime.timedelta(days=date_s)  # 会用作文件名
+            str_date = str(new_date)[2:]
+            wait = WebDriverWait(_driver, timeout=15)  #
+            elements = _driver.find_elements(
+                By.XPATH, '//input[@placeholder="开始日期"]')
+            # _driver.execute_script("arguments[0].click();", elements[0])  # 点击
+
+            input_box = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//input[@placeholder="开始日期"]')))  #
+
+            # from selenium.webdriver.common.keys import Keys
+            for i in range(8):
+                input_box.send_keys(Keys.BACKSPACE)
+            input_box.send_keys(str_date)
+            time.sleep(1)
+            input_box = wait.until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, '//input[@placeholder="结束日期"]')))  # 文件名输入框
+
+            for i in range(8):
+                input_box.send_keys(Keys.BACKSPACE)
+            input_box.send_keys(str_date)
+            time.sleep(2)
+            input_box.send_keys(Keys.ENTER)
+            time.sleep(2)
+            wait.until(EC.presence_of_element_located((By.XPATH, '//button/span[contains(text(), "查询")]')))
+            elements = _driver.find_elements(
+                By.XPATH, '//button/span[contains(text(), "查询")]')
+            _driver.execute_script("arguments[0].click();", elements[0])  # 点击
+            time.sleep(3)
+            wait.until(EC.presence_of_element_located(
+                (By.XPATH,
+                 '//button[@class="el-button el-button--primary el-button--small is-plain"]/span[contains(text(), "下载数据")]')))
+            elements = _driver.find_elements(
+                By.XPATH,
+                '//button[@class="el-button el-button--primary el-button--small is-plain"]/span[contains(text(), "下载数据")]')
+            _driver.execute_script("arguments[0].click();", elements[0])  # 点击
+            time.sleep(3)
+            self.clean_data(date=new_date)
+        _driver.quit()
+
+    def clean_data(self, date):
+        for root, dirs, files in os.walk(D_PATH, topdown=False):
+            for name in files:
+                if '~$' in name or 'DS_Store' in name:
+                    continue
+                if name.endswith('csv'):
+                    pattern = re.findall('[\u4e00-\u9fff]+', name)
+                    if pattern:
+                        continue
+                    pattern = re.findall('^[0-9a-zA-Z_]{5,}-[0-9a-zA-Z_]+-[0-9a-zA-Z_]+-[0-9a-zA-Z_]+', name)
+                    if not pattern:
+                        continue
+                    df = pd.read_csv(os.path.join(root, name), encoding='gb2312', header=0, na_filter=False)
+                    df.insert(loc=0, column='日期', value=date)  # df中插入新列
+                    df.rename(columns={'spuId': 'spu_id'}, inplace=True)
+                    df['数据更新时间'] = pd.to_datetime(df['数据更新时间'], format='%Y-%m-%d %H:%M:%S', errors='ignore')
+                    # df['数据更新时间'] = df['数据更新时间'].apply(lambda x: re.sub('  ', ' ', str(x)) if x else x)
+                    # print(df['数据更新时间'])
+                    # breakpoint()
+                    new_name = f'爱库存_商品榜单_spu_{date}_{date}.csv'
+                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False)
+                    os.remove(os.path.join(root, name))
+
+
+def akucun():
+    akc = AikuCun()
+    akc.get_data(shop_name='aikucun', date_num=3)
+    # akc.clean_data()
+
+    # 新版 数据分类
+    dp = aggregation.DatabaseUpdate(path=D_PATH)
+    dp.new_unzip(is_move=True)
+    dp.cleaning(is_move=False, is_except=['临时文件'])  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
+    # 将 self.datas 更新至数据库
+    dp.upload_df(service_databases=[
+        # {'home_lx': 'mongodb'},
+        # {'home_lx': 'mysql'},
+        {'company': 'mysql'},
+        # {'nas': 'mysql'},
+    ])
+    # 数据分类
+    c = data_clean.DataClean(path=D_PATH, source_path=Source_Path)
+    c.set_up_to_mogo = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
+    c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
+    c.new_unzip(is_move=True, )  # 解压文件
+    c.change_and_sort(is_except=['临时文件'])
+    c.move_all()  # 移到文件到原始文件夹
+
+
+if __name__ == '__main__':
+    pass
+    # test()
+    akucun()

{mdbq-2.2.9 → mdbq-2.3.1}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.2.9
+Version: 2.3.1
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.2.9 → mdbq-2.3.1}/mdbq.egg-info/SOURCES.txt

@@ -43,4 +43,7 @@ mdbq/pbix/__init__.py
 mdbq/pbix/pbix_refresh.py
 mdbq/pbix/refresh_all.py
 mdbq/pbix/refresh_all_old.py
-mdbq/spider/__init__.py
+mdbq/req_post/__init__.py
+mdbq/req_post/req_tb.py
+mdbq/spider/__init__.py
+mdbq/spider/aikucun.py

{mdbq-2.2.9 → mdbq-2.3.1}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.2.9',
+      version='2.3.1',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',