mdbq 2.3.3__py3-none-any.whl → 2.3.5__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -526,7 +526,7 @@ class DatabaseUpdate:
                 elif name.endswith('.csv') and '竞店分析-来源分析-入店搜索词' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     check_remove_file = True
-                elif name.endswith('.csv') and '爱库存_商品榜单' in name:
+                elif name.endswith('.csv') and '爱库存_商品榜单_spu_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     check_remove_file = True
                 # ----------------------- 京东数据处理分界线 -----------------------
@@ -1299,13 +1299,13 @@ if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
     # file_dir(one_file=False, target_service='company')
-    one_file_to_mysql(
-        file='/Users/xigua/Downloads/爱库存_商品榜单_spu_2024-10-17_2024-10-17.csv',
-        db_name='爱库存2',
-        table_name='商品spu榜单',
-        target_service='company',
-        database='mysql'
-    )
+    # one_file_to_mysql(
+    #     file='/Users/xigua/Downloads/爱库存_商品榜单_spu_2024-10-17_2024-10-17.csv',
+    #     db_name='爱库存2',
+    #     table_name='商品spu榜单',
+    #     target_service='company',
+    #     database='mysql'
+    # )
 
     # db_name = '推广数据2'
     # table_name = '权益报表'

mdbq/aggregation/query_data.py

@@ -688,6 +688,53 @@ class MysqlDatasQuery:
         df = pd.concat(_datas, axis=0, ignore_index=True)
         return df
 
+    def aikucun_bd_spu(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            'spi_id': 1,
+            '商品名称': 1,
+            '品牌名称': 1,
+            '商品款号': 1,
+            '一级类目名称': 1,
+            '二级类目名称': 1,
+            '三级类目名称': 1,
+            '转发次数': 1,
+            '转发爱豆人数': 1,
+            '访客量': 1,
+            '浏览量': 1,
+            '下单gmv': 1,
+            '成交gmv': 1,
+            '供货额': 1,
+            '供货价': 1,
+            '销售爱豆人数_成交': 1,
+            '支付人数_交易': 1,
+            '支付人数_成交': 1,
+            '销售量_成交': 1,
+            '销售量_交易': 1,
+            '订单数_成交': 1,
+            '订单数_交易': 1,
+            '成交率_交易': 1,
+            '成交率_成交': 1,
+            '可售库存数': 1,
+            '售罄率': 1,
+            '在架sku数': 1,
+            '可售sku数': 1,
+            'sku数_交易': 1,
+            'sku数_成交': 1,
+            '营销后供货额': 1,
+            '营销后供货价': 1,
+        }
+        projection = {}
+        df = self.download.data_to_df(
+            db_name='爱库存2',
+            table_name='商品spu榜单',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
+
 
 class GroupBy:
     """
@@ -822,6 +869,17 @@ class GroupBy:
             return df
         elif '商品索引表' in table_name:
             return df
+        elif '爱库存_商品spu榜单' in table_name:
+            df.drop_duplicates(
+                subset=[
+                    '日期',
+                    'spu_id',
+                    '访客量',
+                    '浏览量',
+                    '下单gmv',
+                    '成交gmv',
+                ], keep='last', inplace=True, ignore_index=True)
+            return df
         elif '人群报表' in table_name:
             df.rename(columns={
                 '场景名字': '营销场景',
@@ -1713,6 +1771,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': [],
                     '数据主体': sdq.tg_by_day(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '爱库存_商品spu榜单',
+                    '唯一主键': [],
+                    '数据主体': sdq.aikucun_bd_spu(),
+                },
             ]
             for items in data_dict:  # 遍历返回结果
                 db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']

mdbq/clean/data_clean.py

@@ -1142,11 +1142,23 @@ class DataClean:
         shutil.move(os.path.join(path, _name), t2)  # 将文件从下载文件夹移到目标位置
 
     # @try_except
-    def move_all(self, path=None):
+    def move_all(self, path=None, is_except=[]):
         if not path:
             path = self.path
         for root, dirs, files in os.walk(path, topdown=False):
             for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+                # print(is_except, is_continue)
                 def bib(paths, _as_month=None):
                     """闭包函数"""
                     self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
@@ -1336,7 +1348,7 @@ class DataClean:
                 elif name.endswith('.csv') and '京东推广关键词点击成交报表' in name:
                     t_path = str(pathlib.Path(self.source_path, '京东报表/JD推广_关键词报表'))
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '爱库存' in name and '商品榜单' in name:
+                elif name.endswith('.csv') and '爱库存_商品榜单_spu_' in name:
                     t_path = str(pathlib.Path(self.source_path, '爱库存/商品榜单'))
                     bib(t_path, _as_month=True)
                 #  京东分界线   ------- 结束标记

mdbq/spider/aikucun.py

@@ -43,11 +43,12 @@ else:
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
 
 
-def test():
+def get_cookie_aikucun():
     """
     """
     _url = 'https://gray-merc.aikucun.com/index.html'
-    cookie_path = '/Users/xigua/Downloads'
+    cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+    filename_aikucun = 'cookie_aikucun.txt'
     print(_url)
 
     option = webdriver.ChromeOptions()  # 浏览器启动选项
@@ -79,7 +80,6 @@ def test():
         service = Service(chromedriver_path)
     _driver = webdriver.Chrome(service=service, options=option)  # 创建Chrome驱动程序实例
 
-    print('yes')
     # 登录
     _driver.get(_url)
     time.sleep(0.1)
@@ -91,16 +91,16 @@ def test():
     time.sleep(0.1)
 
 
-    _file = os.path.join(cookie_path, f'cookie_.txt')
+    _file = os.path.join(cookie_path, filename_aikucun)
     with open(_file, 'w') as f:
         # 将cookies保存为json格式
         cookies_list = _driver.get_cookies()
-        for cookie in cookies_list:
-            # 该字段有问题所以删除就可以
-            if 'expiry' in cookie:
-                del cookie['expiry']
-            # if 'domain' in cookie:
-            #     cookie['domain'] = '.taobao.com'
+        # for cookie in cookies_list:
+        #     # 该字段有问题所以删除就可以
+        #     if 'expiry' in cookie:
+        #         del cookie['expiry']
+        #     # if 'domain' in cookie:
+        #     #     cookie['domain'] = '.taobao.com'
         cookies_list = json.dumps(cookies_list)
         f.write(cookies_list)
         print(f'cookie已保存: {_file}')
@@ -109,7 +109,8 @@ def test():
 
 class AikuCun:
     def __init__(self):
-        self.url = 'https://gray-merc.aikucun.com/index.html'
+        # self.url = 'https://gray-merc.aikucun.com/index.html'
+        self.sp_url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
         self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
 
     def login(self, shop_name='aikucun'):
@@ -173,7 +174,7 @@ class AikuCun:
         _driver.maximize_window()  # 窗口最大化 方便后续加载数据
 
         # 登录
-        _driver.get(self.url)
+        _driver.get(self.sp_url)
         _driver.delete_all_cookies()  # 首先清除浏览器打开已有的cookies
         name_lists = os.listdir(self.cookie_path)  # cookie 放在主目录下的 cookies 文件夹
         for name in name_lists:
@@ -193,9 +194,10 @@ class AikuCun:
         """
 
         _driver = self.login(shop_name=shop_name)
-        _url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
-        _driver.get(_url)
+
+        _driver.get(self.sp_url)
         time.sleep(3)
+        # breakpoint()
 
         today = datetime.date.today()
         for date_s in range(date_num):
@@ -271,7 +273,7 @@ class AikuCun:
 
 def akucun():
     akc = AikuCun()
-    akc.get_data(shop_name='aikucun', date_num=3)
+    akc.get_data(shop_name='aikucun', date_num=2)
     # akc.clean_data()
 
     # 新版 数据分类
@@ -291,10 +293,10 @@ def akucun():
     c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
     c.new_unzip(is_move=True, )  # 解压文件
     c.change_and_sort(is_except=['临时文件'])
-    c.move_all()  # 移到文件到原始文件夹
+    c.move_all(is_except=['临时文件'])  # 移到文件到原始文件夹
 
 
 if __name__ == '__main__':
     pass
-    # test()
+    # get_cookie_aikucun()
     akucun()

{mdbq-2.3.3.dist-info → mdbq-2.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.3.3
+Version: 2.3.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.3.3.dist-info → mdbq-2.3.5.dist-info}/RECORD

@@ -1,15 +1,15 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=tt0l7ywcG0AFtBpKaiyqeqegWuOwB1-2K8VHbKn9qok,75995
+mdbq/aggregation/aggregation.py,sha256=CS_gMBwPKQo7uId8BrsYNTjtCZKzRVV7gT4PfE2Q46k,76014
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
-mdbq/aggregation/query_data.py,sha256=Za8shm_I9ESzfYUVPRPOh8kk3yVWwvWAvwV0mFVx5mI,85340
+mdbq/aggregation/query_data.py,sha256=-iXeXTTdAEmFLvoOGIKB-6-moszxRCATtNe2QBzucZo,87524
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=vSJsdDFeXpFo8ZRE3fiAsVulaS42aaUrHdvfmAI7KQQ,104230
+mdbq/clean/data_clean.py,sha256=QvbM_mRHvUbwvJu0UpJu4SUQNxLWsFl7QApoq6cPiVU,104788
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=xjlfb-RwW0oLyyHQZFUtmXvCpWI7Y_nhwa0tqW14wZk,19345
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
@@ -40,8 +40,8 @@ mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,
 mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=P5RiXEgwQR44KhBcrgolFPipqILQ4eieDC7MpSWDlkE,18346
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=WJL_9Xik5C_Sk4AaPAwa3ncMR2KHxYh0R3C3CZaUpys,13916
-mdbq-2.3.3.dist-info/METADATA,sha256=gptJFhHqyqxBjxIyHEwaaUXgvpBWsNW1dOvlY4ojZUs,245
-mdbq-2.3.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.3.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.3.3.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=fszwS5jml5-S98u5FurBKVS2L4O-0wXFdU6CsGqbWrE,14104
+mdbq-2.3.5.dist-info/METADATA,sha256=p6gnM-ZXnkZm1VLCgsF3NiAPaV0zsiwF1nGyHJ-qOPY,245
+mdbq-2.3.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.3.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.3.5.dist-info/RECORD,,