mdbq 2.3.3__tar.gz → 2.3.5__tar.gz

{mdbq-2.3.3 → mdbq-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.3.3
+Version: 2.3.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.3.3 → mdbq-2.3.5}/mdbq/aggregation/aggregation.py

@@ -526,7 +526,7 @@ class DatabaseUpdate:
                 elif name.endswith('.csv') and '竞店分析-来源分析-入店搜索词' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     check_remove_file = True
-                elif name.endswith('.csv') and '爱库存_商品榜单' in name:
+                elif name.endswith('.csv') and '爱库存_商品榜单_spu_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     check_remove_file = True
                 # ----------------------- 京东数据处理分界线 -----------------------
@@ -1299,13 +1299,13 @@ if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
     # file_dir(one_file=False, target_service='company')
-    one_file_to_mysql(
-        file='/Users/xigua/Downloads/爱库存_商品榜单_spu_2024-10-17_2024-10-17.csv',
-        db_name='爱库存2',
-        table_name='商品spu榜单',
-        target_service='company',
-        database='mysql'
-    )
+    # one_file_to_mysql(
+    #     file='/Users/xigua/Downloads/爱库存_商品榜单_spu_2024-10-17_2024-10-17.csv',
+    #     db_name='爱库存2',
+    #     table_name='商品spu榜单',
+    #     target_service='company',
+    #     database='mysql'
+    # )
 
     # db_name = '推广数据2'
     # table_name = '权益报表'

{mdbq-2.3.3 → mdbq-2.3.5}/mdbq/aggregation/query_data.py

@@ -688,6 +688,53 @@ class MysqlDatasQuery:
         df = pd.concat(_datas, axis=0, ignore_index=True)
         return df
 
+    def aikucun_bd_spu(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            'spi_id': 1,
+            '商品名称': 1,
+            '品牌名称': 1,
+            '商品款号': 1,
+            '一级类目名称': 1,
+            '二级类目名称': 1,
+            '三级类目名称': 1,
+            '转发次数': 1,
+            '转发爱豆人数': 1,
+            '访客量': 1,
+            '浏览量': 1,
+            '下单gmv': 1,
+            '成交gmv': 1,
+            '供货额': 1,
+            '供货价': 1,
+            '销售爱豆人数_成交': 1,
+            '支付人数_交易': 1,
+            '支付人数_成交': 1,
+            '销售量_成交': 1,
+            '销售量_交易': 1,
+            '订单数_成交': 1,
+            '订单数_交易': 1,
+            '成交率_交易': 1,
+            '成交率_成交': 1,
+            '可售库存数': 1,
+            '售罄率': 1,
+            '在架sku数': 1,
+            '可售sku数': 1,
+            'sku数_交易': 1,
+            'sku数_成交': 1,
+            '营销后供货额': 1,
+            '营销后供货价': 1,
+        }
+        projection = {}
+        df = self.download.data_to_df(
+            db_name='爱库存2',
+            table_name='商品spu榜单',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
+
 
 class GroupBy:
     """
@@ -822,6 +869,17 @@ class GroupBy:
             return df
         elif '商品索引表' in table_name:
             return df
+        elif '爱库存_商品spu榜单' in table_name:
+            df.drop_duplicates(
+                subset=[
+                    '日期',
+                    'spu_id',
+                    '访客量',
+                    '浏览量',
+                    '下单gmv',
+                    '成交gmv',
+                ], keep='last', inplace=True, ignore_index=True)
+            return df
         elif '人群报表' in table_name:
             df.rename(columns={
                 '场景名字': '营销场景',
@@ -1713,6 +1771,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': [],
                     '数据主体': sdq.tg_by_day(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '爱库存_商品spu榜单',
+                    '唯一主键': [],
+                    '数据主体': sdq.aikucun_bd_spu(),
+                },
             ]
             for items in data_dict:  # 遍历返回结果
                 db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']

{mdbq-2.3.3 → mdbq-2.3.5}/mdbq/clean/data_clean.py

@@ -1142,11 +1142,23 @@ class DataClean:
         shutil.move(os.path.join(path, _name), t2)  # 将文件从下载文件夹移到目标位置
 
     # @try_except
-    def move_all(self, path=None):
+    def move_all(self, path=None, is_except=[]):
         if not path:
             path = self.path
         for root, dirs, files in os.walk(path, topdown=False):
             for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+                # print(is_except, is_continue)
                 def bib(paths, _as_month=None):
                     """闭包函数"""
                     self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
@@ -1336,7 +1348,7 @@ class DataClean:
                 elif name.endswith('.csv') and '京东推广关键词点击成交报表' in name:
                     t_path = str(pathlib.Path(self.source_path, '京东报表/JD推广_关键词报表'))
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '爱库存' in name and '商品榜单' in name:
+                elif name.endswith('.csv') and '爱库存_商品榜单_spu_' in name:
                     t_path = str(pathlib.Path(self.source_path, '爱库存/商品榜单'))
                     bib(t_path, _as_month=True)
                 #  京东分界线   ------- 结束标记

{mdbq-2.3.3 → mdbq-2.3.5}/mdbq/spider/aikucun.py

@@ -43,11 +43,12 @@ else:
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
 
 
-def test():
+def get_cookie_aikucun():
     """
     """
     _url = 'https://gray-merc.aikucun.com/index.html'
-    cookie_path = '/Users/xigua/Downloads'
+    cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+    filename_aikucun = 'cookie_aikucun.txt'
     print(_url)
 
     option = webdriver.ChromeOptions()  # 浏览器启动选项
@@ -79,7 +80,6 @@ def test():
         service = Service(chromedriver_path)
     _driver = webdriver.Chrome(service=service, options=option)  # 创建Chrome驱动程序实例
 
-    print('yes')
     # 登录
     _driver.get(_url)
     time.sleep(0.1)
@@ -91,16 +91,16 @@ def test():
     time.sleep(0.1)
 
 
-    _file = os.path.join(cookie_path, f'cookie_.txt')
+    _file = os.path.join(cookie_path, filename_aikucun)
     with open(_file, 'w') as f:
         # 将cookies保存为json格式
         cookies_list = _driver.get_cookies()
-        for cookie in cookies_list:
-            # 该字段有问题所以删除就可以
-            if 'expiry' in cookie:
-                del cookie['expiry']
-            # if 'domain' in cookie:
-            #     cookie['domain'] = '.taobao.com'
+        # for cookie in cookies_list:
+        #     # 该字段有问题所以删除就可以
+        #     if 'expiry' in cookie:
+        #         del cookie['expiry']
+        #     # if 'domain' in cookie:
+        #     #     cookie['domain'] = '.taobao.com'
         cookies_list = json.dumps(cookies_list)
         f.write(cookies_list)
         print(f'cookie已保存: {_file}')
@@ -109,7 +109,8 @@ def test():
 
 class AikuCun:
     def __init__(self):
-        self.url = 'https://gray-merc.aikucun.com/index.html'
+        # self.url = 'https://gray-merc.aikucun.com/index.html'
+        self.sp_url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
         self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
 
     def login(self, shop_name='aikucun'):
@@ -173,7 +174,7 @@ class AikuCun:
         _driver.maximize_window()  # 窗口最大化 方便后续加载数据
 
         # 登录
-        _driver.get(self.url)
+        _driver.get(self.sp_url)
         _driver.delete_all_cookies()  # 首先清除浏览器打开已有的cookies
         name_lists = os.listdir(self.cookie_path)  # cookie 放在主目录下的 cookies 文件夹
         for name in name_lists:
@@ -193,9 +194,10 @@ class AikuCun:
         """
 
         _driver = self.login(shop_name=shop_name)
-        _url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
-        _driver.get(_url)
+
+        _driver.get(self.sp_url)
         time.sleep(3)
+        # breakpoint()
 
         today = datetime.date.today()
         for date_s in range(date_num):
@@ -271,7 +273,7 @@ class AikuCun:
 
 def akucun():
     akc = AikuCun()
-    akc.get_data(shop_name='aikucun', date_num=3)
+    akc.get_data(shop_name='aikucun', date_num=2)
     # akc.clean_data()
 
     # 新版 数据分类
@@ -291,10 +293,10 @@ def akucun():
     c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
     c.new_unzip(is_move=True, )  # 解压文件
     c.change_and_sort(is_except=['临时文件'])
-    c.move_all()  # 移到文件到原始文件夹
+    c.move_all(is_except=['临时文件'])  # 移到文件到原始文件夹
 
 
 if __name__ == '__main__':
     pass
-    # test()
+    # get_cookie_aikucun()
     akucun()

{mdbq-2.3.3 → mdbq-2.3.5}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.3.3
+Version: 2.3.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.3.3 → mdbq-2.3.5}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.3.3',
+      version='2.3.5',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',