mdbq 2.5.7__tar.gz → 2.5.9__tar.gz

{mdbq-2.5.7 → mdbq-2.5.9}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.5.7
-Home-page: https://pypi.org/project/mdbq
+Version: 2.5.9
+Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com
 License: MIT

{mdbq-2.5.7 → mdbq-2.5.9}/mdbq/aggregation/optimize_data.py

@@ -57,8 +57,8 @@ def restart_mongodb():
         subprocess.call(command, shell=True)
 
 
-def op_data(db_name_lists, service_databases=None, days: int = 63, is_mongo=True, is_mysql=True):
-    """ service_databases 这个参数暂时没有用 """
+def op_data(db_name_lists, service_databases=[{'home_lx': 'mysql', 'home_lx': 'mongodb'}], days: int = 63, is_mongo=True, is_mysql=True):
+    """ """
     # for service_database in service_databases:
     #     for service_name, database in service_database.items():
     #         username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
@@ -68,47 +68,48 @@ def op_data(db_name_lists, service_databases=None, days: int = 63, is_mongo=True
     #         ]
     #         s.days = days
     #         s.optimize_list()
+    for service_database in service_databases:
+        for service_name, database in service_database.items():
+            if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
+                # mongodb
+                if is_mongo and database == 'mongodb':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
+                    m.db_name_lists = db_name_lists
+                    m.days = days
+                    m.optimize_list()
+                    if m.client:
+                        m.client.close()
+                        print(f'已关闭 mongodb 连接')
 
-    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
-        # mongodb
-        if is_mongo:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mongodb',
-            )
-            m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
-            m.db_name_lists = db_name_lists
-            m.days = days
-            m.optimize_list()
-            if m.client:
-                m.client.close()
-                print(f'已关闭 mongodb 连接')
+                    if socket.gethostname() == 'xigua_lx':
+                        restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
 
-            if socket.gethostname() == 'xigua_lx':
-                restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
+                # Mysql
+                if is_mysql and database == 'mysql':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+                    s.db_name_lists = db_name_lists
+                    s.days = days
+                    s.optimize_list()
 
-        # Mysql
-        if is_mysql:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='home_lx',
-                database='mysql',
-            )
-            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-            s.db_name_lists = db_name_lists
-            s.days = days
-            s.optimize_list()
-
-    elif socket.gethostname() == 'company':
-        # Mysql
-        if is_mysql:
-            username, password, host, port = get_myconf.select_config_values(
-                target_service='company',
-                database='mysql',
-            )
-            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-            s.db_name_lists = db_name_lists
-            s.days = days
-            s.optimize_list()
+            elif socket.gethostname() == 'company':
+                # Mysql
+                if is_mysql and database == 'mysql':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+                    s.db_name_lists = db_name_lists
+                    s.days = days
+                    s.optimize_list()
 
 
 if __name__ == '__main__':

{mdbq-2.5.7 → mdbq-2.5.9}/mdbq/clean/clean_upload.py

@@ -83,6 +83,141 @@ class DataClean:
             os.makedirs(_save_paths, exist_ok=True)
         _df.to_csv(os.path.join(_save_paths, filenames), encoding=encoding, index=False, header=True)
 
+    def sycm_tm(self, path=None, is_except=[]):
+        """ 天猫 生意参谋数据 """
+        if not path:
+            path = self.path
+        report_names = [
+            {
+                '文件简称': '商品排行',  # 文件名中包含的字符
+                '数据库名': '天猫_生意参谋',
+                '集合名称': '商品排行',
+            },
+            {
+                '文件简称': '店铺来源_来源构成_万里马官方旗舰店',  # 文件名中包含的字符
+                '数据库名': '天猫_生意参谋',
+                '集合名称': '店铺流量来源构成',
+            },
+        ]
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # 这里排除掉非目标报表
+                is_continue = False
+                db_name = None  # 初始化参数
+                collection_name = None
+                for item in report_names:
+                    if item['文件简称'] in name:
+                        db_name = item['数据库名']
+                        collection_name = item['集合名称']
+                        is_continue = True
+                if not is_continue:
+                    continue
+                if name.endswith('.xls') and '商品排行_万里马官方旗舰店' in name:
+                    df = pd.read_excel(os.path.join(root, name), header=4)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        continue
+                    df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
+                    df.replace(to_replace=[','], value='', regex=True, inplace=True)
+                    df.rename(columns={'统计日期': '日期', '商品ID': '商品id'}, inplace=True)
+                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
+                    df.insert(loc=1, column='店铺名称', value=shop_name)
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and '_来源构成_万里马官方旗舰店' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    # print(f'db_name/collection_name 不能为空')
+                    continue
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                    )
+
+    def dmp_tm(self, path=None, is_except=[]):
+        """ 天猫 达摩盘 """
+        if not path:
+            path = self.path
+        report_names = [
+            {
+                '文件简称': '我的人群属性',  # 文件名中包含的字符
+                '数据库名': '达摩盘3',
+                '集合名称': '我的人群属性',
+            },
+        ]
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # 这里排除掉非目标报表
+                is_continue = False
+                db_name = None  # 初始化参数
+                collection_name = None
+                for item in report_names:
+                    if item['文件简称'] in name:
+                        db_name = item['数据库名']
+                        collection_name = item['集合名称']
+                        is_continue = True
+                if not is_continue:
+                    continue
+                if name.endswith('.csv') and '人群属性_万里马官方旗舰店' in name:  # 推广类报表
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        continue
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    # print(f'db_name/collection_name 不能为空')
+                    continue
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                )
+
     def tg_reports(self, path=None, is_except=[]):
         """ 处理天猫淘宝推广类报表 """
         if not path:
@@ -172,7 +307,7 @@ class DataClean:
 
                 # 这里排除掉非推广类报表
                 is_continue = False
-                db_name = None
+                db_name = None  # 初始化参数
                 collection_name = None
                 for item in report_names:
                     if item['文件简称'] in name:
@@ -234,7 +369,7 @@ class DataClean:
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
-                    print(f'db_name/collection_name 不能为空')
+                    # print(f'db_name/collection_name 不能为空')
                     continue
                 self.datas.append(
                     {
@@ -290,7 +425,7 @@ class DataClean:
 
                 # 这里排除掉非目标报表
                 is_continue = False
-                db_name = None
+                db_name = None  # 初始化参数
                 collection_name = None
                 for item in report_names:
                     if item['文件简称'] in name:
@@ -303,7 +438,7 @@ class DataClean:
                 if name.endswith('.csv') and 'baobei' in name:
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2}).csv', name)[0]
+                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2})\W', name)[0]
                     df['日期'] = '-'.join(pattern)
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
                     new_name = f'py_xg_天猫_baobeitrains_{'-'.join(pattern)}.csv'
@@ -327,7 +462,7 @@ class DataClean:
                 elif name.endswith('.csv') and '省份城市分析' in name:
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\.', name)[0]
+                    pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\W', name)[0]  # 注意后面可能有小括号 ...27 (2).csv
                     date = '-'.join(pattern[1:])
                     new_name = f'py_xg_天猫_{pattern[0]}-{date}.csv'
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
@@ -350,9 +485,9 @@ class DataClean:
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and '店铺销售指标' in name:
                     # 生意经, 店铺指标，仅限月数据，实际日指标也可以
-                    name_st = re.findall(r'(.*)\(分日', name)
+                    name_st = re.findall(r'([\u4e00-\u9fa5]+)\(分日', name)
                     if not name_st:
-                        print(f'{name}  已转换的表格')
+                        print(f'{name}  正则提取文件名失败')
                         continue
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
@@ -375,7 +510,7 @@ class DataClean:
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
-                    print(f'db_name/collection_name 不能为空')
+                    # print(f'db_name/collection_name 不能为空')
                     continue
                 self.datas.append(
                     {
@@ -431,7 +566,7 @@ class DataClean:
 
                 # 这里排除掉非目标报表
                 is_continue = False
-                db_name = None
+                db_name = None  # 初始化参数
                 collection_name = None
                 for item in report_names:
                     if item['文件简称'] in name:
@@ -444,7 +579,7 @@ class DataClean:
                 if name.endswith('.csv') and 'baobei' in name:
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2}).csv', name)[0]
+                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2})\W', name)[0]
                     df['日期'] = '-'.join(pattern)
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
                     new_name = f'py_xg_淘宝_baobeitrains_{'-'.join(pattern)}.csv'
@@ -516,7 +651,7 @@ class DataClean:
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
-                    print(f'db_name/collection_name 不能为空')
+                    # print(f'db_name/collection_name 不能为空')
                     continue
                 self.datas.append(
                     {
@@ -582,7 +717,7 @@ class DataClean:
 
                 # 这里排除掉非目标报表
                 is_continue = False
-                db_name = None
+                db_name = None  # 初始化参数
                 collection_name = None
                 for item in report_names:
                     if item['文件简称'] in name:
@@ -633,7 +768,7 @@ class DataClean:
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
-                    print(f'db_name/collection_name 不能为空')
+                    # print(f'db_name/collection_name 不能为空')
                     continue
                 # print(name)
                 self.datas.append(
@@ -663,6 +798,8 @@ class DataClean:
                             break
                 if is_continue:  # 需要排除不做处理的文件或文件夹
                     continue
+                db_name = None  # 初始化参数
+                collection_name = None
 
                 if name.endswith('.xlsx') and '商品素材_' in name:
                     shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
@@ -681,7 +818,7 @@ class DataClean:
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
-                    print(f'db_name/collection_name 不能为空')
+                    # print(f'db_name/collection_name 不能为空')
                     continue
                 self.datas.append(
                     {
@@ -718,6 +855,70 @@ class DataClean:
             os.remove(old_file)  # 如果存在则移除
         shutil.move(os.path.join(path, _name), t2)  # 将文件从下载文件夹移到目标位置
 
+    def move_sycm(self, path=None, is_except=[]):
+        """ 生意参谋 """
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        # print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # print(is_except, is_continue)
+                def bib(paths, _as_month=None):
+                    """闭包函数"""
+                    self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
+
+                if 'py_xg' not in name:  # 排除非目标文件
+                    continue
+
+                if name.endswith('.csv') and '商品排行_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫_生意参谋', '商品排行')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '店铺来源_来源构成_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫_生意参谋', '店铺流量来源')
+                    bib(t_path, _as_month=True)
+
+    def move_dmp(self, path=None, is_except=[]):
+        """ 达摩盘 """
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        # print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # print(is_except, is_continue)
+                def bib(paths, _as_month=None):
+                    """闭包函数"""
+                    self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
+
+                if 'py_xg' not in name:  # 排除非目标文件
+                    continue
+
+                if name.endswith('.csv') and '人群属性_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫_达摩盘', '我的人群属性')
+                    bib(t_path, _as_month=True)
+
+
     # @try_except
     def move_sjy(self, path=None, is_except=[]):
         if not path:
@@ -1142,8 +1343,10 @@ class DataClean:
                 df_to_json.as_json_file()  # 写入 json 文件, 包含数据的 dtypes 信息
 
 
-def main(service_databases=None):
-    # 数据分类
+def main(service_databases=None, is_mysql=False):
+    """
+    is_mysql: 调试时加，False: 是否后续的聚合数据
+    """
 
     if not service_databases:
         service_databases = [
@@ -1153,23 +1356,30 @@ def main(service_databases=None):
             # {'nas': 'mysql'},
         ]
 
-    c = DataClean(
+    cn = DataClean(
         path=upload_path,  # 源文件目录，下载文件夹
         source_path=source_path3,  # 原始文件保存目录
         service_databases=service_databases
     )
-    c.new_unzip(is_move=True)  # 解压文件， is_move 解压后是否删除原 zip 压缩文件
-    c.tg_reports(is_except=['except'])  # 推广报表，天猫淘宝共同清洗
-    c.syj_reports_tm(is_except=['except'])  # 天猫生意经
-    # c.syj_reports_tb(is_except=['except'])  # 淘宝生意经，不可以和天猫同时运行
-    c.jd_reports(is_except=['except'])  # 清洗京东报表
-    c.sp_scene_clean(is_except=['except'])  # 商品素材
-    c.upload_df(service_databases=service_databases)  # 上传数据库
-
-    c.move_sjy(is_except=['临时文件',])  # 生意经，移到文件到原始文件夹
-    c.move_jd(is_except=['临时文件', ])  # 京东，移到文件到原始文件夹
-    c.move_tg_tm(is_except=['临时文件', ])  # 天猫，移到文件到原始文件夹
-    c.move_tg_tb(is_except=['临时文件', ])  # 淘宝店，移到文件到原始文件夹
+    cn.new_unzip(is_move=True)  # 解压文件， is_move 解压后是否删除原 zip 压缩文件
+    cn.sycm_tm(is_except=['except'])  # 天猫生意参谋
+    cn.dmp_tm(is_except=['except'])  # 达摩盘
+    cn.tg_reports(is_except=['except'])  # 推广报表，天猫淘宝共同清洗
+    cn.syj_reports_tm(is_except=['except'])  # 天猫生意经
+    # cn.syj_reports_tb(is_except=['except'])  # 淘宝生意经，不可以和天猫同时运行
+    cn.jd_reports(is_except=['except'])  # 清洗京东报表
+    cn.sp_scene_clean(is_except=['except'])  # 商品素材
+    cn.upload_df(service_databases=service_databases)  # 上传数据库
+
+    cn.move_sycm(is_except=['临时文件', ])  # 生意参谋，移到文件到原始文件夹
+    cn.move_dmp(is_except=['临时文件', ])  # 达摩盘
+    cn.move_sjy(is_except=['临时文件',])  # 生意经，移到文件到原始文件夹
+    cn.move_jd(is_except=['临时文件', ])  # 京东，移到文件到原始文件夹
+    cn.move_tg_tm(is_except=['临时文件', ])  # 天猫，移到文件到原始文件夹
+    cn.move_tg_tb(is_except=['临时文件', ])  # 淘宝店，移到文件到原始文件夹
+
+    if not is_mysql:
+        return
 
     # 更新货品年份基准表， 属性设置 2 - 货品年份基准
     p = products.Products()
@@ -1196,6 +1406,7 @@ def main(service_databases=None):
         days=100,
         is_mongo=True,
         is_mysql=True,
+        service_databases=service_databases
     )
 
     # 数据聚合
@@ -1214,7 +1425,7 @@ def main(service_databases=None):
 
 def test():
     # main_key = '单元报表'
-    path = f'/Users/xigua/数据中心/原始文件2/京东报表/JD流量来源13414124124'
+    path = f'/Users/xigua/数据中心/原始文件2/生意参谋/商品排行qweqeqwe'
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
@@ -1227,30 +1438,34 @@ def test():
             if name.endswith('.csv'):
                 print(name)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                # if '店铺名称' not in df.columns.tolist():
-                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                for item in df.columns.tolist():
-                    if '同比' in item or '环比' in item or '时间' in item:
-                        df.pop(item)
-                date_all = re.findall(r'_(\d{4}-\d{2}-\d{2})_\d{4}-\d{2}-\d{2}', name)[0]
+                if '店铺名称' not in df.columns.tolist():
+                    df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
+                df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
+                df.replace(to_replace=[','], value='', regex=True, inplace=True)
+                if '统计日期' in df.columns.tolist() and '日期' not in df.columns.tolist():
+                    df.rename(columns={'统计日期': '日期', '商品ID': '商品id'}, inplace=True)
+                # shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
+                # df.insert(loc=1, column='店铺名称', value=shop_name)
+
+                date_all = re.findall(r'_(\d{4}-\d{2}-\d{2})_', name)[0]
 
                 date = re.findall(r'_(\d{4}-\d{2})-\d{2}', name)[0]
 
-                new_path = f'/Users/xigua/数据中心/原始文件3/京东报表/店铺来源_三级来源/{date}'
+                new_path = f'/Users/xigua/数据中心/原始文件3/天猫_生意参谋/商品排行/{date}'
                 # new_path = os.path.join(new_path, date)  # 添加 年月分类
                 if not os.path.exists(new_path):
                     os.makedirs(new_path, exist_ok=True)
                 # print(date_all)
 
-                new_name = f'py_xg_京东商智_店铺来源_三级来源_{date_all}.xlsx'
+                new_name = f'py_xg_商品排行_万里马官方旗舰店_{date_all}.csv'
                 # print(os.path.join(new_path, new_name))
                 # breakpoint()
-                # df.to_csv(os.path.join(new_path, new_name), encoding='utf-8_sig', index=False, header=True)
-                try:
-                    df.to_excel(os.path.join(new_path, new_name),
-                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
-                except Exception as e:
-                    print(e)
+                df.to_csv(os.path.join(new_path, new_name), encoding='utf-8_sig', index=False, header=True)
+                # try:
+                #     df.to_excel(os.path.join(new_path, new_name),
+                #                 index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                # except Exception as e:
+                #     print(e)
 
 
 
@@ -1258,11 +1473,12 @@ def test():
 if __name__ == '__main__':
     main(
         service_databases = [
-            # {'company': 'mysql'},
-            {'home_lx': 'mysql'},
+            {'company': 'mysql'},
+            # {'home_lx': 'mysql'},
             # {'home_lx': 'mongodb'},
             # {'nas': 'mysql'},
-        ]
+        ],
+        is_mysql = False,
     )
 
     # c = DataClean(

{mdbq-2.5.7 → mdbq-2.5.9}/mdbq/dataframe/converter.py

@@ -47,7 +47,7 @@ class DataFrameConverter(object):
             try:
                 # 百分比在某些数据库中不兼容, 转换百分比为小数, # 转百分比的列不能含有中文或特殊字符
                 df[col] = df[col].apply(
-                    lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+%', str(x)) else x)
+                    lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+\.?\d*%', str(x)) else x)
             except Exception as e:
                 print(f'留意错误信息: 位于列 -> {col} -> {e}')
 
@@ -93,6 +93,6 @@ if __name__ == '__main__':
     # df = converter.convert_df_cols(df)
     # print(df['a'].dtype)
     # print(df)
-    pattern = 'dfa_dfawr__'
-    pattern = re.sub(r'_+$', '', pattern)
+    pattern = '1540%'
+    pattern = re.findall(r'^\d+\.?\d*%', pattern)
     print(pattern)

{mdbq-2.5.7 → mdbq-2.5.9}/mdbq/mysql/mysql.py

@@ -68,7 +68,7 @@ class MysqlUpload:
 
         return wrapper
 
-    @try_except
+    # @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """
         将 df 写入数据库
@@ -403,6 +403,8 @@ class MysqlUpload:
             return 'INT'
         elif dtype == 'float64':
             res = find_longest_decimal_value(df[col].tolist())  # 取小数位数最长的值
+            if 'e' in str(res):
+                res = round(float(res), 4)
             int_step = len(str(res).split('.')[0])   # 整数位数长度
             f_step = len(str(res).split('.')[1])  # 小数位数长度
 
@@ -415,17 +417,17 @@ class MysqlUpload:
             elif int_step >= 4 and f_step >= 0:
                 return 'decimal(10, 2)'
             elif int_step >= 2 and f_step >= 6:
-                return 'decimal(12, 6)'
+                return 'decimal(12, 4)'
             elif int_step >= 2 and f_step > 4:
-                return 'decimal(10, 6)'
+                return 'decimal(12, 4)'
             elif int_step >= 2 and f_step > 2:
                 return 'decimal(10, 4)'
             elif int_step >= 2 and f_step >= 0:
                 return 'decimal(10, 2)'
             elif int_step >= 1 and f_step >= 6:
-                return 'decimal(10, 6)'
+                return 'decimal(12, 4)'
             elif int_step >= 1 and f_step > 4:
-                return 'decimal(10, 6)'
+                return 'decimal(12, 4)'
             elif int_step >= 1 and f_step > 2:
                 return 'decimal(10, 4)'
             else:

{mdbq-2.5.7 → mdbq-2.5.9}/mdbq/spider/aikucun.py

@@ -8,6 +8,7 @@ import platform
 import re
 import time
 import warnings
+import requests
 import pandas as pd
 from selenium import webdriver
 from selenium.webdriver.support.wait import WebDriverWait
@@ -18,6 +19,7 @@ from mdbq.config import set_support
 from selenium.webdriver.common.keys import Keys
 from mdbq.aggregation import aggregation
 from mdbq.clean import data_clean
+from mdbq.other import ua_sj
 
 warnings.filterwarnings('ignore')
 
@@ -41,6 +43,7 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
     Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
+upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
 
 
 def get_cookie_aikucun():
@@ -84,6 +87,7 @@ def get_cookie_aikucun():
     _driver.get(_url)
     time.sleep(0.1)
     _driver.maximize_window()  # 窗口最大化 方便后续加载数据
+    print(f'请登录并切换到百宝箱，再保存 cookies: \n https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604')
     breakpoint()
 
     d_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
@@ -122,7 +126,7 @@ class AikuCun:
 
     def login(self, shop_name='aikucun'):
         option = webdriver.ChromeOptions()
-        # option.add_argument("--headless")  # 设置无界面模式
+        option.add_argument("--headless")  # 设置无界面模式
         # 调整chrome启动配置
         option.add_argument("--disable-gpu")
         option.add_argument("--no-sandbox")
@@ -154,6 +158,9 @@ class AikuCun:
         option.add_experimental_option('prefs', prefs)
         option.add_experimental_option('excludeSwitches', ['enable-automation'])  # 实验性参数, 左上角小字
 
+        # 修改默认下载文件夹路径
+        option.add_experimental_option("prefs", {"download.default_directory": f'{upload_path}'})
+
         # # 通过excludeSwitches参数禁用默认的启动路径
         # option.add_experimental_option('excludeSwitches', ['enable-automation'])
 
@@ -238,7 +245,7 @@ class AikuCun:
             elements = _driver.find_elements(
                 By.XPATH, '//button/span[contains(text(), "查询")]')
             _driver.execute_script("arguments[0].click();", elements[0])  # 点击
-            time.sleep(3)
+            time.sleep(5)
             wait.until(EC.presence_of_element_located(
                 (By.XPATH,
                  '//button[@class="el-button el-button--primary el-button--small is-plain"]/span[contains(text(), "下载数据")]')))
@@ -246,12 +253,12 @@ class AikuCun:
                 By.XPATH,
                 '//button[@class="el-button el-button--primary el-button--small is-plain"]/span[contains(text(), "下载数据")]')
             _driver.execute_script("arguments[0].click();", elements[0])  # 点击
-            time.sleep(3)
+            time.sleep(5)
             self.clean_data(date=new_date)
         _driver.quit()
 
     def clean_data(self, date):
-        for root, dirs, files in os.walk(D_PATH, topdown=False):
+        for root, dirs, files in os.walk(upload_path, topdown=False):
             for name in files:
                 if '~$' in name or 'DS_Store' in name:
                     continue
@@ -280,11 +287,11 @@ class AikuCun:
 
 def akucun():
     akc = AikuCun()
-    akc.get_data(shop_name='aikucun', date_num=7)
+    akc.get_data(shop_name='aikucun', date_num=10)  # 获取最近 N 天数据，0表示今天
     # akc.clean_data()
 
     # 新版 数据分类
-    dp = aggregation.DatabaseUpdate(path=D_PATH)
+    dp = aggregation.DatabaseUpdate(path=upload_path)
     dp.new_unzip(is_move=True)
     dp.cleaning(is_move=False, is_except=['临时文件'])  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
     # 将 self.datas 更新至数据库
@@ -295,7 +302,7 @@ def akucun():
         # {'nas': 'mysql'},
     ])
     # 数据分类
-    c = data_clean.DataClean(path=D_PATH, source_path=Source_Path)
+    c = data_clean.DataClean(path=upload_path, source_path=Source_Path)
     c.set_up_to_mogo = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
     c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
     c.new_unzip(is_move=True, )  # 解压文件
@@ -303,7 +310,57 @@ def akucun():
     c.move_all(is_except=['临时文件'])  # 移到文件到原始文件夹
 
 
+class AikuCunNew:
+
+    def __init__(self, shop_name,):
+        self.shop_name = shop_name
+        self.today = datetime.date.today()
+        self.headers = {'User-Agent': ua_sj.get_ua()}
+        self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+        self.cookies = {}
+        self.get_cookies()  # 更新 self.cookies 的值
+        self.support_path = set_support.SetSupport(dirname='support').dirname
+        self.start_date = (self.today - datetime.timedelta(days=15)).strftime('%Y-%m-%d')
+        self.end_date = (self.today - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+
+    def akc(self):
+        """
+        """
+        start_date = self.start_date
+        end_date = self.end_date
+        url = 'https://treasurebox.aikucun.com/api/web/merchant/treasure/commodity/list/down?'
+        self.headers.update({'Referer': 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'})
+        now = datetime.datetime.now()
+        timestamp_ms = round(time.mktime(now.timetuple()) * 1000 + now.microsecond / 1000)
+        data = {
+            'time': timestamp_ms,
+            'sign': '2DA6A7580C859B374AE830CAD78BB84B'
+        }
+        res = requests.post(
+            url,
+            headers=self.headers,
+            cookies=self.cookies,
+            params=data
+        )
+        print(res.text)
+
+
+
+    def get_cookies(self):
+        files = os.listdir(self.cookie_path)
+        for file in files:
+            if self.shop_name in file and '~' not in file:
+                with open(os.path.join(self.cookie_path, file), 'r') as f:
+                    cookies_data = json.load(f)
+                break
+        for data in cookies_data:
+            self.cookies.update({data['name']: data['value']})
+
+
 if __name__ == '__main__':
     pass
     # get_cookie_aikucun()
     akucun()
+
+    # a = AikuCunNew(shop_name='aikucun')
+    # a.akc()

{mdbq-2.5.7 → mdbq-2.5.9}/mdbq.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.5.7
-Home-page: https://pypi.org/project/mdbq
+Version: 2.5.9
+Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com
 License: MIT

{mdbq-2.5.7 → mdbq-2.5.9}/setup.py

@@ -3,10 +3,10 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.5.7',
+      version='2.5.9',
       author='xigua, ',
       author_email="2587125111@qq.com",
-      url='https://pypi.org/project/mdbq',
+      url='https://pypi.org/project/mdbsql',
       long_description='''
       世界上最庄严的问题：我能做什么好事？
       ''',