mdbq 2.5.4__py3-none-any.whl → 2.5.6__py3-none-any.whl

mdbq/clean/clean_upload.py

@@ -0,0 +1,1232 @@
+# -*- coding:utf-8 -*-
+import warnings
+import pandas as pd
+from functools import wraps
+import chardet
+import zipfile
+from pyzipper import PyZipFile
+import os
+import platform
+import pathlib
+import json
+from mdbq.mongo import mongo
+from mdbq.mysql import mysql
+from mdbq.config import get_myconf
+from mdbq.aggregation import df_types
+from mdbq.config import products
+from mdbq.aggregation import optimize_data
+from mdbq.aggregation import query_data
+import datetime
+import time
+import re
+import shutil
+import getpass
+
+warnings.filterwarnings('ignore')
+
+
+if platform.system() == 'Windows':
+    # windows版本
+    Data_Path = r'C:\同步空间\BaiduSyncdisk'
+    D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
+    Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
+elif platform.system() == 'Linux':
+    Data_Path = '数据中心'
+    D_PATH = 'Downloads'
+    if not os.path.exists(D_PATH):
+        os.makedirs(D_PATH)
+    Share_Path = ''  # linux 通常是远程服务器，不需要访问共享
+else:
+    Data_Path = f'/Users/{getpass.getuser()}/数据中心'  # 使用Mac独立网络时
+    D_PATH = os.path.join(f'/Users/{getpass.getuser()}/Downloads')
+    Share_Path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
+
+upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
+source_path = os.path.join(Data_Path, '原始文件2')  # 此目录保存下载并清洗过的文件，作为数据库备份
+source_path3 = os.path.join(Data_Path, '原始文件3')  # 此目录保存下载并清洗过的文件，作为数据库备份
+
+
+class DataClean:
+    """ 数据分类 """
+
+    def __init__(self, path, source_path, service_databases):
+        self.path = path  # 数据源位置，下载文件夹
+        self.source_path = source_path  # 原始文件保存目录
+        self.datas = []
+        self.service_databases = service_databases
+
+    @staticmethod
+    def try_except(func):  # 在类内部定义一个异常处理方法
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                print(f'{func.__name__}, {e}')  # 将异常信息返回
+
+        return wrapper
+
+    @staticmethod
+    def get_encoding(file_path):
+        """
+        获取文件的编码方式, 读取速度比较慢，非必要不要使用
+        """
+        with open(file_path, 'rb') as f:
+            f1 = f.read()
+            encod = chardet.detect(f1).get('encoding')
+        return encod
+
+    @staticmethod
+    def save_to_csv(_df, _save_paths, filenames, encoding='utf-8_sig'):
+        if '.csv' not in filenames:
+            filenames = f'{filenames}.csv'
+        if not os.path.exists(_save_paths):
+            os.makedirs(_save_paths, exist_ok=True)
+        _df.to_csv(os.path.join(_save_paths, filenames), encoding=encoding, index=False, header=True)
+
+    def tg_reports(self, path=None, is_except=[]):
+        """ 处理天猫淘宝推广类报表 """
+        if not path:
+            path = self.path
+        report_names = [
+            {
+                '文件简称': 'tg_report_主体报表',
+                '数据库名': '推广数据3',
+                '集合名称': '主体报表',
+            },
+            {
+                '文件简称': 'tg_report_创意报表_创意',
+                '数据库名': '推广数据3',
+                '集合名称': '创意报表_创意',
+            },
+            {
+                '文件简称': 'tg_report_创意报表_素材',
+                '数据库名': '推广数据3',
+                '集合名称': '创意报表_素材',
+            },
+            {
+                '文件简称': 'tg_report_单元报表',
+                '数据库名': '推广数据3',
+                '集合名称': '单元报表',
+            },
+            {
+                '文件简称': 'tg_report_地域报表_省份',
+                '数据库名': '推广数据3',
+                '集合名称': '地域报表_省份',
+            },
+            {
+                '文件简称': 'tg_report_地域报表_城市',
+                '数据库名': '推广数据3',
+                '集合名称': '地域报表_城市',
+            },
+            {
+                '文件简称': 'tg_report_关键词报表',
+                '数据库名': '推广数据3',
+                '集合名称': '关键词报表',
+            },
+            {
+                '文件简称': 'tg_report_计划报表',
+                '数据库名': '推广数据3',
+                '集合名称': '计划报表',
+            },
+            {
+                '文件简称': 'tg_report_权益报表',
+                '数据库名': '推广数据3',
+                '集合名称': '权益报表',
+            },
+            {
+                '文件简称': 'tg_report_人群报表',
+                '数据库名': '推广数据3',
+                '集合名称': '人群报表',
+            },
+            {
+                '文件简称': 'tg_report_营销场景报表',
+                '数据库名': '推广数据3',
+                '集合名称': '营销场景报表',
+            },
+            {
+                '文件简称': 'tg_report_超级直播报表_人群',
+                '数据库名': '推广数据3',
+                '集合名称': '超级直播',
+            },
+            {
+                '文件简称': 'tg_report_品销宝_明星店铺',
+                '数据库名': '推广数据3',
+                '集合名称': '品销宝',
+            }
+        ]
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # 这里排除掉非推广类报表
+                is_continue = False
+                db_name = None
+                collection_name = None
+                for item in report_names:
+                    if item['文件简称'] in name:
+                        db_name = item['数据库名']
+                        collection_name = item['集合名称']
+                        is_continue = True
+                if not is_continue:
+                    continue
+                # 区分淘宝和天猫的报表
+                if '万里马官方旗舰店' in name:
+                    db_name = f'天猫_{db_name}'
+                elif '万里马官方企业店' in name:
+                    db_name = f'淘宝_{db_name}'
+                else:
+                    print(f'报表名称错误，不属于天猫/淘宝店：{name}')
+                    continue
+
+                if name.endswith('.csv'):  # 推广类报表
+                    if '明星店铺' in name:  # 明星店铺可能会先释放 csv 文件
+                        continue
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
+                    df.fillna(0, inplace=True)
+                    date_min = df["日期"].values.min()
+                    date_max = df["日期"].values.max()
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    df.insert(loc=1, column='店铺名称', value=shop_name)
+                    new_name = f'py_xg_{name}'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.xlsx') and '品销宝_明星店铺' in name:
+                    # 品销宝
+                    sheets4 = ['账户', '推广计划', '推广单元', '创意', '品牌流量包', '定向人群']  # 品销宝
+                    file_name4 = os.path.splitext(name)[0]  # 明星店铺报表
+                    for sheet4 in sheets4:
+                        df = pd.read_excel(os.path.join(root, name), sheet_name=sheet4, header=0, engine='openpyxl')
+                        if len(df) == 0:
+                            print(f'{name} 报表数据为空')
+                            os.remove(os.path.join(root, name))
+                            continue
+                        if len(df) < 1:
+                            print(f'{name} 跳过')
+                            continue
+                        else:
+                            shop_name = re.findall(r'明星店铺_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
+                            df.insert(loc=1, column='店铺名称', value=shop_name)
+                            df.insert(loc=2, column='报表类型', value=sheet4)
+                            df.fillna(0, inplace=True)
+                            df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+                            # min_clm = str(df['日期'].min()).split(' ')[0]
+                            # max_clm = str(df['日期'].max()).split(' ')[0]
+                            new_file_name4 = f'{sheet4}_py_xg_{file_name4}.csv'
+                            # 以sheet名进一步创建子文件夹
+                            # root_new = os.path.join(self.source_path, '推广报表/品销宝', sheet4)
+                            self.save_to_csv(df, upload_path, new_file_name4)
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    print(f'db_name/collection_name 不能为空')
+                    continue
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                )
+
+    def syj_reports_tm(self, path=None, is_except=[]):
+        """ 生意经报表 """
+        if not path:
+            path = self.path
+        report_names = [
+            {
+                '文件简称': 'baobei',
+                '数据库名': '天猫_生意经3',
+                '集合名称': '宝贝指标',
+            },
+            {
+                '文件简称': 'order',
+                '数据库名': '天猫_生意经3',
+                '集合名称': '订单指标',
+            },
+            {
+                '文件简称': '省份城市分析',
+                '数据库名': '天猫_生意经3',
+                '集合名称': '省份城市分析',
+            },
+            {
+                '文件简称': '店铺销售指标',
+                '数据库名': '天猫_生意经3',
+                '集合名称': '店铺销售指标',
+            },
+        ]
+
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # 这里排除掉非目标报表
+                is_continue = False
+                db_name = None
+                collection_name = None
+                for item in report_names:
+                    if item['文件简称'] in name:
+                        db_name = item['数据库名']
+                        collection_name = item['集合名称']
+                        is_continue = True
+                if not is_continue:
+                    continue
+
+                if name.endswith('.csv') and 'baobei' in name:
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2}).csv', name)[0]
+                    df['日期'] = '-'.join(pattern)
+                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
+                    new_name = f'py_xg_天猫_baobeitrains_{'-'.join(pattern)}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and 'order' in name:
+                    """ 这里不能使用表格原先的 gb2312， 会报错 """
+                    # encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding='gb18030', header=0, na_filter=False)
+                    pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)[0]
+                    date1 ='-'.join(pattern[1:4])
+                    date2 = '-'.join(pattern[4:7])
+                    df.insert(loc=0, column='日期', value=date1)
+                    df.insert(loc=1, column='数据周期', value=f'{date1}_{date2}')
+                    df.rename(columns={'宝贝标题': '商品标题', '宝贝链接': '商品链接'}, inplace=True)
+                    df['颜色编码'] = df['商家编码'].apply(
+                        lambda x: ''.join(re.findall(r' .*(\d{4})$', str(x))) if x else x)
+                    new_name = f'py_xg_天猫_order_{date1}_{date2}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and '省份城市分析' in name:
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\.', name)[0]
+                    date = '-'.join(pattern[1:])
+                    new_name = f'py_xg_天猫_{pattern[0]}-{date}.csv'
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        os.remove(os.path.join(root, name))
+                        continue
+                    df['省'] = df['省份'].apply(lambda x: x if ' ├─ ' not in x and ' └─ ' not in x else None)
+                    df['城市'] = df[['省份', '省']].apply(lambda x: '汇总' if x['省'] else x['省份'], axis=1)
+                    df['省'].fillna(method='ffill', inplace=True)
+                    df['城市'].replace(to_replace=[' ├─ | └─ '], value='', regex=True, inplace=True)
+                    pov = df.pop('省')
+                    city = df.pop('城市')
+                    df['省+市'] = df['省份']
+                    df['省份'] = pov
+                    df.insert(loc=1, column='城市', value=city)
+                    df.insert(loc=0, column='日期', value=date)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and '店铺销售指标' in name:
+                    # 生意经, 店铺指标，仅限月数据，实际日指标也可以
+                    name_st = re.findall(r'(.*)\(分日', name)
+                    if not name_st:
+                        print(f'{name}  已转换的表格')
+                        continue
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        os.remove(os.path.join(root, name))
+                        continue
+                    df['日期'] = df['日期'].astype(str).apply(
+                        lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', x)[0]) if x else x)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+                    # min_clm = str(df.min()['日期']).split(' ')[0]
+                    # max_clm = str(df.max()['日期']).split(' ')[0]
+                    min_clm = str(df['日期'].min()).split(' ')[0]
+                    max_clm = str(df['日期'].max()).split(' ')[0]
+                    new_name = f'py_xg_天猫_{name_st[0]}-{min_clm}_{max_clm}.csv'  # 保存时将(分日)去掉
+                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    print(f'db_name/collection_name 不能为空')
+                    continue
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                )
+
+    def syj_reports_tb(self, path=None, is_except=[]):
+        """ 淘宝店 生意经报表 """
+        if not path:
+            path = self.path
+        report_names = [
+            {
+                '文件简称': 'baobei',
+                '数据库名': '淘宝_生意经3',
+                '集合名称': '宝贝指标',
+            },
+            {
+                '文件简称': 'order',
+                '数据库名': '淘宝_生意经3',
+                '集合名称': '订单指标',
+            },
+            {
+                '文件简称': '省份城市分析',
+                '数据库名': '淘宝_生意经3',
+                '集合名称': '省份城市分析',
+            },
+            {
+                '文件简称': '店铺销售指标',
+                '数据库名': '淘宝_生意经3',
+                '集合名称': '店铺销售指标',
+            },
+        ]
+
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # 这里排除掉非目标报表
+                is_continue = False
+                db_name = None
+                collection_name = None
+                for item in report_names:
+                    if item['文件简称'] in name:
+                        db_name = item['数据库名']
+                        collection_name = item['集合名称']
+                        is_continue = True
+                if not is_continue:
+                    continue
+
+                if name.endswith('.csv') and 'baobei' in name:
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2}).csv', name)[0]
+                    df['日期'] = '-'.join(pattern)
+                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
+                    new_name = f'py_xg_淘宝_baobeitrains_{'-'.join(pattern)}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and 'order' in name:
+                    """ 这里不能使用表格原先的 gb2312， 会报错 """
+                    # encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding='gb18030', header=0, na_filter=False)
+                    pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)[0]
+                    date1 ='-'.join(pattern[1:4])
+                    date2 = '-'.join(pattern[4:7])
+                    df.insert(loc=0, column='日期', value=date1)
+                    df.insert(loc=1, column='数据周期', value=f'{date1}_{date2}')
+                    df.rename(columns={'宝贝标题': '商品标题', '宝贝链接': '商品链接'}, inplace=True)
+                    df['颜色编码'] = df['商家编码'].apply(
+                        lambda x: ''.join(re.findall(r' .*(\d{4})$', str(x))) if x else x)
+                    new_name = f'py_xg_淘宝_order_{date1}_{date2}.csv'
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and '省份城市分析' in name:
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\.', name)[0]
+                    date = '-'.join(pattern[1:])
+                    new_name = f'py_xg_淘宝_{pattern[0]}-{date}.csv'
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        os.remove(os.path.join(root, name))
+                        continue
+                    df['省'] = df['省份'].apply(lambda x: x if ' ├─ ' not in x and ' └─ ' not in x else None)
+                    df['城市'] = df[['省份', '省']].apply(lambda x: '汇总' if x['省'] else x['省份'], axis=1)
+                    df['省'].fillna(method='ffill', inplace=True)
+                    df['城市'].replace(to_replace=[' ├─ | └─ '], value='', regex=True, inplace=True)
+                    pov = df.pop('省')
+                    city = df.pop('城市')
+                    df['省+市'] = df['省份']
+                    df['省份'] = pov
+                    df.insert(loc=1, column='城市', value=city)
+                    df.insert(loc=0, column='日期', value=date)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and '店铺销售指标' in name:
+                    # 生意经, 店铺指标，仅限月数据，实际日指标也可以
+                    name_st = re.findall(r'(.*)\(分日', name)
+                    if not name_st:
+                        print(f'{name}  已转换的表格')
+                        continue
+                    encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
+                    if len(df) == 0:
+                        print(f'{name} 报表数据为空')
+                        os.remove(os.path.join(root, name))
+                        continue
+                    df['日期'] = df['日期'].astype(str).apply(
+                        lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', x)[0]) if x else x)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+                    # min_clm = str(df.min()['日期']).split(' ')[0]
+                    # max_clm = str(df.max()['日期']).split(' ')[0]
+                    min_clm = str(df['日期'].min()).split(' ')[0]
+                    max_clm = str(df['日期'].max()).split(' ')[0]
+                    new_name = f'py_xg_淘宝_{name_st[0]}-{min_clm}_{max_clm}.csv'  # 保存时将(分日)去掉
+                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
+                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    print(f'db_name/collection_name 不能为空')
+                    continue
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                )
+
+    def jd_reports(self, path=None, is_except=[]):
+        """ 处理京东报表 """
+        if not path:
+            path = self.path
+        report_names = [
+            {
+                '文件简称': '京东推广_点击成交',
+                '数据库名': '京东数据3',
+                '集合名称': '推广数据_京准通',
+            },
+            {
+                '文件简称': '京东推广_搜索词',
+                '数据库名': '京东数据3',
+                '集合名称': '推广数据_搜索词报表',
+            },
+            {
+                '文件简称': '京东推广_关键词',
+                '数据库名': '京东数据3',
+                '集合名称': '推广数据_关键词报表',
+            },
+            {
+                '文件简称': '京东商智_sku_商品明细',
+                '数据库名': '京东数据3',
+                '集合名称': '京东商智_sku_商品明细',
+            },
+            {
+                '文件简称': '京东商智_spu_商品明细',
+                '数据库名': '京东数据3',
+                '集合名称': '京东商智_spu_商品明细',
+            },
+            {
+                '文件简称': '京东商智_店铺来源_三级来源',
+                '数据库名': '京东数据3',
+                '集合名称': '京东商智_店铺来源',
+            },
+        ]
+
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # 这里排除掉非目标报表
+                is_continue = False
+                db_name = None
+                collection_name = None
+                for item in report_names:
+                    if item['文件简称'] in name:
+                        db_name = item['数据库名']
+                        collection_name = item['集合名称']
+                        is_continue = True
+                if not is_continue:
+                    continue
+
+                if name.endswith('.xlsx') and '京东推广_' in name:
+                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    new_name = f'py_xg_{name}'
+                    os.rename(os.path.join(root, name), os.path.join(root, new_name))
+                elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
+                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df.replace(to_replace=['-'], value='', regex=False, inplace=True)
+                    pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
+                    df.insert(loc=0, column='日期', value=pattern)
+                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                    df.fillna(0, inplace=True)
+                    new_name = f'py_xg_{name}'
+                    df.to_excel(os.path.join(upload_path, new_name),
+                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
+                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df.replace(to_replace=['-'], value='', regex=False, inplace=True)
+                    pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
+                    df.insert(loc=0, column='日期', value=pattern)
+                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                    df.fillna(0, inplace=True)
+                    new_name = f'py_xg_{name}'
+                    df.to_excel(os.path.join(upload_path, new_name),
+                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    os.remove(os.path.join(root, name))
+                elif name.endswith('.xlsx') and '京东商智_店铺来源_三级来源' in name:
+                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df.replace(to_replace=['-'], value='', regex=False, inplace=True)
+                    df.rename(columns={'时间': '日期'}, inplace=True)
+                    for col in df.columns.tolist():
+                        if '环比' in col or '同比' in col:
+                            df.drop(col, axis=1, inplace=True)
+                    df.fillna(0, inplace=True)
+                    new_name = f'py_xg_{name}'
+                    df.to_excel(os.path.join(upload_path, new_name),
+                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    print(f'db_name/collection_name 不能为空')
+                    continue
+                # print(name)
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                )
+
+    def sp_scene_clean(self, path=None, is_except=[]):
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
+                    continue
+                if 'py_xg' in name:
+                    continue
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                if name.endswith('.xlsx') and '商品素材_' in name:
+                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
+                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df.insert(loc=1, column='店铺名称', value=shop_name)
+                    new_name = f'py_xg_{name}'
+                    df.to_excel(os.path.join(upload_path, new_name),
+                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    if '官方旗舰店' in name:
+                        db_name = '属性设置3'
+                        collection_name = '商品素材_天猫'
+                    elif '官方企业店' in name:
+                        db_name = '属性设置3'
+                        collection_name = '商品素材_淘宝'
+                    os.remove(os.path.join(root, name))
+
+                # 将数据传入 self.datas 等待更新进数据库
+                if not db_name or not collection_name:
+                    print(f'db_name/collection_name 不能为空')
+                    continue
+                self.datas.append(
+                    {
+                        '数据库名': db_name,
+                        '集合名称': collection_name,
+                        '数据主体': df,
+                        '文件名': name,
+                    }
+                )
+    """
+    {文件分类}
+    将已处理完的文件 分类移到原始文件夹下
+    此处t_path参数定义了子文件夹的生成名称
+    """
+
+    @staticmethod
+    def move_files(path, _name, target_path, _as_month=None):
+        """
+        name: 移动的文件名，
+        target_path: 目标位置
+        """
+        t2 = target_path  # t2 赋值有用, 不能省略
+        if not os.path.exists(t2):  # 如果目录不存在则创建
+            os.makedirs(t2, exist_ok=True)
+        if _as_month:
+            _date = re.findall(r'(\d{4}-\d{2})-\d{2}', str(_name))
+            if _date:
+                _date = _date[0]
+                t2 = pathlib.Path(t2, _date)  # 添加 年月分类
+                if not os.path.exists(t2):
+                    os.makedirs(t2, exist_ok=True)
+        old_file = os.path.join(t2, _name)  # 检查目标位置是否已经存在该文件
+        if os.path.isfile(old_file):
+            os.remove(old_file)  # 如果存在则移除
+        shutil.move(os.path.join(path, _name), t2)  # 将文件从下载文件夹移到目标位置
+
+    # @try_except
+    def move_sjy(self, path=None, is_except=[]):
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        # print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # print(is_except, is_continue)
+                def bib(paths, _as_month=None):
+                    """闭包函数"""
+                    self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
+
+                if 'py_xg' not in name:  # 排除非目标文件
+                    continue
+
+                if '天猫' in name and name.endswith('.csv') and 'baobei' in name:
+                    t_path = os.path.join(self.source_path, '天猫_生意经', '宝贝指标')
+                    bib(t_path, _as_month=True)
+                elif '天猫' in name and name.endswith('.csv') and '省份城市分析' in name:
+                    t_path = os.path.join(self.source_path, '天猫_生意经', '省份城市分析')
+                    bib(t_path, _as_month=True)
+                elif '天猫' in name and name.endswith('.csv') and '店铺销售指标' in name:
+                    t_path = os.path.join(self.source_path, '天猫_生意经', '店铺销售指标')
+                    bib(t_path, _as_month=False)
+                elif '天猫' in name and name.endswith('.csv') and 'order' in name:
+                    t_path = os.path.join(self.source_path, '天猫_生意经', '订单数据')
+                    bib(t_path, _as_month=False)
+                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and 'baobei' in name:
+                    t_path = os.path.join(self.source_path, '淘宝_生意经', '宝贝指标')
+                    bib(t_path, _as_month=True)
+                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and '省份城市分析' in name:
+                    t_path = os.path.join(self.source_path, '淘宝_生意经', '省份城市分析')
+                    bib(t_path, _as_month=True)
+                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and '店铺销售指标' in name:
+                    t_path = os.path.join(self.source_path, '淘宝_生意经', '店铺销售指标')
+                    bib(t_path, _as_month=False)
+                elif '淘宝' in name or '企业店' in name and name.endswith('.csv') and 'order' in name:
+                    t_path = os.path.join(self.source_path, '淘宝_生意经', '订单数据')
+                    bib(t_path, _as_month=False)
+
+    # @try_except
+    def move_jd(self, path=None, is_except=[]):
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        # print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # print(is_except, is_continue)
+                def bib(paths, _as_month=None):
+                    """闭包函数"""
+                    self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
+
+                if 'py_xg' not in name:  # 排除非目标文件
+                    continue
+
+                if name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
+                    t_path = os.path.join(self.source_path, '京东报表', 'spu_商品明细')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
+                    t_path = os.path.join(self.source_path, '京东报表', 'sku_商品明细')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.xlsx') and '京东推广_搜索词' in name:
+                    t_path = os.path.join(self.source_path, '京东报表', '搜索词报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.xlsx') and '京东推广_点击成交' in name:
+                    t_path = os.path.join(self.source_path, '京东报表', '推广报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.xlsx') and '京东推广_关键词点击' in name:
+                    t_path = os.path.join(self.source_path, '京东报表', '关键词报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.xlsx') and '京东商智_店铺来源_三级来源' in name:
+                    t_path = os.path.join(self.source_path, '京东报表', '店铺来源_三级来源')
+                    bib(t_path, _as_month=True)
+
+    # @try_except
+    def move_tg_tm(self, path=None, is_except=[]):
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        # print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+                # print(is_except, is_continue)
+                def bib(paths, _as_month=None):
+                    """闭包函数"""
+                    self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
+
+                if 'py_xg' not in name:  # 排除非目标文件
+                    continue
+
+                if name.endswith('.csv') and 'tg_report_主体报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '主体报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_营销场景报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '营销场景报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_人群报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '人群报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_权益报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '权益报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_计划报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '计划报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_关键词报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '关键词报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_地域报表_省份_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '地域报表_省份')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_地域报表_城市_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '地域报表_城市')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_单元报表_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '单元报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_创意报表_素材粒度_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '创意报表_素材粒度')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_创意报表_创意粒度_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '创意报表_创意粒度')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_超级直播报表_人群_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '超级直播报表_人群')
+                    bib(t_path, _as_month=True)
+
+                elif name.endswith('.csv') and 'tg_report_品销宝_明星店铺_万里马官方旗舰店' in name:
+                    if '账户' in name:
+                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '账户报表')
+                        bib(t_path, _as_month=True)
+                    elif '推广计划' in name:
+                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '推广计划报表')
+                        bib(t_path, _as_month=True)
+                    elif '推广单元' in name:
+                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '推广单元报表')
+                        bib(t_path, _as_month=True)
+                    elif '创意' in name:
+                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '创意报表')
+                        bib(t_path, _as_month=True)
+                    elif '品牌流量包' in name:
+                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '品牌流量包报表')
+                        bib(t_path, _as_month=True)
+                    elif '定向人群' in name:
+                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '定向人群报表')
+                        bib(t_path, _as_month=True)
+                elif name.endswith('xlsx') and '商品素材_万里马官方旗舰店' in name:
+                    t_path = os.path.join(self.source_path, '商品素材', '天猫')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('xlsx') and '商品素材_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '商品素材', '淘宝')
+                    bib(t_path, _as_month=True)
+
+    # @try_except
+    def move_tg_tb(self, path=None, is_except=[]):
+        if not path:
+            path = self.path
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                # print(name)
+                is_continue = False
+                if is_except:
+                    for item in is_except:
+                        # print(item, f'-----', os.path.join(root, name))
+                        if item in os.path.join(root, name):
+                            # print(name)
+                            is_continue = True
+                            break
+                if is_continue:  # 需要排除不做处理的文件或文件夹
+                    continue
+
+                # print(is_except, is_continue)
+                def bib(paths, _as_month=None):
+                    """闭包函数"""
+                    self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
+
+                if 'py_xg' not in name:  # 排除非目标文件
+                    continue
+
+                if name.endswith('.csv') and 'tg_report_主体报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '主体报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_营销场景报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '营销场景报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_人群报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '人群报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_权益报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '权益报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_计划报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '计划报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_关键词报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '关键词报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_地域报表_省份_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '地域报表_省份')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_地域报表_城市_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '地域报表_城市')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_单元报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '单元报表')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_创意报表_素材粒度_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '创意报表_素材粒度')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_创意报表_创意粒度_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '创意报表_创意粒度')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_超级直播报表_万里马官方企业店' in name:
+                    t_path = os.path.join(self.source_path, '淘宝推广报表', '超级直播报表')
+                    bib(t_path, _as_month=True)
+
+    # @try_except
+    def new_unzip(self, path=None, is_move=None):
+        """
+        {解压并移除zip文件}
+        如果是京东的商品明细，处理过程：
+        1. 读取 zip包的文件名
+        2. 组合完整路径，判断文件夹下是否已经有同名文件
+        3. 如果有，则将该同名文件改名，（从文件名中提取日期，重新拼接文件名）
+        4. 然后解压 zip包
+        5. 需要用 _jd_rename 继续重命名刚解压的文件
+        is_move 参数,  是否移除 下载目录的所有zip 文件
+        """
+        if not path:
+            path = self.path
+        res_names = []  # 需要移除的压缩文件
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                if '~$' in name or 'DS_Store' in name:
+                    continue
+                if name.endswith('.zip'):
+                    old_file = os.path.join(root, name)
+                    f = zipfile.ZipFile(old_file, 'r')
+                    if len(f.namelist()) == 1:  # 压缩包只有一个文件的情况
+                        for zip_name in f.namelist():  # 读取zip内的文件名称
+                            # zip_name_1 = zip_name.encode('cp437').decode('utf-8')
+                            try:
+                                zip_name_1 = zip_name.encode('utf-8').decode('utf-8')
+                            except:
+                                zip_name_1 = zip_name.encode('cp437').decode('utf-8')
+                            new_path = os.path.join(root, zip_name_1)  # 拼接解压后的文件路径
+                            if os.path.isfile(new_path) and '全部渠道_商品明细' in new_path:  # 是否存在和包内同名的文件
+                                # 专门处理京东文件, 已过期可删
+                                df = pd.read_excel(new_path)
+                                try:
+                                    pattern1 = re.findall(r'\d{8}_(\d{4})(\d{2})(\d{2})_全部渠道_商品明细',
+                                                          name)
+                                    pattern2 = re.findall(
+                                        r'\d{8}_(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})_全部渠道_商品明细',
+                                        name)
+                                    if pattern1:
+                                        year_date = '-'.join(list(pattern1[0])) + '_' + '-'.join(list(pattern1[0]))
+                                    elif pattern2:
+                                        year_date = '-'.join(list(pattern2[0])[0:3]) + '_' + '-'.join(
+                                            list(pattern2[0])[3:7])
+                                    else:
+                                        year_date = '无法提取日期'
+                                        print(f'{name} 无法从文件名中提取日期，请检查pattern或文件')
+                                    if ('10035975359247' in df['商品ID'].values or '10056642622343' in
+                                            df['商品ID'].values):
+                                        os.rename(new_path,
+                                                  os.path.join(root, 'sku_' + year_date + '_全部渠道_商品明细.xls'))
+                                        f.extract(zip_name_1, root)
+                                    elif ('10021440233518' in df['商品ID'].values or '10022867813485' in
+                                          df['商品ID'].values):
+                                        os.rename(new_path,
+                                                  os.path.join(root, 'spu_' + year_date + '_全部渠道_商品明细.xls'))
+                                        f.extract(zip_name_1, root)
+                                    if is_move:
+                                        os.remove(os.path.join(root, name))
+                                except Exception as e:
+                                    print(e)
+                                    continue
+                            else:
+                                f.extract(zip_name, root)
+                                if zip_name_1 != zip_name:
+                                    os.rename(os.path.join(root, zip_name), os.path.join(root, zip_name_1))
+                                if is_move:
+                                    res_names.append(name)
+                                    # os.remove(os.path.join(root, name))  # 这里不能移除，会提示文件被占用
+                        f.close()
+                    else:  # 压缩包内包含多个文件的情况
+                        f.close()
+                        self.unzip_all(path=old_file, save_path=path)
+
+        if is_move:
+            for name in res_names:
+                os.remove(os.path.join(path, name))
+                print(f'移除{os.path.join(path, name)}')
+
+    @staticmethod
+    def unzip_all(path, save_path):
+        """
+        遍历目录， 重命名有乱码的文件
+        2. 如果压缩包是文件夹， 则保存到新文件夹，并删除有乱码的文件夹
+        3. 删除MAC系统的临时文件夹__MACOSX
+        """
+        with PyZipFile(path) as _f:
+            _f.extractall(save_path)
+            _f.close()
+        for _root, _dirs, _files in os.walk(save_path, topdown=False):
+            for _name in _files:
+                if '~$' in _name or 'DS_Store' in _name:
+                    continue
+                try:
+                    _new_root = _root.encode('cp437').decode('utf-8')
+                    _new_name = _name.encode('cp437').decode('utf-8')
+                except:
+                    _new_root = _root.encode('utf-8').decode('utf-8')
+                    _new_name = _name.encode('utf-8').decode('utf-8')
+                _old = os.path.join(_root, _name)
+                _new = os.path.join(_new_root, _new_name)
+                if _new_root != _root:  # 目录乱码，创建新目录
+                    os.makedirs(_new_root, exist_ok=True)
+                os.rename(_old, _new)
+            try:
+                _new_root = _root.encode('cp437').decode('utf-8')
+            except:
+                _new_root = _root.encode('utf-8').decode('utf-8')
+            if _new_root != _root or '__MACOSX' in _root:
+                shutil.rmtree(_root)
+
+    def upload_df(self, service_databases=None, path=None):
+        """
+        将清洗后的 df 上传数据库, copysh.py 调用
+        """
+        if not service_databases:
+            service_databases = self.service_databases
+        df_to_json = df_types.DataTypes()  # json 文件, 包含数据的 dtypes 信息
+        for service_database in service_databases:
+            for service_name, database in service_database.items():
+                # print(service_name, database)
+                if database == 'mongodb':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    d = mongo.UploadMongo(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                        drop_duplicates=False,
+                    )
+                    for data in self.datas:
+                        db_name, collection_name, df = data['数据库名'], data['集合名称'], data['数据主体']
+                        df_to_json.get_df_types(
+                            df=df,
+                            db_name=db_name,
+                            collection_name=collection_name,
+                            is_file_dtype=True,  # 默认本地文件优先: True
+                        )
+                        d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
+                    if d.client:
+                        d.client.close()
+
+                elif database == 'mysql':
+                    username, password, host, port = get_myconf.select_config_values(
+                        target_service=service_name,
+                        database=database,
+                    )
+                    m = mysql.MysqlUpload(
+                        username=username,
+                        password=password,
+                        host=host,
+                        port=port,
+                    )
+                    for data in self.datas:
+                        df, db_name, collection_name, rt_filename = data['数据主体'], data['数据库名'], data['集合名称'], data['文件名']
+                        df_to_json.get_df_types(
+                            df=df,
+                            db_name=db_name,
+                            collection_name=collection_name,
+                            is_file_dtype=True,  # 默认本地文件优先: True
+                        )
+                        m.df_to_mysql(
+                            df=df,
+                            db_name=db_name,
+                            table_name=collection_name,
+                            move_insert=True,  # 先删除，再插入
+                            df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                            drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                            filename=rt_filename,  # 用来追踪处理进度
+                            service_database=service_database,  # 字典
+                        )
+                df_to_json.as_json_file()  # 写入 json 文件, 包含数据的 dtypes 信息
+
+
+def main(service_databases=None):
+    # 数据分类
+
+    if not service_databases:
+        service_databases = [
+            # {'home_lx': 'mongodb'},
+            {'home_lx': 'mysql'},
+            # {'company': 'mysql'},
+            # {'nas': 'mysql'},
+        ]
+
+    c = DataClean(
+        path=upload_path,  # 源文件目录，下载文件夹
+        source_path=source_path3,  # 原始文件保存目录
+        service_databases=service_databases
+    )
+    c.new_unzip(is_move=True)  # 解压文件， is_move 解压后是否删除原 zip 压缩文件
+    c.tg_reports(is_except=['except'])  # 推广报表，天猫淘宝共同清洗
+    c.syj_reports_tm(is_except=['except'])  # 天猫生意经
+    # c.syj_reports_tb(is_except=['except'])  # 淘宝生意经，不可以和天猫同时运行
+    c.jd_reports(is_except=['except'])  # 清洗京东报表
+    c.sp_scene_clean(is_except=['except'])  # 商品素材
+    c.upload_df(service_databases=service_databases)  # 上传数据库
+
+    c.move_sjy(is_except=['临时文件',])  # 生意经，移到文件到原始文件夹
+    c.move_jd(is_except=['临时文件', ])  # 京东，移到文件到原始文件夹
+    c.move_tg_tm(is_except=['临时文件', ])  # 天猫，移到文件到原始文件夹
+    c.move_tg_tb(is_except=['临时文件', ])  # 淘宝店，移到文件到原始文件夹
+
+    # 更新货品年份基准表， 属性设置 2 - 货品年份基准
+    p = products.Products()
+    p.to_mysql(service_databases=service_databases)
+
+    # 清理所有非聚合数据的库
+    optimize_data.op_data(
+        db_name_lists=[
+            '京东数据2',
+            '推广数据2',
+            '市场数据2',
+            '生意参谋2',
+            '生意经2',
+            '属性设置2',
+            # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+            '京东数据3',
+            '天猫_推广数据3',
+            '淘宝_推广数据3',
+            # '市场数据3',
+            # '生意参谋3',
+            '天猫_生意经3',
+            # '淘宝_生意经3',
+        ],
+        days=100,
+        is_mongo=True,
+        is_mysql=True,
+    )
+
+    # 数据聚合
+    query_data.data_aggregation(service_databases=service_databases, months=3)
+    time.sleep(60)
+
+    # 清理聚合数据, mongodb 中没有聚合数据，所以只需要清理 mysql 即可
+    optimize_data.op_data(
+        db_name_lists=['聚合数据'],
+        days=3650,
+        service_databases=service_databases,
+        is_mongo=False,
+        is_mysql=True,
+    )
+
+
+if __name__ == '__main__':
+    main(
+        service_databases = [
+            # {'company': 'mysql'},
+            {'home_lx': 'mysql'},
+            # {'home_lx': 'mongodb'},
+            # {'nas': 'mysql'},
+        ]
+    )
+
+    # c = DataClean(
+    #     path=upload_path,  # 源文件目录，下载文件夹
+    #     source_path=source_path3,  # 原始文件保存目录
+    #     service_databases=[{'home_lx': 'mysql'},]
+    # )
+    # c.sp_scene_clean(is_except=['except'])  # 商品素材
+    # c.move_tg_tm(is_except=['临时文件', ])  # 天猫，移到文件到原始文件夹

mdbq/dataframe/converter.py

@@ -44,8 +44,12 @@ class DataFrameConverter(object):
                 df.pop(col)  # 等待插入的 df 不能包含 id 列，否则可能跟现有 id 主键冲突
                 continue
 
-            # 百分比在某些数据库中不兼容, 转换百分比为小数
-            df[col] = df[col].apply(lambda x: float(float((str(x).rstrip("%"))) / 100) if str(x).endswith('%') and '~' not in str(x) else x)
+            try:
+                # 百分比在某些数据库中不兼容, 转换百分比为小数, # 转百分比的列不能含有中文或特殊字符
+                df[col] = df[col].apply(
+                    lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+%', str(x)) else x)
+            except Exception as e:
+                print(f'留意错误信息: 位于列 -> {col} -> {e}')
 
             if col.endswith('占比') or col.endswith('率'):
                 df = df.astype({col: float}, errors='raise')

{mdbq-2.5.4.dist-info → mdbq-2.5.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.5.4
+Version: 2.5.6
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.5.4.dist-info → mdbq-2.5.6.dist-info}/RECORD

@@ -9,6 +9,7 @@ mdbq/aggregation/query_data.py,sha256=WKe42Xq1Gi-ELuIT0k2jh3X4-R7heb0ub3Mj3yuCRA
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
+mdbq/clean/clean_upload.py,sha256=q_3kjiE0sU6uV13TW9rVuPmbO01itYhkC4gTnz_nZ5o,64455
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
@@ -19,7 +20,7 @@ mdbq/config/products.py,sha256=hN9UMkM6j76HYMulTYdtr3mOhh9QdpvvrLH14a_mbFY,5980
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=OY5sMFzdF7wkfE59Es-urlZ2oJJY5nkao7009GSyVv4,4110
+mdbq/dataframe/converter.py,sha256=u7rQbIsgVZWOIybJaknduf7wViBdBkyU8mwUo24mDt0,4304
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
@@ -41,7 +42,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=KdihSB3q44jsXUQAldfWRVfCSrEw2MNbM-_BhP_29g4,14448
-mdbq-2.5.4.dist-info/METADATA,sha256=QPhMknQuHoAlaRRB4M7H0h-GINpIWWzG7HOujYCQwHw,243
-mdbq-2.5.4.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-2.5.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.5.4.dist-info/RECORD,,
+mdbq-2.5.6.dist-info/METADATA,sha256=IIsrPFdnbkCfH4ziUSl_U05g3Cvc9vwyySZjLXN6SVU,243
+mdbq-2.5.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.5.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.5.6.dist-info/RECORD,,