mdbq 2.8.7__py3-none-any.whl → 2.8.9__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -7,7 +7,7 @@ import numpy as np
 import chardet
 import zipfile
 import socket
-from pandas.tseries.holiday import next_monday
+
 from pyzipper import PyZipFile
 import os
 import platform
@@ -1201,8 +1201,9 @@ def one_file_to_mysql(file, db_name, table_name):
 
 
 def test():
-    path = os.path.relpath(r'/Users/xigua/Downloads/未命名文件夹')
+    path = os.path.relpath(r'/Users/xigua/Downloads/手淘搜索_本店引流词/2024-05')
 
+    results = []
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
@@ -1210,17 +1211,23 @@ def test():
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 # print(name)
-                if len(df) == 0:
-                    continue
-                # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
-                if '颜色编码' in df.columns.tolist():
-                    print(name)
-                    df.pop('颜色编码')
-                    df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # if len(df) == 0:
+                #     continue
+                # # df.insert(loc=1, column='店铺名称', value='万里马官方旗舰店')
+                # if '店铺名称' not in df.columns.tolist():
+                #     print(name)
+                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                #     df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # pattern = re.findall(r'\d{4}-\d{2}-\d{2}_\d{4}-\d{2}-\d{2}', name)[0]
                 # new_name = f'py_xg_店铺销售指标_万里马官方旗舰店_{pattern}.csv'
                 # df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
                 # os.remove(os.path.join(root, name))
+                results.append(df)
+    df = pd.concat(results)
+    path = '/Users/xigua/Downloads/手淘搜索_本店引流词'
+    filename = 'py_xg_手淘搜索_本店引流词_万里马官方旗舰店_2024-05_合并.csv'
+    df.to_csv(os.path.join(path, filename), encoding='utf-8_sig', index=False, header=True)
+
 
 
 if __name__ == '__main__':
@@ -1237,14 +1244,14 @@ if __name__ == '__main__':
     # )
 
     # # 上传一个目录到指定数据库
-    # db_name = '生意经3'
-    # table_name = '宝贝指标'
+    # db_name = '生意参谋3'
+    # table_name = '手淘搜索_本店引流词'
     # upload_dir(
-    #     path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/生意经/宝贝指标sdff'),
+    #     path=os.path.relpath(r'/Users/xigua/Downloads/手淘搜索_本店引流词'),
     #     db_name=db_name,
     #     collection_name=table_name,
     # )
 
 
-    # test()
+    test()
 

mdbq/aggregation/query_data.py

@@ -2318,5 +2318,5 @@ if __name__ == '__main__':
     data_aggregation(
         months=3,
         is_juhe=True,  # 生成聚合表
-        # less_dict=['多店推广场景_按日聚合'],  # 单独聚合某一个数据库
+        # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
     )

mdbq/clean/clean_upload.py

@@ -133,8 +133,6 @@ class DataClean:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-                if 'py_xg' in name:
-                    continue
                 is_continue = False
                 if is_except:
                     for item in is_except:
@@ -156,46 +154,17 @@ class DataClean:
                         is_continue = True
                 if not is_continue:
                     continue
-                if name.endswith('.xls') and '商品排行_' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=4)
-                    if len(df) == 0:
-                        print(f'{name} 报表数据不能为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
-                    df.replace(to_replace=[','], value='', regex=True, inplace=True)
-                    df.rename(columns={'统计日期': '日期', '商品ID': '商品id'}, inplace=True)
-                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
-                    if '店铺名称' not in df.columns.tolist():
-                        df.insert(loc=1, column='店铺名称', value=shop_name)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
-                elif name.endswith('.xls') and '手淘搜索_本店引流词_' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=5, engine='xlrd')
-                    if len(df) == 0:
-                        print(f'{name} 报表数据不能为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
-                    df.replace(to_replace=[','], value='', regex=True, inplace=True)
-                    df.rename(columns={'统计日期': '日期'}, inplace=True)
-                    shop_name = re.findall(r'本店.*_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
-                    kw_type = re.findall('手淘搜索_本店引流词_([\u4e00-\u9fff]+)_', name)[0]
-                    df.insert(loc=2, column='词类型', value=kw_type)
-                    if '店铺名称' in df.columns.tolist():
-                        df['店铺名称'] = shop_name
-                    else:
-                        df.insert(loc=1, column='店铺名称', value=shop_name)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+                if name.endswith('.csv') and '商品排行_' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    # df = pd.read_excel(os.path.join(root, name), header=4)
+
+                elif name.endswith('.csv') and '手淘搜索_本店引流词_' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    # df = pd.read_excel(os.path.join(root, name), header=5, engine='xlrd')
 
                 elif name.endswith('.csv') and '_来源构成_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+
                 elif name.endswith('.csv') and '爱库存_商品榜单_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if '店铺名称' not in df.columns.tolist():
@@ -205,12 +174,12 @@ class DataClean:
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and '直播分场次效果' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
-                    if '店铺名称' not in df.columns.tolist():
-                        df.insert(loc=1, column='店铺名称', value=shop_name)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+                    # shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
+                    # if '店铺名称' not in df.columns.tolist():
+                    #     df.insert(loc=1, column='店铺名称', value=shop_name)
+                    # new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    # self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
+                    # os.remove(os.path.join(root, name))
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -245,8 +214,6 @@ class DataClean:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-                if 'py_xg' in name:
-                    continue
                 is_continue = False
                 if is_except:
                     for item in is_except:
@@ -270,30 +237,8 @@ class DataClean:
                     continue
                 if name.endswith('.csv') and '人群属性_万里马官方旗舰店' in name:  # 推广类报表
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    if len(df) == 0:
-                        print(f'{name} 报表数据为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and 'dmp人群报表_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    df = df[df['日期'] != '']
-                    if len(df) == 0:
-                        print(f'{name} 报表数据为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    for col in df.columns.tolist():
-                        if '(' in col or ')' in col:
-                            new_col = re.sub(r'\(.*\)', '', col)
-                            df.rename(columns={col: new_col}, inplace=True)
-                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
-                    if '店铺名称' not in df.columns.tolist():
-                        df.insert(loc=1, column='店铺名称', value=shop_name)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -388,8 +333,8 @@ class DataClean:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-                if 'py_xg' in name:
-                    continue
+                # if 'py_xg' in name:
+                #     continue
                 is_continue = False
                 if is_except:
                     for item in is_except:
@@ -420,21 +365,10 @@ class DataClean:
                     print(f'报表名称错误，不属于天猫/淘宝店：{name}')
                     continue
 
-                if name.endswith('.csv'):  # 推广类报表
-                    if '明星店铺' in name:  # 明星店铺可能会先释放 csv 文件
-                        continue
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
-                    df.fillna(0, inplace=True)
-                    date_min = df["日期"].values.min()
-                    date_max = df["日期"].values.max()
-                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
-                    df.insert(loc=1, column='店铺名称', value=shop_name)
-                    new_name = f'py_xg_{name}'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+                if name.endswith('.csv') and '明星店铺' not in name:  # 推广类报表
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                elif name.endswith('.csv') and '品销宝_明星店铺' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 elif name.endswith('.xlsx') and '品销宝_明星店铺' in name:
                     # 品销宝
                     sheets4 = ['账户', '推广计划', '推广单元', '创意', '品牌流量包', '定向人群']  # 品销宝
@@ -469,8 +403,9 @@ class DataClean:
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
-                    # print(f'db_name/collection_name 不能为空')
+                    print(f'db_name/collection_name 不能为空')
                     continue
+                # print(db_name, collection_name)
                 self.datas.append(
                     {
                         '数据库名': db_name,
@@ -511,8 +446,8 @@ class DataClean:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-                if 'py_xg' in name:
-                    continue
+                # if 'py_xg' in name:
+                #     continue
                 is_continue = False
                 if is_except:
                     for item in is_except:
@@ -536,27 +471,12 @@ class DataClean:
                     continue
 
                 if name.endswith('.csv') and 'baobei' in name:
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    p = df.pop('日期')
-                    df.insert(loc=0, column='日期', value=p)
-                    df['日期'] = df['日期'].apply(lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0]) if int(x) > 0 else '')
-                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    new_name = f'py_xg_{name}'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+                    # encoding = self.get_encoding(file_path=os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 elif name.endswith('.csv') and 'order' in name:
                     """ 如果是手动下载的表格，这里不能使用表格原先的 gb2312， 会报错 """
                     # df = pd.read_csv(os.path.join(root, name), encoding='gb18030', header=0, na_filter=False)
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    df.rename(columns={'宝贝标题': '商品标题', '宝贝链接': '商品链接'}, inplace=True)
-                    df['日期'] = df['日期'].apply(lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0]) if int(x) > 0 else '')
-                    df['商品id'] = df.apply(lambda x: re.sub(r'.*id=', '', x['商品链接']), axis=1)
-                    df = df[df['订单号'] != '']
-                    new_name = f'py_xg_{name}'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 elif name.endswith('.csv') and '省份城市分析' in name:
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
@@ -583,158 +503,7 @@ class DataClean:
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and '店铺销售指标' in name:
                     # 生意经, 店铺指标，仅限月数据，实际日指标也可以
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    if len(df) == 0:
-                        print(f'{name} 报表数据为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    df['日期'] = df['日期'].apply(lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0]) if int(x) > 0 else '')
-                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    new_name = f'py_xg_{name}'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
-
-                # 将数据传入 self.datas 等待更新进数据库
-                if not db_name or not collection_name:
-                    # print(f'db_name/collection_name 不能为空')
-                    continue
-                self.datas.append(
-                    {
-                        '数据库名': db_name,
-                        '集合名称': collection_name,
-                        '数据主体': df,
-                        '文件名': name,
-                    }
-                )
-
-    def syj_reports_tb(self, path=None, is_except=[]):
-        """ 淘宝店 生意经报表 """
-        if not path:
-            path = self.path
-        report_names = [
-            {
-                '文件简称': 'baobei',
-                '数据库名': '淘宝_生意经3',
-                '集合名称': '宝贝指标',
-            },
-            {
-                '文件简称': 'order',
-                '数据库名': '淘宝_生意经3',
-                '集合名称': '订单数据',
-            },
-            {
-                '文件简称': '省份城市分析',
-                '数据库名': '淘宝_生意经3',
-                '集合名称': '省份城市分析',
-            },
-            {
-                '文件简称': '店铺销售指标',
-                '数据库名': '淘宝_生意经3',
-                '集合名称': '店铺销售指标',
-            },
-        ]
-
-        for root, dirs, files in os.walk(path, topdown=False):
-            for name in files:
-                if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
-                    continue
-                if 'py_xg' in name:
-                    continue
-                is_continue = False
-                if is_except:
-                    for item in is_except:
-                        if item in os.path.join(root, name):
-                            # print(name)
-                            is_continue = True
-                            break
-                if is_continue:  # 需要排除不做处理的文件或文件夹
-                    continue
-
-                # 这里排除掉非目标报表
-                is_continue = False
-                db_name = None  # 初始化参数
-                collection_name = None
-                for item in report_names:
-                    if item['文件简称'] in name:
-                        db_name = item['数据库名']
-                        collection_name = item['集合名称']
-                        is_continue = True
-                if not is_continue:
-                    continue
-
-                if name.endswith('.csv') and 'baobei' in name:
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    pattern = re.findall(r'-(\d{4})(\d{2})(\d{2})\W', name)[0]
-                    df['日期'] = '-'.join(pattern)
-                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    new_name = f'py_xg_淘宝_baobeitrains_{'-'.join(pattern)}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
-                elif name.endswith('.csv') and 'order' in name:
-                    """ 这里不能使用表格原先的 gb2312， 会报错 """
-                    # encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding='gb18030', header=0, na_filter=False)
-                    pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)[0]
-                    date1 ='-'.join(pattern[1:4])
-                    date2 = '-'.join(pattern[4:7])
-                    df.insert(loc=0, column='日期', value=date1)
-                    df.insert(loc=1, column='数据周期', value=f'{date1}_{date2}')
-                    df.rename(columns={'宝贝标题': '商品标题', '宝贝链接': '商品链接'}, inplace=True)
-                    df['颜色编码'] = df['商家编码'].apply(
-                        lambda x: ''.join(re.findall(r' .*(\d{4})$', str(x))) if x else x)
-                    new_name = f'py_xg_淘宝_order_{date1}_{date2}.csv'
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
-                elif name.endswith('.csv') and '省份城市分析' in name:
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\.', name)[0]
-                    date = '-'.join(pattern[1:])
-                    new_name = f'py_xg_淘宝_{pattern[0]}-{date}.csv'
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    if len(df) == 0:
-                        print(f'{name} 报表数据为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    df['省'] = df['省份'].apply(lambda x: x if ' ├─ ' not in x and ' └─ ' not in x else None)
-                    df['城市'] = df[['省份', '省']].apply(lambda x: '汇总' if x['省'] else x['省份'], axis=1)
-                    df['省'].fillna(method='ffill', inplace=True)
-                    df['城市'].replace(to_replace=[' ├─ | └─ '], value='', regex=True, inplace=True)
-                    pov = df.pop('省')
-                    city = df.pop('城市')
-                    df['省+市'] = df['省份']
-                    df['省份'] = pov
-                    df.insert(loc=1, column='城市', value=city)
-                    df.insert(loc=0, column='日期', value=date)
-                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
-                elif name.endswith('.csv') and '店铺销售指标' in name:
-                    # 生意经, 店铺指标，仅限月数据，实际日指标也可以
-                    name_st = re.findall(r'(.*)\(分日', name)
-                    if not name_st:
-                        print(f'{name}  已转换的表格')
-                        continue
-                    encoding = self.get_encoding(file_path=os.path.join(root, name))
-                    df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
-                    if len(df) == 0:
-                        print(f'{name} 报表数据为空')
-                        os.remove(os.path.join(root, name))
-                        continue
-                    df['日期'] = df['日期'].astype(str).apply(
-                        lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', x)[0]) if x else x)
-                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-                    # min_clm = str(df.min()['日期']).split(' ')[0]
-                    # max_clm = str(df.max()['日期']).split(' ')[0]
-                    min_clm = str(df['日期'].min()).split(' ')[0]
-                    max_clm = str(df['日期'].max()).split(' ')[0]
-                    new_name = f'py_xg_淘宝_{name_st[0]}-{min_clm}_{max_clm}.csv'  # 保存时将(分日)去掉
-                    df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
-                    self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
-                    os.remove(os.path.join(root, name))
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -770,17 +539,17 @@ class DataClean:
                 '集合名称': '推广数据_关键词报表',
             },
             {
-                '文件简称': '京东商智_sku_商品明细',
+                '文件简称': 'sku_商品明细',
                 '数据库名': '京东数据3',
                 '集合名称': '京东商智_sku_商品明细',
             },
             {
-                '文件简称': '京东商智_spu_商品明细',
+                '文件简称': 'spu_商品明细',
                 '数据库名': '京东数据3',
                 '集合名称': '京东商智_spu_商品明细',
             },
             {
-                '文件简称': '京东商智_店铺来源_三级来源',
+                '文件简称': '店铺来源_三级来源',
                 '数据库名': '京东数据3',
                 '集合名称': '京东商智_店铺来源',
             },
@@ -790,8 +559,8 @@ class DataClean:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
-                if 'py_xg' in name:
-                    continue
+                # if 'py_xg' in name:
+                #     continue
                 is_continue = False
                 if is_except:
                     for item in is_except:
@@ -817,47 +586,50 @@ class DataClean:
                 if name.endswith('.csv') and '京东推广_' in name:
                     # df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    new_name = f'py_xg_{name}'
-                    if os.path.isfile(os.path.join(root, new_name)):
-                        os.remove(os.path.join(root, new_name))
-                    os.rename(os.path.join(root, name), os.path.join(root, new_name))
-                elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
-                    df.replace(to_replace=['-'], value='', regex=False, inplace=True)
-                    pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
-                    df.insert(loc=0, column='日期', value=pattern)
-                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                    df.fillna(0, inplace=True)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
-                    # df.to_excel(os.path.join(upload_path, new_name),
-                    #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
-                    os.remove(os.path.join(root, name))
-                elif name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
-                    df.replace(to_replace=['-'], value='', regex=False, inplace=True)
-                    pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
-                    df.insert(loc=0, column='日期', value=pattern)
-                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                    df.fillna(0, inplace=True)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
-                    # df.to_excel(os.path.join(upload_path, new_name),
-                    #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
-                    os.remove(os.path.join(root, name))
-                elif name.endswith('.xlsx') and '京东商智_店铺来源_三级来源' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
-                    df.replace(to_replace=['-'], value='', regex=False, inplace=True)
-                    df.rename(columns={'时间': '日期'}, inplace=True)
-                    for col in df.columns.tolist():
-                        if '环比' in col or '同比' in col:
-                            df.drop(col, axis=1, inplace=True)
-                    df.fillna(0, inplace=True)
-                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
-                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
-                    # df.to_excel(os.path.join(upload_path, new_name),
-                    #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
-                    os.remove(os.path.join(root, name))
+                    # new_name = f'py_xg_{name}'
+                    # if os.path.isfile(os.path.join(root, new_name)):
+                    #     os.remove(os.path.join(root, new_name))
+                    # os.rename(os.path.join(root, name), os.path.join(root, new_name))
+                elif name.endswith('.csv') and 'sku_商品明细' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    # df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
+                    # df.replace(to_replace=['-'], value='', regex=False, inplace=True)
+                    # pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
+                    # df.insert(loc=0, column='日期', value=pattern)
+                    # df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                    # df.fillna(0, inplace=True)
+                    # new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    # df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
+                    # # df.to_excel(os.path.join(upload_path, new_name),
+                    # #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    # os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and 'spu_商品明细' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    # df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
+                    # df.replace(to_replace=['-'], value='', regex=False, inplace=True)
+                    # pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
+                    # df.insert(loc=0, column='日期', value=pattern)
+                    # df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                    # df.fillna(0, inplace=True)
+                    # new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    # df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
+                    # # df.to_excel(os.path.join(upload_path, new_name),
+                    # #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    # os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and '店铺来源_三级来源' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    # df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
+                    # df.replace(to_replace=['-'], value='', regex=False, inplace=True)
+                    # df.rename(columns={'时间': '日期'}, inplace=True)
+                    # for col in df.columns.tolist():
+                    #     if '环比' in col or '同比' in col:
+                    #         df.drop(col, axis=1, inplace=True)
+                    # df.fillna(0, inplace=True)
+                    # new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    # df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
+                    # # df.to_excel(os.path.join(upload_path, new_name),
+                    # #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    # os.remove(os.path.join(root, name))
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -1125,10 +897,10 @@ class DataClean:
                 if 'py_xg' not in name:  # 排除非目标文件
                     continue
 
-                if name.endswith('.csv') and '京东商智_spu_商品明细' in name:
+                if name.endswith('.csv') and 'spu_商品明细' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '京东商智_spu_商品明细')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '京东商智_sku_商品明细' in name:
+                elif name.endswith('.csv') and 'sku_商品明细' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '京东商智_sku_商品明细')
                     bib(t_path, _as_month=True)
                 elif name.endswith('.csv') and '京东推广_搜索词' in name:
@@ -1140,7 +912,7 @@ class DataClean:
                 elif name.endswith('.csv') and '京东推广_关键词点击' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '关键词报表')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '京东商智_店铺来源_三级来源' in name:
+                elif name.endswith('.csv') and '店铺来源_三级来源' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '店铺来源_三级来源')
                     bib(t_path, _as_month=True)
 
@@ -1209,25 +981,10 @@ class DataClean:
                     t_path = os.path.join(self.source_path, '天猫推广报表', '超级短视频_主体')
                     bib(t_path, _as_month=True)
 
-                elif name.endswith('.csv') and 'tg_report_品销宝_明星店铺_万里马官方旗舰店' in name:
-                    if '账户' in name:
-                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '账户报表')
-                        bib(t_path, _as_month=True)
-                    elif '推广计划' in name:
-                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '推广计划报表')
-                        bib(t_path, _as_month=True)
-                    elif '推广单元' in name:
-                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '推广单元报表')
-                        bib(t_path, _as_month=True)
-                    elif '创意' in name:
-                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '创意报表')
-                        bib(t_path, _as_month=True)
-                    elif '品牌流量包' in name:
-                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '品牌流量包报表')
-                        bib(t_path, _as_month=True)
-                    elif '定向人群' in name:
-                        t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '定向人群报表')
-                        bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and 'tg_report_品销宝_明星店铺_' in name:
+                    t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝')
+                    bib(t_path, _as_month=True)
+
                 elif name.endswith('xlsx') and '商品素材_万里马官方旗舰店' in name:
                     t_path = os.path.join(self.source_path, '商品素材')
                     bib(t_path, _as_month=True)
@@ -1452,8 +1209,8 @@ class DataClean:
                 df=df,
                 db_name=db_name,
                 table_name=collection_name,
-                move_insert=True,  # 先删除，再插入
-                df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                move_insert=False,  # 先删除，再插入，新版有多店数据，不可按日期删除
+                df_sql=True,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
                 drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
                 filename=rt_filename,  # 用来追踪处理进度
                 service_database=service_database,  # 字典
@@ -1521,8 +1278,7 @@ def main(is_mysql=False, is_company=False):
     cn.dmp_tm(is_except=['except'])  # 达摩盘
     cn.tg_reports(is_except=['except'])  # 推广报表，天猫淘宝共同清洗
     cn.syj_reports_tm(is_except=['except'])  # 天猫生意经
-    # # 淘宝生意经，不可以和天猫同时运行
-    # cn.syj_reports_tb(is_except=['except'])  # 淘宝生意经，不可以和天猫同时运行
+
     cn.jd_reports(is_except=['except'])  # 清洗京东报表
     cn.sp_scene_clean(is_except=['except'])  # 商品素材
     cn.upload_df()  # 上传数据库

mdbq/company/copysh.py

@@ -301,6 +301,30 @@ class TbFiles:
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         print(f'{now} 同步完成！')
 
+    def refresh_excel(self):
+        # 刷新共享位置的指定文件/文件夹
+        if platform.system() == 'Windows' and socket.gethostname() == 'company':
+            excel_path = os.path.join(self.share_path, 'EXCEL报表')
+            files = os.listdir(excel_path)
+            files = [f'{excel_path}\\{item}' for item in files if item.endswith('.xlsx') or item.endswith('.xls')]
+            r = refresh_all.RefreshAll()
+            for file in files:
+                if '~' in file or 'DS_Store' in file or 'baidu' in file or 'xunlei' in file:
+                    continue
+                if file.endswith('.xlsx') or file.endswith('.xls'):
+                    r.refresh_excel(file=file)
+                time.sleep(5)
+
+            # 临时加的
+            # excel_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\0-WLM_运营周报-1012输出.xlsx'
+            dir_files = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新'
+            files = os.listdir(dir_files)
+            for file in files:
+                if file.endswith('.xlsx') and file.startswith(
+                        '0-WLM_运营周报') and '~' not in file and 'baidu' not in file:
+                    excel_file = os.path.join(dir_files, file)
+                    r.refresh_excel(file=excel_file)
+
     def check_upload_mysql(self):
         # 每天只更新一次
         today = datetime.date.today()
@@ -407,12 +431,15 @@ def main():
                 )
                 # print(conf)
                 myconfig.write_back(datas=conf)  # 写回文件生效
+
+            if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+                t.refresh_excel()
             now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
             print(f'{now}数据完成！')
 
-        t.sleep_minutes = 5  # 同步前休眠时间
-        if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
-            t.tb_file()
+        # t.sleep_minutes = 5  # 同步前休眠时间
+        # if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+        #     t.tb_file()
         time.sleep(600)  # 检测间隔
 
 

mdbq/config/products.py

@@ -34,6 +34,9 @@ class Products:
 
     def update_my_datas(self):
         my_datas = [
+            {
+                '平台': '天猫', '商品id': '848929365673', '上市年份': '2024年11月'
+            },
             {
                 '平台': '天猫', '商品id': '840499705810', '上市年份': '2024年10月'
             },

mdbq/mysql/mysql.py

@@ -180,6 +180,14 @@ class MysqlUpload:
                 # except Exception as e:
                 #     print(f'{e}')
                 #     connection.rollback()
+
+                if cl and db_n and tb_n:
+                    mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name)  # 更新一个表的 dtypes
+                elif cl and db_n:
+                    mysql_types.mysql_all_dtypes(db_name=db_name)  # 更新一个数据库的 dtypes
+                elif cl:
+                    mysql_types.mysql_all_dtypes()  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
+
                 connection.close()
                 return
 

{mdbq-2.8.7.dist-info → mdbq-2.8.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.8.7
+Version: 2.8.9
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.8.7.dist-info → mdbq-2.8.9.dist-info}/RECORD

@@ -1,24 +1,24 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=fLsYQO7LTUtVFaVOxXry4OZviiIpZMcTLplE-HXW9XY,71964
+mdbq/aggregation/aggregation.py,sha256=VtGP7KhUY-NUBA2CXt50hYuGOZ1bU_NeaBBvAUxYgTg,72338
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
-mdbq/aggregation/query_data.py,sha256=rjXFP6dsrbXNdgA840z_xhrR-ka2i-hVdllHD6Yn2O4,100180
+mdbq/aggregation/query_data.py,sha256=iRgPljgOPE7dzhaaVxRXOEOOKQTmWg6sGsDplNLTvQw,100177
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=LRXcvsLqON5NJ5rj7RbeV0750N2Jrjtmr4J7sNPNPoM,81544
+mdbq/clean/clean_upload.py,sha256=FJxoEX-2QKuFhrF1ecl_LdZ1uFnVPx4HigNcXdErB28,66561
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=gWaNgRe_rBHBBphch2AHkGAljZRdfqYBn_FC3m_l11A,20061
+mdbq/company/copysh.py,sha256=UD5BLBe9uMfqjdslyY7-TtGXuJI5jsol-w4kIQFDfQk,21577
 mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/myconfig.py,sha256=EGymTlAimtHIDJ9egCtOehBEPOj6rea504kvsEZu64o,854
-mdbq/config/products.py,sha256=ceCvyH23PSvdappIL3yIyIpo6Cqhw6ZKyVNFoj3XIYc,6087
+mdbq/config/products.py,sha256=Sj4FSb2dZcMKp6ox-FJdIR87QLgMN_TJ7Z6KAWMTWyw,6214
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=cpCmGP4qlkZrJBFdQagUiHXPfuS13J_XcSNeQ97NgFY,45709
+mdbq/mysql/mysql.py,sha256=apcj0WDdbrHr7UzO2kjcesDxDUlWxG4KcIpI1mBuwMk,46152
 mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.8.7.dist-info/METADATA,sha256=cDSdYxyr-otFt0wyENhynpHpw5P800B9oB_GW6bfINs,243
-mdbq-2.8.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.8.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.8.7.dist-info/RECORD,,
+mdbq-2.8.9.dist-info/METADATA,sha256=2_jXKvIIyj0iXk8T4uADVdvpKKF_kxFoWlN75PLoQlo,243
+mdbq-2.8.9.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.8.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.8.9.dist-info/RECORD,,

{mdbq-2.8.7.dist-info → mdbq-2.8.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any