mdbq 1.2.4__py3-none-any.whl → 1.2.6__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -47,10 +47,29 @@ class DatabaseUpdate:
             print(f'1.1.0 初始化时传入了不存在的目录: {self.path}')
             return
 
+        filename = '标题对照表.csv'
+        support_file = set_support.SetSupport(dirname='support').dirname
+        if not os.path.isfile(os.path.join(support_file, filename)):
+            print(f'缺少关键文件支持: {os.path.join(support_file, filename)}')
+            return
+        df = pd.read_csv(os.path.join(support_file, filename), encoding='utf-8_sig', header=0, na_filter=False)
+        datas = df.to_dict('records')  # 转字典
+        # print(datas)
+
         for root, dirs, files in os.walk(self.path, topdown=False):
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.ini' in name or '$RECYCLE.BIN' in name or 'Icon' in name:
                     continue
+
+                db_name = None  # 初始化/重置变量，避免进入下一个循环
+                collection_name = None
+                for data in datas:  # 根据标题对照表适配 db_name 和 collection_name
+                    if data['关键词1'] in name and data['关键词2'] in name:
+                        db_name = data['数据库名']
+                        collection_name = data['数据表']
+                # print(name, db_name, collection_name)
+                # return
+
                 # 只针对 csv, xlsx 文件进行处理
                 if not name.endswith('.csv') and not name.endswith('.xls') and not name.endswith('.xlsx'):
                     continue
@@ -107,8 +126,6 @@ class DatabaseUpdate:
                         shop_name = ''
                     # df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
                     # df.replace(to_replace=[''], value=0, regex=False, inplace=True)
-                    db_name = '推广数据2'
-                    collection_name = '超级直播'
                 elif name.endswith('.xls') and '短直联投' in name:
                     # 短直联投
                     df = pd.read_excel(os.path.join(root, name), sheet_name=None, header=0)
@@ -117,8 +134,6 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     # df.replace(to_replace=[''], value=0, regex=False, inplace=True)
-                    db_name = '推广数据2'
-                    collection_name = '短直联投'
                 elif name.endswith('.xls') and '视频加速推广' in name:
                     # 超级短视频
                     df = pd.read_excel(os.path.join(root, name), sheet_name=None, header=0)
@@ -127,15 +142,11 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     # df.replace(to_replace=[''], value=0, regex=False, inplace=True)
-                    db_name = '推广数据2'
-                    collection_name = '超级短视频'
                 if '人群报表汇总' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=1, na_filter=False)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         continue
-                    db_name = '推广数据2'
-                    collection_name = '达摩盘_dmp报表'
                 # ----------------- 推广报表 分割线 -----------------
                 # ----------------- 推广报表 分割线 -----------------
                 date01 = re.findall(r'(\d{4}-\d{2}-\d{2})_\d{4}-\d{2}-\d{2}', str(name))
@@ -183,8 +194,6 @@ class DatabaseUpdate:
                     if date01[0] != date02[0]:
                         data_lis = date01[0] + '_' + date02[0]
                         df.insert(loc=1, column='数据周期', value=data_lis)
-                    db_name = '生意参谋2'
-                    collection_name = '商品排行'
                 elif name.endswith('.xls') and '参谋店铺整体日报' in name:
                     # 自助取数，店铺日报
                     df = pd.read_excel(os.path.join(root, name), header=7)
@@ -192,8 +201,6 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     df.rename(columns={'统计日期': '日期'}, inplace=True)
-                    db_name = '生意参谋2'
-                    collection_name = '自助取数_整体日报'
                 elif name.endswith('.xls') and '参谋每日流量_自助取数_新版' in name:
                     # 自助取数，每日流量
                     df = pd.read_excel(os.path.join(root, name), header=7)
@@ -211,8 +218,6 @@ class DatabaseUpdate:
                         else '智能场景' if x == '智能场景(原万相台)'
                         else x
                     )
-                    db_name = '生意参谋2'
-                    collection_name = '自助取数_每日流量'
                 elif name.endswith('.xls') and '商品sku' in name:
                     # 自助取数，商品sku
                     df = pd.read_excel(os.path.join(root, name), header=7)
@@ -225,8 +230,6 @@ class DatabaseUpdate:
                         'SKU ID': 'sku id',
                         '商品SKU': '商品sku',
                     }, inplace=True)
-                    db_name = '生意参谋2'
-                    collection_name = '自助取数_商品sku'
                 elif name.endswith('.xls') and '参谋店铺流量来源（月）' in name:
                     # 自助取数，月店铺流量来源
                     df = pd.read_excel(os.path.join(root, name), header=7)
@@ -245,8 +248,6 @@ class DatabaseUpdate:
                         else x
                     )
                     df['日期'] = df['数据周期'].apply(lambda x: re.findall('(.*) ~', x)[0])
-                    db_name = '生意参谋2'
-                    collection_name = '自助取数_店铺流量_月数据'
                 elif name.endswith('.csv') and 'baobei' in name:
                     # 生意经宝贝指标日数据
                     date = re.findall(r's-(\d{4})(\d{2})(\d{2})\.', str(name))
@@ -264,8 +265,6 @@ class DatabaseUpdate:
                     new_date = '-'.join(date[0])
                     df.insert(loc=0, column='日期', value=new_date)
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    db_name = '生意经2'
-                    collection_name = '宝贝指标'
                 elif name.endswith('.csv') and '店铺销售指标' in name:
                     # 生意经, 店铺指标，仅限月数据，实际日指标也可以
                     name_st = re.findall(r'(.*)\(分日', name)
@@ -279,9 +278,7 @@ class DatabaseUpdate:
                     df['日期'] = df['日期'].astype(str).apply(
                         lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', x)[0]) if x else x)
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    db_name = '生意经2'
-                    collection_name = '店铺指标'
-                elif name.endswith('csv') and '省份' in name:
+                elif name.endswith('csv') and '省份城市分析' in name:
                     # 生意经，地域分布, 仅限日数据
                     pattern = re.findall(r'(.*[\u4e00-\u9fa5])(\d{4})(\d{2})(\d{2})\.', name)
                     if not pattern or '省份城市分析2' not in name:
@@ -303,8 +300,6 @@ class DatabaseUpdate:
                     df.insert(loc=0, column='日期', value=date)
                     df['省份'] = pov
                     df['省+市'] = df[['省份', '城市']].apply(lambda x: f'{x["省份"]}-{x["城市"]}', axis=1)
-                    db_name = '生意经2'
-                    collection_name = '地域分布_省份城市分析'
                 elif name.endswith('csv') and 'order' in name:
                     # 生意经，订单数据，仅限月数据
                     pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)
@@ -328,8 +323,6 @@ class DatabaseUpdate:
                     df.rename(columns={'宝贝标题': '商品标题', '宝贝链接': '商品链接'}, inplace=True)
                     df['颜色编码'] = df['商家编码'].apply(
                         lambda x: ''.join(re.findall(r' .*(\d{4})$', str(x))) if x else x)
-                    db_name = '生意经2'
-                    collection_name = '订单数据'
                 elif name.endswith('.xlsx') and '直播间成交订单明细' in name:
                     # 直播间成交订单明细
                     df = pd.read_excel(os.path.join(root, name), header=0)
@@ -338,8 +331,6 @@ class DatabaseUpdate:
                         continue
                     df.rename(columns={'场次ID': '场次id', '商品ID': '商品id'}, inplace=True)
                     df['日期'] = df['支付时间'].apply(lambda x: x.strftime('%Y-%m-%d'))
-                    db_name = '生意参谋2'
-                    collection_name = '直播间成交订单明细'
                 elif name.endswith('.xlsx') and '直播间大盘数据' in name:
                     # 直播间大盘数据
                     df = pd.read_excel(os.path.join(root, name), header=0)
@@ -347,8 +338,6 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     df.rename(columns={'统计日期': '日期'}, inplace=True)
-                    db_name = '生意参谋2'
-                    collection_name = '直播间大盘数据'
                 elif name.endswith('.xls') and '直播业绩-成交拆解' in name:
                     # 直播业绩-成交拆解
                     df = pd.read_excel(os.path.join(root, name), header=5)
@@ -356,44 +345,26 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     df.rename(columns={'统计日期': '日期'}, inplace=True)
-                    db_name = '生意参谋2'
-                    collection_name = '直播业绩'
                 elif name.endswith('.csv') and '淘宝店铺数据' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '市场数据2'
-                    collection_name = '淘宝店铺数据'
                 elif name.endswith('.csv') and '人群洞察' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
                     df = df[df['人群规模'] != '']
                     if len(df) == 0:
                         continue
-                    db_name = '推广数据2'
-                    collection_name = '万相台_人群洞察'
                 elif name.endswith('.csv') and '客户_客户概况_画像' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '生意参谋2'
-                    collection_name = '客户_客户概况_画像'
                 elif name.endswith('.csv') and '市场排行_店铺' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '市场数据2'
-                    collection_name = '市场排行_店铺'
                 elif name.endswith('.csv') and '类目洞察_属性分析' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '市场数据2'
-                    collection_name = '类目洞察_属性分析'
                 elif name.endswith('.csv') and '类目洞察_价格分析' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '市场数据2'
-                    collection_name = '类目洞察_价格分析'
                 elif name.endswith('.csv') and '竞店分析-销售分析' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '市场数据2'
-                    collection_name = '竞店分析_销售分析'
                 elif name.endswith('.csv') and '竞店分析-来源分析' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                    db_name = '市场数据2'
-                    collection_name = '竞店分析_来源分析'
                 # ----------------------- 京东数据处理分界线 -----------------------
                 # ----------------------- 京东数据处理分界线 -----------------------
                 elif name.endswith('.xlsx') and '店铺来源_流量来源' in name:
@@ -416,8 +387,6 @@ class DatabaseUpdate:
                     for col_2024 in cols:  # 京东这个表有字段加了去年日期，删除这些同比数据字段，不然列数量爆炸
                         if '20' in col_2024 and '流量来源' in name:
                             df.drop(col_2024, axis=1, inplace=True)
-                    db_name = '京东数据2'
-                    collection_name = '流量来源_日数据'
                 elif name.endswith('.xlsx') and '全部渠道_商品明细' in name:
                     # 京东商品明细 文件转换
                     date1 = re.findall(r'_(\d{4})(\d{2})(\d{2})_全部', str(name))
@@ -439,7 +408,7 @@ class DatabaseUpdate:
                     df.rename(columns={'商品ID': '商品id'}, inplace=True)
                     df.insert(loc=0, column='日期', value=date1)
                     df['最近上架时间'].loc[0] = df['最近上架时间'].loc[1]  # 填充这一列, 避免上传 mysql 日期类型报错
-                    if 'sku' in new_name:
+                    if 'sku' in new_name:  # 即使有文件对照表，也不能删除这个条件，spu ，sku 是后来加的
                         db_name = '京东数据2'
                         collection_name = 'sku_商品明细'
                     elif 'spu' in new_name:
@@ -455,8 +424,6 @@ class DatabaseUpdate:
                     for col in ['词人气', '搜索点击率']:
                         if col in df.columns.tolist():
                             df[col] = df[col].apply(lambda x: round(x, 6) if x else x)
-                    db_name = '京东数据2'
-                    collection_name = '商品词下排名'
                 elif name.endswith('.xlsx') and '搜索分析-排名定位-商品排名' in name:
                     # 京东商品排名
                     date_in = re.findall(r'(\d{4}-\d{2}-\d{2})-搜索', str(name))[0]
@@ -468,8 +435,6 @@ class DatabaseUpdate:
                     df.rename(columns={'SKU': 'skuid'}, inplace=True)
                     if '点击率' in df.columns.tolist():
                         df['点击率'] = df['点击率'].apply(lambda x: round(x, 6) if x else x)
-                    db_name = '京东数据2'
-                    collection_name = '商品排名'
                 elif name.endswith('.xls') and '竞店概况_竞店详情' in name:
                     # 京东，竞争-竞店概况-竞店详情-全部渠道
                     date01 = re.findall(r'全部渠道_(\d{4})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})', str(name))
@@ -480,9 +445,7 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     df.insert(loc=0, column='日期', value=start_date)
-                    db_name = '京东数据2'
-                    collection_name = '竞店监控_日数据'
-                elif name.endswith('.xls') and '店铺' in name:
+                elif name.endswith('.xls') and 'JD店铺日报_店铺' in name:
                     # 京东 自助报表  店铺日报
                     df = pd.read_excel(os.path.join(root, name), header=0)
                     if len(df) == 0:
@@ -491,8 +454,6 @@ class DatabaseUpdate:
                     df['日期'] = df['日期'].apply(
                         lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0])
                     )
-                    db_name = '京东数据2'
-                    collection_name = '京东_自助取数_店铺日报'
                 elif name.endswith('.xls') and '商家榜单_女包_整体' in name:
                     # 京东 行业 商家榜单
                     date2 = re.findall(r'_\d{8}-\d+', name)
@@ -506,8 +467,6 @@ class DatabaseUpdate:
                         continue
                     df['日期'] = df['日期'].astype(str).apply(lambda x: f'{x[:4]}-{x[4:6]}-{x[6:8]}')
                     df.insert(loc=0, column='类型', value='商家榜单')
-                    db_name = '京东数据2'
-                    collection_name = '商家榜单'
                 elif name.endswith('.xlsx') and '批量SKU导出-批量任务' in name:
                     # 京东 sku 导出
                     df = pd.read_excel(os.path.join(root, name), header=0)
@@ -517,8 +476,6 @@ class DatabaseUpdate:
                     d_time = datetime.datetime.today().strftime('%Y-%m-%d')
                     df.insert(loc=0, column='日期', value=d_time)
                     df['商品链接'] = df['商品链接'].apply(lambda x: f'https://{x}' if x else x)
-                    db_name = '属性设置2'
-                    collection_name = '京东sku商品信息'
                 elif name.endswith('.xlsx') and '批量SPU导出-批量任务' in name:
                     # 京东 spu 导出
                     df = pd.read_excel(os.path.join(root, name), header=0)
@@ -527,8 +484,6 @@ class DatabaseUpdate:
                         continue
                     d_time = datetime.datetime.today().strftime('%Y-%m-%d')
                     df.insert(loc=0, column='日期', value=d_time)
-                    db_name = '属性设置2'
-                    collection_name = '京东spu商品信息'
                 elif name.endswith('.csv') and '万里马箱包推广1_完整点击成交' in name:
                     # 京东推广数据
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
@@ -536,8 +491,6 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     df['日期'] = df['日期'].apply(lambda x: f'{str(x)[:4]}-{str(x)[4:6]}-{str(x)[6:8]}')
-                    db_name = '京东数据2'
-                    collection_name = '推广数据_京准通'
                 elif name.endswith('.csv') and '万里马箱包推广1_京东推广搜索词_pbix同步不要' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
@@ -546,16 +499,12 @@ class DatabaseUpdate:
                     df['日期'] = df['日期'].apply(lambda x: f'{str(x)[:4]}-{str(x)[4:6]}-{str(x)[6:8]}')
                     df['是否品牌词'] = df['搜索词'].str.contains('万里马|wanlima', regex=True)
                     df['是否品牌词'] = df['是否品牌词'].apply(lambda x: '品牌词' if x else '')
-                    db_name = '京东数据2'
-                    collection_name = '推广数据_搜索词报表'
                 elif name.endswith('.xlsx') and '零售明细统计' in name:
                     df = pd.read_excel(os.path.join(root, name), header=0)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         continue
                     df = df[df['缩略图'] != '合计']
-                    db_name = '生意经2'
-                    collection_name = 'e3_零售明细统计'
 
                 # 商品素材，必须保持放在最后处理
                 elif name.endswith('xlsx'):
@@ -599,6 +548,14 @@ class DatabaseUpdate:
             for name in files:
                 if '~$' in name or '.DS' in name or '.localized' in name or '.jpg' in name or '.png' in name:
                     continue
+
+                db_name = None  # 初始化/重置变量，避免进入下一个循环
+                collection_name = None
+                for data in datas:  # 根据标题对照表适配 db_name 和 collection_name
+                    if data['关键词1'] in name and data['关键词2'] in name:
+                        db_name = data['数据库名']
+                        collection_name = data['数据表']
+
                 # df = pd.DataFrame()
                 if name.endswith('.xlsx') and '明星店铺' in name:
                     # 品销宝
@@ -614,8 +571,6 @@ class DatabaseUpdate:
                             # print(f'{name}/{sheet4} 跳过')
                             continue
                         df.insert(loc=1, column='报表类型', value=sheet4)
-                        db_name = '推广数据2'
-                        collection_name = f'品销宝_{sheet4}'
                         self.datas.append(
                             {
                                 '数据库名': db_name,
@@ -895,6 +850,8 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
                         continue
+                    # if '新版' not in name:
+                    #     continue
                     cv = converter.DataFrameConverter()
                     df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
                     try:
@@ -996,14 +953,35 @@ def test():
         # break
 
 
+def test2():
+    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
+    dp.new_unzip(is_move=True)
+    dp.cleaning(is_move=False, )  # 清洗数据, 存入 self.datas
+    dp.upload_df(service_databases=[
+        # {'home_lx': 'mongodb'},
+        {'home_lx': 'mysql'},
+        # {'nas': 'mysql'}
+    ])
+
+
 if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
-    file_dir(one_file=False)
+    # file_dir(one_file=False)
     # one_file_to_mysql(
-    #     file='/Users/xigua/数据中心/原始文件2/京东报表/商品信息导出/spu/京东商品信息_批量SPU导出-批量任务_2024-08-10.csv',
-    #     db_name='京东数据2',
-    #     table_name='京东spu商品信息',
+    #     file='/Users/xigua/数据中心/原始文件2/推广报表/品销宝/账户/账户_明星店铺报表_2023-11-13_2023-12-12.csv',
+    #     db_name='推广数据2',
+    #     table_name='品销宝',
     #     target_service='home_lx',
     #     database='mysql'
     # )
+    # db_name = '生意参谋2'
+    # table_name = '店铺来源_日数据_新版'
+    # upload_dir(
+    #     path='/Users/xigua/数据中心/原始文件2/生意参谋/流量来源',
+    #     db_name=db_name,
+    #     collection_name=table_name,
+    #     dbs={'mysql': True, 'mongodb': False},
+    # )
+
+    test2()

mdbq/aggregation/query_data.py

@@ -166,7 +166,7 @@ class MysqlDatasQuery:
             '加购人数': 1,
         }
         df = self.download.data_to_df(
-            db_name='生意参谋数据2',
+            db_name='生意参谋2',
             table_name='店铺来源_日数据',
             start_date=start_date,
             end_date=end_date,
@@ -240,6 +240,7 @@ class GroupBy:
                 '总成交金额': '成交金额'
             }, inplace=True)
             df = df.astype({
+                '商品id': str,
                 '花费': float,
                 '展现量': int,
                 '点击量': int,
@@ -263,12 +264,13 @@ class GroupBy:
                 )
             else:
                 df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-                    **{'加购量': ('加购量', np.min),
-                       '成交笔数': ('成交笔数', np.min),
-                       '成交金额': ('成交金额', np.min),
-                       '自然流量曝光量': ('自然流量曝光量', np.min),
-                       '直接成交笔数': ('直接成交笔数', np.max),
-                       '直接成交金额': ('直接成交金额', np.max)
+                    **{
+                        '加购量': ('加购量', np.min),
+                        '成交笔数': ('成交笔数', np.min),
+                        '成交金额': ('成交金额', np.min),
+                        '自然流量曝光量': ('自然流量曝光量', np.min),
+                        '直接成交笔数': ('直接成交笔数', np.max),
+                        '直接成交金额': ('直接成交金额', np.max)
                        }
                 )
             df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
@@ -276,10 +278,10 @@ class GroupBy:
                     **{
                         '花费': ('花费', np.sum),
                         '成交笔数': ('成交笔数', np.max),
-                       '成交金额': ('成交金额', np.max),
-                       '自然流量曝光量': ('自然流量曝光量', np.max),
-                       '直接成交笔数': ('直接成交笔数', np.max),
-                       '直接成交金额': ('直接成交金额', np.max)
+                        '成交金额': ('成交金额', np.max),
+                        '自然流量曝光量': ('自然流量曝光量', np.max),
+                        '直接成交笔数': ('直接成交笔数', np.max),
+                        '直接成交金额': ('直接成交金额', np.max)
                        }
                 )
             self.data_tgyj.update(
@@ -290,6 +292,7 @@ class GroupBy:
             return df
         elif '宝贝指标' in table_name:
             """ 聚合时不可以加商家编码，编码有些是空白，有些是 0 """
+            df['宝贝id'] = df['宝贝id'].astype(str)
             df.fillna(0, inplace=True)
             # df = df[(df['销售额'] != 0) | (df['退款额'] != 0)]  # 注释掉, 因为后续使用生意经作为基准合并推广表，需确保所有商品id 齐全
             df = df.groupby(['日期', '宝贝id', '行业类目'], as_index=False).agg(
@@ -320,6 +323,7 @@ class GroupBy:
         elif '店铺来源_日数据' in table_name:
             return df
         elif '商品id编码表' in table_name:
+            df['宝贝id'] = df['宝贝id'].astype(str)
             df.drop_duplicates(subset='宝贝id', keep='last', inplace=True, ignore_index=True)
             # df['行业类目'] = df['行业类目'].apply(lambda x: re.sub(' ', '', x))
             try:
@@ -359,6 +363,7 @@ class GroupBy:
                     table_name: df[['商品id', '商品图片']],
                 }
             )
+            df['商品id'] = df['商品id'].astype(str)
             return df
         elif '商品成本' in table_name:
             df.sort_values(by=['款号', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
@@ -373,7 +378,7 @@ class GroupBy:
             print(f'<{table_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
 
-    @try_except
+    # @try_except
     def performance(self, bb_tg=True):
          # print(self.data_tgyj)
         tg, syj, idbm, pic, cost = (
@@ -390,13 +395,14 @@ class GroupBy:
         df = pd.merge(tg, df, how='left', left_on='商品id', right_on='宝贝id')
         df.drop(labels='宝贝id', axis=1, inplace=True)
         if bb_tg is True:
-             # 生意经合并推广表，完整的数据表，包含全店所有推广、销售数据
+            # 生意经合并推广表，完整的数据表，包含全店所有推广、销售数据
             df = pd.merge(syj, df, how='left', left_on=['日期', '宝贝id'], right_on=['日期', '商品id'])
         else:
             # 推广表合并生意经 , 以推广数据为基准，销售数据不齐全
             df = pd.merge(df, syj, how='left', left_on=['日期', '商品id'], right_on=['日期', '宝贝id'])
         df.drop(labels='宝贝id', axis=1, inplace=True)
         df.drop_duplicates(subset=['日期', '商品id', '花费', '销售额'], keep='last', inplace=True, ignore_index=True)
+        df['成本价'] = df['成本价'].astype('float64')
         df['商品成本'] = df.apply(lambda x: (x['成本价'] + x['销售额']/x['销售量'] * 0.11 + 6) * x['销售量'] if x['销售量'] > 0 else 0, axis=1)
         df['商品毛利'] = df.apply(lambda x: x['销售额'] - x['商品成本'], axis=1)
         df['毛利率'] = df.apply(lambda x: round((x['销售额'] - x['商品成本']) / x['销售额'], 4) if x['销售额'] > 0 else 0, axis=1)
@@ -493,12 +499,13 @@ def data_aggregation(service_databases=[{}]):
     for service_database in service_databases:
         for service_name, database in service_database.items():
             sdq = MysqlDatasQuery(target_service=service_name)  # 实例化数据处理类
-            sdq.months = 1  # 设置数据周期， 1 表示近 2 个月
+            sdq.months = 0  # 设置数据周期， 1 表示近 2 个月
             g = GroupBy()  # 实例化数据聚合类
             # 实例化数据库连接
             username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
             m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
 
+            # 从数据库中获取数据, 返回包含 df 数据的字典
             data_dict = [
                 {
                     '数据库名': '聚合数据',
@@ -531,19 +538,19 @@ def data_aggregation(service_databases=[{}]):
                     '数据主体': sdq.sp_cost(),
                 },
             ]
-            for items in data_dict:
+            for items in data_dict:  # 遍历返回结果
                 db_name, table_name, df = items['数据库名'], items['集合名'], items['数据主体']
                 df = g.groupby(df=df, table_name=table_name, is_maximize=True)  # 2. 聚合数据
                 # g.as_csv(df=df, filename=table_name + '.csv')
-                m.df_to_mysql(df=df, db_name=db_name, table_name=table_name)  # 3. 回传数据库
+                m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, drop_dup=True)  # 3. 回传数据库
             res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_全店商品销售')
+            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_全店商品销售', drop_dup=True)
             res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
-            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_推广商品销售')
+            m.df_to_mysql(df=res, db_name='聚合数据', table_name='_推广商品销售', drop_dup=True)
 
     # optimize_data.op_data(service_databases=service_databases, days=3650)  # 立即启动对聚合数据的清理工作
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}])
+    data_aggregation(service_databases=[{'home_lx': 'mysql'}])
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/mysql/mysql.py

@@ -65,7 +65,8 @@ class MysqlUpload:
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
-        drop_duplicates：仅限于聚合数据使用，其他情况不要设置
+        df_sql: 使用 df.to_sql 函数上传整个表, 不会排重
+        drop_duplicates：仅限于聚合数据使用，其他情况不要设置此参数
         drop_dup: 值为 True 时检查重复数据再插入，反之直接上传
         filename: 传这个参数是方便定位产生错误的文件
         """
@@ -152,7 +153,7 @@ class MysqlUpload:
 
             if df_sql:
                 now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-                print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
+                print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
                 engine = create_engine(
                     f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
                 df.to_sql(
@@ -185,7 +186,7 @@ class MysqlUpload:
 
             # 5. 更新插入数据
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
+            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
 
             datas = df.to_dict(orient='records')
             for data in datas:

mdbq/mysql/s_query.py

@@ -12,6 +12,7 @@ from sqlalchemy import create_engine
 import os
 import calendar
 from mdbq.config import get_myconf
+from mdbq.dataframe import converter
 
 warnings.filterwarnings('ignore')
 """
@@ -84,6 +85,8 @@ class QueryDatas:
 
         if len(df) == 0:
             print(f'database: {db_name}, table: {table_name} 查询的数据为空')
+        cv = converter.DataFrameConverter()
+        df = cv.convert_df_cols(df)
         return df
 
     def columns_to_list(self, db_name, table_name,  columns_name) -> list:

{mdbq-1.2.4.dist-info → mdbq-1.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.2.4
+Version: 1.2.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.2.4.dist-info → mdbq-1.2.6.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=a-lK0R6fEqj5InZZXly-xqfyZ4mCDMqAuEHuTzKMvv0,57467
+mdbq/aggregation/aggregation.py,sha256=mBgIY7afloW8H5qoBy56vCabIQRxVvAhrRZgGbZUxFQ,55791
 mdbq/aggregation/df_types.py,sha256=rHLIgv82PJSFmDvXkZyOJAffXkFyyMyFO23w9tUt8EQ,7525
 mdbq/aggregation/mysql_types.py,sha256=umVixmbFZM63k-QhVWLvOuhcAde4P_oDKbdo8ry2O9w,10633
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
-mdbq/aggregation/query_data.py,sha256=sdMWj0st_VFDfBHmqTrY05k-0yoagdnaiNMoB0otEuk,25255
+mdbq/aggregation/query_data.py,sha256=fg_9OdNSwHbo9vhK1pAKOazHFHZfE9_rBxRyQIWJX9U,25694
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -24,9 +24,8 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/data_types_即将删除.py,sha256=sjBBDKr9674LdjM5N_dwyJACdZPbdB8Beli59jGdgnQ,10378
-mdbq/mysql/mysql.py,sha256=hZWk7rgoeCkNYbGMgJokiPJSoMJ3y-wSVH_ojzh49l8,37464
-mdbq/mysql/s_query.py,sha256=4c24SwbqtnO33o8CgWlTQ_j8sZYl5BRIQkaD9CI-vTY,7901
+mdbq/mysql/mysql.py,sha256=KvUQflP5sYOECTHOs2Fs9ABcQvgPCbBnAX2ZlE3JjgY,37544
+mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
@@ -36,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.2.4.dist-info/METADATA,sha256=ekCyUS5vKgOSEK7iYzdo-70H1ThaVi89ATDo6jnJBSI,245
-mdbq-1.2.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.2.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.2.4.dist-info/RECORD,,
+mdbq-1.2.6.dist-info/METADATA,sha256=_s1z5j_Q_dSi4lrw46NcpwMlgz5TkZnndOmWp4290Mk,245
+mdbq-1.2.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.2.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.2.6.dist-info/RECORD,,

{mdbq-1.2.4.dist-info → mdbq-1.2.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

mdbq/mysql/data_types_/345/215/263/345/260/206/345/210/240/351/231/244.py

@@ -1,243 +0,0 @@
-# -*- coding:utf-8 -*-
-import warnings
-import pandas as pd
-import numpy as np
-import chardet
-import zipfile
-
-from numpy import dtype
-from pandas.tseries.holiday import next_monday
-from pyzipper import PyZipFile
-import os
-import platform
-import json
-import pymysql
-from mdbq.mongo import mongo
-from mdbq.mysql import mysql
-from mdbq.mysql import s_query
-from mdbq.config import get_myconf
-from mdbq.config import set_support
-from mdbq.dataframe import converter
-import datetime
-import time
-import re
-import shutil
-import getpass
-
-from sqlalchemy.dialects.postgresql.pg_catalog import pg_get_serial_sequence
-
-warnings.filterwarnings('ignore')
-"""
-1. 记录 dataframe 或者数据库的列信息(dtypes)
-2. 更新 mysql 中所有数据库的 dtypes 信息到本地 json
-"""
-
-
-class DataTypes:
-    """
-     数据简介: 记录 dataframe 或者数据库的列信息(dtypes)，可以记录其信息或者加载相关信息用于入库使用，
-     第一字段为分类(如 dataframe/mysql)，第二字段为数据库名，第三字段为集合名，第四段列名及其数据类型
-    """
-    def __init__(self):
-        self.datas = {
-            '_json统计':
-                {
-                    '分类': 0,
-                    '数据库量': 0,
-                    '集合数量': 0,
-                    '字段量': 0,
-                    '数据简介': '记录 dataframe 或者数据库的列信息(dtypes)',
-                }
-        }
-        self.path = set_support.SetSupport(dirname='support').dirname
-        self.json_file = os.path.join(self.path, 'mysql_types.json')
-        if not os.path.isdir(self.path):
-            os.makedirs(self.path)
-        if not os.path.isfile(self.json_file):
-            with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
-
-    def json_before(self):
-        """ 本地 json 文件的 dtypes 信息, 初始化更新给 self.datas """
-        with open(self.json_file, 'r', encoding='utf-8_sig') as f:
-            json_ = json.load(f)
-            self.datas.update(json_)
-
-    def df_dtypes_to_json(self, db_name, collection_name, path, df=pd.DataFrame(), is_file_dtype=True):
-        if len(df) == 0:
-            return
-        cv = converter.DataFrameConverter()
-        df = cv.convert_df_cols(df=df)  # 清理 dataframe 列名的不合规字符
-        dtypes = df.dtypes.apply(str).to_dict()
-        dtypes = {'dataframe': {db_name: {collection_name: dtypes}}}
-        self.dtypes_to_json(dtypes=dtypes, cl='dataframe', db_name=db_name, collection_name=collection_name, path=path, is_file_dtype=is_file_dtype)
-
-    def dtypes_to_json(self, cl, dtypes, db_name, collection_name, path, is_file_dtype, ):
-        """ 更新 dataframe 的 dtypes 信息到 json 文件 """
-        if not os.path.exists(path):
-            os.makedirs(path)
-        json_file = os.path.join(path, 'mysql_types.json')
-        if os.path.isfile(json_file):
-            self.json_before(json_file=json_file)  # 更新本地json信息到 self.datas
-
-        if not os.path.isfile(json_file):  # 如果不存在本地 json 文件, 直接返回即可
-            self.datas.update(dtypes)
-            with open(json_file, 'w', encoding='utf-8_sig') as f:
-                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
-        else:  # 存在则读取，并更新 df 的 dtypes
-            if cl in self.datas.keys():
-                if db_name in list(self.datas[cl].keys()):  # ['京东数据2', '天猫数据2', '生意参谋数据2', '生意经2']
-                    if collection_name in list(self.datas[cl][db_name].keys()):
-                        if is_file_dtype:  # 旧数据优先
-                            # # 用 dtypes 更新, 允许手动指定 json 文件里面的数据类型
-                            dtypes[cl][db_name][collection_name].update(self.datas[cl][db_name][collection_name])
-                            # 将 dtypes 更新进去，使 self.datas 包含新旧信息
-                            self.datas[cl][db_name][collection_name].update(dtypes[cl][db_name][collection_name])
-                        else:  # 新数据优先
-                            self.datas[cl][db_name][collection_name].update(dtypes[cl][db_name][collection_name])
-                    else:
-                        if is_file_dtype:  # 旧数据优先
-                            dtypes[cl][db_name].update(self.datas[cl][db_name])
-                            self.datas[cl][db_name].update(dtypes[cl][db_name])
-                        else:
-                            self.datas[cl][db_name].update(dtypes[cl][db_name])
-                else:
-                    # dtypes.update(self.datas)  # 可以注释掉, 因为旧数据 self.datas 是空的
-                    self.datas[cl].update(dtypes[cl])
-            else:
-                self.datas.update(dtypes)
-
-        cif = 0  # 分类
-        dbs = 0  # 数据库
-        collections = 0  # 集合
-        cols = 0  # 字段
-        for k, v in self.datas.items():
-            if k == '_json统计':
-                continue  # 不统计头信息
-            cif += 1
-            for t, g in v.items():
-                dbs += 1
-                for d, j in g.items():
-                    collections += 1
-                    for t, p in j.items():
-                        cols += 1
-        tips = {'分类': cif, '数据库量': dbs, '集合数量': collections, '字段量': cols}
-        self.datas['_json统计'].update(tips)
-        with open(json_file, 'w', encoding='utf-8_sig') as f:
-            json.dump(
-                self.datas,
-                f,
-                ensure_ascii=False,  # 默认True，非ASCII字符将被转义。如为False，则非ASCII字符会以\uXXXX输出
-                sort_keys=True,  # 默认为False。如果为True，则字典的输出将按键排序。
-                indent=4,
-            )
-
-    def mysql_dtypes_to_json(self, db_name, tabel_name, path, is_file_dtype=True):
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
-            database='mysql',
-        )
-        sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-        name_type = sq.dtypes_to_list(db_name=db_name, tabel_name=tabel_name)
-        if name_type:
-            dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
-            dtypes = {'mysql': {db_name: {tabel_name: dtypes}}}
-            self.dtypes_to_json(dtypes=dtypes, cl='mysql', db_name=db_name, collection_name=tabel_name, path=path, is_file_dtype=is_file_dtype)
-        else:
-            print(f'数据库回传数据(name_type)为空')
-
-    def load_dtypes(self, db_name, collection_name, path, cl='dataframe', ):
-        if os.path.isfile(path):
-            self.json_before(json_file=path)  # 更新本地json信息到 self.datas
-        elif os.path.isdir(path):
-            json_file = os.path.join(path, 'mysql_types.json')
-            if os.path.isfile(json_file):
-                self.json_before(json_file=json_file)
-            else:
-                # 如果不存在，则新建文件
-                with open(json_file, 'w', encoding='utf-8_sig') as f:
-                    json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
-                # print(f'不存在的文件: {json_file}')
-                return
-
-        if cl in self.datas.keys():
-            if db_name in list(self.datas[cl].keys()):
-                if collection_name in list(self.datas[cl][db_name].keys()):
-                    return self.datas[cl][db_name][collection_name]
-                else:
-                    print(f'不存在的集合名信息: {collection_name}, 文件位置: {json_file}')
-                    return {}
-            else:
-                print(f'不存在的数据库信息: {db_name}, 文件位置: {json_file}')
-                return {}
-        else:
-            print(f'不存在的数据分类: {cl}, 文件位置: {json_file}')
-            return {}
-
-
-def mysql_all_dtypes(path=None):
-    """
-    更新笔记本 mysql 中所有数据库的 dtypes 信息到本地 json
-    """
-    if not path:
-        path = set_support.SetSupport(dirname='support').dirname
-
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
-    config = {
-        'host': host,
-        'port': port,
-        'user': username,
-        'password': password,
-        'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
-        'cursorclass': pymysql.cursors.DictCursor,
-    }
-
-    connection = pymysql.connect(**config)  # 连接数据库
-    with connection.cursor() as cursor:
-        sql = "SHOW DATABASES;"
-        cursor.execute(sql)
-        db_name_lists = cursor.fetchall()
-        db_name_lists = [item['Database'] for item in db_name_lists]
-        connection.close()
-
-    sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
-    db_name_lists = [item for item in db_name_lists if item not in sys_lists]
-
-    # db_name_lists = [
-    #     '京东数据2',
-    #     '天猫数据2',
-    #     '市场数据2',
-    #     '生意参谋数据2',
-    #     '生意经2',
-    #     '属性设置2',
-    #     '聚合数据',
-    # ]
-    results = []
-    for db_name in db_name_lists:
-        config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**config)  # 连接数据库
-        try:
-            with connection.cursor() as cursor:
-                sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_name}';"
-                sql = "SHOW TABLES;"
-                cursor.execute(sql)
-                table_name = cursor.fetchall()
-                for table in table_name:
-                    for k, v in table.items():
-                        results.append({db_name: v})
-        except:
-            pass
-        finally:
-            connection.close()
-        time.sleep(0.5)
-
-    d = DataTypes()
-    for result in results:
-        for k, v in result.items():
-            d.mysql_dtypes_to_json(db_name=k, tabel_name=v, path=path)
-
-
-if __name__ == '__main__':
-    # mysql_all_dtypes()  # 更新 mysql 中所有数据库的 dtypes 信息到本地 json
-    d = DataTypes()
-