mdbq 1.7.6__py3-none-any.whl → 1.7.8__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1100,12 +1100,12 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
 
-    # db_name = '推广数据2'
-    # table_name = '超级直播'
-    # upload_dir(
-    #     path='/Users/xigua/数据中心/原始文件2/推广报表/超级直播',
-    #     db_name=db_name,
-    #     collection_name=table_name,
-    #     dbs={'mysql': True, 'mongodb': False},
-    # )
+    db_name = '推广数据2'
+    table_name = '营销场景报表'
+    upload_dir(
+        path='/Users/xigua/数据中心/原始文件2/推广报表/营销场景报表',
+        db_name=db_name,
+        collection_name=table_name,
+        dbs={'mysql': True, 'mongodb': False},
+    )
 

mdbq/aggregation/query_data.py

@@ -1048,10 +1048,20 @@ class GroupBy:
             '直接成交金额': float,
             '自然流量曝光量': int,
         }, errors='raise')
+        # tg = tg.groupby(['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
+        #     **{'加购量': ('加购量', np.max),
+        #        '成交笔数': ('成交笔数', np.max),
+        #        '成交金额': ('成交金额', np.max),
+        #        '自然流量曝光量': ('自然流量曝光量', np.max),
+        #        '直接成交笔数': ('直接成交笔数', np.max),
+        #        '直接成交金额': ('直接成交金额', np.max)
+        #        }
+        # )
         df = pd.concat([tg, zb, pxb], axis=0, ignore_index=True)
         df.fillna(0, inplace=True)  # concat 之后要填充空值
         df = df.astype(
             {
+                '商品id': str,
                 '自然流量曝光量': int,
         }
         )
@@ -1249,13 +1259,13 @@ def data_aggregation(service_databases=[{}], months=1):
                 {
                     '数据库名': '聚合数据',
                     '集合名': '天猫生意经_宝贝指标',
-                    '唯一主键': ['日期', '宝贝id'],
+                    '唯一主键': ['日期', '宝贝id'],  # 不能加其他字段做主键，比如销售额，是变动的，不是唯一的
                     '数据主体': sdq.syj(),
                 },
                 {
                     '数据库名': '聚合数据',
                     '集合名': '天猫_店铺来源_日数据',
-                    '唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
+                    '唯一主键': ['日期', '一级来源', '二级来源', '三级来源', '访客数'],
                     '数据主体': sdq.dplyd(),
                 },
                 {
@@ -1327,7 +1337,7 @@ def data_aggregation(service_databases=[{}], months=1):
                 {
                     '数据库名': '聚合数据',
                     '集合名': '天猫_品销宝账户报表',
-                    '唯一主键': ['日期', '报表类型'],
+                    '唯一主键': ['日期', '报表类型', '推广渠道', '营销场景', '花费'],
                     '数据主体': sdq.pxb_zh(),
                 },
             ]
@@ -1347,23 +1357,36 @@ def data_aggregation(service_databases=[{}], months=1):
                     )
                     g.sp_index_datas = pd.DataFrame()  # 重置，不然下个循环会继续刷入数据库
                 # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
-                m.df_to_mysql(
-                    df=df,
-                    db_name=db_name,
-                    table_name=table_name,
-                    # df_sql=True,
-                    drop_duplicates=False,
-                    icm_update=unique_key_list,
-                    service_database=service_database,
-                )  # 3. 回传数据库
+                if '日期' in df.columns.tolist():
+                    m.df_to_mysql(
+                        df=df,
+                        db_name=db_name,
+                        table_name=table_name,
+                        move_insert=True,  # 先删除，再插入
+                        # df_sql=True,
+                        # drop_duplicates=False,
+                        # icm_update=unique_key_list,
+                        service_database=service_database,
+                    )  # 3. 回传数据库
+                else:  # 没有日期列的就用主键排重
+                    m.df_to_mysql(
+                        df=df,
+                        db_name=db_name,
+                        table_name=table_name,
+                        # df_sql=True,
+                        drop_duplicates=False,
+                        icm_update=unique_key_list,
+                        service_database=service_database,
+                    )  # 3. 回传数据库
             res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
                 df=res,
                 db_name='聚合数据',
                 table_name='_全店商品销售',
+                move_insert=True,  # 先删除，再插入
                 # df_sql=True,
-                drop_duplicates=False,
-                icm_update=['日期', '商品id'],  # 设置唯一主键
+                # drop_duplicates=False,
+                # icm_update=['日期', '商品id'],  # 设置唯一主键
                 service_database=service_database,
             )
             res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
@@ -1371,9 +1394,10 @@ def data_aggregation(service_databases=[{}], months=1):
                 df=res,
                 db_name='聚合数据',
                 table_name='_推广商品销售',
+                move_insert=True,  # 先删除，再插入
                 # df_sql=True,
-                drop_duplicates=False,
-                icm_update=['日期', '商品id'],  # 设置唯一主键
+                # drop_duplicates=False,
+                # icm_update=['日期', '商品id'],  # 设置唯一主键
                 service_database=service_database,
             )
 
@@ -1382,9 +1406,10 @@ def data_aggregation(service_databases=[{}], months=1):
                 df=res,
                 db_name='聚合数据',
                 table_name='天猫_推广汇总',
+                move_insert=True,  # 先删除，再插入
                 # df_sql=True,
-                drop_duplicates=False,
-                icm_update=['日期', '商品id'],  # 设置唯一主键
+                # drop_duplicates=False,
+                # icm_update=['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'],  # 设置唯一主键
                 service_database=service_database,
             )
 
@@ -1394,9 +1419,10 @@ def data_aggregation(service_databases=[{}], months=1):
                 df=res,
                 db_name='聚合数据',
                 table_name='_京东_推广商品销售',
+                move_insert=True,  # 先删除，再插入
                 # df_sql=True,
-                drop_duplicates=False,
-                icm_update=['日期', '跟单sku id', '货号', '花费'],  # 设置唯一主键
+                # drop_duplicates=False,
+                # icm_update=['日期', '跟单sku id', '货号', '花费'],  # 设置唯一主键
                 service_database=service_database,
             )
 
@@ -1410,7 +1436,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=1)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

mdbq/company/copysh.py

@@ -20,6 +20,7 @@ from mdbq.config import get_myconf
 from mdbq.config import set_support
 from mdbq.config import products
 from mdbq.mysql import mysql
+from mdbq.pbix import refresh_all
 warnings.filterwarnings('ignore')
 
 
@@ -250,6 +251,16 @@ class TbFiles:
             else:
                 print(f'{src} 所需同步的文件不存在，请检查：pd_list参数')
 
+        excel_path = os.path.join(self.share_path, 'EXCEL报表')
+        files = os.listdir(excel_path)
+        r = refresh_all.RefreshAll()
+        for file in files:
+            if file.endswith('.xlsx'):
+                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                print(f'正在刷新 excel: {file}')
+                r.refresh_excel2(excel_file=os.path.join(excel_path, file))
+            time.sleep(10)
+
         self.before_max_time = self.check_change()  # 重置值, 避免重复同步
 
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

mdbq/mysql/mysql.py

@@ -57,11 +57,14 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
+        move_insert: 根据df 的日期，先移除数据库数据，再插入, df_sql, drop_duplicates, icm_update 都要设置为 False
+        原则上只限于聚合数据使用，原始数据插入时不要设置
+
         df_sql: 这是一个临时参数, 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重，初创表大量上传数据的时候使用
         drop_duplicates: 值为 True 时检查重复数据再插入，反之直接上传，数据量大时会比较慢
         icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用，设置此参数时需将 drop_duplicates 改为 False
@@ -162,20 +165,34 @@ class MysqlUpload:
             elif cl:
                 mysql_types.mysql_all_dtypes(service_database=service_database)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
 
-            # # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
-            # if drop_duplicates and '日期' in df.columns.tolist():
-            #     dates = df['日期'].values.tolist()
-            #     start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
-            #     end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
-            #     sql = f"DELETE FROM `{table_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
-            #     cursor.execute(sql)
-            #     connection.commit()
-
-            # 5. 更新插入数据
+            # 4. 更新插入数据
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
             for service_name, database in service_database.items():
                 print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {service_name}, {self.filename}')
 
+            # 5. 移除指定日期范围内的数据，原则上只限于聚合数据使用，原始数据插入时不要设置
+            if move_insert and '日期' in df.columns.tolist():
+                # 移除数据
+                dates = df['日期'].values.tolist()
+                start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
+                end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+                sql = f"DELETE FROM `{table_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
+                cursor.execute(sql)
+                connection.commit()
+
+                # 插入数据
+                engine = create_engine(
+                    f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
+                df.to_sql(
+                    name=table_name,
+                    con=engine,
+                    if_exists='append',
+                    index=False,
+                    chunksize=1000
+                )
+                connection.close()
+                return
+
             datas = df.to_dict(orient='records')
             for data in datas:
                 # data 是传进来待处理的数据, 不是数据库数据

mdbq/pbix/refresh_all.py

@@ -17,7 +17,7 @@ class RefreshAll:
     def __init__(self):
         self.my_conf = os.path.join(set_support.SetSupport(dirname='support').dirname, '.my_conf')
         self.pbix_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'ref_list.txt')
-        self.excel_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cp_list.txt')
+        self.excel_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'ref_list.txt')
         self.run_py_path = 'run_py'
         self.procname = 'PBIDesktop.exe'
 
@@ -84,6 +84,31 @@ class RefreshAll:
                 except Exception as e:
                     print(e)
 
+    def refresh_excel2(self, excel_file):
+        # 刷新 excel
+        if excel_file.endswith('.xlsx'):
+            try:
+                print(f'正在刷新 >>>{excel_file}')
+                xlapp = win32com.client.Dispatch('Excel.Application')  # 创建Excel程序App
+                xlapp.Visible = False  # 窗口是否可见
+                xlapp.DisplayAlerts = False  # 是否显示警告信息
+                wb = xlapp.Workbooks.Open(excel_file)
+                conjuncts = wb.Connections.Count  # 统计工作簿含有多少外部链接
+                if conjuncts == 0:
+                    wb.Close(SaveChanges=False)
+                    xlapp.Quit()
+                else:
+                    time.sleep(2)
+                    wb.RefreshAll()
+                    xlapp.CalculateUntilAsyncQueriesDone()
+                    time.sleep(2)
+                    wb.Save()
+                    wb.Close(SaveChanges=True)
+                    xlapp.Quit()
+                print('文件刷新 >>>' + excel_file)
+            except Exception as e:
+                print(e)
+
     def pbi(self, path, _timeout=300):
         """
         这是原本属于独立的库模块:  pbix_refresh
@@ -146,7 +171,7 @@ class RefreshAll:
 
 
 if __name__ == '__main__':
-    # r = RefreshAll()
+    r = RefreshAll()
     # r.refresh_pbix()
-    # r.refresh_excel()
+    r.refresh_excel()
     pass

{mdbq-1.7.6.dist-info → mdbq-1.7.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.7.6
+Version: 1.7.8
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.7.6.dist-info → mdbq-1.7.8.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=d7pYUku7Wbxl0tvKKNTG7mppOjGqg0LF62OpfW8fVBk,64120
+mdbq/aggregation/aggregation.py,sha256=sgsetJHK4fOcXvqQCVgJoSIwZQLMznVG3I-MqHlW_fM,64116
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=WxLtzR6s6gIPe6e1hB3xycZirrN83IZ0s0PyI0t2Cls,66792
+mdbq/aggregation/query_data.py,sha256=9UhWwFAhMieC5iOIna_sAhsk2WvSpzGd5aDvkRo_pVE,68486
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=T0WYOKFwNZTNk3temKOw1K2H54kxu9QBJjlTbkMtxNk,94217
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=WCZ92vCJAy6_ZFeOxWL-U9gArIpyga4xts-s1wKsspY,17268
+mdbq/company/copysh.py,sha256=z1jql2UABdKGGPYF6VRhXcBwCYaCBFR91kZwthBlOdU,17754
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=-CFEW0dQh4OIwVgwK-cL0eVp1LN3PjJgN89d4P5TB9I,6011
 mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=cIK_GI6Ggb9LsxsvVUv0AviD7kdyCumk_eQ9MSOwsms,43320
+mdbq/mysql/mysql.py,sha256=UKnBmywqTzc0VJfZGlC-9KzV7I--9P7H-jspUp_IvtU,44071
 mdbq/mysql/s_query.py,sha256=fIQvQKPyV7rvSUuxVWXv9S5FmCnIM4GHKconE1Zn5BA,8378
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -33,9 +33,9 @@ mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
-mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
+mdbq/pbix/refresh_all.py,sha256=sBZ61LKvm-raa9ROnC-AAvPYLU7dbudmuxy__5QCB2A,7176
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.7.6.dist-info/METADATA,sha256=QzujGJTSxIt0YV7h24bB6lTwt0Tw7QNStfyJncdGqxY,245
-mdbq-1.7.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.7.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.7.6.dist-info/RECORD,,
+mdbq-1.7.8.dist-info/METADATA,sha256=GSjzapW9LjD8Jco2uKTvfb6Ex9lWJW1Eo33BpvXLUls,245
+mdbq-1.7.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.7.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.7.8.dist-info/RECORD,,

{mdbq-1.7.6.dist-info → mdbq-1.7.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any