mdbq 2.7.9__tar.gz → 2.8.1__tar.gz

{mdbq-2.7.9 → mdbq-2.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.7.9
+Version: 2.8.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.7.9 → mdbq-2.8.1}/mdbq/aggregation/aggregation.py

@@ -6,6 +6,7 @@ import pandas as pd
 import numpy as np
 import chardet
 import zipfile
+import socket
 from pandas.tseries.holiday import next_monday
 from pyzipper import PyZipFile
 import os
@@ -16,6 +17,7 @@ from mdbq.mysql import mysql
 from mdbq.aggregation import df_types
 from mdbq.config import get_myconf
 from mdbq.config import set_support
+from mdbq.config import myconfig
 from mdbq.dataframe import converter
 import datetime
 import time
@@ -25,14 +27,32 @@ import getpass
 
 warnings.filterwarnings('ignore')
 """
+
+此文件不再更新
+
+
 1. DatabaseUpdate: 程序用于对爬虫下载的原始数据进行清洗并入库;
     数据入库时会较检并更新本地 json 文件的 dtypes 信息;
     若 json 缺失 dtypes 信息, 会按 df 类型自动转换并更新本地 json, 可以手动修改添加本地 json 信息，手动修改优先;
 2. upload_dir: 函数将一个文件夹上传至数据库;
 """
 
+username, password, host, port, service_database = None, None, None, None, None,
+if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    service_database = {'xigua_lx': 'mysql'}
+elif socket.gethostname() in ['company', 'Mac2.local']:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    service_database = {'company': 'mysql'}
+if not username:
+    print(f'找不到主机：')
+
 
-class DatabaseUpdate:
+class DatabaseUpdateBak:
     """
     清洗文件，并入库，被 tg.py 调用
     """
@@ -858,10 +878,6 @@ class DatabaseUpdate:
             for service_name, database in service_database.items():
                 # print(service_name, database)
                 if database == 'mongodb':
-                    username, password, host, port = get_myconf.select_config_values(
-                        target_service=service_name,
-                        database=database,
-                    )
                     d = mongo.UploadMongo(
                         username=username,
                         password=password,
@@ -882,10 +898,6 @@ class DatabaseUpdate:
                         d.client.close()
 
                 elif database == 'mysql':
-                    username, password, host, port = get_myconf.select_config_values(
-                        target_service=service_name,
-                        database=database,
-                    )
                     m = mysql.MysqlUpload(
                         username=username,
                         password=password,
@@ -1058,10 +1070,6 @@ class DatabaseUpdate:
 
         for service_database in service_databases:
             for service_name, database in service_database.items():
-                username, password, host, port = get_myconf.select_config_values(
-                    target_service=service_name,
-                    database=database,
-                )
                 m = mysql.MysqlUpload(
                     username=username,
                     password=password,
@@ -1095,10 +1103,6 @@ class DatabaseUpdate:
         df['日期'] = datetime.datetime.now().strftime('%Y-%m-%d')
         for service_database in service_databases:
             for service_name, database in service_database.items():
-                username, password, host, port = get_myconf.select_config_values(
-                    target_service=service_name,
-                    database=database,
-                )
                 m = mysql.MysqlUpload(
                     username=username,
                     password=password,
@@ -1117,46 +1121,13 @@ class DatabaseUpdate:
                 )
 
 
-def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, target_service='company'):
+def upload_dir(path, db_name, collection_name, json_path=None):
     """ 上传一个文件夹到 mysql 或者 mongodb 数据库 """
     if not os.path.isdir(path):
         print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 函数只接受文件夹路径，不是一个文件夹: {path}')
         return
 
-    if dbs['mongodb']:
-        username, password, host, port = get_myconf.select_config_values(
-            target_service=target_service,
-            database='mongodb',
-        )
-        d = mongo.UploadMongo(
-            username=username,
-            password=password,
-            host=host,
-            port=port,
-            drop_duplicates=False,
-        )
-
-    if dbs['mysql']:
-        username, password, host, port = get_myconf.select_config_values(
-            target_service=target_service,
-            database='mysql',
-        )
-        m = mysql.MysqlUpload(
-            username=username,
-            password=password,
-            host=host,
-            port=port,
-        )
-        # username, password, host, port = get_myconf.select_config_values(
-        #     target_service='nas',
-        #     database='mysql',
-        # )
-        # nas = mysql.MysqlUpload(
-        #     username=username,
-        #     password=password,
-        #     host=host,
-        #     port=port,
-        # )
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
 
     # 从本地 json 文件从读取 df 的数据类型信息
     df_to_json = df_types.DataTypes()
@@ -1197,31 +1168,23 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
                     dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
                     df = df.astype(dtypes)  # 再次更新 df 的数据类型
 
-                if dbs['mongodb']:
-                    d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                if dbs['mysql']:  # drop_duplicates: 值为 True 时检查重复数据再插入
-                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
-                                  move_insert=False,  # 先删除，再插入
-                                  df_sql = True,
-                                  drop_duplicates=False,
-                                  filename=name, count=f'{i}/{count}',
-                                  service_database={target_service: 'mysql'},   # 这个参数是用来设置更新哪台服务器的 types 信息到本地 json 文件
-                                  )
-                    # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
+                m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
+                              move_insert=False,  # 先删除，再插入
+                              df_sql = True,
+                              drop_duplicates=False,
+                              filename=name, count=f'{i}/{count}',
+                              )
+                # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
             except Exception as e:
                 print(name, e)
             i += 1
-    if dbs['mongodb']:
-        if d.client:
-            d.client.close()  # 必须手动关闭数据库连接
 
 
-def one_file_to_mysql(file, db_name, table_name, target_service, database):
+def one_file_to_mysql(file, db_name, table_name):
     """ 上传单个文件到 mysql 数据库 file 参数是一个文件 """
     if not os.path.isfile(file):
         print(f'{os.path.splitext(os.path.basename(__file__))[0]}.one_file_to_mysql: 函数只接受文件, 此文件不存在: {file}')
         return
-    username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
     filename = os.path.basename(file)
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False, float_precision='high')
     # df.replace(to_replace=[','], value='', regex=True, inplace=True)  # 替换掉特殊字符
@@ -1234,121 +1197,62 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
         move_insert=False,
         df_sql=True,
         drop_duplicates=False,
-        service_database={target_service: database},
     )
 
 
-def file_dir(one_file=True, target_service='company'):
-    """
-    按照文件记录对照表上传数据
-    批量上传数据库
-    one_file: 值为 True 时每个文件夹取一个文件上传数据库，反之上传所有文件夹数据
-    """
-    filename = '文件目录对照表.csv'
-    if platform.system() == 'Windows':
-        path = 'C:\\同步空间\\BaiduSyncdisk\\原始文件2'
-    else:
-        path = '/Users/xigua/数据中心/原始文件2'
-
-    support_file = set_support.SetSupport(dirname='support').dirname
-    df = pd.read_csv(os.path.join(support_file, filename), encoding='utf-8_sig', header=0, na_filter=False)
-    datas = df.to_dict('records')  # 转字典
-    for data in datas:
-        # print(data)
-        if data['入库进度'] == 0:
-            sub_path, db_name, table_name = data['子文件夹'], data['数据库名'], data['数据表']
-            if platform.system() == 'Windows':
-                sub_path = sub_path.replace('/', '\\')
-            # print(os.path.join(path, sub_path), db_name, table_name)
-            if one_file:  # 从每个文件夹中取出一个文件上传
-                real_path_list = []
-                for root, dirs, files in os.walk(os.path.join(path, sub_path), topdown=False):
-                    for name in files:
-                        if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
-                            real_path_list.append(os.path.join(root, name))
-                            break
-                for real_path in real_path_list:
-                    one_file_to_mysql(
-                        file=real_path,
-                        db_name=db_name,
-                        table_name=table_name,
-                        target_service=target_service,
-                        database='mysql'
-                    )
-            else:  # 上传全部文件夹
-                upload_dir(
-                    path=os.path.join(path, sub_path),
-                    db_name = db_name,
-                    collection_name = table_name,
-                    dbs={'mysql': True, 'mongodb': False},
-                    target_service=target_service,
-                )
-            data.update({'入库进度': 1})  # 更新进度为已上传
-    # 将进度信息写回文件
-    df = pd.DataFrame.from_dict(datas, orient='columns')
-    df.to_csv(os.path.join(support_file, filename), encoding='utf-8_sig', index=False, header=True)
-
-
 def test():
-    path = os.path.relpath(r'/Users/xigua/数据中心/原始文件3/达摩盘/dmp人群报表')
+    path = os.path.relpath(r'/Users/xigua/Downloads/直播间')
+    results = []
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
                 # print(name)
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                print(name)
+                # print(name)
                 if len(df) == 0:
-                    print(name)
-                    os.remove(os.path.join(root, name))
                     continue
-                df = df[df['日期'] != '']
-                # cols = df.columns.tolist()
-                # if '千次展现花费' not in cols:
-                #     df.insert(loc=14, column='千次展现花费', value=0.0)
-                    # df.rename(columns={'总费用': '花费', '直接购物车数': '直接加购数', '总购物车数': '总加购数'}, inplace=True)
-                # if '类目ID' not in cols:
-                #     df['类目ID'] = 'null'
-                # if '类目名称' not in cols:
-                #     df['类目名称'] = 'null'
-                # if '店铺名称' not in cols:
-                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                # if '全站roi' in cols:
-                #     df.rename(columns={'全站roi': '全站投产比'}, inplace=True)
-                # new_name = f'{os.path.splitext(name)[0]}.csv'
-                df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
-                # breakpoint()
-                # os.remove(os.path.join(root, name))
-                # new_name = f'{os.path.splitext(name)[0]}.xlsx'
-                # df.to_excel(os.path.join(root, name),
-                #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
-        #         break
-        # break
+                results.append(df)
+                # df = df[df['日期'] != '']
+                # df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+    df = pd.concat(results)
+
+    df.rename(columns={
+        '订单id': '计划ID',
+        '投资回报率': '投入产出比',
+        '直接投资回报率': '直接成交投入产出比',
+        '预售金额': '直接成交投入产出比',
+        '直接投资回报率': '总预售成交金额',
+        '预售订单量': '总预售成交笔数',
+        # '商品点击量': '展现量',
+        # '商品点击率': '观看率',
+    }, inplace=True)
+
+    df.to_csv(os.path.join('/Users/xigua/Downloads/', f'py_xg_tg_report_超级直播报表_人群_万里马官方旗舰店_.csv'), encoding='utf-8_sig', index=False, header=True)
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
-    print(username, password, host, port)
-    # file_dir(one_file=False, target_service='company')
+    username = 'root'
+    password = ''
+    host = ''
+    port = ''
 
-    # # 上传 1 个文件到数据库
-    # one_file_to_mysql(
-    #     file=r'C:\同步空间\BaiduSyncdisk\原始文件2\属性设置\电商定价.csv',
-    #     db_name='属性设置3',
-    #     table_name='电商定价',
-    #     target_service='home_lx',
-    #     database='mysql'
-    # )
-    #
-    # 上传一个目录到指定数据库
-    db_name = '京东数据3'
-    table_name = '京东商智_店铺来源'
-    upload_dir(
-        path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/京东报表/店铺来源_三级来asdasdas源'),
-        db_name=db_name,
-        collection_name=table_name,
-        dbs={'mysql': True, 'mongodb': False},
-        target_service='company',
+    # 上传 1 个文件到数据库
+    one_file_to_mysql(
+        file=r'/Users/xi',
+        db_name='推广数据2',
+        table_name='超级直播',
     )
 
+    # # 上传一个目录到指定数据库
+    # db_name = '推广数据2'
+    # table_name = '超级直播'
+    # upload_dir(
+    #     path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/天猫推广报表/超级直播报表_人群/2024-04'),
+    #     db_name=db_name,
+    #     collection_name=table_name,
+    # )
+
+
     # test()
+

{mdbq-2.7.9 → mdbq-2.8.1}/mdbq/aggregation/query_data.py

@@ -669,6 +669,44 @@ class MysqlDatasQuery:
         df_tm_pxb.rename(columns={'报表类型': '营销场景', '消耗': '花费'}, inplace=True)
         df_tm_pxb['营销场景'] = '品销宝'
 
+        # 因为 2024.04.16及之前的营销场景报表不含超级直播，所以在此添加
+        if start_date < pd.to_datetime('2024-04-17'):
+            projection = {
+                '日期': 1,
+                '场景名字': 1,
+                '花费': 1,
+                '展现量': 1,
+                '观看次数': 1,
+                '总购物车数': 1,
+                '总成交笔数': 1,
+                '总成交金额': 1,
+                '店铺名称': 1,
+            }
+            df_tm_living = self.download.data_to_df(
+                db_name='推广数据2',
+                table_name='超级直播',
+                start_date=start_date,
+                end_date=pd.to_datetime('2024-04-16'),  # 只可以取此日期之前的数据
+                projection=projection,
+            )
+            if len(df_tm_living) > 0:
+                df_tm_living.rename(columns={'场景名字': '营销场景'}, inplace=True)
+                df_tm_living = df_tm_living.groupby(
+                    ['日期', '店铺名称', '营销场景', '花费'],
+                    as_index=False).agg(
+                    **{
+                        '展现量': ('展现量', np.max),
+                        '点击量': ('观看次数', np.max),
+                        '加购量': ('总购物车数', np.max),
+                        '成交笔数': ('总成交笔数', np.max),
+                        '成交金额': ('总成交金额', np.max)
+                    }
+                )
+            else:
+                df_tm_living = pd.DataFrame()
+        else:
+            df_tm_living = pd.DataFrame()
+
         projection = {
             '日期': 1,
             '产品线': 1,
@@ -741,7 +779,7 @@ class MysqlDatasQuery:
         df_jd_qzyx = df_jd_qzyx[['日期', '店铺名称', '营销场景', '花费', '展现量', '点击量', '成交笔数', '成交金额']]
         df_jd_qzyx = df_jd_qzyx[df_jd_qzyx['花费'] > 0]
 
-        _datas = [item for item in  [df_tm, df_tb, df_tm_pxb, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
+        _datas = [item for item in  [df_tm, df_tb, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
         df = pd.concat(_datas, axis=0, ignore_index=True)
         return df
 
@@ -2277,7 +2315,7 @@ def main():
 
 if __name__ == '__main__':
     data_aggregation(
-        months=3,
+        months=15,
         is_juhe=True,  # 生成聚合表
-        # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
+        # less_dict=['多店推广场景_按日聚合'],  # 单独聚合某一个数据库
     )

{mdbq-2.7.9 → mdbq-2.8.1}/mdbq/spider/aikucun.py

@@ -96,7 +96,10 @@ def get_cookie_aikucun():
 
     # 将cookies保存为json格式
     cookies_list = _driver.get_cookies()
-
+    for cookie in cookies_list:
+        # 该字段有问题所以删除就可以
+        if 'expiry' in cookie:
+            del cookie['expiry']
     json_file = os.path.join(cookie_path, filename_aikucun)
     with open(json_file, 'w', encoding='utf-8') as f:
         json.dump(cookies_list, f, ensure_ascii=False, sort_keys=True, indent=4)
@@ -216,7 +219,7 @@ class AikuCun:
         today = datetime.date.today()
         for date_s in range(date_num):
             new_date = today - datetime.timedelta(days=date_s)  # 会用作文件名
-            print(f'正在下载爱库存文件 {date_s}/{date_num}: {new_date}')
+            print(f'正在下载爱库存文件 {date_s+1}/{date_num}: {new_date}')
             str_date = str(new_date)[2:]
             wait = WebDriverWait(_driver, timeout=15)  #
             elements = _driver.find_elements(
@@ -361,7 +364,7 @@ class AikuCunNew:
 
 if __name__ == '__main__':
     pass
-    get_cookie_aikucun()
+    # get_cookie_aikucun()
     akucun()
 
     # a = AikuCunNew(shop_name='aikucun')

{mdbq-2.7.9 → mdbq-2.8.1}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.7.9
+Version: 2.8.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.7.9 → mdbq-2.8.1}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.7.9',
+      version='2.8.1',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbq',