mdbq 3.8.2__py3-none-any.whl → 3.8.4__py3-none-any.whl

mdbq/__version__.py

@@ -1,3 +1 @@
-VERSION = (0, 3, 6)
-
-__version__ = '.'.join(map(str, VERSION))
+VERSION = '3.8.3'

mdbq/aggregation/query_data.py

@@ -22,7 +22,9 @@ import sys
 
 """
 
-content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 
@@ -468,7 +470,7 @@ class MysqlDatasQuery:
             '店铺名称': 1,
         }
         __res = []
-        for year in range(2024, datetime.datetime.today().year+1):
+        for year in range(2025, datetime.datetime.today().year+1):
             df = self.download.data_to_df(
                 db_name='推广数据_圣积天猫店',
                 table_name=f'主体报表_{year}',
@@ -2459,6 +2461,7 @@ class MysqlDatasQuery:
         df_tm_living = pd.DataFrame()  # 天猫超级直播
         df_jd = pd.DataFrame()  # 京东推广
         df_jd_qzyx = pd.DataFrame()  # 京东全站推广
+        df_jd_ziying = pd.DataFrame()  # 京东推广
 
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -2525,7 +2528,7 @@ class MysqlDatasQuery:
             )
         # sj圣积
         __res = []
-        for year in range(2024, datetime.datetime.today().year + 1):
+        for year in range(2025, datetime.datetime.today().year + 1):
             df_sj = self.download.data_to_df(
                 db_name='推广数据_圣积天猫店',
                 table_name=f'营销场景报表_{year}',
@@ -2783,7 +2786,52 @@ class MysqlDatasQuery:
             df_jd_qzyx = df_jd_qzyx[['日期', '店铺名称', '营销场景', '花费', '展现量', '点击量', '成交笔数', '成交金额']]
             df_jd_qzyx = df_jd_qzyx[df_jd_qzyx['花费'] > 0]
 
-        _datas = [item for item in  [df_tm, df_tb, df_tb_qzt, df_al, df_sj, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
+        projection = {
+            '日期': 1,
+            '产品线': 1,
+            '触发sku_id': 1,
+            '跟单sku_id': 1,
+            '花费': 1,
+            '展现数': 1,
+            '点击数': 1,
+            '直接订单行': 1,
+            '直接订单金额': 1,
+            '总订单行': 1,
+            '总订单金额': 1,
+            '直接加购数': 1,
+            '总加购数': 1,
+            'spu_id': 1,
+            '店铺名称': 1,
+        }
+        __res = []
+        for year in range(2025, datetime.datetime.today().year + 1):
+            df_jd_ziying = self.download.data_to_df(
+                db_name='京东数据3',
+                table_name=f'推广数据_京准通_自营店_{year}',
+                start_date=start_date,
+                end_date=end_date,
+                projection=projection,
+            )
+            __res.append(df_jd_ziying)
+        df_jd_ziying = pd.concat(__res, ignore_index=True)
+        if len(df_jd_ziying) > 0:
+            df_jd_ziying = df_jd_ziying.groupby(
+                ['日期', '店铺名称', '产品线', '触发sku_id', '跟单sku_id', 'spu_id', '花费', '展现数', '点击数'],
+                as_index=False).agg(
+                **{
+                    '直接订单行': ('直接订单行', np.max),
+                    '直接订单金额': ('直接订单金额', np.max),
+                    '成交笔数': ('总订单行', np.max),
+                    '成交金额': ('总订单金额', np.max),
+                    '直接加购数': ('直接加购数', np.max),
+                    '加购量': ('总加购数', np.max),
+                }
+            )
+            df_jd_ziying = df_jd_ziying[['日期', '店铺名称', '产品线', '花费', '展现数', '点击数', '加购量', '成交笔数', '成交金额']]
+            df_jd_ziying.rename(columns={'产品线': '营销场景', '展现数': '展现量', '点击数': '点击量'}, inplace=True)
+            df_jd_ziying = df_jd_ziying[df_jd_ziying['花费'] > 0]
+
+        _datas = [item for item in  [df_tm, df_tb, df_tb_qzt, df_al, df_sj, df_tm_pxb, df_tm_living, df_jd, df_jd_qzyx, df_jd_ziying] if len(item) > 0]  # 阻止空的 dataframe
         df = pd.concat(_datas, axis=0, ignore_index=True)
         # 超级直播全站推广不包含在营销场景报表中，所以单独添加  2025-03-05
         projection = {

mdbq/log/spider_logging.py

@@ -10,12 +10,11 @@ def setup_logging(reMoveOldHandler=True):
     """
     reMoveOldHandler: 替换根日志记录器的所有现有处理器
     """
-    D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])))
+    dir_path = os.path.expanduser("~")
+    if not os.path.isdir(os.path.join(dir_path, 'logfile')):
+        os.makedirs(os.path.join(dir_path, 'logfile'))
 
-    if not os.path.isdir(os.path.join(D_PATH, 'logfile')):
-        os.makedirs(os.path.join(D_PATH, 'logfile'))
-
-    log_file = os.path.join(D_PATH, 'logfile', 'spider_tg.log')
+    log_file = os.path.join(dir_path, 'logfile', 'spider_tg.log')
     file_handler = RotatingFileHandler(
         filename=log_file,
         maxBytes=3*1024*1024,  # 3MB

mdbq/other/download_sku_picture.py

@@ -47,7 +47,9 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 if not os.path.exists(upload_path):  # 数据中心根目录
     os.makedirs(upload_path)
 
-content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 

mdbq/redis/getredis.py

@@ -17,7 +17,8 @@ import orjson
 logger = logging.getLogger(__name__)
 
 # 创建一个文件处理器，用于将日志写入文件
-dir_path = os.path.realpath(os.path.dirname(sys.argv[0]))
+dir_path = os.path.expanduser("~")
+# config_file = os.path.join(dir_path, 'spd.txt')
 log_file = os.path.join(dir_path, 'logfile', 'redis.log')
 if not os.path.isdir(os.path.join(dir_path, 'logfile')):
     os.mkdir(os.path.join(dir_path, 'logfile'))

mdbq/spider/aikucun.py

@@ -22,8 +22,9 @@ from mdbq.mysql import s_query
 from mdbq.config import config
 from mdbq.other import ua_sj
 
-
-content = config.read_config(file_path=os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'spd.txt'))
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')

{mdbq-3.8.2.dist-info → mdbq-3.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.2
+Version: 3.8.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.2.dist-info → mdbq-3.8.4.dist-info}/RECORD

@@ -1,8 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
+mdbq/__version__.py,sha256=N2YV132Ke9nxG8e_N4DlVVVafsd63H4Sbusd36SCowU,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/optimize_data_bak.py,sha256=wB7prQdZAHyjzXH9V8g8X_JBMdvCCUITN1hVwK72Tdg,952
-mdbq/aggregation/query_data.py,sha256=R8KL_tX6kU3Na2Gy-FGzNaGRZEzCl8bptYgOTSXYDpw,178194
+mdbq/aggregation/query_data.py,sha256=fbNyjRlJF3ycVBfAmj_JXMaauUmj3oIFIzKx2XV2sR4,180368
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -12,26 +11,25 @@ mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
-mdbq/log/spider_logging.py,sha256=59xe4Ckb7m-sBt3GYk8DC_hQg7-jnjBRq1o718r1Ry8,1676
+mdbq/log/spider_logging.py,sha256=KX9TTUn9naZNBACCEFhyTktnWhr5JaSNQLppLGyrm9Y,1645
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=tR6l4Zzn9j6zKaFcy0Ktw2oL8OoX3QB6jDoDp1l2fiM,95474
 mdbq/mysql/s_query.py,sha256=09Dp7DrVXui6dAI6zFDfrsUOdjPblF_oYUpgqbZMhXg,8757
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=xX3jU2Nu2R9CoveM7xevRstg1P40hvR1KL8wqmwt3O0,44809
+mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
-mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=1pTga2iINx0NV2ffl0D-aspZhrZMDQR8SpohAv1acoo,24076
+mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=GBZsCjsN3i1ZW9pAxeMAmb-y3yW3p3HJpjfrnnna5gg,19702
-mdbq-3.8.2.dist-info/METADATA,sha256=pmdQf_CAgoh9rcPTtyIQwrifPpr_WtddcV1sBPhDdj8,363
-mdbq-3.8.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.2.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=cYtLIEvvfKZ0pUeASpKaMY31vOKFa74KNcZQ3iCiL0c,19724
+mdbq-3.8.4.dist-info/METADATA,sha256=DKnJoT-wW88fh5-SnfD6v748tCpgii5KB7gqqe2fdAU,363
+mdbq-3.8.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.8.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.8.4.dist-info/RECORD,,

mdbq/aggregation/optimize_data_bak.py

@@ -1,30 +0,0 @@
-# -*- coding: UTF-8 –*-
-from mdbq.mysql import mysql
-from mdbq.config import default
-import subprocess
-import psutil
-import time
-import platform
-import logging
-"""
-对指定数据库所有冗余数据进行清理
-"""
-targe_host, hostname, local =  default.return_default_host()
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
-if not username:
-    logger.info(f'找不到主机：')
-
-logger = logging.getLogger(__name__)
-
-
-def op_data(db_name_lists, days: int = 63, is_mongo=True, is_mysql=True):
-    # Mysql
-    if is_mysql:
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = db_name_lists
-        s.days = days
-        s.optimize_list()
-
-
-if __name__ == '__main__':
-    op_data(db_name_lists=['聚合数据'], days=10, is_mongo=True, is_mysql=True)

mdbq/pbix/refresh_all_old.py

@@ -1,177 +0,0 @@
-import os
-import sys
-import win32com.client
-import time
-import multiprocessing
-import psutil
-from pywinauto.application import Application
-import warnings
-from mdbq.config import set_support
-
-warnings.filterwarnings('ignore')
-top_path = os.path.realpath(os.path.dirname(sys.argv[0]))  # 程序运行目录, 打包时使用
-sys.path.append(top_path)
-
-
-class RefreshAll:
-    def __init__(self):
-        self.my_conf = os.path.join(set_support.SetSupport(dirname='support').dirname, '.my_conf')
-        self.pbix_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'ref_list.txt')
-        self.excel_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'ref_list.txt')
-        self.run_py_path = 'run_py'
-        self.procname = 'PBIDesktop.exe'
-
-    def refresh_pbix(self):
-        # 刷新 PBI
-        if not os.path.exists(self.pbix_path):
-            print(f'{self.pbix_path}: PBIxx file not found. ')
-            return
-        with open(self.pbix_path, 'r', encoding='utf-8') as f:
-            content = f.readlines()
-            content = [item.strip() for item in content if not item.strip().startswith('#') and not item.strip().startswith('[')]
-            pbix_list = [item for item in content if item]
-
-        if not pbix_list:
-            return
-        for filename in pbix_list:
-            if filename.endswith('.pbix'):
-                newfile = os.path.join(self.run_py_path, filename)
-                """刷新程序"""
-                for es in range(4):
-                    try:
-                        print(f'正在刷新 >>>{filename}')
-                        self.pbi(path=newfile)
-                        print('文件刷新 >>>' + filename)
-                        break
-                    except Exception as e:
-                        print(e)
-                        print('报错的文件 >>>' + filename + ' >> ' + str(es + 1))
-
-    def refresh_excel(self):
-        # 刷新 excel
-        if not os.path.exists(self.excel_path):
-            print(f'{self.excel_path}: Excel file not found. ')
-            return
-        with open(self.excel_path, 'r', encoding='utf-8') as f:
-            content = f.readlines()
-            content = [item.strip() for item in content if not item.strip().startswith('#')]
-            excel_list = [item for item in content if item]
-
-        if not excel_list:
-            return
-        for filename in excel_list:
-            if filename.endswith('.xlsx'):
-                try:
-                    print(f'正在刷新 >>>{filename}')
-                    path = os.path.join(top_path, self.run_py_path, filename)  # 拼接文件路径
-                    xlapp = win32com.client.Dispatch('Excel.Application')  # 创建Excel程序App
-                    xlapp.Visible = False  # 窗口是否可见
-                    xlapp.DisplayAlerts = False  # 是否显示警告信息
-                    wb = xlapp.Workbooks.Open(path)
-                    conjuncts = wb.Connections.Count  # 统计工作簿含有多少外部链接
-                    if conjuncts == 0:
-                        wb.Close(SaveChanges=False)
-                        xlapp.Quit()
-                    else:
-                        time.sleep(2)
-                        wb.RefreshAll()
-                        xlapp.CalculateUntilAsyncQueriesDone()
-                        time.sleep(2)
-                        wb.Save()
-                        wb.Close(SaveChanges=True)
-                        xlapp.Quit()
-                    print('文件刷新 >>>' + filename)
-                except Exception as e:
-                    print(e)
-
-    def refresh_excel2(self, excel_file):
-        # 刷新 excel
-        if excel_file.endswith('.xlsx'):
-            try:
-                print(f'正在刷新 >>>{excel_file}')
-                xlapp = win32com.client.Dispatch('Excel.Application')  # 创建Excel程序App
-                xlapp.Visible = False  # 窗口是否可见
-                xlapp.DisplayAlerts = False  # 是否显示警告信息
-                wb = xlapp.Workbooks.Open(excel_file)
-                conjuncts = wb.Connections.Count  # 统计工作簿含有多少外部链接
-                if conjuncts == 0:
-                    wb.Close(SaveChanges=False)
-                    xlapp.Quit()
-                else:
-                    time.sleep(2)
-                    wb.RefreshAll()
-                    xlapp.CalculateUntilAsyncQueriesDone()
-                    time.sleep(2)
-                    wb.Save()
-                    wb.Close(SaveChanges=True)
-                    xlapp.Quit()
-                print('文件刷新 >>>' + excel_file)
-            except Exception as e:
-                print(e)
-
-    def pbi(self, path, _timeout=300):
-        """
-        这是原本属于独立的库模块:  pbix_refresh
-        path: pbi文件路径
-        _timeout: 刷新等待时间
-        如果连接失败，请将 power bi 文件名的特殊字符(空格等)去掉或改为英文文件名试试
-        """
-        # 关闭已打开的 power bi进程
-        for proc in psutil.process_iter():
-            if proc.name() == self.procname:
-                proc.kill()
-        time.sleep(1)
-
-        # 启动 Power bi
-        os.system('start "" "' + path + '"')
-        time.sleep(5)
-
-        # 通过 connect 方法连接进程
-        app = Application(backend='uia').connect(path=self.procname)
-        # title_re可以通过正则表达式连接：".*Power BI Desktop", 仅限 2023年9月之前的 Power bi版本
-        # 2023年10月之后的版本没有此窗口后缀, 因此需用文件名找窗口
-        # 如果同时打开了其他同名的文件（非 power bi文件时）, 可能会引发错误
-        _filename = os.path.splitext(os.path.basename(path))[0]  # 文件名不含后缀
-        win = app.window(title_re=f'{_filename}.*?')  # 连接 pbi 窗口
-        time.sleep(10)
-        # win.print_control_identifiers()  # 打印窗口全部信息, 推荐使用 inspect 开发工具查询窗口句柄信息
-        num = 0
-        while True:
-            try:
-                win['刷新'].wait("enabled", timeout=_timeout)
-                time.sleep(3)
-                win['刷新'].click()
-                break
-            except:
-                print(f'{path}, 未识别窗口句柄, 连接超时')
-                num += 1
-                if num > 1:
-                    break
-
-        num = 0
-        while True:
-            try:
-                win['保存'].wait("enabled", timeout=_timeout)  # timeout 通过"保存"按键状态, 等待刷新窗口关闭, 时间不要设置太短
-                time.sleep(3)
-                win['保存'].click()
-                break
-            except:
-                print(f'{path}, 未识别窗口句柄, 文件未保存')
-                num += 1
-                if num > 1:
-                    break
-        # win.type_keys("^s")
-        win.wait("enabled", timeout=15)
-        win.close()
-
-        # 关闭进程
-        for proc in psutil.process_iter():
-            if proc.name() == self.procname:
-                proc.kill()
-
-
-if __name__ == '__main__':
-    # r = RefreshAll()
-    # # r.refresh_pbix()
-    # r.refresh_excel()
-    pass