mdbq 2.7.4__py3-none-any.whl → 2.7.6__py3-none-any.whl

mdbq/aggregation/mysql_types.py

@@ -152,11 +152,11 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
     """
     username, password, host, port, service_name = None, None, None, None, None
     conf = myconfig.main()
-    if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
+    if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
         data = conf['Windows']['xigua_lx']['mysql']['local']
         username, password, host, port = data['username'], data['password'], data['host'], data['port']
         service_name = 'xigua_lx'  # 影响 mysql_types_xigua_lx.json 文件名
-    elif socket.gethostname().lower() in ['company', 'Mac2.local']:
+    elif socket.gethostname() in ['company', 'Mac2.local']:
         data = conf['Windows']['company']['mysql']['local']
         username, password, host, port = data['username'], data['password'], data['host'], data['port']
         service_name = 'company'  # 影响 mysql_types_company.json 文件名

mdbq/aggregation/optimize_data.py

@@ -11,12 +11,12 @@ import platform
 对指定数据库所有冗余数据进行清理
 """
 username, password, host, port, service_database = None, None, None, None, None,
-if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
+if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
     conf = myconfig.main()
     data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'mac2.local']:
+elif socket.gethostname() in ['company', 'Mac2.local']:
     conf = myconfig.main()
     data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
@@ -74,7 +74,7 @@ def restart_mongodb():
 def op_data(db_name_lists, days: int = 63, is_mongo=True, is_mysql=True):
     """ """
 
-    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'mac2.local':
+    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
         # Mysql
         if is_mysql:
             s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)

mdbq/aggregation/query_data.py

@@ -26,12 +26,12 @@ import os
 
 """
 username, password, host, port, service_database = None, None, None, None, None,
-if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
+if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
     conf = myconfig.main()
     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'mac2.local']:
+elif socket.gethostname() in ['company', 'Mac2.local']:
     conf = myconfig.main()
     conf_data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
@@ -2277,7 +2277,7 @@ def main():
 
 if __name__ == '__main__':
     data_aggregation(
-        months=1,
+        months=3,
         is_juhe=True,  # 生成聚合表
         # less_dict=['天猫_品销宝账户报表'],  # 单独聚合某一个数据库
     )

mdbq/clean/clean_upload.py

@@ -46,12 +46,12 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 source_path3 = os.path.join(Data_Path, '原始文件3')  # 此目录保存下载并清洗过的文件，作为数据库备份
 
 username, password, host, port, service_database = None, None, None, None, None,
-if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
+if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
     conf = myconfig.main()
     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'mac2.local']:
+elif socket.gethostname() in ['company', 'Mac2.local']:
     conf = myconfig.main()
     conf_data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
@@ -440,7 +440,7 @@ class DataClean:
                     sheets4 = ['账户', '推广计划', '推广单元', '创意', '品牌流量包', '定向人群']  # 品销宝
                     file_name4 = os.path.splitext(name)[0]  # 明星店铺报表
                     for sheet4 in sheets4:
-                        df = pd.read_excel(os.path.join(root, name), sheet_name=sheet4, header=0, engine='xlrd')
+                        df = pd.read_excel(os.path.join(root, name), sheet_name=sheet4, header=0, engine='openpyxl')
                         if len(df) == 0:
                             print(f'{name} 报表数据为空')
                             os.remove(os.path.join(root, name))
@@ -452,6 +452,8 @@ class DataClean:
                             shop_name = re.findall(r'明星店铺_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
                             df.insert(loc=1, column='店铺名称', value=shop_name)
                             df.insert(loc=2, column='报表类型', value=sheet4)
+                            if '访客触达率' not in df.columns.tolist():
+                                df['访客触达率'] = '0'
                             df.fillna(0, inplace=True)
                             df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
                             # min_clm = str(df['日期'].min()).split(' ')[0]
@@ -460,10 +462,21 @@ class DataClean:
                             # 以sheet名进一步创建子文件夹
                             # root_new = os.path.join(self.source_path, '推广报表/品销宝', sheet4)
                             self.save_to_csv(df, upload_path, new_file_name4)
+                        if not db_name or not collection_name:
+                            continue
+                        # 一定要单独处理品销宝报表，因为 1 表包含 6个 sheet，都要添加
+                        self.datas.append(
+                            {
+                                '数据库名': db_name,
+                                '集合名称': collection_name,
+                                '数据主体': df,
+                                '文件名': name,
+                            }
+                        )
                     os.remove(os.path.join(root, name))
 
                 # 将数据传入 self.datas 等待更新进数据库
-                if not db_name or not collection_name:
+                if not db_name or not collection_name or '品销宝' in collection_name:
                     # print(f'db_name/collection_name 不能为空')
                     continue
                 self.datas.append(

mdbq/company/copysh.py

@@ -10,22 +10,43 @@ import shutil
 import time
 import re
 import socket
-from dateutil.utils import today
 from mdbq.bdup import bdup
-from mdbq.aggregation import aggregation
 from mdbq.aggregation import query_data
 from mdbq.aggregation import optimize_data
-from mdbq.config import update_conf
-from mdbq.config import get_myconf
 from mdbq.config import set_support
-from mdbq.config import products
+from mdbq.config import myconfig
 from mdbq.mysql import mysql
 from mdbq.pbix import refresh_all
-from mdbq.other import sku_picture
 from mdbq.clean import clean_upload
 warnings.filterwarnings('ignore')
 
 
+if platform.system() == 'Windows':
+    # windows版本
+    Data_Path = r'C:\同步空间\BaiduSyncdisk'
+    D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
+    Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
+elif platform.system() == 'Linux':
+    Data_Path = '数据中心'
+    D_PATH = 'Downloads'
+    if not os.path.exists(D_PATH):
+        os.makedirs(D_PATH)
+    Share_Path = ''  # linux 通常是远程服务器，不需要访问共享
+else:
+    Data_Path = f'/Users/{getpass.getuser()}/数据中心'  # 使用Mac独立网络时
+    D_PATH = os.path.join(f'/Users/{getpass.getuser()}/Downloads')
+    Share_Path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
+
+upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹，将统一上传百度云备份
+source_path = os.path.join(Data_Path, '原始文件3')  # 此目录保存下载并清洗过的文件，作为数据库备份
+if not os.path.exists(upload_path):  # 数据中心根目录
+    os.makedirs(upload_path)
+if not os.path.exists(Data_Path):  # 数据中心根目录
+    os.makedirs(Data_Path)
+if not os.path.exists(source_path):  # 原始文件
+    os.makedirs(source_path)
+
+
 class TbFiles:
     """
     用于在公司台式机中 定时同步pandas数据源文件到共享
@@ -289,31 +310,6 @@ class TbFiles:
             return False
 
 
-class UpdateMysql:
-    def __init__(self):
-        support_path = set_support.SetSupport(dirname='support').dirname
-        self.my_conf = os.path.join(support_path, '.copysh_conf')
-        self.ch_record = False
-        self.d_path = None
-
-    def check_date(self):
-        """ 检查公司台式机 .copysh_conf 文件中的 ch_record 值，决定是否执行更新"""
-        config = configparser.ConfigParser()  # 初始化configparser类
-        try:
-            config.read(self.my_conf, 'UTF-8')
-            self.ch_record = config.get('database', 'ch_record').lower()
-            self.d_path = config.get('database', 'd_path')
-        except Exception as e:
-            print(e)
-        if self.ch_record == 'false':
-            return False, self.d_path
-        elif self.ch_record == 'true':
-            return True, self.d_path
-        else:
-            print(f'配置可能有误: {self.ch_record}, self.ch_record 值应为: true 或 false')
-            return False, self.d_path
-
-
 def op_data(days: int =100):
 
     # 清理数据库， 除了 聚合数据
@@ -343,28 +339,39 @@ def op_data(days: int =100):
 
 
 def main():
+    if socket.gethostname() != 'company':
+        print(f'只可以在 company 运行')
+        return
     t = TbFiles()
-    u = UpdateMysql()
     while True:
-        res, d_path = u.check_date()  # 文件中的 ch_record 值，决定是否执行更新
-        if res:
-            upload_path = f'windows2/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
+        system = platform.system()  # 本机系统
+        host_name = socket.gethostname()  # 本机名
+        conf = myconfig.main()
+        data = conf[system][host_name]
+        is_download = data['is_download']  # 读取配置, 如果是 Ture 则执行更新
+
+        if is_download:
+            bd_remoto_path = f'windows2/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
             b = bdup.BaiDu()
             # 1. 从百度云下载文件
-            b.download_dir(local_path=d_path, remote_path=upload_path)
+            b.download_dir(local_path=upload_path, remote_path=bd_remoto_path)
 
-            # 2. 对文件进行清洗和上传数据库
             clean_upload.main(
-                    service_databases = [{'company': 'mysql'}],
-                    is_mysql = False,  # 清理聚合数据
+                is_mysql=True,  # 调试时加，False: 是否后续的聚合数据及清理
             )
 
-            # 此操作用于修改 .copysh_conf 文件，将 ch_record 改为 false (更新完成)
-            w = update_conf.UpdateConf()
-            w.update_config(filename='.copysh_conf', option='ch_record', new_value='False')
-            time.sleep(60)
-            # 3. 数据清理和聚合
-            op_data(days=100)
+            #
+            # # 3. 数据清理和聚合
+            # op_data(days=100)
+
+            # 此处不可以使用 data 更新，要使用具体键值，否则数据有覆盖
+            conf['Windows']['company'].update(
+                {
+                    'is_download': False  # 更新完成，下次需 all-datas.py 修改或者手动修改
+                }
+            )
+            # print(conf)
+            myconfig.write_back(datas=conf)  # 写回文件生效
 
         t.sleep_minutes = 5  # 同步前休眠时间
         t.tb_file()

{mdbq-2.7.4.dist-info → mdbq-2.7.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.7.4
+Version: 2.7.6
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.7.4.dist-info → mdbq-2.7.6.dist-info}/RECORD

@@ -3,16 +3,16 @@ mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=nn8RjHuiAH68awM893wXX0Bvg0QpsEQ5RXkxFn7H-fs,77219
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
-mdbq/aggregation/mysql_types.py,sha256=VPaDclx4twRZFfv4JYbsuwqzWWFnBLEPr00j1RvXWDI,10937
-mdbq/aggregation/optimize_data.py,sha256=tJU5yVeh6b8vjyBehJJkMxtQ7cjyo3erZtXzpp0q-Uo,3530
-mdbq/aggregation/query_data.py,sha256=MNokoRaoo2FzbcPQWGfa9UJDrbe0xvYdkESBM5RbM8E,98392
+mdbq/aggregation/mysql_types.py,sha256=o8mbhmApd1XOHttbzYtV46SjKZ2NAcIpO1FuwIyBuf4,10920
+mdbq/aggregation/optimize_data.py,sha256=79uwiM2WqNNFxGpE2wKz742PRq-ZGgFjdOV0vgptHdY,3513
+mdbq/aggregation/query_data.py,sha256=QpfIWX197ICqRnRhLZoKgkX9CuWvx_ryfScSaFdDN-Y,98375
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=QdM8vjKuzsR5W3LXP9-BvsplFq1mNwNR7DvyRYc7k_w,82905
+mdbq/clean/clean_upload.py,sha256=Mlt9h31Wa_VHhTG40jWioRqk8ELZxDEHMurLalYAJ-o,83627
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=sisL5eo3D5HGGYvRw46xGqnqFaI3SxfBnoa-Y7zknus,17541
+mdbq/company/copysh.py,sha256=4VOAvHZfnhsKdeKL0Ho6q2x1OkD6nsZXRLyYdu8YAaE,18011
 mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=4Y5zd64hZUFtll8AdpUc2napDas-La-A6XzAhb2mLv0,17157
-mdbq-2.7.4.dist-info/METADATA,sha256=PNcXI9u1qhqqBYgV2y4hBg943DJOxXRvGq6peBK4bow,243
-mdbq-2.7.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.7.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.7.4.dist-info/RECORD,,
+mdbq-2.7.6.dist-info/METADATA,sha256=g9yVclBoUQygTKxK2ciOGKeyaHjDsdw0um9CiD-HA-k,243
+mdbq-2.7.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.7.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.7.6.dist-info/RECORD,,