mdbq 2.6.5__py3-none-any.whl → 2.6.7__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1329,7 +1329,7 @@ if __name__ == '__main__':
     db_name = '京东数据3'
     table_name = '京东商智_spu_商品明细'
     upload_dir(
-        path='/Users/xigua/数据中心/原始文件3/京东报表/spu_商品明细',
+        path='/Users/xigua/数据中心/原始文件3/京东报表/spu_商品明细qwqw',
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},

mdbq/aggregation/query_data.py

@@ -423,7 +423,6 @@ class MysqlDatasQuery:
             '总订单行': 1,
             '总订单金额': 1,
             '总加购数': 1,
-            '下单新客数（去重）': 1,
             '领券数': 1,
             '商品关注数': 1,
             '店铺关注数': 1,
@@ -493,24 +492,26 @@ class MysqlDatasQuery:
         return pd.to_datetime(start_date), pd.to_datetime(end_date)
 
     @try_except
-    def tm_search(self):
+    def se_search(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
             '日期': 1,
-            '关键词': 1,
+            '店铺名称': 1,
+            '搜索词': 1,
+            '词类型': 1,
             '访客数': 1,
+            '加购人数': 1,
+            '商品收藏人数': 1,
             '支付转化率': 1,
-            '支付金额': 1,
-            '下单金额': 1,
             '支付买家数': 1,
-            '下单买家数': 1,
-            '加购人数': 1,
+            '支付金额': 1,
             '新访客': 1,
-            '店铺名称': 1,
+            '客单价': 1,
+            'uv价值': 1,
         }
         df = self.download.data_to_df(
             db_name='生意参谋3',
-            table_name='店铺来源_手淘搜索',
+            table_name='手淘搜索_本店引流词',
             start_date=start_date,
             end_date=end_date,
             projection=projection,
@@ -1462,7 +1463,6 @@ class GroupBy:
                     '总订单行': ('总订单行', np.max),
                     '总订单金额': ('总订单金额', np.max),
                     '总加购数': ('总加购数', np.max),
-                    '下单新客数': ('下单新客数（去重）', np.max),
                     '领券数': ('领券数', np.max),
                     '商品关注数': ('商品关注数', np.max),
                     '店铺关注数': ('店铺关注数', np.max)
@@ -1476,17 +1476,16 @@ class GroupBy:
             return df
         elif '天猫店铺来源_手淘搜索' in table_name:
             df = df.groupby(
-                ['日期', '关键词', '店铺名称'],
+                ['日期', '店铺名称', '词类型', '搜索词'],
                 as_index=False).agg(
                 **{
                     '访客数': ('访客数', np.max),
-                    '支付转化率': ('支付转化率', np.max),
+                    '加购人数': ('加购人数', np.max),
                     '支付金额': ('支付金额', np.max),
-                    '下单金额': ('下单金额', np.max),
+                    '支付转化率': ('支付转化率', np.max),
                     '支付买家数': ('支付买家数', np.max),
-                    '下单买家数': ('下单买家数', np.max),
-                    '加购人数': ('加购人数', np.max),
-                    '新访客': ('新访客', np.max),
+                    '客单价': ('客单价', np.max),
+                    'uv价值': ('uv价值', np.max)
                 }
             )
             return df
@@ -2077,6 +2076,7 @@ def data_aggregation(service_databases=[{}], months=1, is_juhe=True):
     2. 数据聚合清洗
     3. 统一回传数据库: <聚合数据>  （不再导出为文件）
     公司台式机调用
+    months: 1+，写 0 表示当月数据，但在每月 1 号时可能会因为返回空数据出错
     """
     for service_database in service_databases:
         for service_name, database in service_database.items():
@@ -2183,14 +2183,14 @@ def data_aggregation(service_databases=[{}], months=1, is_juhe=True):
                     '数据库名': '聚合数据',
                     '集合名': '天猫店铺来源_手淘搜索',  # 暂缺
                     '唯一主键': ['日期', '关键词', '访客数'],
-                    '数据主体': sdq.tm_search(),
-                },
-                {
-                    '数据库名': '聚合数据',
-                    '集合名': '生意参谋_直播场次分析',  # 暂缺
-                    '唯一主键': ['场次id'],
-                    '数据主体': sdq.zb_ccfx(),
+                    '数据主体': sdq.se_search(),
                 },
+                # {
+                #     '数据库名': '聚合数据',
+                #     '集合名': '生意参谋_直播场次分析',  # 暂缺
+                #     '唯一主键': ['场次id'],
+                #     '数据主体': sdq.zb_ccfx(),
+                # },
                 {
                     '数据库名': '聚合数据',
                     '集合名': '多店推广场景_按日聚合',
@@ -2304,6 +2304,6 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=0, is_juhe=True)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=24, is_juhe=False)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/clean/clean_upload.py

@@ -151,7 +151,7 @@ class DataClean:
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.xls') and '手淘搜索_本店引流词_' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=5)
+                    df = pd.read_excel(os.path.join(root, name), header=5, engine='xlrd')
                     if len(df) == 0:
                         print(f'{name} 报表数据不能为空')
                         continue
@@ -382,7 +382,7 @@ class DataClean:
                     sheets4 = ['账户', '推广计划', '推广单元', '创意', '品牌流量包', '定向人群']  # 品销宝
                     file_name4 = os.path.splitext(name)[0]  # 明星店铺报表
                     for sheet4 in sheets4:
-                        df = pd.read_excel(os.path.join(root, name), sheet_name=sheet4, header=0, engine='openpyxl')
+                        df = pd.read_excel(os.path.join(root, name), sheet_name=sheet4, header=0, engine='xlrd')
                         if len(df) == 0:
                             print(f'{name} 报表数据为空')
                             os.remove(os.path.join(root, name))
@@ -765,11 +765,11 @@ class DataClean:
                     continue
 
                 if name.endswith('.xlsx') and '京东推广_' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                     new_name = f'py_xg_{name}'
                     os.rename(os.path.join(root, name), os.path.join(root, new_name))
                 elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                     df.replace(to_replace=['-'], value='', regex=False, inplace=True)
                     pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
                     df.insert(loc=0, column='日期', value=pattern)
@@ -780,7 +780,7 @@ class DataClean:
                                 index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                     df.replace(to_replace=['-'], value='', regex=False, inplace=True)
                     pattern = re.findall(r'_(\d{4}-\d{2}-\d{2})', name)[0]
                     df.insert(loc=0, column='日期', value=pattern)
@@ -791,7 +791,7 @@ class DataClean:
                                 index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.xlsx') and '京东商智_店铺来源_三级来源' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
                     df.replace(to_replace=['-'], value='', regex=False, inplace=True)
                     df.rename(columns={'时间': '日期'}, inplace=True)
                     for col in df.columns.tolist():
@@ -870,7 +870,7 @@ class DataClean:
 
                 if name.endswith('.xlsx') and '商品素材_' in name:
                     shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
-                    df = pd.read_excel(os.path.join(root, name), header=0)
+                    df = pd.read_excel(os.path.join(root, name), header=0, engine='xlrd')
                     if '日期' not in df.columns.tolist():
                         df.insert(loc=0, column='日期', value=datetime.datetime.today().strftime('%Y-%m-%d'))
                     if '店铺名称' not in df.columns.tolist():
@@ -1276,7 +1276,7 @@ class DataClean:
                             new_path = os.path.join(root, zip_name_1)  # 拼接解压后的文件路径
                             if os.path.isfile(new_path) and '全部渠道_商品明细' in new_path:  # 是否存在和包内同名的文件
                                 # 专门处理京东文件, 已过期可删
-                                df = pd.read_excel(new_path)
+                                df = pd.read_excel(new_path, engine='xlrd')
                                 try:
                                     pattern1 = re.findall(r'\d{8}_(\d{4})(\d{2})(\d{2})_全部渠道_商品明细',
                                                           name)
@@ -1466,6 +1466,57 @@ def test():
                 #     print(e)
 
 
+def date_table(service_databases=[{}]):
+    """
+    生成 pbix 使用的日期表
+    """
+    start_date = '2022-01-01'  # 日期表的起始日期
+    yesterday = time.strftime('%Y-%m-%d', time.localtime(time.time() - 86400))
+    dic = pd.date_range(start=start_date, end=yesterday)
+    df = pd.DataFrame(dic, columns=['日期'])
+    df.sort_values('日期', ascending=True, ignore_index=True, inplace=True)
+    df.reset_index(inplace=True)
+    # inplace 添加索引到 df
+    p = df.pop('index')
+    df['月2'] = df['日期']
+    df['月2'] = df['月2'].dt.month
+    df['日期'] = df['日期'].dt.date  # 日期格式保留年月日，去掉时分秒
+    df['年'] = df['日期'].apply(lambda x: str(x).split('-')[0] + '年')
+    df['月'] = df['月2'].apply(lambda x: str(x) + '月')
+    # df.drop('月2', axis=1, inplace=True)
+    mon = df.pop('月2')
+    df['日'] = df['日期'].apply(lambda x: str(x).split('-')[2])
+    df['年月'] = df.apply(lambda x: x['年'] + x['月'], axis=1)
+    df['月日'] = df.apply(lambda x: x['月'] + x['日'] + '日', axis=1)
+    df['第n周'] = df['日期'].apply(lambda x: x.strftime('第%W周'))
+    df['索引'] = p
+    df['月索引'] = mon
+    df.sort_values('日期', ascending=False, ignore_index=True, inplace=True)
+
+    for service_database in service_databases:
+        for service_name, database in service_database.items():
+            username, password, host, port = get_myconf.select_config_values(
+                target_service=service_name,
+                database=database,
+            )
+            m = mysql.MysqlUpload(
+                username=username,
+                password=password,
+                host=host,
+                port=port,
+            )
+            m.df_to_mysql(
+                df=df,
+                db_name='聚合数据',
+                table_name='日期表',
+                move_insert=True,  # 先删除，再插入
+                df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                filename=None,  # 用来追踪处理进度
+                service_database=service_database,  # 用来追踪处理进度
+            )
+
+
 def main(service_databases=None, is_mysql=False):
     """
     is_mysql: 调试时加，False: 是否后续的聚合数据
@@ -1505,6 +1556,8 @@ def main(service_databases=None, is_mysql=False):
     if not is_mysql:
         return
 
+    # 更新日期表
+    date_table(service_databases=service_databases)
     # 更新货品年份基准表， 属性设置 2 - 货品年份基准
     p = products.Products()
     p.to_mysql(service_databases=service_databases)
@@ -1544,16 +1597,16 @@ def main(service_databases=None, is_mysql=False):
 
 
 if __name__ == '__main__':
-    main(
-        service_databases = [
-            {'company': 'mysql'},
-            # {'home_lx': 'mysql'},
-            # {'home_lx': 'mongodb'},
-            # {'nas': 'mysql'},
-        ],
-        is_mysql = False,  # 清理聚合数据
-    )
-
+    # main(
+    #     service_databases = [
+    #         {'company': 'mysql'},
+    #         # {'home_lx': 'mysql'},
+    #         # {'home_lx': 'mongodb'},
+    #         # {'nas': 'mysql'},
+    #     ],
+    #     is_mysql = False,  # 清理聚合数据
+    # )
+    date_table(service_databases=[{'company': 'mysql'}])
     # c = DataClean(
     #     path=upload_path,  # 源文件目录，下载文件夹
     #     source_path=source_path3,  # 原始文件保存目录
@@ -1564,3 +1617,4 @@ if __name__ == '__main__':
 
 
     # test()
+

mdbq/company/copysh.py

@@ -22,6 +22,7 @@ from mdbq.config import products
 from mdbq.mysql import mysql
 from mdbq.pbix import refresh_all
 from mdbq.other import sku_picture
+from mdbq.clean import clean_upload
 warnings.filterwarnings('ignore')
 
 
@@ -338,13 +339,16 @@ def op_data(days: int =100):
         # 清理所有非聚合数据的库
         optimize_data.op_data(
             db_name_lists=[
-                '京东数据2',
+                '京东数据3',
+                '属性设置3',
                 '推广数据2',
-                '市场数据2',
-                '生意参谋2',
+                '生意参谋3',
+                '推广数据_淘宝店',
+                '爱库存2'
+                '生意参谋3',
                 '生意经2',
-                '属性设置2',
                 # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+                '达摩盘3',
             ],
             days=days,
         )
@@ -363,34 +367,17 @@ def main():
     while True:
         res, d_path = u.check_date()  # 文件中的 ch_record 值，决定是否执行更新
         if res:
-            upload_path = f'windows/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
+            upload_path = f'windows2/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
             b = bdup.BaiDu()
+            # 从百度云下载文件
             b.download_dir(local_path=d_path, remote_path=upload_path)
 
-            dp = aggregation.DatabaseUpdate(path=d_path)
-            dp.new_unzip(is_move=True)
-            dp.cleaning(is_move=True, is_except=[])  # 公司台式机需要移除自身下载的文件
-            dp.upload_df(service_databases=[{'company': 'mysql'}])
-            dp.date_table(service_databases=[{'company': 'mysql'}])  # 因为日期表不受 days 参数控制，因此单独更新日期表
-            dp.other_table(service_databases=[{'company': 'mysql'}])  # 上传 support 文件夹下的 主推商品.csv
-            # 更新货品年份基准表， 属性设置 2 - 货品年份基准
-            p = products.Products()
-            p.to_mysql(service_databases=[
-                # {'home_lx': 'mysql'},
-                {'company': 'mysql'}
-            ]
+            # 对文件进行清洗和上传数据库
+            clean_upload.main(
+                    service_databases = [{'company': 'mysql'}],
+                    is_mysql = False,  # 清理聚合数据
             )
 
-            if datetime.datetime.now().day in [1, 3, 7, 9, 12, 16, 19, 22, 25, 27]:
-                sku_picture.download_spu(
-                    service_name='company',
-                    database='mysql',
-                    db_name='属性设置2',
-                    table_name='商品spu素材下载记录',
-                    col_name='商品图片',
-                    save_path=os.path.join(f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\天猫报表\\其他文件', '商品id_商家编码_图片'),
-                )
-
             # 此操作用于修改 .copysh_conf 文件，将 ch_record 改为 false (更新完成)
             w = update_conf.UpdateConf()
             w.update_config(filename='.copysh_conf', option='ch_record', new_value='False')

mdbq/company/copysh_bak.py

@@ -0,0 +1,417 @@
+# -*- coding: UTF-8 –*-
+import os
+import platform
+import warnings
+import getpass
+import sys
+import configparser
+import datetime
+import shutil
+import time
+import re
+import socket
+from dateutil.utils import today
+from mdbq.bdup import bdup
+from mdbq.aggregation import aggregation
+from mdbq.aggregation import query_data
+from mdbq.aggregation import optimize_data
+from mdbq.config import update_conf
+from mdbq.config import get_myconf
+from mdbq.config import set_support
+from mdbq.config import products
+from mdbq.mysql import mysql
+from mdbq.pbix import refresh_all
+from mdbq.other import sku_picture
+warnings.filterwarnings('ignore')
+
+
+class TbFiles:
+    """
+    用于在公司台式机中 定时同步pandas数据源文件到共享
+    """
+    def __init__(self):
+
+        support_path = set_support.SetSupport(dirname='support').dirname
+
+        self.my_conf = os.path.join(support_path, '.copysh_conf')
+        self.path1 = os.path.join(support_path, 'tb_list.txt')
+        self.path2 = os.path.join(support_path, 'cp_list.txt')
+        self.d_path = None
+        self.data_path = None
+        self.share_path = None
+        self.before_max_time = []
+        self.sleep_minutes = 30
+        self.tomorrow = datetime.date.today()
+
+    def check_change(self):
+        """ 检查 source_path 的所有文件修改日期, 函数返回最新修改日期 """
+        source_path = os.path.join(self.data_path, 'pandas数据源')
+        if not os.path.exists(source_path):
+            return
+        results = []
+        for root, dirs, files in os.walk(source_path, topdown=False):
+            for name in files:
+                if '~$' in name or 'baiduyun' in name or name.startswith('.') or 'Icon' in name or 'xunlei' in name:
+                    continue  # 排除这些文件的变动
+                # stat_info = os.path.getmtime(os.path.join(root, name))
+                _c = os.stat(os.path.join(root, name)).st_mtime  # 读取文件的元信息 >>>文件修改时间
+                c_time = datetime.datetime.fromtimestamp(_c)  # 格式化修改时间
+                results.append(c_time)
+        return max(results).strftime('%Y%m%d%H%M%S')
+
+    def check_conf(self):
+        if not os.path.isfile(self.my_conf):
+            self.set_conf()  # 添加配置文件
+            print('因缺少配置文件, 已自动初始化')
+        config = configparser.ConfigParser()  # 初始化configparser类
+        try:
+            config.read(self.my_conf, 'UTF-8')
+            self.d_path = config.get('database', 'd_path')
+            self.data_path = config.get('database', 'data_path')
+            self.share_path = config.get('database', 'share_path')
+            if self.d_path is None or self.data_path is None or self.share_path is None:
+                self.set_conf()
+                print('配置文件部分值不完整, 已自动初始化')
+            if not os.path.exists(self.d_path) or not os.path.exists(self.data_path) or not os.path.exists(self.share_path):
+                self.set_conf()
+                print('配置文件异常(可能跨系统), 已自动初始化')
+        except Exception as e:
+            print(e)
+            print('配置文件部分值缺失, 已自动初始化')
+            self.set_conf()
+        sys.path.append(self.share_path)
+
+    def set_conf(self):
+        if platform.system() == 'Windows':
+            self.d_path = os.path.join('C:\\Users', getpass.getuser(), 'Downloads')
+            self.data_path = os.path.join('C:\\同步空间', 'BaiduSyncdisk')
+            self.share_path = os.path.join('\\\\192.168.1.198', '时尚事业部\\01.运营部\\天猫报表')  # 共享文件根目录
+        elif platform.system() == 'Darwin':
+            self.d_path = os.path.join('/Users', getpass.getuser(), 'Downloads')
+            self.data_path = os.path.join('/Users', getpass.getuser(), '数据中心')
+            self.share_path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
+        else:
+            self.d_path = 'Downloads'
+            self.data_path = os.path.join(getpass.getuser(), '数据中心')
+            self.share_path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
+
+        if not os.path.exists(self.share_path):
+            self.share_path = re.sub('时尚事业部', '时尚事业部-1', self.share_path)
+
+        with open(self.my_conf, 'w+', encoding='utf-8') as f:
+            f.write('[database]\n')
+            f.write(f'# 配置文件\n')
+            f.write('# 下载目录\n')
+            f.write(f'd_path = {self.d_path}\n\n')
+            f.write('# 数据中心目录\n')
+            f.write(f'data_path = {self.data_path}\n\n')
+            f.write('# 共享目录\n')
+            f.write(f'share_path = {self.share_path}\n\n')
+            f.write('# 公司台式机中，用于触发下载百度云文件，更新至本机数据库\n')
+            f.write(f'ch_record = False\n\n')
+        print('目录初始化!')
+
+    def tb_file(self):
+
+        self.check_conf()  # 检查配置文件
+
+        now_max_time = self.check_change()
+        if now_max_time in  self.before_max_time:
+            return  # 不更新
+        else:
+            self.before_max_time = []  # 重置变量，以免越来越占内存
+            self.before_max_time.append(now_max_time)
+
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
+        res = self.check_upload_mysql()
+        if not res:
+            print(f'检测到源文件修改, 但今日已经同步过, 不再同步')
+            return
+        print(f'{now}pandas数据源文件修改, 触发同步 ({self.sleep_minutes}分钟后开始)')
+
+        if not os.path.exists(self.data_path):
+            print(f'{self.data_path}: 本地目录不存在或配置文件异常, 无法同步此目录')
+            return None
+        if not os.path.exists(self.share_path):
+            print(f'{self.share_path}: 本机未连接共享或配置文件异常, 无法同步')
+            return None
+
+        time.sleep(self.sleep_minutes*60)  # 开始同步前休眠时间
+        recent_time = 48  # 同步近N小时内更新过的文件，单位：小时
+        tb_list = []
+        pd_list = []
+        try:
+            with open(self.path1, 'r', encoding='utf-8') as f:
+                content = f.readlines()
+                content = [item.strip() for item in content if not item.strip().startswith('#')]
+                tb_list = [item for item in content if item]
+
+            with open(self.path2, 'r', encoding='utf-8') as f:
+                content = f.readlines()
+                content = [item.strip() for item in content if not item.strip().startswith('#')]
+                pd_list = [item for item in content if item]
+        except Exception as e:
+            print(e)
+
+        source_path = os.path.join(self.data_path, 'pandas数据源')  # \BaiduSyncdisk\pandas数据源
+        target_path = os.path.join(self.share_path, 'pandas数据源')  # \01.运营部\天猫报表\pandas数据源
+
+        if not os.path.exists(target_path):  # 检查共享主目录,创建目录
+            os.makedirs(target_path, exist_ok=True)
+
+        # 删除共享的副本
+        file_list = os.listdir(self.share_path)
+        for file_1 in file_list:
+            if '副本_' in file_1 or 'con' in file_1:  # or '.DS' in file_1
+                try:
+                    os.remove(os.path.join(self.share_path, file_1))
+                    print(f'移除: {os.path.join(self.share_path, file_1)}')
+                except Exception as e:
+                    print(e)
+                    print(f'移除失败：{os.path.join(self.share_path, file_1)}')
+        file_list2 = os.listdir(target_path)  # 删除乱七八糟的临时文件
+        for file_1 in file_list2:
+            if '.DS' in file_1 or 'con' in file_1:
+                try:
+                    os.remove(os.path.join(target_path, file_1))
+                    print(f'移除: {os.path.join(target_path, file_1)}')
+                except Exception as e:
+                    print(e)
+
+        # 删除 run_py的 副本
+        del_p = os.path.join(self.data_path, '自动0备份', 'py', '数据更新', 'run_py')
+        for file_1 in os.listdir(del_p):
+            if '副本_' in file_1:
+                try:
+                    os.remove(os.path.join(del_p, file_1))
+                    print(f'移除: {os.path.join(del_p, file_1)}')
+                except Exception as e:
+                    print(e)
+                    print(f'移除失败：{os.path.join(del_p, file_1)}')
+
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        print(f'{now} 正在同步文件...')
+        # 复制 run_py的文件到共享
+        for file_1 in tb_list:
+            s = os.path.join(del_p, file_1)
+            t = os.path.join(self.share_path, file_1)
+            try:
+                shutil.copy2(s, t)
+                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
+                print(f'{now}复制: {s}')
+            except Exception as e:
+                print(e)
+                s1 = os.path.join(del_p, f'副本_{file_1}')
+                t1 = os.path.join(self.share_path, f'副本_{file_1}')
+                shutil.copy2(s, s1)  # 创建副本
+                shutil.copy2(s1, t1)  # 复制副本到共享
+                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
+                print(f'{now}已创建副本 -->> {s1}')
+
+        # 同步 pandas 文件到共享
+        now_time = time.time()
+        for filenames in pd_list:
+            src = os.path.join(source_path, filenames)  # 原位置，可能是文件或文件夹
+            dst = os.path.join(target_path, filenames)  # 目标位置，可能是文件或文件夹
+            if os.path.isdir(src):  # 如果是文件夹
+                for root, dirs, files in os.walk(src, topdown=False):
+                    for name in files:
+                        if '~$' in name or 'DS_Store' in name:
+                            continue
+                        if name.endswith('csv') or name.endswith('xlsx') or name.endswith('pbix') or name.endswith(
+                                'xls'):
+                            new_src = os.path.join(root, name)
+                            # share_path = dst + '\\' + new_src.split(src)[1]  # 拼接目标路径
+                            share_path = os.path.join(f'{dst}{new_src.split(src)[1]}')  # 拼接目标路径
+                            ls_paths = os.path.dirname(os.path.abspath(share_path))  # 获取上级目录，用来创建
+                            if not os.path.exists(ls_paths):  # 目录不存在则创建
+                                os.makedirs(ls_paths, exist_ok=True)
+                            c_stat = os.stat(new_src).st_mtime  # 读取文件的元信息 >>>文件修改时间
+                            if now_time - c_stat < recent_time * 3600:  # 仅同步近期更新的文件
+                                # res_name = os.path.basename(new_src)
+                                try:
+                                    shutil.copy2(new_src, share_path)
+                                    now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
+                                    print(f'{now}复制文件: {new_src}')
+                                except Exception as e:
+                                    print(e)
+            elif os.path.isfile(src) and 'DS_Store' not in src:  # 如果是文件
+                if src.endswith('csv') or src.endswith('xlsx') or src.endswith('pbix') or src.endswith('xls'):
+                    c_stat = os.stat(src).st_mtime  # 读取文件的元信息 >>>文件修改时间
+                    if now_time - c_stat < recent_time * 3600:
+                        ls_paths = os.path.dirname(os.path.abspath(src))  # 获取上级目录，用来创建
+                        if not os.path.exists(ls_paths):  # 目录不存在则创建
+                            os.makedirs(ls_paths, exist_ok=True)
+                        # new_name = os.path.basename(src)
+                        try:
+                            shutil.copy2(src, dst)
+                            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ')
+                            print(f'{now}复制文件: {src}')
+                        except Exception as e:
+                            print(e)
+            else:
+                print(f'{src} 所需同步的文件不存在，请检查：pd_list参数')
+
+        # 刷新共享位置的指定文件/文件夹
+        excel_path = os.path.join(self.share_path, 'EXCEL报表')
+        files = os.listdir(excel_path)
+        files = [f'{excel_path}\\{item}' for item in files if item.endswith('.xlsx') or item.endswith('.xls')]
+        r = refresh_all.RefreshAll()
+        for file in files:
+            if '~' in file or 'DS_Store' in file or 'baidu' in file or 'xunlei' in file:
+                continue
+            if file.endswith('.xlsx') or file.endswith('.xls'):
+                r.refresh_excel(file=file)
+            time.sleep(5)
+
+        # 临时加的
+        # excel_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\0-WLM_运营周报-1012输出.xlsx'
+        dir_files = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新'
+        files = os.listdir(dir_files)
+        for file in files:
+            if file.endswith('.xlsx') and file.startswith('0-WLM_运营周报') and '~' not in file and 'baidu' not in file:
+                excel_file = os.path.join(dir_files, file)
+                r.refresh_excel(file=excel_file)
+
+        self.before_max_time = self.check_change()  # 重置值, 避免重复同步
+
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        print(f'{now} 同步完成！')
+
+    def check_upload_mysql(self):
+        # 每天只更新一次
+        today = datetime.date.today()
+        if today == self.tomorrow:
+            self.tomorrow = today + datetime.timedelta(days=1)
+            return True
+        else:
+            return False
+
+
+class UpdateMysql:
+    def __init__(self):
+        support_path = set_support.SetSupport(dirname='support').dirname
+        self.my_conf = os.path.join(support_path, '.copysh_conf')
+        self.ch_record = False
+        self.d_path = None
+
+    def check_date(self):
+        """ 检查公司台式机 .copysh_conf 文件中的 ch_record 值，决定是否执行更新"""
+        config = configparser.ConfigParser()  # 初始化configparser类
+        try:
+            config.read(self.my_conf, 'UTF-8')
+            self.ch_record = config.get('database', 'ch_record').lower()
+            self.d_path = config.get('database', 'd_path')
+        except Exception as e:
+            print(e)
+        if self.ch_record == 'false':
+            return False, self.d_path
+        elif self.ch_record == 'true':
+            return True, self.d_path
+        else:
+            print(f'配置可能有误: {self.ch_record}, self.ch_record 值应为: true 或 false')
+            return False, self.d_path
+
+
+def op_data(days: int =100):
+
+    # 清理数据库， 除了 聚合数据
+    if socket.gethostname() == 'company':  # 公司台式机自身运行
+        # # Mysql
+        # username, password, host, port = get_myconf.select_config_values(
+        #     target_service='company',
+        #     database='mysql',
+        # )
+        # s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        # s.db_name_lists = [
+        #     '京东数据2',
+        #     '推广数据2',
+        #     '市场数据2',
+        #     '生意参谋2',
+        #     '生意经2',
+        #     '属性设置2',
+        #     # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+        # ]
+        # s.days = days
+        # s.optimize_list()
+
+        # 清理所有非聚合数据的库
+        optimize_data.op_data(
+            db_name_lists=[
+                '京东数据2',
+                '推广数据2',
+                '市场数据2',
+                '生意参谋2',
+                '生意经2',
+                '属性设置2',
+                # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+            ],
+            days=days,
+        )
+
+        # 数据聚合
+        query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=3,)
+        time.sleep(60)
+
+        # 清理聚合数据
+        optimize_data.op_data(db_name_lists=['聚合数据'], days=3650, )
+
+
+def main():
+    t = TbFiles()
+    u = UpdateMysql()
+    while True:
+        res, d_path = u.check_date()  # 文件中的 ch_record 值，决定是否执行更新
+        if res:
+            upload_path = f'windows/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
+            b = bdup.BaiDu()
+            b.download_dir(local_path=d_path, remote_path=upload_path)
+
+            dp = aggregation.DatabaseUpdate(path=d_path)
+            dp.new_unzip(is_move=True)
+            dp.cleaning(is_move=True, is_except=[])  # 公司台式机需要移除自身下载的文件
+            dp.upload_df(service_databases=[{'company': 'mysql'}])
+            dp.date_table(service_databases=[{'company': 'mysql'}])  # 因为日期表不受 days 参数控制，因此单独更新日期表
+            dp.other_table(service_databases=[{'company': 'mysql'}])  # 上传 support 文件夹下的 主推商品.csv
+            # 更新货品年份基准表， 属性设置 2 - 货品年份基准
+            p = products.Products()
+            p.to_mysql(service_databases=[
+                # {'home_lx': 'mysql'},
+                {'company': 'mysql'}
+            ]
+            )
+
+            if datetime.datetime.now().day in [1, 3, 7, 9, 12, 16, 19, 22, 25, 27]:
+                sku_picture.download_spu(
+                    service_name='company',
+                    database='mysql',
+                    db_name='属性设置2',
+                    table_name='商品spu素材下载记录',
+                    col_name='商品图片',
+                    save_path=os.path.join(f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\天猫报表\\其他文件', '商品id_商家编码_图片'),
+                )
+
+            # 此操作用于修改 .copysh_conf 文件，将 ch_record 改为 false (更新完成)
+            w = update_conf.UpdateConf()
+            w.update_config(filename='.copysh_conf', option='ch_record', new_value='False')
+            time.sleep(60)
+            op_data(days=100)  # 数据清理和聚合
+
+        t.sleep_minutes = 5  # 同步前休眠时间
+        t.tb_file()
+        time.sleep(600)  # 检测间隔
+
+
+if __name__ == '__main__':
+    main()
+    # # 聚合数据，并清理聚合数据
+    # query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=1)
+
+    # sku_picture.download_spu(
+    #     service_name='company',
+    #     database='mysql',
+    #     db_name='属性设置2',
+    #     table_name='商品spu素材下载记录',
+    #     col_name='商品图片',
+    #     save_path=os.path.join(f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\天猫报表\\其他文件', '商品id_商家编码_图片'),
+    # )

mdbq/dataframe/converter.py

@@ -37,6 +37,7 @@ class DataFrameConverter(object):
         df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
         df.replace(to_replace=['"'], value='', regex=True, inplace=True)
         cols = df.columns.tolist()
+
         df.reset_index(inplace=True, drop=True)  # 重置索引，避免下面的 df.loc[0, col] 会出错
 
         for col in cols:
@@ -81,7 +82,9 @@ class DataFrameConverter(object):
                         df[col] = df[col].apply(lambda x: pd.to_datetime(x))
                     except:
                         pass
-            new_col = col.lower()
+            new_col = re.sub(r'[（）()-,，$%&~^、* ]', '_', col.lower())
+            new_col = re.sub(r'_{2,}', '_', new_col)
+            new_col = re.sub(r'_+$', '', new_col)
             df.rename(columns={col: new_col}, inplace=True)
         df.fillna(0, inplace=True)
         return df

{mdbq-2.6.5.dist-info → mdbq-2.6.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.6.5
+Version: 2.6.7
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.6.5.dist-info → mdbq-2.6.7.dist-info}/RECORD

@@ -1,18 +1,19 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=5WnLHNResPSMNNFYqt2trvw3PQM3XCHQD-XMZkfMYBM,76602
+mdbq/aggregation/aggregation.py,sha256=aAAYq3-I4dqqXFFGwznihDl9ELajfi1NTIFdPFJ0Z_0,76606
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=gdScrgTAb6RbXHZy1LitX7lggMGn1GTLhkYSgztfwew,4903
-mdbq/aggregation/query_data.py,sha256=0NGYmfl1klQryriHu4V6_Twi9WPERHbl56X3kUqmZaY,102619
+mdbq/aggregation/query_data.py,sha256=Ob5PHdAzYF4gcZ85YX4R91-XKB53Gs48OKzAXGNK-6M,102603
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=I9aJL-674ISOi5ZAbeGViRRKlcMW2bXQ2TGGBOQvzh4,81148
+mdbq/clean/clean_upload.py,sha256=js0lngM43eAFUQ_J3RvqE4HFMO-9VAWTUbNY1H98LZo,83703
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
+mdbq/company/copysh.py,sha256=6RR2wbpUXHCrbdPzBJZOvPx_mhWWCtqeZwZ7x5B-r5s,18781
+mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
@@ -20,7 +21,7 @@ mdbq/config/products.py,sha256=hN9UMkM6j76HYMulTYdtr3mOhh9QdpvvrLH14a_mbFY,5980
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=KNHxk3dNw1ycOpcnTg83yHrV9B3pvoYwK3Wc_bzk2NE,4314
+mdbq/dataframe/converter.py,sha256=3n3_FKBxv7bFWeRcmv9CfiApFXuvvbRwZxTwR-SLGzU,4461
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
@@ -42,7 +43,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=4Y5zd64hZUFtll8AdpUc2napDas-La-A6XzAhb2mLv0,17157
-mdbq-2.6.5.dist-info/METADATA,sha256=mUyIb-qC1-GsTA2eIp_1_-oUUkJa8rXo9eLDw9PmUko,245
-mdbq-2.6.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.6.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.6.5.dist-info/RECORD,,
+mdbq-2.6.7.dist-info/METADATA,sha256=85v7mF0rMpWyIE8IJqMd8h8LUXHctsnE6qcQWhl4ENs,245
+mdbq-2.6.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.6.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.6.7.dist-info/RECORD,,