mdbq 2.7.0__py3-none-any.whl → 2.7.1__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1,5 +1,7 @@
 # -*- coding:utf-8 -*-
 import warnings
+from unittest.mock import inplace
+
 import pandas as pd
 import numpy as np
 import chardet
@@ -1118,7 +1120,7 @@ class DatabaseUpdate:
 def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, target_service='company'):
     """ 上传一个文件夹到 mysql 或者 mongodb 数据库 """
     if not os.path.isdir(path):
-        print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 路径不存在或错误: {path}')
+        print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 函数只接受文件夹路径，不是一个文件夹: {path}')
         return
 
     if dbs['mongodb']:
@@ -1288,29 +1290,40 @@ def file_dir(one_file=True, target_service='company'):
 
 
 def test():
-    path = '/Users/xigua/数据中心/原始文件2/京东报表/JD商品明细spu'
+    path = os.path.relpath(r'C:\Users\Administrator\Downloads\JD商品明细sku')
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
+                print(name)
+                # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                df['最近上架时间'].loc[0] = df['最近上架时间'].loc[1]
-                # print(df[['日期', '最近上架时间']])
+                cols = df.columns.tolist()
+                if '店铺名称' not in cols:
+                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                if '曝光量' in cols:
+                    df.rename(columns={
+                        '曝光量': '搜索曝光量',
+                        '点击次数': '搜索点击次数',
+                        '点击率': '搜索点击率',
+                    }, inplace=True)
+                if '取消金额' in cols:
+                    df.rename(columns={
+                        '取消金额': '取消及售后退款金额',
+                        '取消商品件数': '取消及售后退款件数',
+                        '取消单量': '取消及售后退款单量',
+                    }, inplace=True)
+                if '取消及售后退款金额' not in cols:
+                    df['取消及售后退款金额'] = '0.0'
+                    df['取消及售后退款件数'] = 0
+                    df['取消及售后退款单量'] = 0
                 df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # new_name = f'{os.path.splitext(name)[0]}.xlsx'
+                # df.to_excel(os.path.join(root, name),
+                #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
         #         break
         # break
 
 
-def test2():
-    dp = DatabaseUpdate(path='/Users/xigua/Downloads')
-    dp.new_unzip(is_move=True)
-    dp.cleaning(is_move=False, )  # 清洗数据, 存入 self.datas
-    dp.upload_df(service_databases=[
-        # {'home_lx': 'mongodb'},
-        {'company': 'mysql'},
-        # {'nas': 'mysql'}
-    ], path=None, service_name=None)
-
-
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
@@ -1326,24 +1339,15 @@ if __name__ == '__main__':
     # )
 
     # 上传一个目录到指定数据库
-    db_name = '推广数据2'
-    table_name = '营销场景报表'
+    db_name = '京东数据3'
+    table_name = '京东商智_spu_商品明细'
     upload_dir(
-        path='/Users/xigua/数据中心/原始文件3/天猫推广报表/营销场景报表',
+        path=os.path.relpath(r'C:\同步空间\BaiduSyncdisk\原始文件3\京东报表\京东商智_spu_商品明细'),
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},
-        target_service='company',
+        target_service='home_lx',
     )
 
 
-    # # 新版 数据分类
-    # dp = DatabaseUpdate(path='/Users/xigua/Downloads')
-    # dp.new_unzip(is_move=True)
-    # dp.cleaning(is_move=False)  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # # 将 self.datas 更新至数据库
-    # # dp.upload_df(service_databases=[
-    # #     # {'home_lx': 'mongodb'},
-    # #     {'company': 'mysql'},
-    # #     # {'nas': 'mysql'},
-    # # ])
+    # test()

mdbq/aggregation/query_data.py

@@ -2316,7 +2316,7 @@ def main():
 if __name__ == '__main__':
     data_aggregation(
         service_databases=[{'company': 'mysql'}],
-        months=12,
-        is_juhe=True,  # 立即启动对聚合数据的清理工作
-        # less_dict=['营销场景报表'],  # 单独聚合某一个数据库
+        months=1,
+        is_juhe=False,  # 立即启动对聚合数据的清理工作
+        # less_dict=['生意参谋_直播场次分析'],  # 单独聚合某一个数据库
     )

mdbq/clean/clean_upload.py

@@ -145,6 +145,7 @@ class DataClean:
                     df = pd.read_excel(os.path.join(root, name), header=4)
                     if len(df) == 0:
                         print(f'{name} 报表数据不能为空')
+                        os.remove(os.path.join(root, name))
                         continue
                     df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
                     df.replace(to_replace=[','], value='', regex=True, inplace=True)
@@ -159,6 +160,7 @@ class DataClean:
                     df = pd.read_excel(os.path.join(root, name), header=5, engine='xlrd')
                     if len(df) == 0:
                         print(f'{name} 报表数据不能为空')
+                        os.remove(os.path.join(root, name))
                         continue
                     df.replace(to_replace=['-'], value=0, regex=False, inplace=True)
                     df.replace(to_replace=[','], value='', regex=True, inplace=True)
@@ -255,6 +257,7 @@ class DataClean:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
+                        os.remove(os.path.join(root, name))
                         continue
                     new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
@@ -263,6 +266,7 @@ class DataClean:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
+                        os.remove(os.path.join(root, name))
                         continue
                     for col in df.columns.tolist():
                         if '(' in col or ')' in col:
@@ -563,6 +567,7 @@ class DataClean:
                     name_st = re.findall(r'([\u4e00-\u9fa5]+)\(分日', name)
                     if not name_st:
                         print(f'{name}  正则提取文件名失败')
+                        os.remove(os.path.join(root, name))
                         continue
                     encoding = self.get_encoding(file_path=os.path.join(root, name))
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
@@ -802,8 +807,9 @@ class DataClean:
                 if not is_continue:
                     continue
 
-                if name.endswith('.xlsx') and '京东推广_' in name:
-                    df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
+                if name.endswith('.csv') and '京东推广_' in name:
+                    # df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     new_name = f'py_xg_{name}'
                     os.rename(os.path.join(root, name), os.path.join(root, new_name))
                 elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
@@ -813,9 +819,10 @@ class DataClean:
                     df.insert(loc=0, column='日期', value=pattern)
                     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
                     df.fillna(0, inplace=True)
-                    new_name = f'py_xg_{name}'
-                    df.to_excel(os.path.join(upload_path, new_name),
-                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
+                    # df.to_excel(os.path.join(upload_path, new_name),
+                    #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
                     df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
@@ -824,9 +831,10 @@ class DataClean:
                     df.insert(loc=0, column='日期', value=pattern)
                     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
                     df.fillna(0, inplace=True)
-                    new_name = f'py_xg_{name}'
-                    df.to_excel(os.path.join(upload_path, new_name),
-                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
+                    # df.to_excel(os.path.join(upload_path, new_name),
+                    #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.xlsx') and '京东商智_店铺来源_三级来源' in name:
                     df = pd.read_excel(os.path.join(root, name), header=0, engine='openpyxl')
@@ -836,9 +844,10 @@ class DataClean:
                         if '环比' in col or '同比' in col:
                             df.drop(col, axis=1, inplace=True)
                     df.fillna(0, inplace=True)
-                    new_name = f'py_xg_{name}'
-                    df.to_excel(os.path.join(upload_path, new_name),
-                                index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False, header=True)
+                    # df.to_excel(os.path.join(upload_path, new_name),
+                    #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
                     os.remove(os.path.join(root, name))
 
                 # 将数据传入 self.datas 等待更新进数据库
@@ -1119,10 +1128,10 @@ class DataClean:
                     continue
 
                 if name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
-                    t_path = os.path.join(self.source_path, '京东报表', 'spu_商品明细')
+                    t_path = os.path.join(self.source_path, '京东报表', '京东商智_spu_商品明细')
                     bib(t_path, _as_month=True)
                 elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
-                    t_path = os.path.join(self.source_path, '京东报表', 'sku_商品明细')
+                    t_path = os.path.join(self.source_path, '京东报表', '京东商智_sku_商品明细')
                     bib(t_path, _as_month=True)
                 elif name.endswith('.xlsx') and '京东推广_搜索词' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '搜索词报表')
@@ -1587,7 +1596,7 @@ def main(service_databases=None, is_mysql=False):
     cn.dmp_tm(is_except=['except'])  # 达摩盘
     cn.tg_reports(is_except=['except'])  # 推广报表，天猫淘宝共同清洗
     cn.syj_reports_tm(is_except=['except'])  # 天猫生意经
-    """ 淘宝生意经，不可以和天猫同时运行 """
+    # # 淘宝生意经，不可以和天猫同时运行
     # cn.syj_reports_tb(is_except=['except'])  # 淘宝生意经，不可以和天猫同时运行
     cn.jd_reports(is_except=['except'])  # 清洗京东报表
     cn.sp_scene_clean(is_except=['except'])  # 商品素材
@@ -1646,8 +1655,8 @@ def main(service_databases=None, is_mysql=False):
 if __name__ == '__main__':
     main(
         service_databases = [
-            {'company': 'mysql'},
-            # {'home_lx': 'mysql'},
+            # {'company': 'mysql'},
+            {'home_lx': 'mysql'},
             # {'home_lx': 'mongodb'},
             # {'nas': 'mysql'},
         ],

mdbq/company/copysh.py

@@ -321,7 +321,7 @@ def op_data(days: int =100):
         # 清理所有非聚合数据的库
         optimize_data.op_data(
             db_name_lists=[
-                '京东数据3',
+                '京东数据2',
                 '属性设置3',
                 '推广数据2',
                 '推广数据_淘宝店',
@@ -367,7 +367,6 @@ def main():
             op_data(days=100)
 
         t.sleep_minutes = 5  # 同步前休眠时间
-        # 4. 同步共享文件
         t.tb_file()
         time.sleep(600)  # 检测间隔
 

{mdbq-2.7.0.dist-info → mdbq-2.7.1.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.7.0
-Home-page: https://pypi.org/project/mdbsql
+Version: 2.7.1
+Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com
 License: MIT

{mdbq-2.7.0.dist-info → mdbq-2.7.1.dist-info}/RECORD

@@ -1,18 +1,18 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=v_5VM-InaDDvDNjAy_b8xsc38-vf78GkqoXjoe6MZ8U,76569
+mdbq/aggregation/aggregation.py,sha256=wR_rHSu3srNBZPKng-7c3L_FKAuj6cL7GVwTCOAleH4,77125
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=gdScrgTAb6RbXHZy1LitX7lggMGn1GTLhkYSgztfwew,4903
-mdbq/aggregation/query_data.py,sha256=gqvKDgg3jrfCcI1VudrnQLJgKHUThZVTqS1zOQ5wgMk,102766
+mdbq/aggregation/query_data.py,sha256=m7Y2xSazPYKvy51yPK6n_Izsv5cjV83oHsiNc7N4fyA,102779
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=X5WcWm7kkGZDMpk8p0vMq-SFIcrSL1DmVCYWbxYmLVI,86644
+mdbq/clean/clean_upload.py,sha256=_weFInJnBNZxqErIBHt_10SoMLLT5PIV_j_6n84Q_Y8,87490
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=3ZYm_rTE8nXcbgZlyHsa0y-RAkZ8vcmwkxMy_Jj4F2k,17574
+mdbq/company/copysh.py,sha256=sisL5eo3D5HGGYvRw46xGqnqFaI3SxfBnoa-Y7zknus,17541
 mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -44,7 +44,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=4Y5zd64hZUFtll8AdpUc2napDas-La-A6XzAhb2mLv0,17157
-mdbq-2.7.0.dist-info/METADATA,sha256=i_0WznHsXfCR0sToIhC5S4mv3hv1qMUOFFdws6FocOg,245
-mdbq-2.7.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.7.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.7.0.dist-info/RECORD,,
+mdbq-2.7.1.dist-info/METADATA,sha256=Dtp6f3EYkLh9ML8akrYeEZ0h6qzcdL1XkYfv2CHkHnM,243
+mdbq-2.7.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.7.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.7.1.dist-info/RECORD,,

{mdbq-2.7.0.dist-info → mdbq-2.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any