mdbq 2.7.2__tar.gz → 2.7.4__tar.gz

{mdbq-2.7.2 → mdbq-2.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.7.2
+Version: 2.7.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.7.2 → mdbq-2.7.4}/mdbq/aggregation/aggregation.py

@@ -1290,24 +1290,35 @@ def file_dir(one_file=True, target_service='company'):
 
 
 def test():
-    path = os.path.relpath(r'C:\同步空间\BaiduSyncdisk\原始文件3\京东报表\京准通_全sdfsdf站营销')
+    path = os.path.relpath(r'/Users/xigua/数据中心/原始文件3/达摩盘/dmp人群报表')
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
                 # print(name)
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                print(name, len(df))
+                print(name)
                 if len(df) == 0:
                     print(name)
                     os.remove(os.path.join(root, name))
                     continue
-                cols = df.columns.tolist()
-                if '店铺名称' not in cols:
-                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                if '全站roi' in cols:
-                    df.rename(columns={'全站roi': '全站投产比'}, inplace=True)
+                df = df[df['日期'] != '']
+                # cols = df.columns.tolist()
+                # if '千次展现花费' not in cols:
+                #     df.insert(loc=14, column='千次展现花费', value=0.0)
+                    # df.rename(columns={'总费用': '花费', '直接购物车数': '直接加购数', '总购物车数': '总加购数'}, inplace=True)
+                # if '类目ID' not in cols:
+                #     df['类目ID'] = 'null'
+                # if '类目名称' not in cols:
+                #     df['类目名称'] = 'null'
+                # if '店铺名称' not in cols:
+                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                # if '全站roi' in cols:
+                #     df.rename(columns={'全站roi': '全站投产比'}, inplace=True)
+                # new_name = f'{os.path.splitext(name)[0]}.csv'
                 df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # breakpoint()
+                # os.remove(os.path.join(root, name))
                 # new_name = f'{os.path.splitext(name)[0]}.xlsx'
                 # df.to_excel(os.path.join(root, name),
                 #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
@@ -1328,16 +1339,16 @@ if __name__ == '__main__':
     #     target_service='home_lx',
     #     database='mysql'
     # )
-
+    #
     # 上传一个目录到指定数据库
-    db_name = '推广数据2'
-    table_name = '品销宝'
+    db_name = '京东数据3'
+    table_name = '京东商智_店铺来源'
     upload_dir(
-        path=os.path.relpath(r'C:\同步空间\BaiduSyncdisk\原始文件3\天猫推广报表\品销宝'),
+        path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/京东报表/店铺来源_三级来asdasdas源'),
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},
-        target_service='home_lx',
+        target_service='company',
     )
 
     # test()

{mdbq-2.7.2 → mdbq-2.7.4}/mdbq/aggregation/optimize_data.py

@@ -10,12 +10,13 @@ import platform
 """
 对指定数据库所有冗余数据进行清理
 """
+username, password, host, port, service_database = None, None, None, None, None,
 if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
     conf = myconfig.main()
     data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'Mac2.local']:
+elif socket.gethostname().lower() in ['company', 'mac2.local']:
     conf = myconfig.main()
     data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
@@ -73,7 +74,7 @@ def restart_mongodb():
 def op_data(db_name_lists, days: int = 63, is_mongo=True, is_mysql=True):
     """ """
 
-    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
+    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'mac2.local':
         # Mysql
         if is_mysql:
             s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)

{mdbq-2.7.2 → mdbq-2.7.4}/mdbq/aggregation/query_data.py

@@ -25,13 +25,13 @@ import os
 3.  在 data_aggregation 类中添加 data_dict 字典键值，回传数据到数据库
 
 """
-
+username, password, host, port, service_database = None, None, None, None, None,
 if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
     conf = myconfig.main()
     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'Mac2.local']:
+elif socket.gethostname().lower() in ['company', 'mac2.local']:
     conf = myconfig.main()
     conf_data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']

{mdbq-2.7.2 → mdbq-2.7.4}/mdbq/clean/clean_upload.py

@@ -45,12 +45,13 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 # source_path = os.path.join(Data_Path, '原始文件2')  # 此目录保存下载并清洗过的文件，作为数据库备份
 source_path3 = os.path.join(Data_Path, '原始文件3')  # 此目录保存下载并清洗过的文件，作为数据库备份
 
+username, password, host, port, service_database = None, None, None, None, None,
 if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
     conf = myconfig.main()
     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'Mac2.local']:
+elif socket.gethostname().lower() in ['company', 'mac2.local']:
     conf = myconfig.main()
     conf_data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
@@ -278,6 +279,7 @@ class DataClean:
                     os.remove(os.path.join(root, name))
                 elif name.endswith('.csv') and 'dmp人群报表_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    df = df[df['日期'] != '']
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         os.remove(os.path.join(root, name))
@@ -1140,22 +1142,22 @@ class DataClean:
                 if 'py_xg' not in name:  # 排除非目标文件
                     continue
 
-                if name.endswith('.xlsx') and '京东商智_spu_商品明细' in name:
+                if name.endswith('.csv') and '京东商智_spu_商品明细' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '京东商智_spu_商品明细')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.xlsx') and '京东商智_sku_商品明细' in name:
+                elif name.endswith('.csv') and '京东商智_sku_商品明细' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '京东商智_sku_商品明细')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.xlsx') and '京东推广_搜索词' in name:
+                elif name.endswith('.csv') and '京东推广_搜索词' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '搜索词报表')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.xlsx') and '京东推广_点击成交' in name:
+                elif name.endswith('.csv') and '京东推广_点击成交' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '推广报表')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.xlsx') and '京东推广_关键词点击' in name:
+                elif name.endswith('.csv') and '京东推广_关键词点击' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '关键词报表')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.xlsx') and '京东商智_店铺来源_三级来源' in name:
+                elif name.endswith('.csv') and '京东商智_店铺来源_三级来源' in name:
                     t_path = os.path.join(self.source_path, '京东报表', '店铺来源_三级来源')
                     bib(t_path, _as_month=True)
 
@@ -1583,10 +1585,4 @@ def main(is_mysql=False):
 
 
 if __name__ == '__main__':
-    main(is_mysql=True)
-
-    # conf = myconfig.main()
-    # data = conf['Windows']['xigua_lx']['mysql']['local']
-    # db_list = conf['Windows']['xigua_lx']['mysql']['数据库集']
-    # db_list = [item for item in db_list if item != '聚合数据']
-    # print(db_list)
+    main(is_mysql=False)

{mdbq-2.7.2 → mdbq-2.7.4}/mdbq/config/products.py

@@ -13,12 +13,13 @@ from numpy.ma.core import product
 天猫货品年份基准对照
 用于聚合数据，通过此数据表进一步可确定商品上架年月
 """
+username, password, host, port, service_database = None, None, None, None, None,
 if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
     conf = myconfig.main()
     data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']
     service_database = {'xigua_lx': 'mysql'}
-elif socket.gethostname().lower() in ['company', 'Mac2.local']:
+elif socket.gethostname().lower() in ['company', 'mac2.local']:
     conf = myconfig.main()
     data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = data['username'], data['password'], data['host'], data['port']

{mdbq-2.7.2 → mdbq-2.7.4}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.7.2
+Version: 2.7.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.7.2 → mdbq-2.7.4}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.7.2',
+      version='2.7.4',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbq',