mdbq 1.9.6__py3-none-any.whl → 1.9.8__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -596,13 +596,15 @@ class DatabaseUpdate:
                         continue
                     df.insert(loc=0, column='日期', value=start_date)
                     check_remove_file = True
-                elif name.endswith('.xls') and 'JD店铺日报_店铺' in name:
+                elif name.endswith('.xls') and ('JD店铺日报_店铺' in name or '店铺_20' in name):
                     # 京东 自助报表  店铺日报
                     df = pd.read_excel(os.path.join(root, name), header=0)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
                         check_remove_file = True
                         continue
+                    if '访客数-全部渠道' not in df.columns.tolist():  # 识别是否真的京东日报
+                        continue
                     df['日期'] = df['日期'].apply(
                         lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0])
                     )
@@ -723,6 +725,9 @@ class DatabaseUpdate:
                     except Exception as e:
                         print(f'{name},  {e}')
                 if len(df) > 0:
+                    if not db_name or not collection_name:
+                        print(f'从本地csv文件中，根据文件标题匹配数据库名和数据表，结果存在空值，db_name: {db_name}, collection_name: {collection_name}')
+                    # else:
                     # 将数据传入 self.datas 等待更新进数据库
                     self.datas.append(
                         {
@@ -761,14 +766,19 @@ class DatabaseUpdate:
                             # print(f'{name}/{sheet4} 跳过')
                             continue
                         df.insert(loc=1, column='报表类型', value=sheet4)
-                        self.datas.append(
-                            {
-                                '数据库名': db_name,
-                                '集合名称': collection_name,
-                                '数据主体': df,
-                                '文件名': name,
-                            }
-                        )
+                        if not db_name or not collection_name:
+                            print(
+                                f'从本地csv文件中，根据文件标题匹配数据库名和数据表，结果存在空值，db_name: {db_name}, collection_name: {collection_name}')
+                        else:
+                            # 将数据传入 self.datas 等待更新进数据库
+                            self.datas.append(
+                                {
+                                    '数据库名': db_name,
+                                    '集合名称': collection_name,
+                                    '数据主体': df,
+                                    '文件名': name,
+                                }
+                            )
                     if is_move:
                         os.remove(os.path.join(root, name))
 

mdbq/aggregation/df_types.py

@@ -118,6 +118,12 @@ class DataTypes:
 
     def as_json_file(self):
         """ 保存为本地 json 文件 """
+        self.datas = {k: 'null' if v is None else v for k, v in self.datas.items()}  # 替换字典中，值存在空值的值
+        self.datas = {k if k != None else 'null': v for k, v in self.datas.items()}  # 替换字典中，键存在空值的键
+        if 'null' in str(self.datas):
+            print(f'self.datas 数据中存在空值，可能有未匹配的数据库名或数据表名，请检查 《标题对照表.csv》，已取消写入 df_types.json ')
+            print('self.datas: ', self.datas)
+            return
         with open(self.json_file, 'w', encoding='utf-8_sig') as f:
             json.dump(
                 self.datas,

mdbq/clean/data_clean.py

@@ -895,13 +895,15 @@ class DataClean:
                             m.df_to_mysql(df=df, db_name='京东数据1', tabel_name='京东_竞店监控_日数据')
                         os.remove(os.path.join(root, name))
 
-                    elif name.endswith('.xls') and '店铺' in name:
+                    elif name.endswith('.xls') and ('JD店铺日报_店铺' in name or '店铺_20' in name):
                         # 京东 自助报表  店铺日报
                         df = pd.read_excel(os.path.join(root, name), header=0)
                         if len(df) == 0:
                             print(f'{name} 报表数据为空')
                             os.remove(os.path.join(root, name))
                             continue
+                        if '访客数-全部渠道' not in df.columns.tolist():  # 识别是否真的京东日报
+                            continue
                         df['日期'] = df['日期'].apply(
                             lambda x: '-'.join(re.findall(r'(\d{4})(\d{2})(\d{2})', str(x))[0])
                         )

mdbq/company/copysh.py

@@ -377,3 +377,4 @@ if __name__ == '__main__':
     main()
     # # 聚合数据，并清理聚合数据
     # query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=1)
+

{mdbq-1.9.6.dist-info → mdbq-1.9.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.9.6
+Version: 1.9.8
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.9.6.dist-info → mdbq-1.9.8.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=KxAxlhtYtJKjTgWWmff5BlO8PD_FpGmMxzxVYASrFQ8,71917
-mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
+mdbq/aggregation/aggregation.py,sha256=ZDMYHZXuVx9MIiTYDjGUHLNBvj98h6-q6x2BrlBdaNw,72786
+mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
 mdbq/aggregation/query_data.py,sha256=32NjVVYLnfFkzD8TflmNVhpdQTLRRUrb9toMGApSOC8,72379
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=VI_f9mQ2tHExBytQCCkajbPVpC0yNEOaVCtySe_OW40,100789
+mdbq/clean/data_clean.py,sha256=y83uqOyM6nL0d3ClUqYMjE23ghBEkhz9uv19qrxA8NA,100980
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=4PGjvmPzvrmstOaAwHQGFXIGCWqqNXZEOYf1QdUvMlI,17762
+mdbq/company/copysh.py,sha256=VUaaJPXPYPHWwnkdK77PWz_dAXZyEmYBA9Df1yROHAc,17764
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=-CFEW0dQh4OIwVgwK-cL0eVp1LN3PjJgN89d4P5TB9I,6011
 mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.9.6.dist-info/METADATA,sha256=fdqv73OvAg866cfG7kXzJDdg27mvNxWTgFzulIi1B6Y,245
-mdbq-1.9.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.9.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.9.6.dist-info/RECORD,,
+mdbq-1.9.8.dist-info/METADATA,sha256=ZtwrpFJzCLnABnvOHyrVFqDt346SgQIh2SsqIaioSF8,245
+mdbq-1.9.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.9.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.9.8.dist-info/RECORD,,