mdbq 3.3.4__py3-none-any.whl → 3.3.7__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -1253,8 +1253,8 @@ class MysqlDatasQuery:
             '三级来源索引': 'smallint',
         }
         # df.to_csv('/Users/xigua/Downloads/ll.csv', index=False, header=True, encoding='utf-8_sig')
-        min_date = df['日期'].min()
-        max_date = df['日期'].max()
+        min_date = df['日期'].min().strftime("%Y-%m-%d")
+        max_date = df['日期'].max().strftime("%Y-%m-%d")
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(

mdbq/mongo/mongo.py

@@ -8,11 +8,23 @@ import pandas as pd
 import numpy as np
 import pymongo
 from functools import wraps
+import socket
+import platform
 from concurrent.futures import ThreadPoolExecutor
-from mdbq.config import get_myconf
+from mdbq.config import myconfig
 from mdbq.dataframe import converter
 
 warnings.filterwarnings('ignore')
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['remoto']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
 
 
 def rename_col(username, password, host, db_name, collection_name, old_name, new_name, port: int = 27017,):
@@ -679,30 +691,21 @@ class OptimizeDatas:
 
 
 def upload_one_dir():
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
+    if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+        conf = myconfig.main()
+        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
+        username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+            'port']
+    else:
+        conf = myconfig.main()
+        conf_data = conf['Windows']['company']['mysql']['remoto']
+        username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+            'port']
+
     p = UploadMongo(username=username, password=password, host=host, port=port, drop_duplicates=False)
     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
     print(f'{now}数据处理中...')
 
-    p.db_name = ''
-    p.collection_name = f''
-    path = os.path.join('C:\\同步空间', 'BaiduSyncdisk', '原始文件2', r'京东报表', 'JD流量来源')
-
-    for root, dirs, files in os.walk(path, topdown=False):
-        for name in files:
-            if '按天_' not in name:
-                continue
-            if name.endswith('.csv') and 'baidu' not in name:
-                df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                for col in df.columns.tolist():
-                    if '日期' in col:
-                        df[col] = df[col].apply(lambda x: pd.to_datetime(x) if x else pd.to_datetime('2099-01-01'))
-                p.df_to_mongo(df=df)
-    if p.client:
-        p.client.close()
-    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-    print(f'{now}数据完成!')
-
 
 def main():
     pass
@@ -710,7 +713,6 @@ def main():
 
 if __name__ == '__main__':
     # main()
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
     print(username, password, host, port)
 
     # for db_name in [

mdbq/mysql/mysql.py

@@ -128,7 +128,7 @@ class MysqlUpload:
 
         return wrapper
 
-    def keep_connect(self, _config, max_try: int=5):
+    def keep_connect(self, _db_name, _config, max_try: int=5):
         attempts = 1
         while attempts <= max_try:
             try:
@@ -137,8 +137,8 @@ class MysqlUpload:
             except Exception as e:
                 print(f'连接失败，正在重试: {attempts}/{max_try} {e}')
                 attempts += 1
-                time.sleep(10)
-        print(f'连接失败，重试次数超限')
+                time.sleep(20)
+        print(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
         return None
 
     def cover_doc_dtypes(self, dict_data):
@@ -215,7 +215,7 @@ class MysqlUpload:
             return
 
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -236,7 +236,7 @@ class MysqlUpload:
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
         # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -382,7 +382,7 @@ class MysqlUpload:
                     print(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
 
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -403,7 +403,7 @@ class MysqlUpload:
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
         # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -749,7 +749,7 @@ class MysqlUpload:
                 [dtypes.update({k: inside_v}) for inside_k, inside_v in set_typ.items() if k == inside_k]
 
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -770,7 +770,7 @@ class MysqlUpload:
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
         # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -1049,7 +1049,7 @@ class MysqlUpload:
             print(f'未指定文件名: filename')
             return
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         # try:
@@ -1061,7 +1061,7 @@ class MysqlUpload:
                 return
         self.config.update({'database': db_name})
         # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -1094,7 +1094,7 @@ class MysqlUpload:
         df = pd.DataFrame()
 
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         try:
@@ -1116,7 +1116,7 @@ class MysqlUpload:
         # 读取数据
         self.config.update({'database': db_name})
         # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         try:
@@ -1240,7 +1240,7 @@ class OptimizeDatas:
 
         return wrapper
 
-    def keep_connect(self, _config, max_try: int=5):
+    def keep_connect(self, _db_name, _config, max_try: int=5):
         attempts = 1
         while attempts <= max_try:
             try:
@@ -1249,8 +1249,8 @@ class OptimizeDatas:
             except Exception as e:
                 print(f'连接失败，正在重试: {attempts}/{max_try} {e}')
                 attempts += 1
-                time.sleep(10)
-        print(f'连接失败，重试次数超限')
+                time.sleep(20)
+        print(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
         return None
 
     def optimize_list(self):
@@ -1300,7 +1300,7 @@ class OptimizeDatas:
                 #     continue
                 self.config.update({'database': self.db_name})  # 添加更新 config 字段
                 # self.connection = pymysql.connect(**self.config)
-                self.connection = self.keep_connect(_config=self.config, max_try=5)
+                self.connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
                 if not self.connection:
                     return
                 with self.connection.cursor() as cursor:
@@ -1455,7 +1455,7 @@ class OptimizeDatas:
     def database_list(self):
         """ 获取所有数据库 """
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -1467,7 +1467,7 @@ class OptimizeDatas:
     def table_list(self, db_name):
         """ 获取指定数据库的所有数据表 """
         # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
         if not connection:
             return
         try:
@@ -1486,7 +1486,7 @@ class OptimizeDatas:
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
         # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         with connection.cursor() as cursor:
@@ -1501,7 +1501,7 @@ class OptimizeDatas:
         """
         self.config.update({'database': db_name})  # 添加更新 config 字段
         # connection = pymysql.connect(**self.config)
-        connection = self.keep_connect(_config=self.config, max_try=5)
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
         if not connection:
             return
         try:
@@ -1537,7 +1537,7 @@ class OptimizeDatas:
             for key, table_name in table_dict.items():
                 self.config.update({'database': self.db_name})  # 添加更新 config 字段
                 # self.connection = pymysql.connect(**self.config)
-                self.connection = self.keep_connect(_config=self.config, max_try=5)
+                self.connection = self.keep_connect(_db_name=self.db_name, _config=self.config, max_try=5)
                 if not self.connection:
                     return
                 with self.connection.cursor() as cursor:

mdbq/mysql/recheck_mysql.py

@@ -156,7 +156,7 @@ class ReCheckMysql:
 
 
 def recheck_csv():
-    path = '/Users/xigua/数据中心/原始文件2/推广报表34324234'
+    path = ''
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~' in name or 'baidu' in name or 'Ds_' in name or 'xunlei' in name:

mdbq/spider/aikucun.py

@@ -17,8 +17,6 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
 from mdbq.config import set_support
 from selenium.webdriver.common.keys import Keys
-from mdbq.aggregation import aggregation
-from mdbq.clean import data_clean
 from mdbq.other import ua_sj
 from mdbq.mysql import mysql
 from mdbq.config import myconfig
@@ -46,7 +44,7 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
     Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
-upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
+upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
 m_engine = mysql.MysqlUpload(username='', password='', host='', port=0, charset='utf8mb4')
 company_engine = mysql.MysqlUpload(username='', password='', host='', port=0, charset='utf8mb4')
@@ -458,26 +456,6 @@ class AikuCun:
 def akucun(headless=True, date_num=10):
     akc = AikuCun()
     akc.get_data(shop_name='aikucun', date_num=date_num, headless=headless)  # 获取最近 N 天数据，0表示今天
-    # akc.clean_data()
-
-    # # 新版 数据分类
-    # dp = aggregation.DatabaseUpdate(path=upload_path)
-    # dp.new_unzip(is_move=True)
-    # dp.cleaning(is_move=False, is_except=['临时文件'])  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # # 将 self.datas 更新至数据库
-    # dp.upload_df(service_databases=[
-    #     # {'home_lx': 'mongodb'},
-    #     # {'home_lx': 'mysql'},
-    #     {'company': 'mysql'},
-    #     # {'nas': 'mysql'},
-    # ])
-    # # 数据分类
-    # c = data_clean.DataClean(path=upload_path, source_path=Source_Path)
-    # c.set_up_to_mogo = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
-    # c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
-    # c.new_unzip(is_move=True, )  # 解压文件
-    # c.change_and_sort(is_except=['临时文件'])
-    # c.move_all(is_except=['临时文件'])  # 移到文件到原始文件夹
 
 
 class AikuCunNew:
@@ -529,7 +507,7 @@ class AikuCunNew:
 
 if __name__ == '__main__':
     get_cookie_aikucun()  # 登录并获取 cookies
-    akucun(date_num=10, headless=True)  # 下载数据
+    akucun(date_num=5, headless=True)  # 下载数据
 
     # a = AikuCunNew(shop_name='aikucun')
     # a.akc()

{mdbq-3.3.4.dist-info → mdbq-3.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.3.4
+Version: 3.3.7
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.3.4.dist-info → mdbq-3.3.7.dist-info}/RECORD

@@ -2,32 +2,23 @@ mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2kzY,80264
-mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
-mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
-mdbq/aggregation/query_data.py,sha256=4C9BmMUV4x1YvezIySO5Nh8tqGqg1z4GVnuYDGU5OJs,167595
+mdbq/aggregation/query_data.py,sha256=_5mnSFHV6xAFs_1YF_H2zMOdJeMavgga4lZQ_qpqxPQ,167637
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
-mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=yMAb6tV9XHhFJbRrCOeaPfszApJ9y5M4-hQGuBSXNqE,67799
-mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
-mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=eFu6focRqm2Njn_XN1KW2ZYJiTv6EYgsdBCLokobyxQ,21572
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/myconfig.py,sha256=EGymTlAimtHIDJ9egCtOehBEPOj6rea504kvsEZu64o,854
 mdbq/config/products.py,sha256=Ab6eaAUMUtjRL8z9NvYukyCjp3nAi4OYISY_IdPhAJ0,6279
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
-mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
-mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
+mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=pTeZD0KgVNJnRLaMXP11wz8lBBq2_xkAqIuoRmnEMWA,85259
-mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
+mdbq/mysql/mysql.py,sha256=ZK6E-idQWrURtoimc0uG8B1tnhtSFQXDJHfu8sWeJg4,85675
+mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
 mdbq/mysql/s_query.py,sha256=MbIprZ4yJDAZ9AahZPzl7hqS695Vs0P-AJNwAtA_EEc,9287
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -40,11 +31,9 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
-mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=BKVa0xbTkyhIH5kQgOdyPDtwFPScbMNAFYup_-fFF9Y,24809
-mdbq-3.3.4.dist-info/METADATA,sha256=5HCgLOtVbO2JXkq0HhNkTOo2wqWf4MFVh78NWT_ruPk,243
-mdbq-3.3.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.3.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.3.4.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=UFY-TwlvquEYK58rTdRuv5Wx3KA21m-bIrwvvfPRyOk,23749
+mdbq-3.3.7.dist-info/METADATA,sha256=aMT3CEx_q-0vG6CncxkAXv1PPk2RsieIZxrkYyk1jPE,243
+mdbq-3.3.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.3.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.3.7.dist-info/RECORD,,

mdbq/aggregation/df_types.py

@@ -1,188 +0,0 @@
-# -*- coding:utf-8 -*-
-import warnings
-import pandas as pd
-import numpy as np
-import chardet
-import zipfile
-
-from numpy import dtype
-from pandas.tseries.holiday import next_monday
-from pyzipper import PyZipFile
-import os
-import platform
-import json
-import pymysql
-from mdbq.mongo import mongo
-from mdbq.mysql import mysql
-from mdbq.mysql import s_query
-from mdbq.config import get_myconf
-from mdbq.config import set_support
-from mdbq.dataframe import converter
-import datetime
-import time
-import re
-import shutil
-import getpass
-
-from sqlalchemy.dialects.postgresql.pg_catalog import pg_get_serial_sequence
-
-warnings.filterwarnings('ignore')
-"""
-1. 记录 dataframe 或者数据库的列信息(dtypes)
-2. 更新 mysql 中所有数据库的 dtypes 信息到本地 json
-"""
-
-
-class DataTypes:
-    """
-     数据简介: 记录 dataframe 或者数据库的列信息(dtypes)，可以记录其信息或者加载相关信息用于入库使用，
-     第一字段为分类(如 dataframe/mysql)，第二字段为数据库名，第三字段为集合名，第四段列名及其数据类型
-    """
-    def __init__(self, path=None):
-        self.datas = {
-            "json统计":
-                {
-                    "字段量": 0,
-                    "数据库量": 0,
-                    "集合数量": 0
-                }
-        }
-        self.path = path
-        if not self.path:
-            self.path = set_support.SetSupport(dirname='support').dirname
-        self.json_file = os.path.join(self.path, 'df_types.json')
-        if not os.path.isdir(self.path):
-            os.makedirs(self.path)
-        if not os.path.isfile(self.json_file):
-            with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
-        self.json_before()
-
-    def json_before(self):
-        """ 本地 json 文件的 dtypes 信息, 初始化更新给 self.datas """
-        with open(self.json_file, 'r', encoding='utf-8_sig') as f:
-            json_ = json.load(f)
-            self.datas.update(json_)
-
-    def get_df_types(self, db_name, collection_name, df=pd.DataFrame(), is_file_dtype=True):
-        """
-        读取 df 的 dtypes, 并更新本地 json 文件
-        期间会 清理不合规的列名, 并对数据类型进行转换(尝试将 object 类型转为 int 或 float)
-        返回: df 的 dtypes, 后续使用示例: df = df.astype(dtypes, errors='ignore')
-        is_file_dtype=True: 默认情况下以旧 json 优先, 即允许手动指定 json 文件里面的数据类型
-        """
-        if len(df) == 0:
-            return
-        cv = converter.DataFrameConverter()
-        df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值
-        dtypes = df.dtypes.apply(str).to_dict()
-        dtypes = {db_name: {collection_name: dtypes}}
-
-        if not self.datas:  # 如果不存在本地 json 文件, 直接返回即可
-            self.datas.update(dtypes)
-            return self.datas[db_name][collection_name]
-        else:  # 存在则读取，并更新 df 的 dtypes
-            if db_name in list(self.datas.keys()):  # ['京东数据2', '推广数据2', '生意参谋2', '生意经2']
-                if collection_name in list(self.datas[db_name].keys()):
-                    if is_file_dtype:  # 旧数据优先
-                        # # 用 dtypes 更新, 允许手动指定 json 文件里面的数据类型
-                        dtypes[db_name][collection_name].update(self.datas[db_name][collection_name])
-                        # 将 dtypes 更新进去，使 self.datas 包含新旧信息
-                        self.datas[db_name][collection_name].update(dtypes[db_name][collection_name])
-                    else:  # 新数据优先
-                        self.datas[db_name][collection_name].update(dtypes[db_name][collection_name])
-                else:
-                    if is_file_dtype:  # 旧数据优先
-                        dtypes[db_name].update(self.datas[db_name])
-                        self.datas[db_name].update(dtypes[db_name])
-                    else:
-                        self.datas[db_name].update(dtypes[db_name])
-            else:
-                # dtypes.update(self.datas)  # 可以注释掉, 因为旧数据 self.datas 是空的
-                self.datas.update(dtypes)
-            dbs = 0
-            collections = 0
-            cols = 0
-            # self.datas.pop('json统计')
-            for k, v in self.datas.items():
-                if k == 'json统计':
-                    continue
-                dbs += 1
-                for d, j in v.items():
-                    collections += 1
-                    for t, p in j.items():
-                        cols += 1
-            tips = {'json统计': {'数据库量': dbs, '集合数量': collections, '字段量': cols}}
-            self.datas.update(tips)
-            return self.datas[db_name][collection_name]  # 返回 df 的 dtypes
-
-    def as_json_file(self):
-        """ 保存为本地 json 文件 """
-        self.datas = {k: 'null' if v is None else v for k, v in self.datas.items()}  # 替换字典中，值存在空值的值
-        self.datas = {k if k != None else 'null': v for k, v in self.datas.items()}  # 替换字典中，键存在空值的键
-        if 'null' in str(self.datas):
-            print(f'self.datas 数据中存在空值，可能有未匹配的数据库名或数据表名，请检查 《标题对照表.csv》，已取消写入 df_types.json ')
-            print('self.datas: ', self.datas)
-            return
-        with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-            json.dump(
-                self.datas,
-                f,
-                ensure_ascii=False,  # 默认True，非ASCII字符将被转义。如为False，则非ASCII字符会以\uXXXX输出
-                sort_keys=True,  # 默认为False。如果为True，则字典的输出将按键排序。
-                indent=4,
-            )
-        time.sleep(1)
-
-    def df_dtypes_to_json(self, db_name, collection_name, path, df=pd.DataFrame(), is_file_dtype=True):
-        if len(df) == 0:
-            return
-        cv = converter.DataFrameConverter()
-        df = cv.convert_df_cols(df=df)  # 清理 dataframe 列名的不合规字符
-        dtypes = df.dtypes.apply(str).to_dict()
-        dtypes = {'dataframe': {db_name: {collection_name: dtypes}}}
-        self.dtypes_to_json(dtypes=dtypes, cl='dataframe', db_name=db_name, collection_name=collection_name, path=path, is_file_dtype=is_file_dtype)
-
-    def load_dtypes(self, db_name, collection_name):
-        if db_name in list(self.datas.keys()):
-            if collection_name in list(self.datas[db_name].keys()):
-                return self.datas[db_name][collection_name]
-            else:
-                print(f'不存在的集合名信息: {collection_name}, 文件位置: {self.json_file}')
-                return {}
-        else:
-            print(f'不存在的数据库信息: {db_name}, 文件位置: {self.json_file}')
-            return {}
-
-
-def update_df_types_to_json(file, db_name, collection_name, is_file_dtype=True):
-    """ 更新一个文件的 dtype 信息到 json 文件 """
-    df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
-    df_to_json = DataTypes()
-    df_to_json.get_df_types(
-        df=df,
-        db_name=db_name,
-        collection_name=collection_name,
-        is_file_dtype=is_file_dtype,  # 日常需开启文件优先, 正常不要让新文件修改 json 已有的类型
-    )
-    df_to_json.as_json_file()
-    print(f'json文件已存储: {df_to_json.json_file}')
-
-
-def test_load_dtypes(db_name, collection_name):
-    d = DataTypes()
-    res = d.load_dtypes(db_name=db_name, collection_name=collection_name)
-    print(res)
-
-
-if __name__ == '__main__':
-    file = '/Users/xigua/数据中心/pandas数据源/店铺日报.csv'
-    update_df_types_to_json(
-        file=file,
-        db_name='pandas数据源',
-        collection_name='店铺日报',
-        is_file_dtype=True,
-    )
-    # test_load_dtypes(db_name='pandas数据源', collection_name='店铺日报')
-
-