mdbq 1.0.8__py3-none-any.whl → 1.0.9__py3-none-any.whl

mdbq/aggregation/mysql_types.py

@@ -133,19 +133,19 @@ class DataTypes:
         if cl in self.datas.keys():
             if db_name in list(self.datas[cl].keys()):
                 if table_name in list(self.datas[cl][db_name].keys()):
-                    return self.datas[cl][db_name][table_name]
+                    return self.datas[cl][db_name][table_name], None, None, None
                 else:
                     print(f'不存在的集合名信息: {table_name}, 文件位置: {self.json_file}')
-                    mysql_all_dtypes(db_name=db_name, table_name=table_name)   # 更新一个表的 dtypes
-                    return {}
+                    # mysql_all_dtypes(db_name=db_name, table_name=table_name)   # 更新一个表的 dtypes
+                    return {}, cl, db_name, table_name
             else:
                 print(f'不存在的数据库信息: {db_name}, 文件位置: {self.json_file}')
-                mysql_all_dtypes(db_name=db_name)  # 更新一个数据库的 dtypes
-                return {}
+                # mysql_all_dtypes(db_name=db_name)  # 更新一个数据库的 dtypes
+                return {}, cl, db_name, None
         else:
             print(f'不存在的数据分类: {cl}, 文件位置: {self.json_file}')
-            mysql_all_dtypes()  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
-            return {}
+            # mysql_all_dtypes()  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
+            return {}, cl, None, None  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
 
 
 def mysql_all_dtypes(db_name=None, table_name=None, path=None):
@@ -155,7 +155,7 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
     if not path:
         path = set_support.SetSupport(dirname='support').dirname
 
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
+    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
     config = {
         'host': host,
         'port': port,

mdbq/mysql/mysql.py

@@ -48,7 +48,7 @@ class MysqlUpload:
                 print(f'{db_name}: {table_name} 传入的 df 数据长度为0')
                 return
         else:
-            print(f'{db_name}: {table_name} 传入的 df 不是 dataframe 结构')
+            print(f'{db_name}: {table_name} 传入的 df 不是有效的 dataframe 结构')
             return
         cv = converter.DataFrameConverter()
         df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值
@@ -82,7 +82,7 @@ class MysqlUpload:
                 print(f'创建 mysql 表: {table_name}')
 
             # 2. 列数据类型转换，将 df 数据类型转换为 mysql 的数据类型
-            dtypes = self.convert_dtypes(df=df, db_name=db_name, table_name=table_name)
+            dtypes, cl, db_n, tb_n = self.convert_dtypes(df=df, db_name=db_name, table_name=table_name)
 
             #  有特殊字符不需转义
             sql = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{db_name}' AND TABLE_NAME = '{table_name}';"
@@ -109,6 +109,14 @@ class MysqlUpload:
                         pass
             connection.commit()  # 提交事务
 
+            # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
+            if cl and db_n and tb_n:
+                mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name)  # 更新一个表的 dtypes
+            elif cl and db_n:
+                mysql_types.mysql_all_dtypes(db_name=db_name)  # 更新一个数据库的 dtypes
+            elif cl:
+                mysql_types.mysql_all_dtypes()  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
+
             # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
             if drop_duplicates and '日期' in df.columns.tolist():
                 dates = df['日期'].values.tolist()
@@ -123,24 +131,21 @@ class MysqlUpload:
             print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}')
             datas = df.to_dict(orient='records')
             for data in datas:
-                try:
-                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名转义
-                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f"'{item}'" for item in data.values()])  # 值要加单引号 ''
-                    condition = []
-                    for k, v in data.items():
-                        condition += [f"`{k}` = '{v}'"]
-                    condition = ' AND '.join(condition)  # 构建查询条件
-                    # print(condition)
-
-                    sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
+                cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名转义
+                # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                values = ', '.join([f"'{item}'" for item in data.values()])  # 值要加单引号 ''
+                condition = []
+                for k, v in data.items():
+                    condition += [f"`{k}` = '{v}'"]
+                condition = ' AND '.join(condition)  # 构建查询条件
+                # print(condition)
+
+                sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
+                cursor.execute(sql)
+                result = cursor.fetchall()  # 获取查询结果, 如果有结果返回 list，没有则返回空元组 tuple
+                if not result:  # 数据不存在则插入
+                    sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                     cursor.execute(sql)
-                    result = cursor.fetchall()  # 获取查询结果, 如果有结果返回 list，没有则返回空元组 tuple
-                    if not result:  # 数据不存在则插入
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                        cursor.execute(sql)
-                except:
-                    pass
         connection.commit()  # 提交事务
 
     def convert_dtypes(self, df, db_name, table_name):
@@ -155,7 +160,7 @@ class MysqlUpload:
         # path = set_support.SetSupport(dirname='support').dirname
         d = mysql_types.DataTypes()
         # 从本地文件中读取 dtype 信息
-        dtypes = d.load_dtypes(cl='mysql', db_name=db_name, table_name=table_name)
+        dtypes, cl, db_n, tb_n = d.load_dtypes(cl='mysql', db_name=db_name, table_name=table_name)
         # 可能会因为没有 json 文件, 返回 None
         if dtypes:
             # 按照文件记录更新 dtypes
@@ -171,7 +176,7 @@ class MysqlUpload:
         dtypes.update({col: self.convert_dtype_to_sql(df=df, col=col, dtype=df[col].dtype) for col in col_not_exist})
         # 至此 df 中全部列类型已经转换完成
         # 返回结果, 示例: {'上市年份': 'mediumtext', '商品id': 'mediumtext', '平台': 'mediumtext'}
-        return dtypes
+        return dtypes, cl, db_n, tb_n  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
 
     def convert_dtype_to_sql(self, df, col, dtype):
         """ 按照以下规则转换DataFrame列的数据类型为 MYSQL 专有的数据类型 """
@@ -199,7 +204,29 @@ class MysqlUpload:
                 return 'mediumtext'
             return 'INT'
         elif dtype == 'float64':
-            return 'double'  # mysql 中不要使用 float 类型，会影响计算结果
+            # step = len(str(max(df[col].tolist())))  # 数字长度包含小数点
+            int_step = len(str(max(df[col].tolist())).split('.')[0])   # 整数位数长度
+            f_step = len(str(max(df[col].tolist())).split('.')[1])  # 小数位数长度
+            if int_step >= 12:
+                return 'mediumtext'  # mysql 中不要使用 float 和 double 类型，会影响计算结果
+            elif int_step >= 8 and f_step >= 0:
+                return 'decimal(16, 2)'
+            elif int_step >= 6 and f_step >= 0:
+                return 'decimal(10, 2)'
+            elif int_step >= 4 and f_step >= 0:
+                return 'decimal(10, 2)'
+            elif int_step >= 2 and f_step >= 6:
+                return 'decimal(12, 4)'
+            elif int_step >= 2 and f_step >= 4:
+                return 'decimal(10, 4)'
+            elif int_step >= 2 and f_step >= 0:
+                return 'decimal(10, 2)'
+            elif int_step >= 1 and f_step >= 6:
+                return 'decimal(10, 6)'
+            elif int_step >= 1 and f_step >= 4:
+                return 'decimal(10, 4)'
+            else:
+                return 'decimal(10, 2)'
         elif dtype == 'object':
             return 'mediumtext'
         else:
@@ -647,8 +674,8 @@ def download_datas(table_name, save_path, start_date):
         df.to_csv(path, index=False, encoding='utf-8_sig', header=True)
 
 
-def one_file_to_mysql(file, db_name, table_name):
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
+def one_file_to_mysql(file, db_name, table_name, target_service, database):
+    username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
 
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
     m = MysqlUpload(username=username, password=password, host=host, port=port)
@@ -659,11 +686,13 @@ if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
     print(username, password, host, port)
 
-    file = '/Users/xigua/数据中心/原始文件2/属性设置/城市等级.csv'
+    file = '/Users/xigua/Downloads/余额查询.csv'
     one_file_to_mysql(
         file=file,
-        db_name='属性设置2',
-        table_name='城市等级',
+        db_name='test',
+        table_name='余额查询',
+        target_service='company',
+        database='mysql'
     )
 
 

{mdbq-1.0.8.dist-info → mdbq-1.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.0.8
+Version: 1.0.9
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.0.8.dist-info → mdbq-1.0.9.dist-info}/RECORD

@@ -3,7 +3,7 @@ mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=L1IyrfdSfCThg7sa2mFQ4hZWnHRhkIBF91FCQIEGAn0,53076
 mdbq/aggregation/df_types.py,sha256=T35KML0sdch8GzIwo7CxSIrt72YVElBeCrsKQx4dX_0,7531
-mdbq/aggregation/mysql_types.py,sha256=ijo9J2aMB5SnLQYkCFZW5OsGOkN9YFQdwTOEH5wEdds,10137
+mdbq/aggregation/mysql_types.py,sha256=xLqoiaMiuC4sF6Jr-BTd4wsBwCuKCNaaeHwTAJEMV_w,10313
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
 mdbq/aggregation/query_data.py,sha256=9mer6idzVkJ4F9SzYjmBbXeUlfNg95PIMU10K58hBc4,25290
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
@@ -25,7 +25,7 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=q0B4wXDSTtXg_vMN7MPh6zdxl6tT68tM74LmdVNQQek,31892
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/data_types_即将删除.py,sha256=sjBBDKr9674LdjM5N_dwyJACdZPbdB8Beli59jGdgnQ,10378
-mdbq/mysql/mysql.py,sha256=4i2ceTcpY9L539H6n-3W9jwH2Y5pJZjXVTCVWETge-c,32473
+mdbq/mysql/mysql.py,sha256=yxia60rwfc4uFsewBdinSfrxIgfLUsN4PqJlZSGwv-g,34146
 mdbq/mysql/s_query.py,sha256=4c24SwbqtnO33o8CgWlTQ_j8sZYl5BRIQkaD9CI-vTY,7901
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.0.8.dist-info/METADATA,sha256=O-d8cPfUvgX1pq_eGbZOnp9V7gkaAD0Q0CHnSqCYgqg,245
-mdbq-1.0.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.0.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.0.8.dist-info/RECORD,,
+mdbq-1.0.9.dist-info/METADATA,sha256=9YaZkzP2cboALNY2oqjGrvAtBV-r2mtDa11G43Tz_iU,245
+mdbq-1.0.9.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.0.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.0.9.dist-info/RECORD,,