mdbq 0.4.4__py3-none-any.whl → 0.4.6__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1035,5 +1035,5 @@ if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
 
-    main()
-    # upload()
+    # main()
+    upload()

mdbq/aggregation/query_data.py

@@ -525,5 +525,5 @@ def data_aggregation(service_databases=[{}]):
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'home_lx': 'mysql'}])
+    data_aggregation(service_databases=[{'company': 'mysql'}])
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/dataframe/converter.py

@@ -9,6 +9,51 @@ class DataFrameConverter(object):
         self.df = df
 
     def convert_df_cols(self, df=pd.DataFrame({})):
+        """
+        清理 dataframe 非法值
+        对数据类型进行转换(尝试将 object 类型转为 int 或 float)
+        """
+        if len(df) == 0:
+            df = self.df
+            if len(df) == 0:
+                return
+        # dtypes = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
+        df.replace([np.inf, -np.inf], 0, inplace=True)  # 清理一些非法值
+        df.replace(to_replace=['\\N', '-', '--', '', 'nan'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
+        df.replace(to_replace=[','], value='', regex=True, inplace=True)
+        df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
+        df.replace(to_replace=['"'], value='', regex=True, inplace=True)
+        cols = df.columns.tolist()
+
+        for col in cols:
+            # df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
+            # 百分比在某些数据库中不兼容, 转换百分比为小数
+            df[col] = df[col].apply(lambda x: float(float((str(x).rstrip("%"))) / 100) if str(x).endswith('%') and '~' not in str(x) else x)
+            # 尝试转换合适的数据类型
+            if df[col].dtype == 'object':
+                try:
+                    # df[col] = df[col].astype(int)  # 尝试转换 int
+                    df[col] = df[col].apply(lambda x: int(x) if '_' not in str(x) else x)
+                except:
+                    # df[col] = df[col].astype('float64', errors='ignore')    # 尝试转换 float, 报错则忽略
+                    try:
+                        df[col] = df[col].apply(lambda x: float(x) if '_' not in str(x) else x)
+                    except:
+                        pass
+            if df[col].dtype == 'float':  # 对于小数类型, 保留 6 位小数
+                df[col] = df[col].apply(lambda x: round(float(x), 6) if x != 0 else x)
+            # 清理列名, 在 mysql 里面列名不能含有某些特殊字符
+            if '日期' in col or '时间' in col:
+                try:
+                    df[col] = df[col].apply(lambda x: pd.to_datetime(x))
+                except:
+                    pass
+            new_col = col.lower()
+            df.rename(columns={col: new_col}, inplace=True)
+        df.fillna(0, inplace=True)
+        return df
+
+    def convert_df_cols_bak(self, df=pd.DataFrame({})):
         """
         清理 dataframe 列名的不合规字符(mysql)
         对数据类型进行转换(尝试将 object 类型转为 int 或 float)

mdbq/mysql/data_types.py

@@ -166,9 +166,9 @@ class DataTypes:
             return {}
 
 
-def mysql_all_dtypes(path = '/Users/xigua/数据中心/自动0备份/py/数据更新/support'):
+def mysql_all_dtypes(path=None):
     """
-    更新 mysql 中所有数据库的 dtypes 信息到本地 json
+    更新笔记本 mysql 中所有数据库的 dtypes 信息到本地 json
     """
     if not os.path.isdir(path):
         path = set_support.SetSupport(dirname='support').dirname

mdbq/mysql/mysql.py

@@ -9,6 +9,7 @@ import warnings
 import pymysql
 import numpy as np
 import pandas as pd
+from more_itertools.more import iequals
 from sqlalchemy import create_engine
 import os
 import calendar
@@ -63,7 +64,7 @@ class MysqlUpload:
             return False
 
     # @try_except
-    def df_to_mysql(self, df, tabel_name, db_name='远程数据源'):
+    def df_to_mysql_bak(self, df, tabel_name, db_name='远程数据源'):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -182,6 +183,106 @@ class MysqlUpload:
         finally:
             connection.close()
 
+    def df_to_mysql(self, df, tabel_name, db_name='远程数据源', drop_duplicates=False):
+        """
+        将 df 写入数据库
+        db_name: 数据库名称
+        tabel_name: 集合/表名称
+        drop_duplicates：仅限于聚合数据使用，其他情况不要设置
+        """
+        cv = converter.DataFrameConverter()
+        df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值
+
+        connection = pymysql.connect(**self.config)  # 连接数据库
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            database_exists = cursor.fetchone()
+            if not database_exists:
+                # 如果数据库不存在，则新建
+                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
+                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
+                if '192.168.1.100' in str(self.host):
+                    sql = f"CREATE DATABASE `{db_name}`"
+                else:
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                cursor.execute(sql)
+                connection.commit()
+                print(f"创建Database: {db_name}")
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            # 1. 查询表, 不存在则创建一个空表
+            sql = f"SHOW TABLES LIKE '{tabel_name}';"  # 有特殊字符不需转义
+            cursor.execute(sql)
+            if not cursor.fetchone():
+                sql = f"CREATE TABLE IF NOT EXISTS `{tabel_name}` (id INT AUTO_INCREMENT PRIMARY KEY)"
+                cursor.execute(sql)
+                print(f'创建 mysql 表: {tabel_name}')
+
+            # 2. 列数据类型转换
+            dtypes = self.convert_dtypes(df=df, db_name=db_name, tabel_name=tabel_name)
+
+            #  有特殊字符不需转义
+            sql = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{db_name}' AND TABLE_NAME = '{tabel_name}';"
+            cursor.execute(sql)
+            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]
+            cols = df.columns.tolist()
+            col_not_exist = [col for col in cols if col not in col_exist]
+            # 检查列，不存在则新建列
+            if col_not_exist:  # 数据表中不存在的列
+                for col in col_not_exist:
+                    try:
+                        #  创建列，需转义
+                        sql = f"ALTER TABLE `{tabel_name}` ADD COLUMN `{col}` {dtypes[col]} DEFAULT NULL;"
+                        cursor.execute(sql)
+                        print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                        # 创建索引
+                        if col == '日期':
+                            cursor.execute(f"SHOW INDEXES FROM `{tabel_name}` WHERE `Column_name` = '{col}'")
+                            result = cursor.fetchone()  # 检查索引是否存在
+                            if not result:
+                                cursor.execute(f"CREATE INDEX index_name ON `{tabel_name}`(`{col}`)")
+                    except:
+                        pass
+            connection.commit()  # 提交事务
+
+            # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
+            if drop_duplicates and '日期' in df.columns.tolist():
+                dates = df['日期'].values.tolist()
+                start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
+                end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+                sql = f"DELETE FROM `{tabel_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
+                cursor.execute(sql)
+                connection.commit()
+
+            # 5. 更新插入数据
+            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
+            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{tabel_name}')
+            datas = df.to_dict(orient='records')
+            for data in datas:
+                try:
+                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f"'{item}'" for item in data.values()])  # 值要加单引号 ''
+                    condition = []
+                    for k, v in data.items():
+                        condition += [f"`{k}` = '{v}'"]
+                    condition = ' AND '.join(condition)  # 构建查询条件
+                    # print(condition)
+
+                    sql = f"SELECT {cols} FROM `{tabel_name}` WHERE {condition}"
+                    cursor.execute(sql)
+                    result = cursor.fetchall()  # 获取查询结果, 如果有结果返回 list，没有则返回空元组 tuple
+                    if not result:  # 数据不存在则插入
+                        sql = f"INSERT INTO `{tabel_name}` ({cols}) VALUES ({values});"
+                        cursor.execute(sql)
+                except:
+                    pass
+        connection.commit()  # 提交事务
+
     def convert_dtypes(self, df, db_name, tabel_name):
         """
         根据本地已经存在的记录着 mysql dtypes 的 json 文件转换 df 的类型为 mysql 专有的数据类型
@@ -240,7 +341,7 @@ class MysqlUpload:
                 return 'mediumtext'
             return 'INT'
         elif dtype == 'float64':
-            return 'FLOAT'
+            return 'double'  # mysql 中不要使用 float 类型，会影响计算结果
         elif dtype == 'object':
             return 'mediumtext'
         else:
@@ -689,8 +790,13 @@ def download_datas(tabel_name, save_path, start_date):
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
+    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
     print(username, password, host, port)
 
+    df = pd.read_csv('/Users/xigua/Downloads/余额查询.csv', encoding='utf-8_sig', header=0, na_filter=False)
+    # df = df.to_dict(orient='records')
+    m = MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql_new(df=df, db_name='te2- %s t', tabel_name='测 -sdf @%试 表')
+
 
 

{mdbq-0.4.4.dist-info → mdbq-0.4.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.4.4
+Version: 0.4.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.4.4.dist-info → mdbq-0.4.6.dist-info}/RECORD

@@ -1,9 +1,9 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=6j6gl0fiHuNtepP-NM38h_-1v_3a62AFtH_a55NXjq0,58254
+mdbq/aggregation/aggregation.py,sha256=UGrhmhlu0oz-96yiaHez6EEacRJ1aALKanHbSWrcGww,58254
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
-mdbq/aggregation/query_data.py,sha256=0i9aF63O-E_SGdJ_09-pAOvAGOzJBIQOzv29dLlwruA,24491
+mdbq/aggregation/query_data.py,sha256=5lzvEokjHuKtlaSBYjOFH8VA2MTtX8R3MwEUNs03qKg,24491
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -16,14 +16,14 @@ mdbq/config/products.py,sha256=tFqSfFSXyZXcof0gAeHq0Ftn4F5i9ucoMyIqZ1H_D2Q,4260
 mdbq/config/set_support.py,sha256=LJLEbUFrv8y-GVskiwOI8A9uRaCEAUa0Yfjugt4yLp0,768
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=h_BDc6oNmMCVFOUzZJq4nXNGDyJFJyycpWqlyrv7U04,3089
+mdbq/dataframe/converter.py,sha256=cD9u9eaDkOcxMaiZH6Wq_0Jp9PLsoPJOmrys7yZpGvI,5535
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=q0B4wXDSTtXg_vMN7MPh6zdxl6tT68tM74LmdVNQQek,31892
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/data_types.py,sha256=OMofgYtNddSJPQmtW4_5tm6f6Ny1P33RtasdikfE7As,9845
-mdbq/mysql/mysql.py,sha256=Dp9pXINt7H-FFmn84mpvOJQ41Y9i1CqrC_1lcWvYAQM,33607
+mdbq/mysql/data_types.py,sha256=N7_SyRviE7H25grmMydLT3W3PLk6s3gIK4i36hut4Ms,9791
+mdbq/mysql/mysql.py,sha256=ItSxepU7XPnzv5SHC2X4jBhqZPF3H6CgdWQe7IgI8bQ,39583
 mdbq/mysql/s_query.py,sha256=6-8O9MHhi3-7n3isJ7t2kTCYL2mSBC_HrxSQmXM5UtI,7901
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -34,7 +34,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-0.4.4.dist-info/METADATA,sha256=oxBpPHqo0B2xYJGI6qcmcW6E3uPM0oJvCsA4RaqDeeo,245
-mdbq-0.4.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-0.4.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-0.4.4.dist-info/RECORD,,
+mdbq-0.4.6.dist-info/METADATA,sha256=Ko9S2D8tJWAkE46mRmNzCiOHeOuJ6RI7TwI002tHEfA,245
+mdbq-0.4.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-0.4.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-0.4.6.dist-info/RECORD,,