mdbq 0.4.5__tar.gz → 0.4.6__tar.gz

{mdbq-0.4.5 → mdbq-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.4.5
+Version: 0.4.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.4.5 → mdbq-0.4.6}/mdbq/aggregation/aggregation.py

@@ -1035,5 +1035,5 @@ if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
 
-    main()
-    # upload()
+    # main()
+    upload()

{mdbq-0.4.5 → mdbq-0.4.6}/mdbq/aggregation/query_data.py

@@ -525,5 +525,5 @@ def data_aggregation(service_databases=[{}]):
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'home_lx': 'mysql'}])
+    data_aggregation(service_databases=[{'company': 'mysql'}])
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

{mdbq-0.4.5 → mdbq-0.4.6}/mdbq/dataframe/converter.py

@@ -9,6 +9,51 @@ class DataFrameConverter(object):
         self.df = df
 
     def convert_df_cols(self, df=pd.DataFrame({})):
+        """
+        清理 dataframe 非法值
+        对数据类型进行转换(尝试将 object 类型转为 int 或 float)
+        """
+        if len(df) == 0:
+            df = self.df
+            if len(df) == 0:
+                return
+        # dtypes = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
+        df.replace([np.inf, -np.inf], 0, inplace=True)  # 清理一些非法值
+        df.replace(to_replace=['\\N', '-', '--', '', 'nan'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
+        df.replace(to_replace=[','], value='', regex=True, inplace=True)
+        df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
+        df.replace(to_replace=['"'], value='', regex=True, inplace=True)
+        cols = df.columns.tolist()
+
+        for col in cols:
+            # df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
+            # 百分比在某些数据库中不兼容, 转换百分比为小数
+            df[col] = df[col].apply(lambda x: float(float((str(x).rstrip("%"))) / 100) if str(x).endswith('%') and '~' not in str(x) else x)
+            # 尝试转换合适的数据类型
+            if df[col].dtype == 'object':
+                try:
+                    # df[col] = df[col].astype(int)  # 尝试转换 int
+                    df[col] = df[col].apply(lambda x: int(x) if '_' not in str(x) else x)
+                except:
+                    # df[col] = df[col].astype('float64', errors='ignore')    # 尝试转换 float, 报错则忽略
+                    try:
+                        df[col] = df[col].apply(lambda x: float(x) if '_' not in str(x) else x)
+                    except:
+                        pass
+            if df[col].dtype == 'float':  # 对于小数类型, 保留 6 位小数
+                df[col] = df[col].apply(lambda x: round(float(x), 6) if x != 0 else x)
+            # 清理列名, 在 mysql 里面列名不能含有某些特殊字符
+            if '日期' in col or '时间' in col:
+                try:
+                    df[col] = df[col].apply(lambda x: pd.to_datetime(x))
+                except:
+                    pass
+            new_col = col.lower()
+            df.rename(columns={col: new_col}, inplace=True)
+        df.fillna(0, inplace=True)
+        return df
+
+    def convert_df_cols_bak(self, df=pd.DataFrame({})):
         """
         清理 dataframe 列名的不合规字符(mysql)
         对数据类型进行转换(尝试将 object 类型转为 int 或 float)

{mdbq-0.4.5 → mdbq-0.4.6}/mdbq/mysql/mysql.py

@@ -9,6 +9,7 @@ import warnings
 import pymysql
 import numpy as np
 import pandas as pd
+from more_itertools.more import iequals
 from sqlalchemy import create_engine
 import os
 import calendar
@@ -63,7 +64,7 @@ class MysqlUpload:
             return False
 
     # @try_except
-    def df_to_mysql(self, df, tabel_name, db_name='远程数据源'):
+    def df_to_mysql_bak(self, df, tabel_name, db_name='远程数据源'):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -182,6 +183,106 @@ class MysqlUpload:
         finally:
             connection.close()
 
+    def df_to_mysql(self, df, tabel_name, db_name='远程数据源', drop_duplicates=False):
+        """
+        将 df 写入数据库
+        db_name: 数据库名称
+        tabel_name: 集合/表名称
+        drop_duplicates：仅限于聚合数据使用，其他情况不要设置
+        """
+        cv = converter.DataFrameConverter()
+        df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值
+
+        connection = pymysql.connect(**self.config)  # 连接数据库
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            database_exists = cursor.fetchone()
+            if not database_exists:
+                # 如果数据库不存在，则新建
+                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
+                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
+                if '192.168.1.100' in str(self.host):
+                    sql = f"CREATE DATABASE `{db_name}`"
+                else:
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                cursor.execute(sql)
+                connection.commit()
+                print(f"创建Database: {db_name}")
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            # 1. 查询表, 不存在则创建一个空表
+            sql = f"SHOW TABLES LIKE '{tabel_name}';"  # 有特殊字符不需转义
+            cursor.execute(sql)
+            if not cursor.fetchone():
+                sql = f"CREATE TABLE IF NOT EXISTS `{tabel_name}` (id INT AUTO_INCREMENT PRIMARY KEY)"
+                cursor.execute(sql)
+                print(f'创建 mysql 表: {tabel_name}')
+
+            # 2. 列数据类型转换
+            dtypes = self.convert_dtypes(df=df, db_name=db_name, tabel_name=tabel_name)
+
+            #  有特殊字符不需转义
+            sql = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{db_name}' AND TABLE_NAME = '{tabel_name}';"
+            cursor.execute(sql)
+            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]
+            cols = df.columns.tolist()
+            col_not_exist = [col for col in cols if col not in col_exist]
+            # 检查列，不存在则新建列
+            if col_not_exist:  # 数据表中不存在的列
+                for col in col_not_exist:
+                    try:
+                        #  创建列，需转义
+                        sql = f"ALTER TABLE `{tabel_name}` ADD COLUMN `{col}` {dtypes[col]} DEFAULT NULL;"
+                        cursor.execute(sql)
+                        print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                        # 创建索引
+                        if col == '日期':
+                            cursor.execute(f"SHOW INDEXES FROM `{tabel_name}` WHERE `Column_name` = '{col}'")
+                            result = cursor.fetchone()  # 检查索引是否存在
+                            if not result:
+                                cursor.execute(f"CREATE INDEX index_name ON `{tabel_name}`(`{col}`)")
+                    except:
+                        pass
+            connection.commit()  # 提交事务
+
+            # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
+            if drop_duplicates and '日期' in df.columns.tolist():
+                dates = df['日期'].values.tolist()
+                start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
+                end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+                sql = f"DELETE FROM `{tabel_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
+                cursor.execute(sql)
+                connection.commit()
+
+            # 5. 更新插入数据
+            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
+            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{tabel_name}')
+            datas = df.to_dict(orient='records')
+            for data in datas:
+                try:
+                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名转义
+                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                    values = ', '.join([f"'{item}'" for item in data.values()])  # 值要加单引号 ''
+                    condition = []
+                    for k, v in data.items():
+                        condition += [f"`{k}` = '{v}'"]
+                    condition = ' AND '.join(condition)  # 构建查询条件
+                    # print(condition)
+
+                    sql = f"SELECT {cols} FROM `{tabel_name}` WHERE {condition}"
+                    cursor.execute(sql)
+                    result = cursor.fetchall()  # 获取查询结果, 如果有结果返回 list，没有则返回空元组 tuple
+                    if not result:  # 数据不存在则插入
+                        sql = f"INSERT INTO `{tabel_name}` ({cols}) VALUES ({values});"
+                        cursor.execute(sql)
+                except:
+                    pass
+        connection.commit()  # 提交事务
+
     def convert_dtypes(self, df, db_name, tabel_name):
         """
         根据本地已经存在的记录着 mysql dtypes 的 json 文件转换 df 的类型为 mysql 专有的数据类型
@@ -240,7 +341,7 @@ class MysqlUpload:
                 return 'mediumtext'
             return 'INT'
         elif dtype == 'float64':
-            return 'FLOAT'
+            return 'double'  # mysql 中不要使用 float 类型，会影响计算结果
         elif dtype == 'object':
             return 'mediumtext'
         else:
@@ -689,8 +790,13 @@ def download_datas(tabel_name, save_path, start_date):
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
+    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
     print(username, password, host, port)
 
+    df = pd.read_csv('/Users/xigua/Downloads/余额查询.csv', encoding='utf-8_sig', header=0, na_filter=False)
+    # df = df.to_dict(orient='records')
+    m = MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql_new(df=df, db_name='te2- %s t', tabel_name='测 -sdf @%试 表')
+
 
 

{mdbq-0.4.5 → mdbq-0.4.6}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.4.5
+Version: 0.4.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.4.5 → mdbq-0.4.6}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='0.4.5',
+      version='0.4.6',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',