mdbq 1.3.1__tar.gz → 1.3.3__tar.gz

{mdbq-1.3.1 → mdbq-1.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.3.1
+Version: 1.3.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.3.1 → mdbq-1.3.3}/mdbq/aggregation/aggregation.py

@@ -599,11 +599,11 @@ class DatabaseUpdate:
             }
         )
 
-    def upload_df(self, service_databases=[{}]):
+    def upload_df(self, service_databases=[{}], path=None, system_name=None):
         """
         将清洗后的 df 上传数据库
         """
-        df_to_json = df_types.DataTypes()  # json 文件, 包含数据的 dtypes 信息
+        df_to_json = df_types.DataTypes(path=path, system_name=system_name)  # json 文件, 包含数据的 dtypes 信息
         for service_database in service_databases:
             for service_name, database in service_database.items():
                 # print(service_name, database)
@@ -803,7 +803,7 @@ class DatabaseUpdate:
         return df
 
 
-def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}):
+def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, system_name=None):
     """ 上传一个文件夹到 mysql 或者 mongodb 数据库 """
     if not os.path.isdir(path):
         print(f'{os.path.splitext(os.path.basename(__file__))[0]}.upload_dir: 函数只接受文件夹路径，不是一个文件夹: {path}')
@@ -845,7 +845,7 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
         )
 
     # 从本地 json 文件从读取 df 的数据类型信息
-    df_to_json = df_types.DataTypes()
+    df_to_json = df_types.DataTypes(path=json_path, system_name=system_name)
     dtypes = df_to_json.load_dtypes(
         db_name=db_name,
         collection_name=collection_name,
@@ -976,13 +976,13 @@ def test2():
         # {'home_lx': 'mongodb'},
         {'home_lx': 'mysql'},
         # {'nas': 'mysql'}
-    ])
+    ], path=None, system_name=None)
 
 
 if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
-    # file_dir(one_file=False)
+    file_dir(one_file=False)
     # one_file_to_mysql(
     #     file='/Users/xigua/数据中心/原始文件2/推广报表/品销宝/账户/账户_明星店铺报表_2023-11-13_2023-12-12.csv',
     #     db_name='推广数据2',
@@ -1001,11 +1001,12 @@ if __name__ == '__main__':
 
     # test2()
 
-    file = '/Users/xigua/Downloads/余额查询.csv'
-    df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
-    m.df_to_mysql(df=df, db_name='test', table_name='增量更新测试',
-                  drop_dup=False,
-                  icm_update=['日期', '推广费余额']
-                  )
+    # file = ''
+    # df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
+    # username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
+    # m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    # m.df_to_mysql(df=df, db_name='test', table_name='增量更新测试',
+    #               drop_dup=False,
+    #               # icm_update=['日期', '推广费余额'],
+    #               system_name='company',
+    #               )

{mdbq-1.3.1 → mdbq-1.3.3}/mdbq/aggregation/df_types.py

@@ -48,7 +48,11 @@ class DataTypes:
                 }
         }
         self.path = set_support.SetSupport(dirname='support').dirname
-        self.json_file = os.path.join(self.path, 'df_types.json')
+        self.system_name = None
+        if self.system_name:
+            self.json_file = os.path.join(self.path, f'{self.system_name}_df_types.json')
+        else:
+            self.json_file = os.path.join(self.path, 'df_types.json')
         if not os.path.isdir(self.path):
             os.makedirs(self.path)
         if not os.path.isfile(self.json_file):

{mdbq-1.3.1 → mdbq-1.3.3}/mdbq/aggregation/mysql_types.py

@@ -38,7 +38,7 @@ class DataTypes:
      数据简介: 记录 dataframe 或者数据库的列信息(dtypes)，可以记录其信息或者加载相关信息用于入库使用，
      第一字段为分类(如 dataframe/mysql)，第二字段为数据库名，第三字段为集合名，第四段列名及其数据类型
     """
-    def __init__(self):
+    def __init__(self, path=None, system_name=None):
         self.datas = {
             '_json统计':
                 {
@@ -46,11 +46,16 @@ class DataTypes:
                     '数据库量': 0,
                     '集合数量': 0,
                     '字段量': 0,
-                    '数据简介': '记录 dataframe 或者数据库的列信息(dtypes)',
+                    '数据简介': '记录数据库各表的数据类型信息',
                 }
         }
-        self.path = set_support.SetSupport(dirname='support').dirname
-        self.json_file = os.path.join(self.path, 'mysql_types.json')
+        self.path = path
+        if not self.path:
+            self.path = set_support.SetSupport(dirname='support').dirname
+        self.system_name = system_name
+        if not self.system_name:
+            self.system_name = 'home_lx'
+        self.json_file = os.path.join(self.path, f'mysql_types_{self.system_name}.json')
         if not os.path.isdir(self.path):
             os.makedirs(self.path)
         if not os.path.isfile(self.json_file):
@@ -149,12 +154,10 @@ class DataTypes:
             return {}, cl, None, None  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
 
 
-def mysql_all_dtypes(db_name=None, table_name=None, path=None):
+def mysql_all_dtypes(db_name=None, table_name=None, path=None, system_name=None):
     """
     更新笔记本 mysql 中所有数据库的 dtypes 信息到本地 json
     """
-    if not path:
-        path = set_support.SetSupport(dirname='support').dirname
 
     username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
     config = {
@@ -205,8 +208,7 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
             connection.close()
         time.sleep(0.5)
 
-    d = DataTypes()
-    d.json_file = os.path.join(path, f'mysql_types.json')  # # json 保存位置
+    d = DataTypes(path=path, system_name=system_name)
     for result in results:
         for db_n, table_n in result.items():
             # print(db_n, table_n, db_name, table_name)
@@ -218,7 +220,6 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
                     continue
             # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
             print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
-            # d.mysql_dtypes_to_json(db_name=db_n, table_name=table_n, path=path)
             sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
             # 获取数据表的指定列, 返回列表
             # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
@@ -242,5 +243,5 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
 if __name__ == '__main__':
     # 更新 mysql 中所有数据库的 dtypes 信息到本地 json
     mysql_all_dtypes(
-        path='/Users/xigua/Downloads',
+        path='/Users/xigua/Downloads', system_name='company',
     )

{mdbq-1.3.1 → mdbq-1.3.3}/mdbq/aggregation/query_data.py

@@ -495,7 +495,7 @@ class GroupBy:
         df.to_excel(os.path.join(path, filename + '.xlsx'), index=index, header=header, engine=engine, freeze_panes=freeze_panes)
         
         
-def data_aggregation(service_databases=[{}], months=1):
+def data_aggregation(service_databases=[{}], months=1, system_name=None,):
     """
     1. 从数据库中读取数据
     2. 数据聚合清洗
@@ -558,7 +558,8 @@ def data_aggregation(service_databases=[{}], months=1):
                     db_name=db_name,
                     table_name=table_name,
                     drop_dup=False,
-                    icm_update=unique_key_list
+                    icm_update=unique_key_list,
+                    system_name=system_name,
                 )  # 3. 回传数据库
             res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
@@ -566,7 +567,8 @@ def data_aggregation(service_databases=[{}], months=1):
                 db_name='聚合数据',
                 table_name='_全店商品销售',
                 drop_dup=False,
-                icm_update=['日期', '商品id']  # 设置唯一主键
+                icm_update=['日期', '商品id'],  # 设置唯一主键
+                system_name = system_name,
             )
             res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
@@ -574,12 +576,13 @@ def data_aggregation(service_databases=[{}], months=1):
                 db_name='聚合数据',
                 table_name='_推广商品销售',
                 drop_dup=False,
-                icm_update=['日期', '商品id']  # 设置唯一主键
+                icm_update=['日期', '商品id'],  # 设置唯一主键
+                system_name=system_name,
             )
 
     # optimize_data.op_data(service_databases=service_databases, days=3650)  # 立即启动对聚合数据的清理工作
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

{mdbq-1.3.1 → mdbq-1.3.3}/mdbq/mysql/mysql.py

@@ -57,7 +57,7 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], df_sql=False, drop_dup=True, drop_duplicates=False, filename=None, count=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], df_sql=False, drop_dup=True, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -115,44 +115,42 @@ class MysqlUpload:
         connection = pymysql.connect(**self.config)  # 重新连接数据库
         with connection.cursor() as cursor:
             # 1. 查询表, 不存在则创建一个空表
-            sql = f"SHOW TABLES LIKE '{table_name}';"  # 有特殊字符不需转义
-            cursor.execute(sql)
+            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
+            cursor.execute(sql, (table_name))
             if not cursor.fetchone():
-                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY)"
+                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
                 cursor.execute(sql)
                 print(f'创建 mysql 表: {table_name}')
 
             # 2. 列数据类型转换，将 df 数据类型转换为 mysql 的数据类型
-            dtypes, cl, db_n, tb_n = self.convert_dtypes(df=df, db_name=db_name, table_name=table_name)
+            dtypes, cl, db_n, tb_n = self.convert_dtypes(df=df, db_name=db_name, table_name=table_name, path=json_path, system_name=system_name)
 
             #  有特殊字符不需转义
-            sql = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{db_name}' AND TABLE_NAME = '{table_name}';"
-            cursor.execute(sql)
+            sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
+            cursor.execute(sql, (db_name, table_name))
             col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]
             cols = df.columns.tolist()
             col_not_exist = [col for col in cols if col not in col_exist]
             # 检查列，不存在则新建列
             if col_not_exist:  # 数据表中不存在的列
                 for col in col_not_exist:
-                    try:
-                        #  创建列，需转义
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
-                        cursor.execute(sql)
-                        print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
-
-                        # 创建索引
-                        if col == '日期':
-                            cursor.execute(f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = '{col}'")
-                            result = cursor.fetchone()  # 检查索引是否存在
-                            if not result:
-                                cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`)")
-                    except Exception as e:
-                        print(f'{self.filename}: {e}')
+                    #  创建列，需转义
+                    sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
+                    cursor.execute(sql)
+                    print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                    # 创建索引
+                    if col == '日期':
+                        sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
+                        cursor.execute(sql, (col))
+                        result = cursor.fetchone()  # 检查索引是否存在
+                        if not result:
+                            cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`)")
             connection.commit()  # 提交事务
 
             if df_sql:
                 now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-                print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
+                print(f'{now}正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
                 engine = create_engine(
                     f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
                 df.to_sql(
@@ -169,11 +167,11 @@ class MysqlUpload:
             # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
             # ⚠️ mysql_all_dtypes 函数默认只读取 home_lx 的数据库信息，不会读取其他系统
             if cl and db_n and tb_n:
-                mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name)  # 更新一个表的 dtypes
+                mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name, system_name=system_name)  # 更新一个表的 dtypes
             elif cl and db_n:
-                mysql_types.mysql_all_dtypes(db_name=db_name)  # 更新一个数据库的 dtypes
+                mysql_types.mysql_all_dtypes(db_name=db_name, system_name=system_name)  # 更新一个数据库的 dtypes
             elif cl:
-                mysql_types.mysql_all_dtypes()  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
+                mysql_types.mysql_all_dtypes(system_name=system_name)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
 
             # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
             if drop_duplicates and '日期' in df.columns.tolist():
@@ -195,20 +193,21 @@ class MysqlUpload:
                 try:
                     cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
                     # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f"'{item}'" for item in data.values()])  # 值要加单引号 ''
+                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
                     condition = []
                     for k, v in data.items():
-                        condition += [f"`{k}` = '{v}'"]
+                        condition += [f'`{k}` = "{v}"']
                     condition = ' AND '.join(condition)  # 构建查询条件
                     # print(condition)
 
                     if drop_dup:  # 查重插入
-                        sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
+                        sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
                         # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                        # print(sql)
                         cursor.execute(sql)
                         result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
                         if not result:  # 数据不存在则插入
-                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
                             cursor.execute(sql)
                         # else:
                         #     print(f'重复数据不插入: {condition[:50]}...')
@@ -224,7 +223,7 @@ class MysqlUpload:
                         unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
                         condition = []
                         for up_col in icm_update:
-                            condition += [f"`{up_col}` = '{data[up_col]}'"]
+                            condition += [f'`{up_col}` = "{data[up_col]}"']
                         condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
                         sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
                         # print(sql)
@@ -251,7 +250,7 @@ class MysqlUpload:
                                         not_change_col += [item for item in update_col if item != col]
                                 # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
                                 if change_values:  # change_values 有数据返回，表示值需要更新
-                                    not_change_values = [f"`{col}` = '{str(data[col])}'" for col in not_change_col]
+                                    not_change_values = [f'`{col}` = "{str(data[col])}"' for col in not_change_col]
                                     not_change_values = ' AND '.join(not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
                                     # print(change_values, not_change_values)
                                     condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
@@ -283,7 +282,7 @@ class MysqlUpload:
                     #     # cursor.execute(sql)
 
                     else:
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
                         cursor.execute(sql)
                 except Exception as e:
                     # print(data)
@@ -293,7 +292,7 @@ class MysqlUpload:
         connection.commit()  # 提交事务
         connection.close()
 
-    def convert_dtypes(self, df, db_name, table_name):
+    def convert_dtypes(self, df, db_name, table_name, path=None, system_name=None):
         """
         根据本地 json 转换 df 的类型为 mysql 专有的数据类型
         可能不存在本地 json 文件 (函数按指定规则转换并更新 json)
@@ -301,7 +300,7 @@ class MysqlUpload:
         """
         cols = df.columns.tolist()
         # path = set_support.SetSupport(dirname='support').dirname
-        d = mysql_types.DataTypes()
+        d = mysql_types.DataTypes(path=path, system_name=system_name)
         # 从本地文件中读取 dtype 信息
         dtypes, cl, db_n, tb_n = d.load_dtypes(cl='mysql', db_name=db_name, table_name=table_name)
         # 可能会因为没有 json 文件, 返回 None

{mdbq-1.3.1 → mdbq-1.3.3}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.3.1
+Version: 1.3.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.3.1 → mdbq-1.3.3}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.3.1',
+      version='1.3.3',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',