mdbq 1.1.7__tar.gz → 1.1.9__tar.gz

{mdbq-1.1.7 → mdbq-1.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.7
+Version: 1.1.9
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.7 → mdbq-1.1.9}/mdbq/aggregation/aggregation.py

@@ -917,8 +917,8 @@ def upload(path, db_name, collection_name):
                         dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
                         df = df.astype(dtypes)
 
-                    # d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, filename=name, count=f'{i}/{count}')
+                    d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
+                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_dup=False, filename=name, count=f'{i}/{count}')
                     # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
                 except Exception as e:
                     print(name, e)
@@ -935,7 +935,7 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
     m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename)
 
 
-def file_dir():
+def file_dir(one_file=True):
     """
     按照文件记录对照表
     批量上传数据库
@@ -954,20 +954,26 @@ def file_dir():
             if platform.system() == 'Windows':
                 sub_path = sub_path.replace('/', '\\')
             # print(os.path.join(path, sub_path), db_name, table_name)
-            # 从每个文件夹中取出一个文件
-            real_path_list = []
-            for root, dirs, files in os.walk(os.path.join(path, sub_path), topdown=False):
-                for name in files:
-                    if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
-                        real_path_list.append(os.path.join(root, name))
-                        break
-            for real_path in real_path_list:
-                one_file_to_mysql(
-                    file=real_path,
-                    db_name=db_name,
-                    table_name=table_name,
-                    target_service='home_lx',
-                    database='mysql'
+            if one_file:  # 从每个文件夹中取出一个文件
+                real_path_list = []
+                for root, dirs, files in os.walk(os.path.join(path, sub_path), topdown=False):
+                    for name in files:
+                        if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
+                            real_path_list.append(os.path.join(root, name))
+                            break
+                for real_path in real_path_list:
+                    one_file_to_mysql(
+                        file=real_path,
+                        db_name=db_name,
+                        table_name=table_name,
+                        target_service='home_lx',
+                        database='mysql'
+                    )
+            else:  # 上传全部文件夹
+                upload(
+                    path=os.path.join(path, sub_path),
+                    db_name = db_name,
+                    collection_name = table_name,
                 )
             data.update({'入库进度': 1})  # 更新进度
     df = pd.DataFrame.from_dict(datas, orient='columns')
@@ -995,4 +1001,4 @@ if __name__ == '__main__':
     #     db_name = '生意经2',
     #     collection_name = 'e3_零售明细统计',
     # )
-    file_dir()
+    file_dir(one_file=True)

{mdbq-1.1.7 → mdbq-1.1.9}/mdbq/mysql/mysql.py

@@ -37,7 +37,7 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_duplicates=False, filename=None, count=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_dup=True, drop_duplicates=False, filename=None, count=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -144,15 +144,19 @@ class MysqlUpload:
                     condition = ' AND '.join(condition)  # 构建查询条件
                     # print(condition)
 
-                    sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
-                    # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                    cursor.execute(sql)
-                    result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
-                    if not result:  # 数据不存在则插入
+                    if drop_dup:
+                        sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
+                        # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                        cursor.execute(sql)
+                        result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
+                        if not result:  # 数据不存在则插入
+                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                            cursor.execute(sql)
+                        # else:
+                        #     print(f'重复数据不插入: {condition[:50]}...')
+                    else:
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                         cursor.execute(sql)
-                    # else:
-                    #     print(f'重复数据不插入: {condition[:50]}...')
                 except Exception as e:
                     # print(data)
                     # print(values)

{mdbq-1.1.7 → mdbq-1.1.9}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.7
+Version: 1.1.9
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.7 → mdbq-1.1.9}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.1.7',
+      version='1.1.9',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',