mdbq 1.1.7__py3-none-any.whl → 1.1.8__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -917,8 +917,8 @@ def upload(path, db_name, collection_name):
                         dtypes = {k: dtypes[k] for k in intersection_keys}  # 使用交集的键创建新字典
                         df = df.astype(dtypes)
 
-                    # d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, filename=name, count=f'{i}/{count}')
+                    d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
+                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_dup=False, filename=name, count=f'{i}/{count}')
                     # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
                 except Exception as e:
                     print(name, e)

mdbq/mysql/mysql.py

@@ -37,7 +37,7 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_duplicates=False, filename=None, count=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_dup=True, drop_duplicates=False, filename=None, count=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -144,15 +144,19 @@ class MysqlUpload:
                     condition = ' AND '.join(condition)  # 构建查询条件
                     # print(condition)
 
-                    sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
-                    # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                    cursor.execute(sql)
-                    result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
-                    if not result:  # 数据不存在则插入
+                    if drop_dup:
+                        sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
+                        # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                        cursor.execute(sql)
+                        result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
+                        if not result:  # 数据不存在则插入
+                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                            cursor.execute(sql)
+                        # else:
+                        #     print(f'重复数据不插入: {condition[:50]}...')
+                    else:
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                         cursor.execute(sql)
-                    # else:
-                    #     print(f'重复数据不插入: {condition[:50]}...')
                 except Exception as e:
                     # print(data)
                     # print(values)

{mdbq-1.1.7.dist-info → mdbq-1.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.7
+Version: 1.1.8
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.7.dist-info → mdbq-1.1.8.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=mz80JHBlnkqKruTl1zvutTe_ed9afgdgZAAjeaJMOrI,57143
+mdbq/aggregation/aggregation.py,sha256=Y0dxVaq_1raS9ZKloDvbudty86Ot-aW0XNZ_yJL6uPE,57157
 mdbq/aggregation/df_types.py,sha256=rHLIgv82PJSFmDvXkZyOJAffXkFyyMyFO23w9tUt8EQ,7525
 mdbq/aggregation/mysql_types.py,sha256=kzUAGM4FYp77tA_dvHjZNGyPoUAOU1WY5QD0uanh9I4,10418
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
@@ -25,7 +25,7 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/data_types_即将删除.py,sha256=sjBBDKr9674LdjM5N_dwyJACdZPbdB8Beli59jGdgnQ,10378
-mdbq/mysql/mysql.py,sha256=tO-VGCiXBzJh488a6i7lbg9LW6eJu7WUke52NAYmUyw,35897
+mdbq/mysql/mysql.py,sha256=bilmjcbrxk9x1gmBZdAYlV8b8IV8N_1w0Kc0Z7OBkDw,36139
 mdbq/mysql/s_query.py,sha256=4c24SwbqtnO33o8CgWlTQ_j8sZYl5BRIQkaD9CI-vTY,7901
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.1.7.dist-info/METADATA,sha256=S4uJYKa8wM2jz5iU3UyPPdeatDfUW-CdOW79R__WAk0,245
-mdbq-1.1.7.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.1.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.1.7.dist-info/RECORD,,
+mdbq-1.1.8.dist-info/METADATA,sha256=6T_PVKB7lW2vNRBJ05V3BURpr4LM9udieCFxcdNGfzk,245
+mdbq-1.1.8.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.1.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.1.8.dist-info/RECORD,,