wcp-library 1.2.5__py3-none-any.whl → 1.2.7__py3-none-any.whl

wcp_library/async_sql/oracle.py

@@ -187,6 +187,28 @@ class AsyncOracleConnection(object):
                 rows = await cursor.fetchall()
         return rows
 
+    @retry
+    async def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName: str, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        df = dfObj[match_cols]
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = :{column}")
+        params = ' AND '.join(param_list)
+
+        main_dict = df.to_dict('records')
+        query = f"""DELETE FROM {outputTableName} WHERE {params}"""
+        await self.execute_many(query, main_dict)
+
     @retry
     async def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """
@@ -209,14 +231,6 @@ class AsyncOracleConnection(object):
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
 
-        # if remove_nan:
-        #     for val, item in enumerate(main_dict):
-        #         for sub_item, value in item.items():
-        #             if pd.isna(value):
-        #                 main_dict[val][sub_item] = None
-        #             else:
-        #                 main_dict[val][sub_item] = value
-
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, bind)
         await self.execute_many(query, main_dict)
 

wcp_library/async_sql/postgres.py

@@ -170,6 +170,28 @@ class AsyncPostgresConnection(object):
             rows = await cursor.fetchall()
         return rows
 
+    @retry
+    async def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName: str, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        df = dfObj[match_cols]
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = %({column})s")
+        params = ' AND '.join(param_list)
+
+        main_dict = df.to_dict('records')
+        query = """DELETE FROM {} WHERE {}""".format(outputTableName, params)
+        await self.execute_many(query, main_dict)
+
     @retry
     async def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """

wcp_library/sql/oracle.py

@@ -191,6 +191,28 @@ class OracleConnection(object):
         self._session_pool.release(connection)
         return rows
 
+    @retry
+    def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName: str, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        df = dfObj[match_cols]
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = :{column}")
+        params = ' AND '.join(param_list)
+
+        main_dict = df.to_dict('records')
+        query = f"""DELETE FROM {outputTableName} WHERE {params}"""
+        self.execute_many(query, main_dict)
+
     @retry
     def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """
@@ -213,14 +235,6 @@ class OracleConnection(object):
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
 
-        # if remove_nan:
-        #     for val, item in enumerate(main_dict):
-        #         for sub_item, value in item.items():
-        #             if pd.isna(value):
-        #                 main_dict[val][sub_item] = None
-        #             else:
-        #                 main_dict[val][sub_item] = value
-
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, bind)
         self.execute_many(query, main_dict)
 

wcp_library/sql/postgres.py

@@ -149,7 +149,8 @@ class PostgresConnection(object):
         """
 
         with self._session_pool.connection() as connection:
-            connection.executemany(query, dictionary)
+            cursor = connection.cursor()
+            cursor.executemany(query, dictionary)
 
     @retry
     def fetch_data(self, query: SQL | str, packed_data=None):
@@ -170,6 +171,28 @@ class PostgresConnection(object):
             rows = cursor.fetchall()
         return rows
 
+    @retry
+    def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName: str, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        df = dfObj[match_cols]
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = %({column})s")
+        params = ' AND '.join(param_list)
+
+        main_dict = df.to_dict('records')
+        query = """DELETE FROM {} WHERE {}""".format(outputTableName, params)
+        self.execute_many(query, main_dict)
+
     @retry
     def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """
@@ -192,14 +215,6 @@ class PostgresConnection(object):
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
 
-        # if remove_nan:
-        #     for val, item in enumerate(main_dict):
-        #         for sub_item, value in item.items():
-        #             if pd.isna(value):
-        #                 main_dict[val][sub_item] = None
-        #             else:
-        #                 main_dict[val][sub_item] = value
-
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, params)
         self.execute_many(query, main_dict)
 

{wcp_library-1.2.5.dist-info → wcp_library-1.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wcp-library
-Version: 1.2.5
+Version: 1.2.7
 Summary: Common utilites for internal development at WCP
 Home-page: https://github.com/Whitecap-DNA/WCP-Library
 Author: Mitch-Petersen

{wcp_library-1.2.5.dist-info → wcp_library-1.2.7.dist-info}/RECORD

@@ -4,8 +4,8 @@ wcp_library/async_credentials/api.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 wcp_library/async_credentials/oracle.py,sha256=MvC7wsHnSyAyPGnRVlH0dvvW2xk5ap_IAByoHX4rUWY,5436
 wcp_library/async_credentials/postgres.py,sha256=3wjqtLH0Nc-U80iscYnz5DV-9M2X4IjPW51C5MI2_tI,5297
 wcp_library/async_sql/__init__.py,sha256=m4eWmUGYUEDomhhOp1ScB2uSIXFsNUNO589o5RwWdyM,1035
-wcp_library/async_sql/oracle.py,sha256=W4_zXc3XxNnMZYXmyCCqx4dx1SQ6J8Tf_ifht_d8btI,7509
-wcp_library/async_sql/postgres.py,sha256=gFkgSpJQ7Giwjtjbh-38szhtFHPAt8WbvtZEp7UZ3oM,6704
+wcp_library/async_sql/oracle.py,sha256=TyKiArqbAR3b2cd2933UuH_EhnOx6he90yVVsQnrtjU,7923
+wcp_library/async_sql/postgres.py,sha256=HTCyJUvGHedmU_KKh756MSMZxzsxe8GLCrJpWITDaQU,7451
 wcp_library/credentials/__init__.py,sha256=HRmg7mqcATeclIz3lZQjSR4nmK6aY6MK9-QXEYZoFrw,1857
 wcp_library/credentials/ftp.py,sha256=FRxKVWifz7olQIrSjDgkTqk7rmc7Zdwmkfq7b62DQY8,4965
 wcp_library/credentials/oracle.py,sha256=IJVGd10LH5LUNKUSFAbApi0sjR1AbloMJRHTuR9zFrQ,5095
@@ -18,8 +18,8 @@ wcp_library/informatica.py,sha256=IXZtk_9X1XLbOEwFrsyOwTgajQKvtXgANBHmuTOP3Kk,40
 wcp_library/logging.py,sha256=e6gG7HFgUrMajUZs4Gs0atFfOJJmdmxN0GerfynNWlY,2061
 wcp_library/selenium_helper.py,sha256=rlphTXsUgnbaXZknY5nfQqxFhnc7UmrpzhV3hQ-cv7k,2509
 wcp_library/sql/__init__.py,sha256=CLlBEBrWVAwE79bUxuQiwikSrYH8m9QRYSJ2l0-ofsY,1003
-wcp_library/sql/oracle.py,sha256=a0wc9zwBdzhZCZFRWxZlokDJEEt-HqhqmwRbtp2T8ZA,7342
-wcp_library/sql/postgres.py,sha256=397Ii0ImjEo_e_QuWJRazfs0WpT9cGrnc6sBPyr4UyY,6596
-wcp_library-1.2.5.dist-info/METADATA,sha256=rBqolzSee-7JQLpWwpnMPDLHUdowhoQFN-gkLT4PvvM,1513
-wcp_library-1.2.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-wcp_library-1.2.5.dist-info/RECORD,,
+wcp_library/sql/oracle.py,sha256=hAnjn6lBYVjH10LI4x7KMd7dVmPONOHa9PQx6tvhc1U,7744
+wcp_library/sql/postgres.py,sha256=jHI7auaB2BYzaL1ztauFwIJP79M7ox6UyYJgqo73GDw,7048
+wcp_library-1.2.7.dist-info/METADATA,sha256=mo5dHBapD5bG3q5b16fU5JYLOGOwKnJvTuXZBt6vqmY,1513
+wcp_library-1.2.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+wcp_library-1.2.7.dist-info/RECORD,,