wcp-library 1.2.4__tar.gz → 1.2.6__tar.gz

{wcp_library-1.2.4 → wcp_library-1.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wcp-library
-Version: 1.2.4
+Version: 1.2.6
 Summary: Common utilites for internal development at WCP
 Home-page: https://github.com/Whitecap-DNA/WCP-Library
 Author: Mitch-Petersen

{wcp_library-1.2.4 → wcp_library-1.2.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wcp-library"
-version = "1.2.4"
+version = "1.2.6"
 description = "Common utilites for internal development at WCP"
 authors = ["Mitch-Petersen <mitch.petersen@wcap.ca>"]
 readme = "README.md"

{wcp_library-1.2.4 → wcp_library-1.2.6}/wcp_library/async_sql/oracle.py

@@ -187,6 +187,27 @@ class AsyncOracleConnection(object):
                 rows = await cursor.fetchall()
         return rows
 
+    @retry
+    async def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = :{column}")
+        params = ' AND '.join(param_list)
+
+        main_dict = dfObj.to_dict('records')
+        query = f"""DELETE FROM {outputTableName} WHERE {params}"""
+        await self.execute_many(query, main_dict)
+
     @retry
     async def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """
@@ -209,14 +230,6 @@ class AsyncOracleConnection(object):
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
 
-        # if remove_nan:
-        #     for val, item in enumerate(main_dict):
-        #         for sub_item, value in item.items():
-        #             if pd.isna(value):
-        #                 main_dict[val][sub_item] = None
-        #             else:
-        #                 main_dict[val][sub_item] = value
-
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, bind)
         await self.execute_many(query, main_dict)
 

{wcp_library-1.2.4 → wcp_library-1.2.6}/wcp_library/async_sql/postgres.py

@@ -170,6 +170,27 @@ class AsyncPostgresConnection(object):
             rows = await cursor.fetchall()
         return rows
 
+    @retry
+    async def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = %({column})s")
+        params = ' AND '.join(param_list)
+
+        main_dict = dfObj.to_dict('records')
+        query = """DELETE FROM {} WHERE {}""".format(outputTableName, params)
+        await self.execute_many(query, main_dict)
+
     @retry
     async def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """

{wcp_library-1.2.4 → wcp_library-1.2.6}/wcp_library/credentials/postgres.py

@@ -13,7 +13,7 @@ class PostgresCredentialManager:
         self.password_url = URL("https://vault.wcap.ca/api/passwords/")
         self.api_key = passwordState_api_key
         self.headers = {"APIKey": self.api_key, 'Reason': 'Python Script Access'}
-        self._password_list_id = 207
+        self._password_list_id = 210
 
     def _get_credentials(self) -> dict:
         """

{wcp_library-1.2.4 → wcp_library-1.2.6}/wcp_library/sql/oracle.py

@@ -191,6 +191,27 @@ class OracleConnection(object):
         self._session_pool.release(connection)
         return rows
 
+    @retry
+    def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = :{column}")
+        params = ' AND '.join(param_list)
+
+        main_dict = dfObj.to_dict('records')
+        query = f"""DELETE FROM {outputTableName} WHERE {params}"""
+        self.execute_many(query, main_dict)
+
     @retry
     def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """
@@ -213,14 +234,6 @@ class OracleConnection(object):
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
 
-        # if remove_nan:
-        #     for val, item in enumerate(main_dict):
-        #         for sub_item, value in item.items():
-        #             if pd.isna(value):
-        #                 main_dict[val][sub_item] = None
-        #             else:
-        #                 main_dict[val][sub_item] = value
-
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, bind)
         self.execute_many(query, main_dict)
 

{wcp_library-1.2.4 → wcp_library-1.2.6}/wcp_library/sql/postgres.py

@@ -149,7 +149,8 @@ class PostgresConnection(object):
         """
 
         with self._session_pool.connection() as connection:
-            connection.executemany(query, dictionary)
+            cursor = connection.cursor()
+            cursor.executemany(query, dictionary)
 
     @retry
     def fetch_data(self, query: SQL | str, packed_data=None):
@@ -170,6 +171,27 @@ class PostgresConnection(object):
             rows = cursor.fetchall()
         return rows
 
+    @retry
+    def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        match_cols = ', '.join(match_cols)
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = %({column})s")
+        params = ' AND '.join(param_list)
+
+        main_dict = dfObj.to_dict('records')
+        query = """DELETE FROM {} WHERE {}""".format(outputTableName, params)
+        self.execute_many(query, main_dict)
+
     @retry
     def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
         """
@@ -192,14 +214,6 @@ class PostgresConnection(object):
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
 
-        # if remove_nan:
-        #     for val, item in enumerate(main_dict):
-        #         for sub_item, value in item.items():
-        #             if pd.isna(value):
-        #                 main_dict[val][sub_item] = None
-        #             else:
-        #                 main_dict[val][sub_item] = value
-
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, params)
         self.execute_many(query, main_dict)