wcp-library 1.2.9__py3-none-any.whl → 1.3.1__py3-none-any.whl

wcp_library/sql/postgres.py

@@ -4,9 +4,9 @@ from typing import Optional
 import numpy as np
 import pandas as pd
 from psycopg.sql import SQL
-from psycopg_pool import ConnectionPool
+from psycopg_pool import AsyncConnectionPool, ConnectionPool
 
-from wcp_library.sql import retry
+from wcp_library.sql import retry, async_retry
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +37,35 @@ def _connect_warehouse(username: str, password: str, hostname: str, port: int, d
     return session_pool
 
 
+async def _async_connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
+                             max_connections: int) -> AsyncConnectionPool:
+    """
+    Create Warehouse Connection
+
+    :param username: username
+    :param password: password
+    :param hostname: hostname
+    :param port: port
+    :param database: database
+    :param min_connections:
+    :param max_connections:
+    :return: session_pool
+    """
+
+    url = f"postgres://{username}:{password}@{hostname}:{port}/{database}"
+
+    session_pool = AsyncConnectionPool(
+        conninfo=url,
+        min_size=min_connections,
+        max_size=max_connections,
+    )
+    await session_pool.open()
+    return session_pool
+
+
+"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
+
+
 class PostgresConnection(object):
     """
     SQL Connection Class
@@ -252,3 +281,214 @@ class PostgresConnection(object):
         """
 
         self._session_pool.close()
+
+
+class AsyncPostgresConnection(object):
+    """
+    SQL Connection Class
+
+    :return: None
+    """
+
+    def __init__(self, min_connections: int = 2, max_connections: int = 5):
+        self._username: Optional[str] = None
+        self._password: Optional[str] = None
+        self._hostname: Optional[str] = None
+        self._port: Optional[int] = None
+        self._database: Optional[str] = None
+        self._session_pool: Optional[AsyncConnectionPool] = None
+
+        self.min_connections = min_connections
+        self.max_connections = max_connections
+
+        self._retry_count = 0
+        self.retry_limit = 50
+        self.retry_error_codes = ['08001', '08004']
+
+    @async_retry
+    async def _connect(self) -> None:
+        """
+        Connect to the warehouse
+
+        :return: None
+        """
+
+        self._session_pool = await _connect_warehouse(self._username, self._password, self._hostname, self._port,
+                                                      self._database, self.min_connections, self.max_connections)
+
+    async def set_user(self, credentials_dict: dict) -> None:
+        """
+        Set the user credentials and connect
+
+        :param credentials_dict: dictionary of connection details
+        :return: None
+        """
+
+        self._username: Optional[str] = credentials_dict['UserName']
+        self._password: Optional[str] = credentials_dict['Password']
+        self._hostname: Optional[str] = credentials_dict['Host']
+        self._port: Optional[int] = int(credentials_dict['Port'])
+        self._database: Optional[str] = credentials_dict['Database']
+
+        await self._connect()
+
+    async def close_connection(self) -> None:
+        """
+        Close the connection
+
+        :return: None
+        """
+
+        await self._session_pool.close()
+
+    @async_retry
+    async def execute(self, query: SQL | str) -> None:
+        """
+        Execute the query
+
+        :param query: query
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            await connection.execute(query)
+
+    @async_retry
+    async def safe_execute(self, query: SQL | str, packed_values: dict) -> None:
+        """
+        Execute the query without SQL Injection possibility, to be used with external facing projects.
+
+        :param query: query
+        :param packed_values: dictionary of values
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            await connection.execute(query, packed_values)
+
+    @async_retry
+    async def execute_multiple(self, queries: list[list[SQL | str, dict]]) -> None:
+        """
+        Execute multiple queries
+
+        :param queries: list of queries
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            for item in queries:
+                query = item[0]
+                packed_values = item[1]
+                if packed_values:
+                    await connection.execute(query, packed_values)
+                else:
+                    await connection.execute(query)
+
+    @async_retry
+    async def execute_many(self, query: SQL | str, dictionary: list[dict]) -> None:
+        """
+        Execute many queries
+
+        :param query: query
+        :param dictionary: dictionary of values
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            await connection.executemany(query, dictionary)
+
+    @async_retry
+    async def fetch_data(self, query: SQL | str, packed_data=None):
+        """
+        Fetch the data from the query
+
+        :param query: query
+        :param packed_data: packed data
+        :return: rows
+        """
+
+        async with self._session_pool.connection() as connection:
+            cursor = connection.cursor()
+            if packed_data:
+                await cursor.execute(query, packed_data)
+            else:
+                await cursor.execute(query)
+            rows = await cursor.fetchall()
+        return rows
+
+    @async_retry
+    async def remove_matching_data(self, dfObj: pd.DataFrame, outputTableName: str, match_cols: list) -> None:
+        """
+        Remove matching data from the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param match_cols: list of columns
+        :return: None
+        """
+
+        df = dfObj[match_cols]
+        param_list = []
+        for column in match_cols:
+            param_list.append(f"{column} = %({column})s")
+        if len(param_list) > 1:
+            params = ' AND '.join(param_list)
+        else:
+            params = param_list[0]
+
+        main_dict = df.to_dict('records')
+        query = """DELETE FROM {} WHERE {}""".format(outputTableName, params)
+        await self.execute_many(query, main_dict)
+
+    @async_retry
+    async def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
+        """
+        Export the DataFrame to the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param columns: list of columns
+        :param remove_nan: remove NaN values
+        :return: None
+        """
+
+        col = ', '.join(columns)
+        param_list = []
+        for column in columns:
+            param_list.append(f"%({column})s")
+        params = ', '.join(param_list)
+
+        if remove_nan:
+            dfObj = dfObj.replace({np.nan: None})
+        main_dict = dfObj.to_dict('records')
+        for record in main_dict:
+            for key in record:
+                if record[key] == '':
+                    record[key] = None
+
+        query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, params)
+        await self.execute_many(query, main_dict)
+
+    @async_retry
+    async def truncate_table(self, tableName: str) -> None:
+        """
+        Truncate the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        truncateQuery = """TRUNCATE TABLE {}""".format(tableName)
+        await self.execute(truncateQuery)
+
+    @async_retry
+    async def empty_table(self, tableName: str) -> None:
+        """
+        Empty the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        deleteQuery = """DELETE FROM {}""".format(tableName)
+        await self.execute(deleteQuery)

{wcp_library-1.2.9.dist-info → wcp_library-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wcp-library
-Version: 1.2.9
+Version: 1.3.1
 Summary: Common utilites for internal development at WCP
 Home-page: https://github.com/Whitecap-DNA/WCP-Library
 Author: Mitch-Petersen

wcp_library-1.3.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+wcp_library/__init__.py,sha256=hwLbcu00uI6L_xjXO9-I0YcODl2WtIOkdNLoDcXv7zk,591
+wcp_library/credentials/__init__.py,sha256=HRmg7mqcATeclIz3lZQjSR4nmK6aY6MK9-QXEYZoFrw,1857
+wcp_library/credentials/credential_manager_asynchronous.py,sha256=oTuo-TJpOWU52eaW6fhQL6QZ3zZvz_1WSpuSeA9thco,5728
+wcp_library/credentials/credential_manager_synchronous.py,sha256=ouPNLt20FvocuMoFx49mbDL7-Moj_WrlpR7k97mx-i4,5512
+wcp_library/credentials/ftp.py,sha256=O4oSPtCCv_0w6sLITFnY9EpN6-K2XxeibbGB0VnTHJ4,2589
+wcp_library/credentials/oracle.py,sha256=m0WtmSyUdKUfsz1SPkRgc7A080rK6cq7jVoQ0YcWJ50,2867
+wcp_library/credentials/postgres.py,sha256=tCCWdc10lgdu6zbU-Hv0ZxGw9rn6ZvDvFkMZqm9qfBo,2571
+wcp_library/emailing.py,sha256=xqNly6Tmj-pvwl5bdys3gauZFDd4SuWCQYzGFNemv2Q,2496
+wcp_library/ftp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wcp_library/ftp/ftp.py,sha256=EpyW0J2QIGxP8zVGD4VarA0hi4C2XAPDPF-0j2sRdpI,4350
+wcp_library/ftp/sftp.py,sha256=hykXGLGdxe7DYAxFdTwjPjTEOYuIpSMyK3NOiTQNUK0,4176
+wcp_library/informatica.py,sha256=IXZtk_9X1XLbOEwFrsyOwTgajQKvtXgANBHmuTOP3Kk,4064
+wcp_library/logging.py,sha256=e6gG7HFgUrMajUZs4Gs0atFfOJJmdmxN0GerfynNWlY,2061
+wcp_library/selenium_helper.py,sha256=rlphTXsUgnbaXZknY5nfQqxFhnc7UmrpzhV3hQ-cv7k,2509
+wcp_library/sql/__init__.py,sha256=s2psmwkq_ZU23iGWvXjJrLu0hD1fB6CDv6RHcK7y828,1917
+wcp_library/sql/oracle.py,sha256=TGiTC5L5UcM5QcHFajgn43NI8HygOGIEAtLmLbVFp2I,15772
+wcp_library/sql/postgres.py,sha256=ybk7WudcuT3_SmdkWrl01Z8rmiishHqvB1SCI9nng7g,14392
+wcp_library-1.3.1.dist-info/METADATA,sha256=jBdqrTJokOY93ioq_ZLIfykk6_V6Mdvx0qFQAaWxaWE,1513
+wcp_library-1.3.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+wcp_library-1.3.1.dist-info/RECORD,,

wcp_library/async_credentials/__init__.py

@@ -1,49 +0,0 @@
-import secrets
-import string
-
-
-class MissingCredentialsError(KeyError):
-    pass
-
-
-async def generate_password(length: int=12, use_nums: bool=True, use_special: bool=True, special_chars_override: list=None, force_num: bool=True, force_spec: bool=True) -> str:
-    """
-    Function to generate a random password
-
-    :param length:
-    :param use_nums: Allows the use of numbers
-    :param use_special: Allows the use of special characters
-    :param special_chars_override: List of special characters to use
-    :param force_num: Requires the password to contain at least one number
-    :param force_spec: Requires the password to contain at least one special character
-    :return: Password
-    """
-
-    letters = string.ascii_letters
-    digits = string.digits
-    if special_chars_override:
-        special_chars = special_chars_override
-    else:
-        special_chars = string.punctuation
-
-    alphabet = letters
-    if use_nums:
-        alphabet += digits
-    if use_special:
-        alphabet += special_chars
-
-    pwd = ''
-    for i in range(length):
-        pwd += ''.join(secrets.choice(alphabet))
-
-    if (use_nums and force_num) and (use_special and force_spec):
-        while pwd[0].isdigit() or not any(char.isdigit() for char in pwd) or not any(char in pwd for char in special_chars):
-            pwd = await generate_password(length, use_nums, use_special, special_chars_override, force_num, force_spec)
-    elif use_nums and force_num:
-        while pwd[0].isdigit() or not any(char.isdigit() for char in pwd):
-            pwd = await generate_password(length, use_nums, use_special, special_chars_override, force_num, force_spec)
-    elif use_special and force_spec:
-        while not any(char in pwd for char in special_chars):
-            pwd = await generate_password(length, use_nums, use_special, special_chars_override, force_num, force_spec)
-
-    return pwd

wcp_library/async_credentials/postgres.py

@@ -1,130 +0,0 @@
-import logging
-
-import aiohttp
-from yarl import URL
-
-from wcp_library.async_credentials import MissingCredentialsError
-
-logger = logging.getLogger(__name__)
-
-
-class AsyncPostgresCredentialManager:
-    def __init__(self, passwordState_api_key: str):
-        self.password_url = URL("https://vault.wcap.ca/api/passwords/")
-        self.api_key = passwordState_api_key
-        self.headers = {"APIKey": self.api_key, 'Reason': 'Python Script Access'}
-        self._password_list_id = 207
-
-    async def _get_credentials(self) -> dict:
-        """
-        Get all credentials from the password list
-
-        :return:
-        """
-
-        logger.debug("Getting credentials from PasswordState")
-        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
-        async with aiohttp.ClientSession() as session:
-            async with session.get(str(url), headers=self.headers) as response:
-                passwords = await response.json()
-
-        if not passwords:
-            raise MissingCredentialsError("No credentials found in this Password List")
-
-        password_dict = {}
-        for password in passwords:
-            password_info = {'PasswordID': password['PasswordID'], 'UserName': password['UserName'], 'Password': password['Password']}
-            for field in password['GenericFieldInfo']:
-                password_info[field['DisplayName']] = field['Value'].lower() if field['DisplayName'].lower() == 'username' else field['Value']
-            password_dict[password['UserName'].lower()] = password_info
-        logger.debug("Credentials retrieved")
-        return password_dict
-
-    async def get_credentials(self, username: str) -> dict:
-        """
-        Get the credentials for a specific username
-
-        :param username:
-        :return:
-        """
-
-        logger.debug(f"Getting credentials for {username}")
-        credentials = await self._get_credentials()
-
-        try:
-            return_credential = credentials[username.lower()]
-        except KeyError:
-            raise MissingCredentialsError(f"Credentials for {username} not found in this Password List")
-        logger.debug(f"Credentials for {username} retrieved")
-        return return_credential
-
-    async def update_credential(self, credentials_dict: dict) -> bool:
-        """
-        Update username and password in PasswordState
-
-        Credentials dictionary must have the following keys:
-            - PasswordID
-            - UserName
-            - Password
-
-        The dictionary can be obtained from the get_credentials method
-
-        :param credentials_dict:
-        :return:
-        """
-
-        logger.debug(f"Updating credentials for {credentials_dict['UserName']}")
-        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
-        async with aiohttp.ClientSession() as session:
-            async with session.get(str(url), headers=self.headers) as response:
-                passwords = await response.json()
-
-        relevant_credential_entry = [x for x in passwords if x['UserName'] == credentials_dict['UserName']][0]
-        for field in relevant_credential_entry['GenericFieldInfo']:
-            if field['DisplayName'] in credentials_dict:
-                credentials_dict[field['GenericFieldID']] = credentials_dict[field['DisplayName']]
-                credentials_dict.pop(field['DisplayName'])
-
-        async with aiohttp.ClientSession() as session:
-            async with session.put(str(self.password_url), json=credentials_dict, headers=self.headers) as response:
-                if response.status == 200:
-                    logger.debug(f"Credentials for {credentials_dict['UserName']} updated")
-                    return True
-                else:
-                    logger.error(f"Failed to update credentials for {credentials_dict['UserName']}")
-                    return False
-
-    async def new_credentials(self, credentials_dict: dict) -> bool:
-        """
-        Create a new credential entry
-
-        Credentials dictionary must have the following keys:
-            - UserName
-            - Password
-            - Host
-            - Port
-            - Database
-
-        :param credentials_dict:
-        :return:
-        """
-
-        data = {
-            "PasswordListID": self._password_list_id,
-            "Title": credentials_dict['UserName'].upper() if "Title" not in credentials_dict else credentials_dict['Title'].upper(),
-            "Notes": credentials_dict['Notes'] if 'Notes' in credentials_dict else None,
-            "UserName": credentials_dict['UserName'].lower(),
-            "Password": credentials_dict['Password'],
-            "GenericField1": credentials_dict['Host'],
-            "GenericField2": credentials_dict['Port'],
-            "GenericField3": credentials_dict['Database']
-        }
-
-        async with aiohttp.ClientSession() as session:
-            async with session.post(str(self.password_url), json=data, headers=self.headers) as response:
-                if response.status == 201:
-                    logger.debug(f"New credentials for {credentials_dict['UserName']} created")
-                    return True
-                else:
-                    logger.error(f"Failed to create new credentials for {credentials_dict['UserName']}")
-                    return False

wcp_library/async_sql/__init__.py

@@ -1,35 +0,0 @@
-import asyncio
-import logging
-from functools import wraps
-
-import oracledb
-import psycopg
-
-logger = logging.getLogger(__name__)
-
-
-def retry(f: callable) -> callable:
-    """
-    Decorator to retry a function
-
-    :param f: function
-    :return: function
-    """
-
-    @wraps(f)
-    async def wrapper(self, *args, **kwargs):
-        self._retry_count = 0
-        while True:
-            try:
-                return await f(self, *args, **kwargs)
-            except (oracledb.OperationalError, psycopg.OperationalError) as e:
-                error_obj, = e.args
-                if error_obj.full_code in self.retry_error_codes and self._retry_count < self.retry_limit:
-                    self._retry_count += 1
-                    logger.debug(f"{self._db_service} connection error")
-                    logger.debug(error_obj.message)
-                    logger.info("Waiting 5 minutes before retrying Oracle connection")
-                    await asyncio.sleep(300)
-                else:
-                    raise e
-    return wrapper