wcp-library 1.0.0__py3-none-any.whl

wcp_library/async_sql/postgres.py

@@ -0,0 +1,217 @@
+import logging
+from typing import Optional
+
+import pandas as pd
+from psycopg.sql import SQL
+from psycopg_pool import AsyncConnectionPool
+
+from WCP_Library.async_sql import retry
+
+logger = logging.getLogger(__name__)
+
+
+async def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str) -> AsyncConnectionPool:
+    """
+    Create Warehouse Connection
+
+    :param username: username
+    :param password: password
+    :param hostname: hostname
+    :param port: port
+    :param database: database
+    :return: session_pool
+    """
+
+    url = f"postgres://{username}:{password}@{hostname}:{port}/{database}"
+
+    session_pool = AsyncConnectionPool(
+        conninfo=url,
+        min_size=2,
+        max_size=5,
+    )
+    await session_pool.open()
+    return session_pool
+
+
+class SQLConnection(object):
+    """
+    SQL Connection Class
+
+    :return: None
+    """
+
+    def __init__(self):
+        self._username: Optional[str] = None
+        self._password: Optional[str] = None
+        self._hostname: Optional[str] = None
+        self._port: Optional[int] = None
+        self._database: Optional[str] = None
+        self._session_pool: Optional[AsyncConnectionPool] = None
+
+        self._retry_count = 0
+        self.retry_limit = 50
+        self.retry_error_codes = ['08001', '08004']
+
+    @retry
+    async def _connect(self) -> None:
+        """
+        Connect to the warehouse
+
+        :return: None
+        """
+
+        self._session_pool = await connect_warehouse(self._username, self._password, self._hostname, self._port, self._database)
+
+    async def set_user(self, credentials_dict: dict) -> None:
+        """
+        Set the user credentials and connect
+
+        :param credentials_dict: dictionary of connection details
+        :return: None
+        """
+
+        self._username: Optional[str] = credentials_dict['UserName']
+        self._password: Optional[str] = credentials_dict['Password']
+        self._hostname: Optional[str] = credentials_dict['Host']
+        self._port: Optional[int] = int(credentials_dict['Port'])
+        self._database: Optional[str] = credentials_dict['Database']
+
+        await self._connect()
+
+    async def close_connection(self) -> None:
+        """
+        Close the connection
+
+        :return: None
+        """
+
+        await self._session_pool.close()
+
+    @retry
+    async def execute(self, query: SQL | str) -> None:
+        """
+        Execute the query
+
+        :param query: query
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            await connection.execute(query)
+
+    @retry
+    async def safe_execute(self, query: SQL | str, packed_values: dict) -> None:
+        """
+        Execute the query without SQL Injection possibility, to be used with external facing projects.
+
+        :param query: query
+        :param packed_values: dictionary of values
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            await connection.execute(query, packed_values)
+
+    @retry
+    async def execute_multiple(self, queries: list[list[SQL | str, dict]]) -> None:
+        """
+        Execute multiple queries
+
+        :param queries: list of queries
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            for item in queries:
+                query = item[0]
+                packed_values = item[1]
+                if packed_values:
+                    await connection.execute(query, packed_values)
+                else:
+                    await connection.execute(query)
+
+    @retry
+    async def execute_many(self, query: SQL | str, dictionary: list[dict]) -> None:
+        """
+        Execute many queries
+
+        :param query: query
+        :param dictionary: dictionary of values
+        :return: None
+        """
+
+        async with self._session_pool.connection() as connection:
+            await connection.executemany(query, dictionary)
+
+    @retry
+    async def fetch_data(self, query: SQL | str, packed_data=None):
+        """
+        Fetch the data from the query
+
+        :param query: query
+        :param packed_data: packed data
+        :return: rows
+        """
+
+        async with self._session_pool.connection() as connection:
+            cursor = connection.cursor()
+            if packed_data:
+                await cursor.execute(query, packed_data)
+            else:
+                await cursor.execute(query)
+            rows = await cursor.fetchall()
+        return rows
+
+    @retry
+    async def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
+        """
+        Export the DataFrame to the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param columns: list of columns
+        :param remove_nan: remove NaN values
+        :return: None
+        """
+
+        col = ', '.join(columns)
+        param_list = []
+        for column in columns:
+            param_list.append(f"%({column})s")
+        params = ', '.join(param_list)
+
+        main_dict = dfObj.to_dict('records')
+        if remove_nan:
+            for val, item in enumerate(main_dict):
+                for sub_item, value in item.items():
+                    if pd.isna(value):
+                        main_dict[val][sub_item] = None
+                    else:
+                        main_dict[val][sub_item] = value
+
+        query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, params)
+        await self.execute_many(query, main_dict)
+
+    @retry
+    async def truncate_table(self, tableName: str) -> None:
+        """
+        Truncate the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        truncateQuery = """TRUNCATE TABLE {}""".format(tableName)
+        await self.execute(truncateQuery)
+
+    @retry
+    async def empty_table(self, tableName: str) -> None:
+        """
+        Empty the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        deleteQuery = """DELETE FROM {}""".format(tableName)
+        await self.execute(deleteQuery)

wcp_library/credentials/__init__.py

@@ -0,0 +1,49 @@
+import secrets
+import string
+
+
+class MissingCredentialsError(KeyError):
+    pass
+
+
+def generate_password(length: int=12, use_nums: bool=True, use_special: bool=True, special_chars_override: list=None, force_num: bool=True, force_spec: bool=True) -> str:
+    """
+    Function to generate a random password
+
+    :param length:
+    :param use_nums: Allows the use of numbers
+    :param use_special: Allows the use of special characters
+    :param special_chars_override: List of special characters to use
+    :param force_num: Requires the password to contain at least one number
+    :param force_spec: Requires the password to contain at least one special character
+    :return: Password
+    """
+
+    letters = string.ascii_letters
+    digits = string.digits
+    if special_chars_override:
+        special_chars = special_chars_override
+    else:
+        special_chars = string.punctuation
+
+    alphabet = letters
+    if use_nums:
+        alphabet += digits
+    if use_special:
+        alphabet += special_chars
+
+    pwd = ''
+    for i in range(length):
+        pwd += ''.join(secrets.choice(alphabet))
+
+    if (use_nums and force_num) and (use_special and force_spec):
+        while pwd[0].isdigit() or not any(char.isdigit() for char in pwd) or not any(char in pwd for char in special_chars):
+            pwd = generate_password(length, use_nums, use_special, special_chars_override, force_num, force_spec)
+    elif use_nums and force_num:
+        while pwd[0].isdigit() or not any(char.isdigit() for char in pwd):
+            pwd = generate_password(length, use_nums, use_special, special_chars_override, force_num, force_spec)
+    elif use_special and force_spec:
+        while not any(char in pwd for char in special_chars):
+            pwd = generate_password(length, use_nums, use_special, special_chars_override, force_num, force_spec)
+
+    return pwd

wcp_library/credentials/oracle.py

@@ -0,0 +1,125 @@
+import logging
+
+import requests
+from yarl import URL
+
+from WCP_Library.credentials import MissingCredentialsError
+
+logger = logging.getLogger(__name__)
+
+
+class OracleCredentialManager:
+    def __init__(self, passwordState_api_key: str):
+        self.password_url = URL("https://vault.wcap.ca/api/passwords/")
+        self.api_key = passwordState_api_key
+        self.headers = {"APIKey": self.api_key, 'Reason': 'Python Script Access'}
+        self._password_list_id = 207
+
+    def _get_credentials(self) -> dict:
+        """
+        Get all credentials from the password list
+
+        :return: Dictionary of credentials
+        """
+
+        logger.debug("Getting credentials from PasswordState")
+        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
+        passwords = requests.get(str(url), headers=self.headers).json()
+
+        if not passwords:
+            raise MissingCredentialsError("No credentials found in this Password List")
+
+        password_dict = {}
+        for password in passwords:
+            password_info = {'PasswordID': password['PasswordID'], 'UserName': password['UserName'], 'Password': password['Password']}
+            for field in password['GenericFieldInfo']:
+                password_info[field['DisplayName']] = field['Value'].lower() if field['DisplayName'].lower() == 'username' else field['Value']
+            password_dict[password['UserName'].lower()] = password_info
+        logger.debug("Credentials retrieved")
+        return password_dict
+
+    def get_credentials(self, username: str) -> dict:
+        """
+        Get the credentials for a specific username
+
+        :param username:
+        :return: Dictionary of credentials
+        """
+
+        logger.debug(f"Getting credentials for {username}")
+        credentials = self._get_credentials()
+
+        try:
+            return_credential = credentials[username.lower()]
+        except KeyError:
+            raise MissingCredentialsError(f"Credentials for {username} not found in this Password List")
+        logger.debug(f"Credentials for {username} retrieved")
+        return return_credential
+
+    def update_credential(self, credentials_dict: dict) -> bool:
+        """
+        Update the credentials for a specific username
+
+        Credentials dictionary must have the following keys:
+            - PasswordID
+            - UserName
+            - Password
+
+        The dictionary should be obtained from the get_credentials method and modified accordingly
+
+        :param credentials_dict:
+        :return: True if successful, False otherwise
+        """
+
+        logger.debug(f"Updating credentials for {credentials_dict['UserName']}")
+        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
+        passwords = requests.get(str(url), headers=self.headers).json()
+
+        relevant_credential_entry = [x for x in passwords if x['UserName'] == credentials_dict['UserName']][0]
+        for field in relevant_credential_entry['GenericFieldInfo']:
+            if field['DisplayName'] in credentials_dict:
+                credentials_dict[field['GenericFieldID']] = credentials_dict[field['DisplayName']]
+                credentials_dict.pop(field['DisplayName'])
+
+        response = requests.put(str(self.password_url), json=credentials_dict, headers=self.headers)
+        if response.status_code == 200:
+            logger.debug(f"Credentials for {credentials_dict['UserName']} updated")
+            return True
+        else:
+            logger.error(f"Failed to update credentials for {credentials_dict['UserName']}")
+            return False
+
+    def new_credentials(self, credentials_dict: dict) -> bool:
+        """
+        Create a new credential entry
+
+        Credentials dictionary must have the following keys:
+            - UserName
+            - Password
+            - Host
+            - Port
+            - Service or SID
+
+        :param credentials_dict:
+        :return: True if successful, False otherwise
+        """
+
+        data = {
+            "PasswordListID": self._password_list_id,
+            "Title": credentials_dict['UserName'].upper() if "Title" not in credentials_dict else credentials_dict['Title'].upper(),
+            "Notes": credentials_dict['Notes'] if 'Notes' in credentials_dict else None,
+            "UserName": credentials_dict['UserName'].lower(),
+            "Password": credentials_dict['Password'],
+            "GenericField1": credentials_dict['Host'],
+            "GenericField2": credentials_dict['Port'],
+            "GenericField3": credentials_dict['Service'] if 'Service' in credentials_dict else None,
+            "GenericField4": credentials_dict['SID'] if 'SID' in credentials_dict else None
+        }
+
+        response = requests.post(str(self.password_url), json=data, headers=self.headers)
+        if response.status_code == 201:
+            logger.debug(f"New credentials for {credentials_dict['UserName']} created")
+            return True
+        else:
+            logger.error(f"Failed to create new credentials for {credentials_dict['UserName']}")
+            return False

wcp_library/credentials/postgres.py

@@ -0,0 +1,124 @@
+import logging
+
+import requests
+from yarl import URL
+
+from WCP_Library.credentials import MissingCredentialsError
+
+logger = logging.getLogger(__name__)
+
+
+class PostgresCredentialManager:
+    def __init__(self, passwordState_api_key: str):
+        self.password_url = URL("https://vault.wcap.ca/api/passwords/")
+        self.api_key = passwordState_api_key
+        self.headers = {"APIKey": self.api_key, 'Reason': 'Python Script Access'}
+        self._password_list_id = 207
+
+    def _get_credentials(self) -> dict:
+        """
+        Get all credentials from the password list
+
+        :return: Dictionary of credentials
+        """
+
+        logger.debug("Getting credentials from PasswordState")
+        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
+        passwords = requests.get(str(url), headers=self.headers).json()
+
+        if not passwords:
+            raise MissingCredentialsError("No credentials found in this Password List")
+
+        password_dict = {}
+        for password in passwords:
+            password_info = {'PasswordID': password['PasswordID'], 'UserName': password['UserName'], 'Password': password['Password']}
+            for field in password['GenericFieldInfo']:
+                password_info[field['DisplayName']] = field['Value'].lower() if field['DisplayName'].lower() == 'username' else field['Value']
+            password_dict[password['UserName'].lower()] = password_info
+        logger.debug("Credentials retrieved")
+        return password_dict
+
+    def get_credentials(self, username: str) -> dict:
+        """
+        Get the credentials for a specific username
+
+        :param username:
+        :return: Dictionary of credentials
+        """
+
+        logger.debug(f"Getting credentials for {username}")
+        credentials = self._get_credentials()
+
+        try:
+            return_credential = credentials[username.lower()]
+        except KeyError:
+            raise MissingCredentialsError(f"Credentials for {username} not found in this Password List")
+        logger.debug(f"Credentials for {username} retrieved")
+        return return_credential
+
+    def update_credential(self, credentials_dict: dict) -> bool:
+        """
+        Update the credentials for a specific username
+
+        Credentials dictionary must have the following keys:
+            - PasswordID
+            - UserName
+            - Password
+
+        The dictionary should be obtained from the get_credentials method and modified accordingly
+
+        :param credentials_dict:
+        :return: True if successful, False otherwise
+        """
+
+        logger.debug(f"Updating credentials for {credentials_dict['UserName']}")
+        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
+        passwords = requests.get(str(url), headers=self.headers).json()
+
+        relevant_credential_entry = [x for x in passwords if x['UserName'] == credentials_dict['UserName']][0]
+        for field in relevant_credential_entry['GenericFieldInfo']:
+            if field['DisplayName'] in credentials_dict:
+                credentials_dict[field['GenericFieldID']] = credentials_dict[field['DisplayName']]
+                credentials_dict.pop(field['DisplayName'])
+
+        response = requests.put(str(self.password_url), json=credentials_dict, headers=self.headers)
+        if response.status_code == 200:
+            logger.debug(f"Credentials for {credentials_dict['UserName']} updated")
+            return True
+        else:
+            logger.error(f"Failed to update credentials for {credentials_dict['UserName']}")
+            return False
+
+    def new_credentials(self, credentials_dict: dict) -> bool:
+        """
+        Create a new credential entry
+
+        Credentials dictionary must have the following keys:
+            - UserName
+            - Password
+            - Host
+            - Port
+            - Database
+
+        :param credentials_dict:
+        :return: True if successful, False otherwise
+        """
+
+        data = {
+            "PasswordListID": self._password_list_id,
+            "Title": credentials_dict['UserName'].upper() if "Title" not in credentials_dict else credentials_dict['Title'].upper(),
+            "Notes": credentials_dict['Notes'] if 'Notes' in credentials_dict else None,
+            "UserName": credentials_dict['UserName'].lower(),
+            "Password": credentials_dict['Password'],
+            "GenericField1": credentials_dict['Host'],
+            "GenericField2": credentials_dict['Port'],
+            "GenericField3": credentials_dict['Database']
+        }
+
+        response = requests.post(str(self.password_url), json=data, headers=self.headers)
+        if response.status_code == 201:
+            logger.debug(f"New credentials for {credentials_dict['UserName']} created")
+            return True
+        else:
+            logger.error(f"Failed to create new credentials for {credentials_dict['UserName']}")
+            return False

wcp_library/emailing.py

@@ -0,0 +1,90 @@
+import smtplib
+from email import encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.utils import formatdate
+from pathlib import Path
+
+
+def send_email(sender: str, recipients: list, subject: str, message=None):
+    """
+    Function to send an email
+
+    :param sender:
+    :param recipients:
+    :param subject:
+    :param message:
+    :return:
+    """
+
+    msg = MIMEMultipart()
+    msg['From'] = sender
+    msg['To'] = ", ".join(recipients)
+    msg['Date'] = formatdate(localtime=True)
+    msg['Subject'] = subject
+    msg.attach(message)
+
+    smtpServer = 'mail.wcap.ca'
+    server = smtplib.SMTP(smtpServer, 25)
+    server.ehlo()
+    server.sendmail(sender, recipients, msg.as_string())
+    server.quit()
+
+
+def email_reporting(subject: str, message: str):
+    """
+    Function to email the reporting team from the Python email
+
+    :param subject:
+    :param message:
+    :return:
+    """
+
+    msg = MIMEMultipart()
+    msg['From'] = "Python@wcap.ca"
+    msg['To'] = "Reporting@wcap.ca"
+    msg['Date'] = formatdate(localtime=True)
+    msg['Subject'] = subject
+    msg.attach(message)
+
+    smtpServer = 'mail.wcap.ca'
+    server = smtplib.SMTP(smtpServer, 25)
+    server.ehlo()
+    server.sendmail("Python@wcap.ca", 'Reporting@wcap.ca', msg.as_string())
+    server.quit()
+
+
+def email_with_attachments(sender: str, recipients: list, subject: str, message=None, attachments: list[Path]=None):
+    """
+    Function to send an email with attachments
+
+    File paths must be passed as a list of Path (pathlib.Path) objects
+
+    :param sender:
+    :param recipients:
+    :param subject:
+    :param message:
+    :param attachments:
+    :return:
+    """
+
+    msg = MIMEMultipart()
+    msg['From'] = sender
+    msg['To'] = ", ".join(recipients)
+    msg['Date'] = formatdate(localtime=True)
+    msg['Subject'] = subject
+    msg.attach(message)
+
+    for attachment in attachments:
+        part = MIMEBase('application', "octet-stream")
+        with open(attachment, 'rb') as file:
+            part.set_payload(file.read())
+        encoders.encode_base64(part)
+        part.add_header('Content-Disposition', 'attachment; filename={}'.format(attachment.name))
+        msg.attach(part)
+
+    smtpServer = 'mail.wcap.ca'
+    server = smtplib.SMTP(smtpServer, 25)
+    server.ehlo()
+    server.sendmail(sender, recipients, msg.as_string())
+    server.quit()