wcp-library 1.0.0__py3-none-any.whl

wcp_library/informatica.py

@@ -0,0 +1,112 @@
+import requests
+import json
+import sys
+
+
+# Taken from https://www.mydatahack.com/running-jobs-with-informatica-cloud-rest-api/
+def get_session_id(username, password, logging):
+    """Authenticate with username and password and
+       retrieve icSessionId and serverUrl that are used for Subsequent API calls"""
+    session_id = ''
+    data = {'@type': 'login', 'username': username, 'password': password}
+    url = "https://dm-us.informaticacloud.com/ma/api/v2/user/login"
+    headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
+    # We need to pass data in string instead of dict so that the data gets posted directly.
+    r = requests.post(url, data=json.dumps(data), headers=headers)
+
+    logging.info('\tAPI Login Response Status Code: ' + str(r.status_code))
+
+    if r.status_code == 200:
+        session_id = r.json()["icSessionId"]
+        server_url = r.json()["serverUrl"]
+        logging.info('\tSession Id: ' + session_id)
+        logging.info('\tServer URL: ' + server_url)
+    else:
+        logging.info('API Login call failed:')
+        logging.info(r.headers)
+        logging.info(r.json())
+        sys.exit(1)
+
+    return session_id, server_url
+
+
+def get_tasks(session_id, server_url, taskType, logging):
+    """ Use this method to get a list of tasks of a specified type. This may be used to determine the TaskID of a task.
+            Task Types: https://jsapi.apiary.io/apis/cloudrestapi/reference/job/list-of-tasks/login.html
+                AVS-Contact validation task
+                DMASK-Data masking task
+                DQA-Data assessment task
+                DRS-Data replication task
+                DSS-Data synchronization task
+                MTT-Mapping configuration task
+                PCS-PowerCenter task"""
+    task_list_url = server_url + "/api/v2/task?type=" + taskType
+    headers = {'icSessionId': session_id}
+    r = requests.get(task_list_url, headers=headers)
+
+    if r.status_code == 200:
+        logging.info('\tRetrieved list of all Tasks')
+        response_dict = json.loads(r.content)
+        return response_dict
+
+    else:
+        logging.info('\tFailed to get list of Tasks: ' + str(r.status_code))
+        return {}
+
+
+def get_task_id(response_dict, taskName, logging):
+    for d in response_dict:
+        if d['name'] == taskName:
+            id = d['id']
+            logging.info('\tTaskID: ' + id)
+            return id
+
+    logging.info('\tCould not find TaskID for the Task Name specified')
+    return ""
+
+
+def get_all_mapping_details(session_id, server_url, logging):
+    mapping_details_url = server_url + "/api/v2/mapping"
+    headers = {'icSessionId': session_id, 'HTTP': '1.0', 'Accept': 'application/json'}
+    r = requests.get(mapping_details_url, headers=headers)
+
+    if r.status_code == 200:
+        response_dict = json.loads(r.content)
+        return response_dict
+
+    else:
+        logging.info('\tFailed to get Mappings: ' + str(r.status_code))
+        return {}
+
+
+def get_singular_mapping_details(session_id, server_url, logging, mappingID):
+    mapping_details_url = server_url + "/api/v2/mapping/" + mappingID
+    headers = {'icSessionId': session_id, 'Accept': 'application/json'}
+    r = requests.get(mapping_details_url, headers=headers)
+
+    if r.status_code == 200:
+        mapping_deets_dict = json.loads(r.content)
+        return mapping_deets_dict
+
+    else:
+        logging.info('\tFailed to get Mapping details for mapping ' + mappingID + ': ' + str(r.status_code))
+        return {}
+
+
+def get_connection_details(session_id, server_url, logging):
+
+    # source_dict = {}
+    # target_dict = {}
+
+    connections_url = server_url + "/api/v2/connection"
+    # target_connections_url = server_url + "/api/v2/mapping"
+    headers = {'icSessionId': session_id, 'content-type': 'application/json'}
+    r = requests.get(connections_url, headers=headers)
+
+    if r.status_code == 200:
+        response_dict = json.loads(r.content)
+        return response_dict
+
+    else:
+        logging.info('\tFailed to get Mappings: ' + str(r.status_code))
+        return {}

wcp_library/logging.py

@@ -0,0 +1,51 @@
+import logging
+import sys
+
+from WCP_Library import application_path
+
+
+def create_log(level: int, iterations: int, project_name: str, mode: str = "w",
+               format: str = "%(asctime)s:%(levelname)s:%(module)s:%(filename)s:%(lineno)d:%(message)s"):
+    """
+    Create log file.
+
+    Log levels: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET
+
+    format help: https://docs.python.org/3/library/logging.html#logrecord-attributes
+
+    :param level: Logging level to output to log file.
+    :param iterations: Number of log files to keep.
+    :param project_name: Name of the project. (Used as the log file name)
+    :param mode: Mode to open the log file. (Default: "w")
+    :param format: Log Format (Default: "%(asctime)s:%(levelname)s:%(module)s:%(filename)s:%(lineno)d:%(message)s")
+    :return:
+    """
+
+
+    for i in range(iterations, 0, -1):
+        if (application_path / (project_name + f"_{i}.log")).exists():
+            (application_path / (project_name + f"_{i}.log")).rename((application_path / (project_name + f"_{i+1}.log")))
+    if (application_path / (project_name + ".log")).exists():
+        (application_path / (project_name + ".log")).rename((application_path / (project_name + "_1.log")))
+    if (application_path / (project_name + f"_{iterations + 1}.log")).exists():
+        (application_path / (project_name + f"_{iterations + 1}.log")).unlink()
+
+
+    logging.basicConfig(
+        filename=(application_path / (project_name + ".log")),
+        level=level,
+        format=format,
+        filemode=mode
+    )
+
+    MIN_LEVEL = logging.DEBUG
+    stdout_hdlr = logging.StreamHandler(sys.stdout)
+    stderr_hdlr = logging.StreamHandler(sys.stderr)
+    stdout_hdlr.setLevel(MIN_LEVEL)
+    stderr_hdlr.setLevel(max(MIN_LEVEL, logging.WARNING))
+
+    rootLogger = logging.getLogger()
+    rootLogger.addHandler(stdout_hdlr)
+    rootLogger.addHandler(stderr_hdlr)
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)

wcp_library/sql/__init__.py

@@ -0,0 +1,35 @@
+import logging
+from functools import wraps
+from time import sleep
+
+import oracledb
+import psycopg
+
+logger = logging.getLogger(__name__)
+
+
+def retry(f: callable) -> callable:
+    """
+    Decorator to retry a function
+
+    :param f: function
+    :return: function
+    """
+
+    @wraps(f)
+    def wrapper(self, *args, **kwargs):
+        self._retry_count = 0
+        while True:
+            try:
+                return f(self, *args, **kwargs)
+            except (oracledb.OperationalError, psycopg.OperationalError) as e:
+                error_obj, = e.args
+                if error_obj.full_code in self.retry_error_codes and self._retry_count < self.retry_limit:
+                    self._retry_count += 1
+                    logger.debug("Oracle connection error")
+                    logger.debug(error_obj.message)
+                    logger.info("Waiting 5 minutes before retrying Oracle connection")
+                    sleep(300)
+                else:
+                    raise e
+    return wrapper

wcp_library/sql/oracle.py

@@ -0,0 +1,249 @@
+import logging
+from typing import Optional
+
+import pandas as pd
+import oracledb
+from oracledb import ConnectionPool
+
+from WCP_Library.sql import retry
+
+logger = logging.getLogger(__name__)
+
+
+def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str) -> ConnectionPool:
+    """
+    Create Warehouse Connection
+
+    :param username: username
+    :param password: password
+    :param hostname: hostname
+    :param port: port
+    :param database: database
+    :return: session_pool
+    """
+
+    dsn = oracledb.makedsn(hostname, port, sid=database)
+    session_pool = oracledb.create_pool(
+        user=username,
+        password=password,
+        dsn=dsn,
+        min=2,
+        max=5,
+        increment=1,
+        threaded=True,
+        encoding="UTF-8"
+    )
+    return session_pool
+
+
+class SQLConnection(object):
+    """
+    SQL Connection Class
+
+    :return: None
+    """
+
+    def __init__(self):
+        self._username: Optional[str] = None
+        self._password: Optional[str] = None
+        self._hostname: Optional[str] = None
+        self._port: Optional[int] = None
+        self._database: Optional[str] = None
+        self._sid: Optional[str] = None
+        self._session_pool: Optional[ConnectionPool] = None
+
+        self._retry_count = 0
+        self.retry_limit = 50
+        self.retry_error_codes = ['ORA-01033', 'DPY-6005', 'DPY-4011']
+
+    @retry
+    def _connect(self) -> None:
+        """
+        Connect to the warehouse
+
+        :return: None
+        """
+
+        sid_or_service = self._database if self._database else self._sid
+
+        self._session_pool = connect_warehouse(self._username, self._password, self._hostname, self._port, sid_or_service)
+
+    def set_user(self, credentials_dict: dict) -> None:
+        """
+        Set the user credentials and connect
+
+        :param credentials_dict: dictionary of connection details
+        :return: None
+        """
+
+        if not any([self._database, self._sid]):
+            raise ValueError("Either Service or SID must be provided")
+
+        self._username: Optional[str] = credentials_dict['UserName']
+        self._password: Optional[str] = credentials_dict['Password']
+        self._hostname: Optional[str] = credentials_dict['Host']
+        self._port: Optional[int] = int(credentials_dict['Port'])
+        self._database: Optional[str] = credentials_dict['Service'] if 'Service' in credentials_dict else None
+        self._sid: Optional[str] = credentials_dict['SID'] if 'SID' in credentials_dict else None
+
+        self._connect()
+
+    def close_connection(self) -> None:
+        """
+        Close the connection
+
+        :return: None
+        """
+
+        self._session_pool.close()
+
+    @retry
+    def execute(self, query: str) -> None:
+        """
+        Execute the query
+
+        :param query: query
+        :return: None
+        """
+
+        connection = self._session_pool.acquire()
+        cursor = connection.cursor()
+        cursor.execute(query)
+        connection.commit()
+        self._session_pool.release(connection)
+
+    @retry
+    def safe_execute(self, query: str, packed_values: dict) -> None:
+        """
+        Execute the query without SQL Injection possibility, to be used with external facing projects.
+
+        :param query: query
+        :param packed_values: dictionary of values
+        :return: None
+        """
+
+        connection = self._session_pool.acquire()
+        cursor = connection.cursor()
+        cursor.execute(query, packed_values)
+        connection.commit()
+        self._session_pool.release(connection)
+
+    @retry
+    def execute_multiple(self, queries: list[list[str, dict]]) -> None:
+        """
+        Execute multiple queries
+
+        :param queries: list of queries
+        :return: None
+        """
+
+        connection = self._session_pool.acquire()
+        cursor = connection.cursor()
+        for item in queries:
+            query = item[0]
+            packed_values = item[1]
+            if packed_values:
+                cursor.execute(query, packed_values)
+            else:
+                cursor.execute(query)
+        connection.commit()
+        self._session_pool.release(connection)
+
+    @retry
+    def execute_many(self, query: str, dictionary: list[dict]) -> None:
+        """
+        Execute many queries
+
+        :param query: query
+        :param dictionary: dictionary of values
+        :return: None
+        """
+
+        connection = self._session_pool.acquire()
+        cursor = connection.cursor()
+        cursor.executemany(query, dictionary)
+        connection.commit()
+        self._session_pool.release(connection)
+
+    @retry
+    def fetch_data(self, query: str, packed_data=None):
+        """
+        Fetch the data from the query
+
+        :param query: query
+        :param packed_data: packed data
+        :return: rows
+        """
+
+        connection = self._session_pool.acquire()
+        cursor = connection.cursor()
+        if packed_data:
+            cursor.execute(query, packed_data)
+        else:
+            cursor.execute(query)
+        rows = cursor.fetchall()
+        self._session_pool.release(connection)
+        return rows
+
+    @retry
+    def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
+        """
+        Export the DataFrame to the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param columns: list of columns
+        :param remove_nan: remove NaN values
+        :return: None
+        """
+
+        col = ', '.join(columns)
+        bindList = []
+        for column in columns:
+            bindList.append(':' + column)
+        bind = ', '.join(bindList)
+
+        main_dict = dfObj.to_dict('records')
+        if remove_nan:
+            for val, item in enumerate(main_dict):
+                for sub_item, value in item.items():
+                    if pd.isna(value):
+                        main_dict[val][sub_item] = None
+                    else:
+                        main_dict[val][sub_item] = value
+
+        query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, bind)
+        self.execute_many(query, main_dict)
+
+    @retry
+    def truncate_table(self, tableName: str) -> None:
+        """
+        Truncate the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        truncateQuery = """TRUNCATE TABLE {}""".format(tableName)
+        self.execute(truncateQuery)
+
+    @retry
+    def empty_table(self, tableName: str) -> None:
+        """
+        Empty the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        deleteQuery = """DELETE FROM {}""".format(tableName)
+        self.execute(deleteQuery)
+
+    def __del__(self) -> None:
+        """
+        Destructor
+
+        :return: None
+        """
+
+        self._session_pool.close()

wcp_library/sql/postgres.py

@@ -0,0 +1,226 @@
+import logging
+from typing import Optional
+
+import pandas as pd
+from psycopg.sql import SQL
+from psycopg_pool import ConnectionPool
+
+from WCP_Library.sql import retry
+
+logger = logging.getLogger(__name__)
+
+
+def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str) -> ConnectionPool:
+    """
+    Create Warehouse Connection
+
+    :param username: username
+    :param password: password
+    :param hostname: hostname
+    :param port: port
+    :param database: database
+    :return: session_pool
+    """
+
+    url = f"postgres://{username}:{password}@{hostname}:{port}/{database}"
+
+    session_pool = ConnectionPool(
+        conninfo=url,
+        min_size=2,
+        max_size=5,
+        open=True
+    )
+    return session_pool
+
+
+class SQLConnection(object):
+    """
+    SQL Connection Class
+
+    :return: None
+    """
+
+    def __init__(self):
+        self._username: Optional[str] = None
+        self._password: Optional[str] = None
+        self._hostname: Optional[str] = None
+        self._port: Optional[int] = None
+        self._database: Optional[str] = None
+        self._session_pool: Optional[ConnectionPool] = None
+
+        self._retry_count = 0
+        self.retry_limit = 50
+        self.retry_error_codes = ['08001', '08004']
+
+    @retry
+    def _connect(self) -> None:
+        """
+        Connect to the warehouse
+
+        :return: None
+        """
+
+        self._session_pool = connect_warehouse(self._username, self._password, self._hostname, self._port, self._database)
+
+    def set_user(self, credentials_dict: dict) -> None:
+        """
+        Set the user credentials and connect
+
+        :param credentials_dict: dictionary of connection details
+        :return: None
+        """
+
+        self._username: Optional[str] = credentials_dict['UserName']
+        self._password: Optional[str] = credentials_dict['Password']
+        self._hostname: Optional[str] = credentials_dict['Host']
+        self._port: Optional[int] = int(credentials_dict['Port'])
+        self._database: Optional[str] = credentials_dict['Database']
+
+        self._connect()
+
+    def close_connection(self) -> None:
+        """
+        Close the connection
+
+        :return: None
+        """
+
+        self._session_pool.close()
+
+    @retry
+    def execute(self, query: SQL | str) -> None:
+        """
+        Execute the query
+
+        :param query: query
+        :return: None
+        """
+
+        with self._session_pool.connection() as connection:
+            connection.execute(query)
+
+    @retry
+    def safe_execute(self, query: SQL | str, packed_values: dict) -> None:
+        """
+        Execute the query without SQL Injection possibility, to be used with external facing projects.
+
+        :param query: query
+        :param packed_values: dictionary of values
+        :return: None
+        """
+
+        with self._session_pool.connection() as connection:
+            connection.execute(query, packed_values)
+
+    @retry
+    def execute_multiple(self, queries: list[list[SQL | str, dict]]) -> None:
+        """
+        Execute multiple queries
+
+        :param queries: list of queries
+        :return: None
+        """
+
+        with self._session_pool.connection() as connection:
+            for item in queries:
+                query = item[0]
+                packed_values = item[1]
+                if packed_values:
+                    connection.execute(query, packed_values)
+                else:
+                    connection.execute(query)
+
+    @retry
+    def execute_many(self, query: SQL | str, dictionary: list[dict]) -> None:
+        """
+        Execute many queries
+
+        :param query: query
+        :param dictionary: dictionary of values
+        :return: None
+        """
+
+        with self._session_pool.connection() as connection:
+            connection.executemany(query, dictionary)
+
+    @retry
+    def fetch_data(self, query: SQL | str, packed_data=None):
+        """
+        Fetch the data from the query
+
+        :param query: query
+        :param packed_data: packed data
+        :return: rows
+        """
+
+        with self._session_pool.connection() as connection:
+            cursor = connection.cursor()
+            if packed_data:
+                cursor.execute(query, packed_data)
+            else:
+                cursor.execute(query)
+            rows = cursor.fetchall()
+        return rows
+
+    @retry
+    def export_DF_to_warehouse(self, dfObj: pd.DataFrame, outputTableName: str, columns: list, remove_nan=False) -> None:
+        """
+        Export the DataFrame to the warehouse
+
+        :param dfObj: DataFrame
+        :param outputTableName: output table name
+        :param columns: list of columns
+        :param remove_nan: remove NaN values
+        :return: None
+        """
+
+        col = ', '.join(columns)
+        param_list = []
+        for column in columns:
+            param_list.append(f"%({column})s")
+        params = ', '.join(param_list)
+
+        main_dict = dfObj.to_dict('records')
+        if remove_nan:
+            for val, item in enumerate(main_dict):
+                for sub_item, value in item.items():
+                    if pd.isna(value):
+                        main_dict[val][sub_item] = None
+                    else:
+                        main_dict[val][sub_item] = value
+
+        query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, params)
+        self.execute_many(query, main_dict)
+
+    @retry
+    def truncate_table(self, tableName: str) -> None:
+        """
+        Truncate the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        truncateQuery = """TRUNCATE TABLE {}""".format(tableName)
+        self.execute(truncateQuery)
+
+    @retry
+    def empty_table(self, tableName: str) -> None:
+        """
+        Empty the table
+
+        :param tableName: table name
+        :return: None
+        """
+
+        deleteQuery = """DELETE FROM {}""".format(tableName)
+        self.execute(deleteQuery)
+
+    def __del__(self) -> None:
+        """
+        Destructor
+
+        :return: None
+        """
+
+        self._session_pool.close()