wcp-library 1.3.2__tar.gz → 1.3.4__tar.gz

{wcp_library-1.3.2 → wcp_library-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wcp-library
-Version: 1.3.2
+Version: 1.3.4
 Summary: Common utilites for internal development at WCP
 Home-page: https://github.com/Whitecap-DNA/WCP-Library
 Author: Mitch-Petersen
@@ -19,6 +19,7 @@ Requires-Dist: psycopg (>=3.2.3,<4.0.0)
 Requires-Dist: psycopg-binary (>=3.2.3,<4.0.0)
 Requires-Dist: psycopg-pool (>=3.2.3,<4.0.0)
 Requires-Dist: pycryptodome (>=3.21.0,<4.0.0)
+Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: selenium (>=4.27.1,<5.0.0)
 Requires-Dist: webdriver-manager (>=4.0.2,<5.0.0)

{wcp_library-1.3.2 → wcp_library-1.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wcp-library"
-version = "1.3.2"
+version = "1.3.4"
 description = "Common utilites for internal development at WCP"
 authors = ["Mitch-Petersen <mitch.petersen@wcap.ca>"]
 readme = "README.md"
@@ -24,6 +24,7 @@ requests = "^2.32.3"
 selenium = "^4.27.1"
 webdriver-manager = "^4.0.2"
 yarl = "^1.17.1"
+pytz = "^2024.2"
 
 [build-system]
 requires = ["poetry-core"]

wcp_library-1.3.4/wcp_library/informatica.py

@@ -0,0 +1,220 @@
+import logging
+import json
+import time
+from datetime import datetime
+from typing import Optional
+
+import requests
+from yarl import URL
+
+from wcp_library.time import convert_tz
+
+logger = logging.getLogger(__name__)
+
+
+class InformaticaError(Exception):
+    pass
+
+
+class InformaticaSession:
+    def __init__(self, username: str, password: str):
+        self.username: str = username
+        self.password: str = password
+        self._session_id: Optional[str] = None
+        self._server_url: Optional[URL] = None
+
+        self._get_session_id()
+
+    def _get_session_id(self) -> None:
+        """
+        Authenticate with username and password
+
+        :return: icSessionId, serverUrl
+        """
+
+        data = {'@type': 'login', 'username': self.username, 'password': self.password}
+        url = "https://dm-us.informaticacloud.com/ma/api/v2/user/login"
+        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
+        response = requests.post(url, data=json.dumps(data), headers=headers)
+
+        logger.debug('\tInformatica API Login Response Status Code: ' + str(response.status_code))
+
+        if response.status_code == 200:
+            logger.info('\tInformatica API Login Successful')
+            self._session_id = response.json()["icSessionId"]
+            self._server_url = URL(response.json()["serverUrl"])
+        else:
+            raise InformaticaError(f'\tInformatica API Login call failed: {response.status_code}')
+
+    def get_tasks(self, task_type: str) -> dict:
+        """
+        Use this method to get a list of tasks of a specified type. This may be used to determine the TaskID of a task.
+        Task Types: https://jsapi.apiary.io/apis/cloudrestapi/reference/job/list-of-tasks/login.html
+            AVS-Contact validation task
+            DMASK-Data masking task
+            DQA-Data assessment task
+            DRS-Data replication task
+            DSS-Data synchronization task
+            MTT-Mapping configuration task
+            PCS-PowerCenter task
+
+        :param taskType: Task Type
+        :return: Task List
+        """
+
+        task_list_url = self._server_url / "api/v2/task"
+        headers = {'icSessionId': self._session_id}
+        response = requests.get(str(task_list_url), headers=headers, params={'type': task_type})
+
+        logger.debug('\tRetrieved list of all Tasks')
+
+        if response.status_code == 200:
+            return json.loads(response.content)
+        else:
+            raise InformaticaError(f'\tFailed to get list of Tasks: {response.status_code}')
+
+    def get_task_id(self, task_name: str, task_type: str) -> str:
+        """
+        Use this method to get the TaskID of a specified task. This may be used to run a task.
+
+        :param task_name: Task Name
+        :param task_type: Task Type
+        :return: Task ID
+        """
+
+        tasks = self.get_tasks(task_type)
+        for task in tasks:
+            if task['name'] == task_name:
+                return task['id']
+
+        raise InformaticaError(f'\tFailed to find TaskID for the Task Name specified: {task_name}')
+
+
+    def is_task_running(self, task_id: str) -> tuple[bool, datetime]:
+        """
+        Use this method to determine if a task is currently running.
+
+        :param task_id: Task ID
+        :return: Tuple of running status and startTime
+        """
+
+        task_status_url = self._server_url / f"api/v2/task/{task_id}/status"
+        headers = {'icSessionId': self._session_id}
+        response = requests.get(str(task_status_url), headers=headers)
+
+        logger.debug(f'\tRetrieved status of Task {task_id}')
+
+        if response.status_code == 200:
+            task_status = json.loads(response.content)
+            utc_time = datetime.strptime(task_status['startTimeUTC'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            local_time = convert_tz(utc_time, 'UTC')
+            return task_status['status'] == 'RUNNING', local_time
+        else:
+            raise InformaticaError(f'\tFailed to get status of Task: {response.status_code}')
+
+    def run_job(self, task_id: str, task_type: str) -> str:
+        """
+        Use this method to run a task.
+
+        :param task_id: Task ID
+        :param task_type: Task Type
+        :return: Run ID
+        """
+
+        job_start_url = self._server_url / "api/v2/job"
+        headers = {'Content-Type': 'application/json', 'icSessionId': self._session_id, 'Accept': 'application/json'}
+        data = {'@type': 'job', 'taskId': task_id, 'taskType': task_type}
+        response = requests.post(str(job_start_url), data=json.dumps(data), headers=headers)
+
+        if response.status_code == 200:
+            logger.info('Starting Informatica Job...')
+            response_dict = json.loads(response.content)
+            runID = response_dict['runId']
+            return runID
+
+        else:
+            raise InformaticaError(f"Failed to start Informatica Job: {response.status_code}")
+
+    def wait_until_job_finish(self, run_id: str) -> datetime:
+        """
+        Use this method to wait until a job finishes running.
+
+        :param run_id:
+        :return: End Time
+        """
+
+        job_status_url = self._server_url / f"api/v2/activity/activityLog"
+        headers = {'icSessionId': self._session_id}
+
+        while True:
+            response = requests.get(str(job_status_url), headers=headers, params={'runId': run_id})
+            if response.status_code == 200:
+                response_dict = json.loads(response.content)
+
+                if not response_dict['endTimeUtc']:
+                    time.sleep(30)
+                    continue
+                if response_dict['state'] == 1:
+                    logger.info('\tJob completed successfully')
+                elif response_dict['state'] == 2:
+                    logger.info('\tJob completed with errors')
+                elif response_dict['state'] == 3:
+                    raise InformaticaError('Job failed')
+                return convert_tz(datetime.strptime(response_dict['endTimeUtc'], '%Y-%m-%dT%H:%M:%S.%fZ'), 'UTC')
+            else:
+                raise InformaticaError(f"Failed to get job status: {response.status_code}")
+
+    def get_connection_details(self) -> dict:
+        """
+        Use this method to get a list of connections.
+
+        :return: Connection List
+        """
+
+        connections_url = self._server_url / "api/v2/connection"
+        headers = {'icSessionId': self._session_id, 'Accept': 'application/json'}
+        response = requests.get(str(connections_url), headers=headers)
+
+        logger.debug('\tRetrieved list of all Connections')
+
+        if response.status_code == 200:
+            return json.loads(response.content)
+        else:
+            raise InformaticaError(f'\tFailed to get list of Connections: {response.status_code}')
+
+    def get_mapping_details(self, mapping_id: str) -> dict:
+        """
+        Use this method to get details of a specific mapping.
+
+        :param mapping_id:
+        :return:
+        """
+
+        mapping_details_url = self._server_url / f"api/v2/mapping/{mapping_id}"
+        headers = {'icSessionId': self._session_id, 'Accept': 'application/json'}
+        response = requests.get(str(mapping_details_url), headers=headers)
+
+        logger.debug(f'\tRetrieved details of Mapping {mapping_id}')
+
+        if response.status_code == 200:
+            return json.loads(response.content)
+        else:
+            raise InformaticaError(f'\tFailed to get details of Mapping {mapping_id}: {response.status_code}')
+
+    def get_all_mapping_details(self) -> dict:
+        """
+        Use this method to get details of all mappings.
+
+        :return:
+        """
+
+        mapping_details_url = self._server_url / "api/v2/mapping"
+        headers = {'icSessionId': self._session_id, 'Accept': 'application/json'}
+        response = requests.get(str(mapping_details_url), headers=headers)
+
+        logger.debug('\tRetrieved details of all Mappings')
+
+        if response.status_code == 200:
+            return json.loads(response.content)
+        else:
+            raise InformaticaError(f'\tFailed to get details of all Mappings: {response.status_code}')

{wcp_library-1.3.2 → wcp_library-1.3.4}/wcp_library/sql/postgres.py

@@ -245,6 +245,10 @@ class PostgresConnection(object):
         if remove_nan:
             dfObj = dfObj.replace({np.nan: None})
         main_dict = dfObj.to_dict('records')
+        for record in main_dict:
+            for key in record:
+                if record[key] == '':
+                    record[key] = None
 
         query = """INSERT INTO {} ({}) VALUES ({})""".format(outputTableName, col, params)
         self.execute_many(query, main_dict)

wcp_library-1.3.4/wcp_library/time.py

@@ -0,0 +1,68 @@
+from datetime import datetime
+
+import pytz
+
+
+def get_current_time(aware: bool=False, tz: str='Canada/Mountain') -> datetime:
+    """
+    Get the current time (Mountain Time)
+
+    you can find a list of timezones by printing pytz.all_timezones
+
+    :param aware:
+    :param tz:
+    :return:
+    """
+
+    tz = pytz.timezone(tz)
+    current_time = datetime.now(tz)
+
+    if not aware:
+        return current_time.replace(tzinfo=None)
+    return current_time
+
+
+def convert_tz(time: datetime, original_tz: str, aware: bool=False, tz: str='Canada/Mountain') -> datetime:
+    """
+    Convert time to a different timezone
+
+    you can find a list of timezones by printing pytz.all_timezones
+
+    :param time:
+    :param original_tz:
+    :param aware:
+    :param tz:
+    :return:
+    """
+
+    time = time.replace(tzinfo=pytz.timezone(original_tz))
+    converted_time = time.astimezone(pytz.timezone(tz))
+    return converted_time if aware else converted_time.replace(tzinfo=None)
+
+def get_utc_timestamp(time: datetime, original_tz: str='Canada/Mountain') -> int:
+    """
+    Get the UTC timestamp of a datetime object
+
+    you can find a list of timezones by printing pytz.all_timezones
+
+    :param time:
+    :param original_tz:
+    :return:
+    """
+
+    converted_time = convert_tz(time, original_tz, aware=False, tz='UTC')
+    return int(converted_time.timestamp())
+
+def get_local_timestamp(time: datetime, original_tz: str='Canada/Mountain') -> int:
+    """
+    Get the local timestamp of a datetime object
+
+    you can find a list of timezones by printing pytz.all_timezones
+
+    :param time:
+    :param original_tz:
+    :return:
+    """
+
+    converted_time = convert_tz(time, original_tz, aware=False)
+    return int(converted_time.timestamp())

wcp_library-1.3.2/wcp_library/informatica.py

@@ -1,112 +0,0 @@
-import requests
-import json
-import sys
-
-
-# Taken from https://www.mydatahack.com/running-jobs-with-informatica-cloud-rest-api/
-def get_session_id(username, password, logging):
-    """Authenticate with username and password and
-       retrieve icSessionId and serverUrl that are used for Subsequent API calls"""
-    session_id = ''
-    data = {'@type': 'login', 'username': username, 'password': password}
-    url = "https://dm-us.informaticacloud.com/ma/api/v2/user/login"
-    headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
-    # We need to pass data in string instead of dict so that the data gets posted directly.
-    r = requests.post(url, data=json.dumps(data), headers=headers)
-
-    logging.info('\tAPI Login Response Status Code: ' + str(r.status_code))
-
-    if r.status_code == 200:
-        session_id = r.json()["icSessionId"]
-        server_url = r.json()["serverUrl"]
-        logging.info('\tSession Id: ' + session_id)
-        logging.info('\tServer URL: ' + server_url)
-    else:
-        logging.info('API Login call failed:')
-        logging.info(r.headers)
-        logging.info(r.json())
-        sys.exit(1)
-
-    return session_id, server_url
-
-
-def get_tasks(session_id, server_url, taskType, logging):
-    """ Use this method to get a list of tasks of a specified type. This may be used to determine the TaskID of a task.
-            Task Types: https://jsapi.apiary.io/apis/cloudrestapi/reference/job/list-of-tasks/login.html
-                AVS-Contact validation task
-                DMASK-Data masking task
-                DQA-Data assessment task
-                DRS-Data replication task
-                DSS-Data synchronization task
-                MTT-Mapping configuration task
-                PCS-PowerCenter task"""
-    task_list_url = server_url + "/api/v2/task?type=" + taskType
-    headers = {'icSessionId': session_id}
-    r = requests.get(task_list_url, headers=headers)
-
-    if r.status_code == 200:
-        logging.info('\tRetrieved list of all Tasks')
-        response_dict = json.loads(r.content)
-        return response_dict
-
-    else:
-        logging.info('\tFailed to get list of Tasks: ' + str(r.status_code))
-        return {}
-
-
-def get_task_id(response_dict, taskName, logging):
-    for d in response_dict:
-        if d['name'] == taskName:
-            id = d['id']
-            logging.info('\tTaskID: ' + id)
-            return id
-
-    logging.info('\tCould not find TaskID for the Task Name specified')
-    return ""
-
-
-def get_all_mapping_details(session_id, server_url, logging):
-    mapping_details_url = server_url + "/api/v2/mapping"
-    headers = {'icSessionId': session_id, 'HTTP': '1.0', 'Accept': 'application/json'}
-    r = requests.get(mapping_details_url, headers=headers)
-
-    if r.status_code == 200:
-        response_dict = json.loads(r.content)
-        return response_dict
-
-    else:
-        logging.info('\tFailed to get Mappings: ' + str(r.status_code))
-        return {}
-
-
-def get_singular_mapping_details(session_id, server_url, logging, mappingID):
-    mapping_details_url = server_url + "/api/v2/mapping/" + mappingID
-    headers = {'icSessionId': session_id, 'Accept': 'application/json'}
-    r = requests.get(mapping_details_url, headers=headers)
-
-    if r.status_code == 200:
-        mapping_deets_dict = json.loads(r.content)
-        return mapping_deets_dict
-
-    else:
-        logging.info('\tFailed to get Mapping details for mapping ' + mappingID + ': ' + str(r.status_code))
-        return {}
-
-
-def get_connection_details(session_id, server_url, logging):
-
-    # source_dict = {}
-    # target_dict = {}
-
-    connections_url = server_url + "/api/v2/connection"
-    # target_connections_url = server_url + "/api/v2/mapping"
-    headers = {'icSessionId': session_id, 'content-type': 'application/json'}
-    r = requests.get(connections_url, headers=headers)
-
-    if r.status_code == 200:
-        response_dict = json.loads(r.content)
-        return response_dict
-
-    else:
-        logging.info('\tFailed to get Mappings: ' + str(r.status_code))
-        return {}