wcp-library 1.1.0__tar.gz → 1.1.3__tar.gz

{wcp_library-1.1.0 → wcp_library-1.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wcp-library
-Version: 1.1.0
+Version: 1.1.3
 Summary: Common utilites for internal development at WCP
 Home-page: https://github.com/Whitecap-DNA/WCP-Library
 Author: Mitch-Petersen
@@ -10,8 +10,10 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.10.10,<4.0.0)
+Requires-Dist: ftputil (>=5.1.0,<6.0.0)
 Requires-Dist: oracledb (>=2.5.0,<3.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: pip-system-certs (>=4.0,<5.0)
 Requires-Dist: psycopg (>=3.2.3,<4.0.0)
 Requires-Dist: psycopg-binary (>=3.2.3,<4.0.0)
@@ -32,7 +34,7 @@ Commonly used functions for the D&A team.
 `pip install WCP-Library`
 
 ## Usage
-See Wiki for more information.
+See [Wiki](https://github.com/Whitecap-DNA/WCP-Library/wiki/) for more information.
 
 ## Authors and acknowledgment
 Mitch Petersen

{wcp_library-1.1.0 → wcp_library-1.1.3}/README.md

@@ -7,7 +7,7 @@ Commonly used functions for the D&A team.
 `pip install WCP-Library`
 
 ## Usage
-See Wiki for more information.
+See [Wiki](https://github.com/Whitecap-DNA/WCP-Library/wiki/) for more information.
 
 ## Authors and acknowledgment
 Mitch Petersen

{wcp_library-1.1.0 → wcp_library-1.1.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wcp-library"
-version = "1.1.0"
+version = "1.1.3"
 description = "Common utilites for internal development at WCP"
 authors = ["Mitch-Petersen <mitch.petersen@wcap.ca>"]
 readme = "README.md"
@@ -11,8 +11,10 @@ packages = [{include = "wcp_library"}]
 [tool.poetry.dependencies]
 python = "^3.12"
 aiohttp = "^3.10.10"
+ftputil = "^5.1.0"
 oracledb = "^2.5.0"
 pandas = "^2.2.3"
+paramiko = "^3.5.0"
 pip-system-certs = "^4.0"
 psycopg = "^3.2.3"
 psycopg-binary = "^3.2.3"

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/async_sql/oracle.py

@@ -10,8 +10,8 @@ from wcp_library.async_sql import retry
 logger = logging.getLogger(__name__)
 
 
-async def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
-                            max_connections: int) -> AsyncConnectionPool:
+async def _connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
+                             max_connections: int) -> AsyncConnectionPool:
     """
     Create Warehouse Connection
 
@@ -71,8 +71,8 @@ class AsyncOracleConnection(object):
 
         sid_or_service = self._database if self._database else self._sid
 
-        self._session_pool = await connect_warehouse(self._username, self._password, self._hostname, self._port,
-                                                     sid_or_service, self.min_connections, self.max_connections)
+        self._session_pool = await _connect_warehouse(self._username, self._password, self._hostname, self._port,
+                                                      sid_or_service, self.min_connections, self.max_connections)
 
     async def set_user(self, credentials_dict: dict) -> None:
         """

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/async_sql/postgres.py

@@ -10,8 +10,8 @@ from wcp_library.async_sql import retry
 logger = logging.getLogger(__name__)
 
 
-async def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
-                            max_connections: int) -> AsyncConnectionPool:
+async def _connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
+                             max_connections: int) -> AsyncConnectionPool:
     """
     Create Warehouse Connection
 
@@ -66,8 +66,8 @@ class AsyncPostgresConnection(object):
         :return: None
         """
 
-        self._session_pool = await connect_warehouse(self._username, self._password, self._hostname, self._port,
-                                                     self._database, self.min_connections, self.max_connections)
+        self._session_pool = await _connect_warehouse(self._username, self._password, self._hostname, self._port,
+                                                      self._database, self.min_connections, self.max_connections)
 
     async def set_user(self, credentials_dict: dict) -> None:
         """

wcp_library-1.1.3/wcp_library/credentials/ftp.py

@@ -0,0 +1,124 @@
+import logging
+
+import requests
+from yarl import URL
+
+from wcp_library.credentials import MissingCredentialsError
+
+logger = logging.getLogger(__name__)
+
+
+class FTPCredentialManager:
+    def __init__(self, passwordState_api_key: str):
+        self.password_url = URL("https://vault.wcap.ca/api/passwords/")
+        self.api_key = passwordState_api_key
+        self.headers = {"APIKey": self.api_key, 'Reason': 'Python Script Access'}
+        self._password_list_id = 208
+
+    def _get_credentials(self) -> dict:
+        """
+        Get all credentials from the password list
+
+        :return: Dictionary of credentials
+        """
+
+        logger.debug("Getting credentials from PasswordState")
+        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
+        passwords = requests.get(str(url), headers=self.headers).json()
+
+        if not passwords:
+            raise MissingCredentialsError("No credentials found in this Password List")
+
+        password_dict = {}
+        for password in passwords:
+            password_info = {'PasswordID': password['PasswordID'], 'UserName': password['UserName'], 'Password': password['Password']}
+            for field in password['GenericFieldInfo']:
+                password_info[field['DisplayName']] = field['Value'].lower() if field['DisplayName'].lower() == 'username' else field['Value']
+            password_dict[password['UserName'].lower()] = password_info
+        logger.debug("Credentials retrieved")
+        return password_dict
+
+    def get_credentials(self, username: str) -> dict:
+        """
+        Get the credentials for a specific username
+
+        :param username:
+        :return: Dictionary of credentials
+        """
+
+        logger.debug(f"Getting credentials for {username}")
+        credentials = self._get_credentials()
+
+        try:
+            return_credential = credentials[username.lower()]
+        except KeyError:
+            raise MissingCredentialsError(f"Credentials for {username} not found in this Password List")
+        logger.debug(f"Credentials for {username} retrieved")
+        return return_credential
+
+    def update_credential(self, credentials_dict: dict) -> bool:
+        """
+        Update the credentials for a specific username
+
+        Credentials dictionary must have the following keys:
+            - PasswordID
+            - UserName
+            - Password
+
+        The dictionary should be obtained from the get_credentials method and modified accordingly
+
+        :param credentials_dict:
+        :return: True if successful, False otherwise
+        """
+
+        logger.debug(f"Updating credentials for {credentials_dict['UserName']}")
+        url = (self.password_url / str(self._password_list_id)).with_query("QueryAll")
+        passwords = requests.get(str(url), headers=self.headers).json()
+
+        relevant_credential_entry = [x for x in passwords if x['UserName'] == credentials_dict['UserName']][0]
+        for field in relevant_credential_entry['GenericFieldInfo']:
+            if field['DisplayName'] in credentials_dict:
+                credentials_dict[field['GenericFieldID']] = credentials_dict[field['DisplayName']]
+                credentials_dict.pop(field['DisplayName'])
+
+        response = requests.put(str(self.password_url), json=credentials_dict, headers=self.headers)
+        if response.status_code == 200:
+            logger.debug(f"Credentials for {credentials_dict['UserName']} updated")
+            return True
+        else:
+            logger.error(f"Failed to update credentials for {credentials_dict['UserName']}")
+            return False
+
+    def new_credentials(self, credentials_dict: dict) -> bool:
+        """
+        Create a new credential entry
+
+        Credentials dictionary must have the following keys:
+            - UserName
+            - Password
+            - Host
+            - Port
+            - FTP/SFTP (FTP or SFTP)
+
+        :param credentials_dict:
+        :return: True if successful, False otherwise
+        """
+
+        data = {
+            "PasswordListID": self._password_list_id,
+            "Title": credentials_dict['UserName'].upper() if "Title" not in credentials_dict else credentials_dict['Title'].upper(),
+            "Notes": credentials_dict['Notes'] if 'Notes' in credentials_dict else None,
+            "UserName": credentials_dict['UserName'].lower(),
+            "Password": credentials_dict['Password'],
+            "GenericField1": credentials_dict['Host'],
+            "GenericField2": credentials_dict['Port'],
+            "GenericField3": credentials_dict['FTP/SFTP']
+        }
+
+        response = requests.post(str(self.password_url), json=data, headers=self.headers)
+        if response.status_code == 201:
+            logger.debug(f"New credentials for {credentials_dict['UserName']} created")
+            return True
+        else:
+            logger.error(f"Failed to create new credentials for {credentials_dict['UserName']}")
+            return False

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/credentials/oracle.py

@@ -122,4 +122,4 @@ class OracleCredentialManager:
             return True
         else:
             logger.error(f"Failed to create new credentials for {credentials_dict['UserName']}")
-            return False
+            return False

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/credentials/postgres.py

@@ -121,4 +121,4 @@ class PostgresCredentialManager:
             return True
         else:
             logger.error(f"Failed to create new credentials for {credentials_dict['UserName']}")
-            return False
+            return False

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/emailing.py

@@ -7,7 +7,7 @@ from email.utils import formatdate
 from pathlib import Path
 
 
-def send_email(sender: str, recipients: list, subject: str, message=None):
+def send_email(sender: str, recipients: list, subject: str, message: str=None) -> None:
     """
     Function to send an email
 
@@ -32,7 +32,7 @@ def send_email(sender: str, recipients: list, subject: str, message=None):
     server.quit()
 
 
-def email_reporting(subject: str, message: str):
+def email_reporting(subject: str, message: str) -> None:
     """
     Function to email the reporting team from the Python email
 
@@ -55,7 +55,7 @@ def email_reporting(subject: str, message: str):
     server.quit()
 
 
-def email_with_attachments(sender: str, recipients: list, subject: str, message=None, attachments: list[Path]=None):
+def email_with_attachments(sender: str, recipients: list, subject: str, message: str=None, attachments: list[Path]=None) -> None:
     """
     Function to send an email with attachments
 

wcp_library-1.1.3/wcp_library/ftp/ftp.py

@@ -0,0 +1,122 @@
+import ftplib
+import logging
+import re
+from pathlib import Path
+from typing import Optional
+
+import ftputil
+
+logger = logging.getLogger(__name__)
+
+
+class FTP:
+    def __init__(self, host: Optional[str]=None, port: Optional[int]=21, password_vault_dict: Optional[dict]=None):
+        self.host: str = host if not password_vault_dict else password_vault_dict['Host']
+        self.port: int = port if not password_vault_dict else password_vault_dict['Port']
+        self._username: Optional[str] = None if not password_vault_dict else password_vault_dict['UserName']
+        self._password: Optional[str] = None if not password_vault_dict else password_vault_dict['Password']
+
+        self._ftp_factory: ftputil.session.session_factory = ftputil.session.session_factory(base_class=ftplib.FTP_TLS,
+                                                                                             port=self.port,
+                                                                                             encrypt_data_channel=True,
+                                                                                             encoding="UTF-8")
+        self.ftp_connection: Optional[ftputil.FTPHost] = None if not (self._username and self._password) \
+            else ftputil.FTPHost(self.host, self._username, self._password, session_factory=self._ftp_factory)
+
+    def login(self, username: str, password: str) -> None:
+        """
+        Login to the FTP server
+
+        :param username:
+        :param password:
+        :return:
+        """
+
+        logger.debug(f"Logging into {self.host} with username {username}")
+        self.host = ftputil.FTPHost(self.host, username, password, session_factory=self._ftp_factory)
+
+    def download(self, remote_file: Path, local_file: Path) -> None:
+        """
+        Download a file from the FTP server
+
+        :param remote_file:
+        :param local_file:
+        :return:
+        """
+
+        logger.debug(f"Downloading {remote_file} to {local_file}")
+        self.ftp_connection.download(remote_file, local_file)
+
+    def download_files(self, local_dir: Path, regex_pattern: str='*') -> None:
+        """
+        Download files from the FTP server matching the regex pattern
+
+        :param local_dir:
+        :param regex_pattern:
+        :return:
+        """
+
+        logger.debug(f"Downloading files from FTP server matching {regex_pattern} to {local_dir}")
+        files = self.list_files()
+        for file in files:
+            if re.match(regex_pattern, file.name):
+                logger.debug(f"Downloading {file} to {local_dir / file.name}")
+                self.ftp_connection.download(file, local_dir / file.name)
+
+    def list_files(self) -> list[Path]:
+        """
+        List files on the FTP server
+
+        Returns a list of file paths
+
+        :return:
+        """
+
+        logger.debug(f"Listing files on {self.ftp_connection.curdir}")
+        return [Path(x) for x in self.ftp_connection.listdir(self.ftp_connection.curdir) if self.ftp_connection.path.isfile(x)]
+
+    def list_dirs(self) -> list[Path]:
+        """
+        List directories on the FTP server
+
+        Returns a list of directory paths
+
+        :return:
+        """
+
+        logger.debug(f"Listing directories on {self.ftp_connection.curdir}")
+        return [Path(x) for x in self.ftp_connection.listdir(self.ftp_connection.curdir) if self.ftp_connection.path.isdir(x)]
+
+    def change_dir(self, remote_dir: Path) -> None:
+        """
+        Change the directory on the FTP server
+
+        :param remote_dir:
+        :return:
+        """
+
+        logger.debug(f"Changing FTP directory to {remote_dir}")
+        self.ftp_connection.chdir(remote_dir)
+
+    def upload(self, local_file: Path, remote_file: Path) -> None:
+        """
+        Upload a file to the FTP server
+
+        :param local_file:
+        :param remote_file:
+        :return:
+        """
+
+        logger.debug(f"Uploading {local_file} to {remote_file}")
+        self.ftp_connection.upload(local_file, remote_file)
+
+    def close(self) -> None:
+        """
+        Close the FTP connection
+
+        :return:
+        """
+
+        logger.debug(f"Closing FTP connection")
+        self.ftp_connection.close()
+        self.ftp_connection = None

wcp_library-1.1.3/wcp_library/ftp/sftp.py

@@ -0,0 +1,122 @@
+import logging
+import re
+import stat
+from pathlib import Path
+from typing import Optional
+
+import paramiko
+
+logger = logging.getLogger(__name__)
+
+
+class SFTP:
+    def __init__(self, host: str, port: int=22):
+        self.host: str = host
+        self.port: int = port
+        self._username: Optional[str] = None
+        self._password: Optional[str] = None
+
+        self.ssh = paramiko.SSHClient()
+        # AutoAddPolicy automatically adds the hostname and new host key to the local HostKeys object
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        self.sftp_connection: Optional[paramiko.SFTP] = None
+
+    def login(self, username: str, password: str) -> None:
+        """
+        Login to the SFTP server
+
+        :param username:
+        :param password:
+        :return:
+        """
+
+        logger.debug(f"Logging into {self.host} with username {username}")
+        self.ssh.connect(self.host, self.port, username, password)
+        self.sftp_connection = self.ssh.open_sftp()
+
+    def download(self, remote_file: Path, local_file: Path) -> None:
+        """
+        Download a file from the SFTP server
+
+        :param remote_file:
+        :param local_file:
+        :return:
+        """
+
+        logger.debug(f"Downloading {remote_file} to {local_file}")
+        self.sftp_connection.get(str(remote_file), local_file)
+
+    def download_files(self, local_dir: Path, regex_pattern: str='*') -> None:
+        """
+        Download files from the SFTP server matching the regex pattern
+
+        :param local_dir:
+        :param regex_pattern:
+        :return:
+        """
+
+        logger.debug(f"Downloading files from FTP server matching {regex_pattern} to {local_dir}")
+        files = self.list_files()
+        for file in files:
+            if re.match(regex_pattern, file.name):
+                logger.debug(f"Downloading {file} to {local_dir / file.name}")
+                self.sftp_connection.get(str(file), local_dir / file.name)
+
+    def list_files(self) -> list[Path]:
+        """
+        List files on the SFTP server
+
+        Returns a list of file paths
+
+        :return:
+        """
+
+        logger.debug(f"Listing files on {self.sftp_connection.getcwd()}")
+        return [Path(x) for x in self.sftp_connection.listdir() if stat.S_ISREG(self.sftp_connection.lstat(x).st_mode)]
+
+    def list_dirs(self) -> list[Path]:
+        """
+        List directories on the SFTP server
+
+        Returns a list of directory paths
+
+        :return:
+        """
+
+        logger.debug(f"Listing directories on {self.sftp_connection.getcwd()}")
+        return [Path(x) for x in self.sftp_connection.listdir() if stat.S_ISDIR(self.sftp_connection.lstat(x).st_mode)]
+
+    def change_dir(self, remote_dir: Path) -> None:
+        """
+        Change the directory on the SFTP server
+
+        :param remote_dir:
+        :return:
+        """
+
+        logger.debug(f"Changing FTP directory to {remote_dir}")
+        self.sftp_connection.chdir(str(remote_dir))
+
+    def upload(self, local_file: Path, remote_file: Path) -> None:
+        """
+        Upload a file to the SFTP server
+
+        :param local_file:
+        :param remote_file:
+        :return:
+        """
+
+        logger.debug(f"Uploading {local_file} to {remote_file}")
+        self.sftp_connection.put(local_file, str(remote_file))
+
+    def close(self):
+        """
+        Close the SFTP connection
+
+        :return:
+        """
+
+        logger.debug("Closing FTP connection")
+        self.sftp_connection.close()
+        self.sftp_connection = None

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/logging.py

@@ -4,7 +4,7 @@ import sys
 from wcp_library import application_path
 
 
-def create_log(level: int, iterations: int, project_name: str, mode: str = "w",
+def create_log(file_level: int, console_level: int, iterations: int, project_name: str, mode: str = "w",
                format: str = "%(asctime)s:%(levelname)s:%(module)s:%(filename)s:%(lineno)d:%(message)s"):
     """
     Create log file.
@@ -13,7 +13,8 @@ def create_log(level: int, iterations: int, project_name: str, mode: str = "w",
 
     format help: https://docs.python.org/3/library/logging.html#logrecord-attributes
 
-    :param level: Logging level to output to log file.
+    :param file_level: Logging level to output to log file.
+    :param console_level: Logging level to output to console.
     :param iterations: Number of log files to keep.
     :param project_name: Name of the project. (Used as the log file name)
     :param mode: Mode to open the log file. (Default: "w")
@@ -33,12 +34,12 @@ def create_log(level: int, iterations: int, project_name: str, mode: str = "w",
 
     logging.basicConfig(
         filename=(application_path / (project_name + ".log")),
-        level=level,
+        level=file_level,
         format=format,
         filemode=mode
     )
 
-    MIN_LEVEL = logging.DEBUG
+    MIN_LEVEL = console_level
     stdout_hdlr = logging.StreamHandler(sys.stdout)
     stderr_hdlr = logging.StreamHandler(sys.stderr)
     stdout_hdlr.setLevel(MIN_LEVEL)
@@ -48,4 +49,4 @@ def create_log(level: int, iterations: int, project_name: str, mode: str = "w",
     rootLogger.addHandler(stdout_hdlr)
     rootLogger.addHandler(stderr_hdlr)
     logger = logging.getLogger(__name__)
-    logger.setLevel(logging.DEBUG)
+    logger.setLevel(console_level)

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/sql/oracle.py

@@ -10,8 +10,8 @@ from wcp_library.sql import retry
 logger = logging.getLogger(__name__)
 
 
-def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
-                            max_connections: int) -> ConnectionPool:
+def _connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
+                       max_connections: int) -> ConnectionPool:
     """
     Create Warehouse Connection
 
@@ -72,8 +72,8 @@ class OracleConnection(object):
 
         sid_or_service = self._database if self._database else self._sid
 
-        self._session_pool = connect_warehouse(self._username, self._password, self._hostname, self._port,
-                                               sid_or_service, self.min_connections, self.max_connections)
+        self._session_pool = _connect_warehouse(self._username, self._password, self._hostname, self._port,
+                                                sid_or_service, self.min_connections, self.max_connections)
 
     def set_user(self, credentials_dict: dict) -> None:
         """

{wcp_library-1.1.0 → wcp_library-1.1.3}/wcp_library/sql/postgres.py

@@ -10,8 +10,8 @@ from wcp_library.sql import retry
 logger = logging.getLogger(__name__)
 
 
-def connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
-                            max_connections: int) -> ConnectionPool:
+def _connect_warehouse(username: str, password: str, hostname: str, port: int, database: str, min_connections: int,
+                       max_connections: int) -> ConnectionPool:
     """
     Create Warehouse Connection
 
@@ -66,7 +66,7 @@ class PostgresConnection(object):
         :return: None
         """
 
-        self._session_pool = connect_warehouse(self._username, self._password, self._hostname, self._port,
+        self._session_pool = _connect_warehouse(self._username, self._password, self._hostname, self._port,
                                                self._database, self.min_connections, self.max_connections)
 
     def set_user(self, credentials_dict: dict) -> None: