bw-essentials-core 0.0.1__py3-none-any.whl

bw_essentials/constants/services.py

@@ -0,0 +1,19 @@
+from enum import Enum
+
+
+class Services(Enum):
+    """
+    Enumeration of available services.
+    """
+    REBALANCE_BUSINESS = 'Rebalance_Business'
+    REBALANCE = 'Rebalance'
+    MASTER_DATA = 'Master_Data'
+    MARKET_PRICER = "Market_Pricer"
+    BROKER = 'Broker'
+    USER_PORTFOLIO = 'User_Portfolio'
+    TRADE_PLACEMENT = 'Trade_Placement'
+    CONTENT = 'Portfolio_Content'
+    NOTIFICATION = 'Notification'
+    USER_REPORTING = 'User_Reporting'
+    PAYMENT = 'Payment'
+    MODEL_PORTFOLIO = "Model_Portfolio"

bw_essentials/data_loch/data_loch.py

@@ -0,0 +1,282 @@
+"""
+Module for interacting with LakeFS and S3.
+
+This module defines the `LakeFS` class, which provides a high-level interface for managing
+version-controlled data in LakeFS using an underlying S3-compatible object store.
+
+Key Features:
+- Upload and download files to/from LakeFS.
+- Sync entire directories from LakeFS to local and vice versa.
+- Check file existence in LakeFS.
+- Commit changes to branches in LakeFS.
+- Retrieve the latest or N latest files from a given LakeFS path.
+- Delete LakeFS directory paths (prefixes).
+
+Requires:
+- `lakefs` client
+- `S3Utils` from `bw_essentials.s3_utils`
+
+Intended for automation of LakeFS operations in data pipelines or services.
+"""
+
+import concurrent
+import logging
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from importlib.util import spec_from_file_location, module_from_spec
+from pathlib import Path
+
+from lakefs.client import Client
+from bw_essentials.s3_utils.s3_utils import S3Utils
+
+logger = logging.getLogger(__name__)
+
+
+class DataLoch:
+    """
+    LakeFS class for managing versioned data operations on LakeFS over S3.
+    """
+
+    def __init__(self, access_key, secret_key, host):
+        """
+        Initialize a LakeFS instance with S3 and LakeFS client setup.
+
+        Args:
+            access_key (str): Access key for authentication with LakeFS/S3.
+            secret_key (str): Secret key for authentication with LakeFS/S3.
+            host (str): LakeFS server URL or S3 endpoint.
+        """
+        logger.info("Initializing LakeFS instance")
+        self.access_key = self._get_env_var("DATA_LAKE_ACCESS_KEY")
+        self.secret_key = self._get_env_var("DATA_LAKE_SECRET_KEY")
+        self.host = self._get_env_var("DATA_LAKE_HOST_URL")
+        self.client = self._get_lakefs_client()
+        self.s3 = S3Utils()
+
+    def _get_env_var(self, key: str) -> str:
+        """
+        Fetch a required variable from bw_config.py located in the root directory.
+
+        Raises:
+            FileNotFoundError: If bw_config.py is not found.
+            AttributeError: If the requested key is not defined in the config.
+
+        Returns:
+            str: The value of the config variable.
+        """
+        config_path = os.path.join(os.getcwd(), "bw_config.py")
+
+        if not os.path.exists(config_path):
+            raise FileNotFoundError("`bw_config.py` file not found in the root directory. "
+                                    "Please ensure the config file exists.")
+
+        spec = spec_from_file_location("bw_config", config_path)
+        bw_config = module_from_spec(spec)
+        sys.modules["bw_config"] = bw_config
+        spec.loader.exec_module(bw_config)
+
+        if not hasattr(bw_config, key):
+            raise AttributeError(f"`{key}` not found in bw_config.py. Please define it in the config.")
+
+        return getattr(bw_config, key)
+
+    def _get_lakefs_client(self):
+        """
+        Get LakeFS client.
+
+        Returns:
+            lakefs.client.Client: Authenticated LakeFS client.
+        """
+        logger.info("Initializing LakeFS client")
+        return Client(host=self.host, username=self.access_key, password=self.secret_key)
+
+    def _get_branch(self, repository, branch):
+        """
+        Return LakeFS branch object.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+
+        Returns:
+            lakefs.Branch: LakeFS branch object.
+        """
+        from lakefs import Repository
+        return Repository(repository, client=self.client).branch(branch_id=branch)
+
+    def _get_meta_data(self, repository, branch):
+        """
+        Return metadata dictionary for commit operation.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+
+        Returns:
+            dict: Metadata for commit.
+        """
+        return {
+            'source': os.getcwd(),
+            'branch': branch,
+            'repository': repository,
+            'timestamp': str(datetime.now())
+        }
+
+    def upload_data(self, repository, branch, local_file_path, file_name, commit=True):
+        """
+        Upload a single file to a LakeFS branch and optionally commit the change.
+
+        Args:
+            repository (str): LakeFS repository name.
+            branch (str): Target branch in the repository.
+            local_file_path (str): Full local path of the file to upload.
+            file_name (str): File name to use in LakeFS.
+            commit (bool): Whether to commit the change after upload. Defaults to True.
+        """
+        logger.info(f"Uploading file {file_name} from {local_file_path}")
+        branch_obj = self._get_branch(repository, branch)
+        self.s3.upload_file(repository, local_file_path, f"{branch}/{file_name}", "application/octet-stream")
+        if commit:
+            commit_response = branch_obj.commit(
+                message=f"{file_name=} uploaded to data lake.",
+                metadata=self._get_meta_data(repository, branch)
+            )
+            logger.info(f"Changes committed: {commit_response=}")
+
+    def download_data(self, repository, branch, local_file_path, file_name):
+        """
+        Download a single file from a LakeFS branch to a local path.
+
+        Args:
+            repository (str): LakeFS repository name.
+            branch (str): Branch in the repository.
+            local_file_path (str): Full local path to save the file.
+            file_name (str): Name of the file in LakeFS.
+        """
+        logger.info(f"Downloading file {file_name} to {local_file_path}")
+        self.s3.download_file(repository, f"{branch}/{file_name}", local_file_path)
+
+    def file_exists(self, repository, branch, filename):
+        """
+        Check whether a file exists in a LakeFS repository branch.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+            filename (str): File name to check.
+
+        Returns:
+            bool: True if file exists, False otherwise.
+        """
+        try:
+            self.s3.s3_instance.head_object(Bucket=repository, Key=f"{branch}/{filename}")
+            return True
+        except Exception:
+            return False
+
+    def sync_dir(self, repository, branch, server_path, local_path=None):
+        """
+        Sync a directory from LakeFS (S3) to the local file system.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+            server_path (str): Path under the branch to sync from.
+            local_path (str, optional): Local destination directory. Defaults to current dir.
+        """
+        logger.info(f"Syncing directory from {server_path} to {local_path}")
+        prefix = f"{branch}/{server_path}"
+        args = []
+        for obj in self.s3.list_bucket_objects(repository, prefix):
+            rel_path = os.path.relpath(obj['Key'], prefix)
+            target = rel_path if not local_path else os.path.join(local_path, rel_path)
+            if os.path.exists(target) or obj['Key'].endswith('/'):
+                continue
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            args.append((repository, obj['Key'], target))
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            [executor.submit(self.s3.download_file, repo, key, target) for repo, key, target in args]
+        logger.info("Directory sync complete")
+
+    def upload_dir(self, repository, branch, local_path, server_path):
+        """
+        Upload all files from a local directory to LakeFS and commit them.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Target branch.
+            local_path (str): Local directory path.
+            server_path (str): Target path in LakeFS under the branch.
+        """
+        logger.info(f"Uploading directory {local_path} to {server_path}")
+        directory = Path(local_path)
+        args = [
+            (repository, branch, str(file), f"{server_path}/{file.name}", False)
+            for file in directory.iterdir() if file.is_file()
+        ]
+        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+            [executor.submit(self.upload_data, repo, br, local, output, commit)
+             for repo, br, local, output, commit in args]
+        branch_obj = self._get_branch(repository, branch)
+        commit_response = branch_obj.commit(
+            message=f"{local_path} uploaded to data lake.",
+            metadata=self._get_meta_data(repository, branch)
+        )
+        logger.info(f"Directory uploaded and committed: {commit_response}")
+
+    def get_latest_file(self, repository, branch, server_path, local_file_path):
+        """
+        Get and download the latest modified file from a LakeFS path.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+            server_path (str): Path in LakeFS under the branch.
+            local_file_path (str): Local destination path to save the file.
+        """
+        logger.info(f"Fetching latest file from {server_path}")
+        prefix = f"{branch}/{server_path}"
+        key = self.s3.get_latest_file_by_prefix(repository, prefix)
+        if key:
+            self.s3.download_file(repository, key, local_file_path)
+
+    def get_latest_n_files(self, repository, branch, server_path, local_file_path, count=1):
+        """
+        Download the latest N files from a given LakeFS path.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+            server_path (str): Path in LakeFS under the branch.
+            local_file_path (str): Local directory to store downloaded files.
+            count (int, optional): Number of recent files to download. Defaults to 1.
+        """
+        logger.info(f"Fetching latest {count} files from {server_path}")
+        prefix = f"{branch}/{server_path}"
+        keys = self.s3.get_latest_n_files(repository, prefix, count=count)
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+        args = []
+        for key in keys:
+            rel_path = os.path.relpath(key, prefix)
+            target = os.path.join(local_file_path, rel_path)
+            if not os.path.exists(target):
+                args.append((repository, key, target))
+        with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+            [executor.submit(self.s3.download_file, repo, key, target) for repo, key, target in args]
+        logger.info("Downloaded latest files")
+
+    def delete_dir(self, repository, branch, server_path):
+        """
+        Delete all files under a given path (prefix) in a LakeFS branch.
+
+        Args:
+            repository (str): Repository name.
+            branch (str): Branch name.
+            server_path (str): Path in LakeFS to delete under the branch.
+        """
+        logger.info(f"Deleting directory {server_path}")
+        prefix = f"{branch}/{server_path}"
+        self.s3.delete_files_by_prefix(repository, prefix)
+        logger.info(f"Deleted all files under prefix {prefix}")

bw_essentials/email_client/email_client.py

@@ -0,0 +1,243 @@
+"""
+email_client.py
+
+This module defines the EmailClient class for sending emails using SMTP. It provides
+methods to send emails with or without file attachments and supports multiple recipients
+and CC addresses. The email body can be plain text or HTML.
+
+Usage:
+    client = EmailClient(
+        smtp_host="smtp.example.com",
+        smtp_port=587,
+        smtp_username="user",
+        smtp_password="pass",
+        sender_email="from@example.com",
+        sender_name="My Service"
+    )
+
+    client.send_email_without_attachment(
+        to_addresses=["to@example.com"],
+        cc_addresses=["cc@example.com"],
+        subject="Test Email",
+        body="Hello, this is a test email!"
+    )
+
+    client.send_email_with_attachment(
+        to_addresses="to@example.com",
+        cc_addresses=None,
+        subject="Report",
+        body="<h1>Monthly Report</h1>",
+        attachment_path="/path/to/report.csv"
+    )
+"""
+
+import logging
+import os
+import smtplib
+import sys
+from importlib.util import spec_from_file_location, module_from_spec
+from typing import Optional, List, Union
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email import encoders
+
+logger = logging.getLogger(__name__)
+
+
+class EmailClient:
+    """
+    A reusable and configurable email client for sending emails using SMTP.
+
+    Attributes:
+        smtp_host (str): SMTP server hostname or IP address.
+        smtp_port (int): SMTP server port (commonly 587 for TLS).
+        smtp_username (str): SMTP username for authentication.
+        smtp_password (str): SMTP password for authentication.
+        sender_email (str): Email address used as the sender.
+        sender_name (str): Display name for the sender.
+    """
+
+    def __init__(
+        self,
+        sender_email: str,
+        sender_name: str,
+    ):
+        """
+        Initialize the EmailClient with SMTP server credentials.
+
+        Args:
+            sender_email (str): Sender's email address.
+            sender_name (str): Display name to show in "From" field.
+        """
+        self.smtp_host = self._get_env_var("SMTP_HOST")
+        self.smtp_port = self._get_env_var("SMTP_PORT")
+        self.smtp_username = self._get_env_var("SMTP_USERNAME")
+        self.smtp_password = self._get_env_var("SMTP_PASSWORD")
+        self.sender_email = sender_email
+        self.sender_name = sender_name
+        logger.info("EmailClient initialized with SMTP host: %s", self.smtp_host)
+
+    def _get_env_var(self, key: str) -> str:
+        """
+        Fetch a required variable from bw_config.py located in the root directory.
+
+        Raises:
+            FileNotFoundError: If bw_config.py is not found.
+            AttributeError: If the requested key is not defined in the config.
+
+        Returns:
+            str: The value of the config variable.
+        """
+        config_path = os.path.join(os.getcwd(), "bw_config.py")
+
+        if not os.path.exists(config_path):
+            raise FileNotFoundError("`bw_config.py` file not found in the root directory. "
+                                    "Please ensure the config file exists.")
+
+        spec = spec_from_file_location("bw_config", config_path)
+        bw_config = module_from_spec(spec)
+        sys.modules["bw_config"] = bw_config
+        spec.loader.exec_module(bw_config)
+
+        if not hasattr(bw_config, key):
+            raise AttributeError(f"`{key}` not found in bw_config.py. Please define it in the config.")
+
+        return getattr(bw_config, key)
+
+    def send_email_without_attachment(
+        self,
+        to_addresses: Union[str, List[str]],
+        cc_addresses: Union[str, List[str], None],
+        subject: str,
+        body: str,
+        is_html: bool = True,
+    ):
+        """
+        Send an email without any attachments.
+
+        Args:
+            to_addresses (Union[str, List[str]]): One or more recipient email addresses.
+            cc_addresses (Union[str, List[str], None]): One or more CC addresses (optional).
+            subject (str): Email subject line.
+            body (str): Email body content.
+            is_html (bool): If True, body is interpreted as HTML; otherwise, plain text.
+        """
+        logger.info("Preparing to send email without attachment | Subject: %s", subject)
+        self._send_email(to_addresses, cc_addresses, subject, body, None, is_html)
+
+    def send_email_with_attachment(
+        self,
+        to_addresses: Union[str, List[str]],
+        cc_addresses: Union[str, List[str], None],
+        subject: str,
+        body: str,
+        attachment_path: str,
+        is_html: bool = True,
+    ):
+        """
+        Send an email with a file attachment.
+
+        Args:
+            to_addresses (Union[str, List[str]]): One or more recipient email addresses.
+            cc_addresses (Union[str, List[str], None]): One or more CC addresses (optional).
+            subject (str): Email subject line.
+            body (str): Email body content.
+            attachment_path (str): Full path to the file to attach.
+            is_html (bool): If True, body is interpreted as HTML; otherwise, plain text.
+        """
+        logger.info(
+            "Preparing to send email with attachment | Subject: %s | Attachment: %s",
+            subject, attachment_path
+        )
+        self._send_email(to_addresses, cc_addresses, subject, body, attachment_path, is_html)
+
+    def _send_email(
+        self,
+        to_addresses: Union[str, List[str]],
+        cc_addresses: Union[str, List[str], None],
+        subject: str,
+        body: str,
+        attachment_path: Optional[str],
+        is_html: bool,
+    ):
+        """
+        Internal helper method to construct and send email messages.
+
+        Args:
+            to_addresses (Union[str, List[str]]): One or more recipient email addresses.
+            cc_addresses (Union[str, List[str], None]): CC addresses.
+            subject (str): Subject of the email.
+            body (str): Email body content.
+            attachment_path (Optional[str]): File path for the attachment, if any.
+            is_html (bool): True if the body is HTML-formatted.
+        """
+        msg = MIMEMultipart()
+        msg['From'] = f'"{self.sender_name}" <{self.sender_email}>'
+        msg['To'] = self._format_addresses(to_addresses)
+        msg['Cc'] = self._format_addresses(cc_addresses) if cc_addresses else ""
+        msg['Subject'] = subject
+
+        msg.attach(MIMEText(body, 'html' if is_html else 'plain'))
+        logger.debug("Email headers and body constructed")
+
+        if attachment_path:
+            if not os.path.exists(attachment_path):
+                logger.warning("Attachment file not found: %s", attachment_path)
+            else:
+                try:
+                    with open(attachment_path, 'rb') as attachment:
+                        part = MIMEBase('application', 'octet-stream')
+                        part.set_payload(attachment.read())
+                        encoders.encode_base64(part)
+                        part.add_header('Content-Disposition', 'attachment',
+                                        filename=os.path.basename(attachment_path))
+                        msg.attach(part)
+                        logger.info("Attachment added: %s", os.path.basename(attachment_path))
+                except Exception as e:
+                    logger.exception("Failed to read attachment file")
+                    raise e
+
+        try:
+            with smtplib.SMTP(self.smtp_host, self.smtp_port) as server:
+                server.starttls()
+                server.login(self.smtp_username, self.smtp_password)
+                recipients = self._get_recipient_list(to_addresses, cc_addresses)
+                server.sendmail(self.sender_email, recipients, msg.as_string())
+                logger.info("Email sent to %s with CC %s", to_addresses, cc_addresses)
+        except Exception as e:
+            logger.exception("Failed to send email")
+            raise e
+
+    def _format_addresses(self, addresses: Union[str, List[str]]) -> str:
+        """
+        Convert a string or list of email addresses to a comma-separated string.
+
+        Args:
+            addresses (Union[str, List[str]]): Email addresses.
+
+        Returns:
+            str: Comma-separated email addresses.
+        """
+        if isinstance(addresses, list):
+            return ", ".join(addresses)
+        return addresses
+
+    def _get_recipient_list(
+        self,
+        to_addresses: Union[str, List[str]],
+        cc_addresses: Union[str, List[str], None]
+    ) -> List[str]:
+        """
+        Merge TO and CC addresses into a single list for email sending.
+
+        Args:
+            to_addresses (Union[str, List[str]]): Main recipients.
+            cc_addresses (Union[str, List[str], None]): CC recipients.
+
+        Returns:
+            List[str]: List of all email recipients.
+        """
+        to_list = to_addresses if isinstance(to_addresses, list) else [to_addresses]
+        cc_list = cc_addresses if isinstance(cc_addresses, list) else [cc_addresses] if cc_addresses else []
+        return to_list + cc_list