pmdb_utils 0.3.0__py3-none-any.whl

pmdb_utils/__init__.py

@@ -0,0 +1,4 @@
+from .dataacceslayer import StorageClient
+from .key_vault_client import KeyVaultClient
+from .api_related import retry_request
+__all__ = ["StorageClient", "KeyVaultClient", "retry_request"]

pmdb_utils/api_related.py

@@ -0,0 +1,64 @@
+# pip3 install requests
+import requests
+import time
+def retry_request(
+    method="GET",
+    url=None,
+    status_forcelist=[],
+    total=3,
+    delay=10,
+    **kwargs,
+):
+    """
+    Sends an HTTP request with retry logic for specified status codes or connection errors.
+
+    Parameters:
+    method (str): The HTTP method to use (e.g., 'GET', 'POST'). Default is 'GET'.
+    url (str): The URL to send the request to. Default is None.
+    status_forcelist (list): A list of HTTP status codes that should trigger a retry. Default is an empty list.
+    total (int): The total number of retry attempts. Default is 3.
+    delay (int): The delay (in seconds) between retries. Default is 10.
+    **kwargs: Additional arguments to pass to the `requests.request` method.
+
+    Returns:
+    response (requests.Response): The final response object if successful.
+    last_response (requests.Response): The last response object after retries if the request fails.
+
+    Notes:
+    - If a `requests.exceptions.ConnectionError` occurs, the function will retry.
+    - If the response status code is in `status_forcelist`, the function will retry after waiting for `delay` seconds.
+    - After exhausting all retries, the last response object is returned.
+    - Inspired by https://www.zenrows.com/blog/python-requests-retry#code-your-retry-wrapper
+
+    Example:
+        response = retry_request(
+            method="GET",
+            url="https://httpstat.us/406",
+            total=5,
+            delay=2,
+            status_forcelist=[406]
+        )
+    """
+    # Store the last response in an empty variable
+    last_response = None
+
+    # Implement retry logic
+    for _ in range(total):
+        try:
+            response = requests.request(method, url, **kwargs)
+            if response.status_code in status_forcelist:
+                # Track the last response
+                last_response = response
+                print(f" Got status code {response.status_code} for url {url} will retry in {delay} seconds retry number {_+1}/{total}")
+                time.sleep(delay)
+                # Retry request
+                continue
+            else:
+                return response
+
+        except requests.exceptions.ConnectionError:
+            pass
+
+    # Log the response after the retry
+    return last_response
+

pmdb_utils/dataacceslayer/__init__.py

@@ -0,0 +1,3 @@
+from .storageclient import StorageClient
+
+__all__ = ["StorageClient"]

pmdb_utils/dataacceslayer/storageclient.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+from urllib.parse import urlparse
+
+
+from typing import Mapping, Optional
+from minio import Minio
+from minio.commonconfig import CopySource
+
+
+class StorageClient:
+    """
+    Storage client to read/write tabular data to/from storage backends.
+
+    Parameters
+    ----------
+    file_path : str, optional
+        Default path to read/write if none is provided to .read()/.write().
+    file_type : str | None
+        Explicit type like 'delta', 'parquet', 'csv'. If None, inferred from file extension.
+    storage_options : Mapping[str, str] | None
+        Extra options for remote storage backends (e.g., S3 credentials).
+    """
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        file_type: Optional[str] = None,
+        storage_options: Optional[Mapping[str, str]] = None,
+    ):
+        self.file_path = file_path
+        self.file_type = file_type.lower() if file_type else None
+        self.storage_options = storage_options
+        self.client = None
+        self._set_client()
+
+    # ---------------------------
+    # client_config
+    # ---------------------------
+    def _set_client(self):
+        # TODO: Must be a better way to do this to handle different venders
+        endpoint_url = self.storage_options[
+            "endpoint_url"
+        ]  # e.g. "https://minio.mycorp.local:9000"
+        parsed = urlparse(endpoint_url)
+
+        self.client = Minio(
+            parsed.netloc,
+            access_key=self.storage_options["aws_access_key_id"],
+            secret_key=self.storage_options["aws_secret_access_key"],
+            secure=parsed.scheme == "https",
+        )
+
+    def fput_object(self, bucket_name: str, object_name: str, file_path: str, **kwargs):
+        return self.client.fput_object(bucket_name, object_name, file_path, **kwargs)
+
+    def list_objects(self, bucket_name: str, prefix: str, recursive: bool):
+        return self.client.list_objects(bucket_name, prefix, recursive)
+
+    def get_object(self, bucket_name: str, object_name: str, **kwargs):
+        return self.client.get_object(
+            bucket_name=bucket_name, object_name=object_name, **kwargs
+        )
+
+    def remove_object(self, bucket_name: str, object_name: str, **kwargs):
+        return self.client.remove_object(
+            bucket_name=bucket_name, object_name=object_name, **kwargs
+        )
+
+    def move_object(
+        self, bucket_name: str, object_name: str, new_object_name: str, **kwargs
+    ):
+        source = CopySource(bucket_name, object_name)
+        self.client.copy_object(
+            bucket_name,
+            new_object_name,
+            source,
+        )
+        self.client.remove_object(bucket_name, object_name)
+
+    def move_folder(
+        self, bucket_name: str, folder_name: str, new_folder_name: str, **kwargs
+    ):
+        objects = self.client.list_objects(
+            bucket_name, prefix=folder_name, recursive=True
+        )
+        for obj in objects:
+            new_object_name = obj.object_name.replace(folder_name, new_folder_name, 1)
+            # Skip if source and destination are the same
+            if obj.object_name == new_object_name:
+                continue
+            source = CopySource(bucket_name, obj.object_name)
+            self.client.copy_object(
+                bucket_name,
+                new_object_name,
+                source,
+            )
+            self.client.remove_object(bucket_name, obj.object_name)

pmdb_utils/key_vault_client.py

@@ -0,0 +1,45 @@
+from azure.identity import ClientSecretCredential
+from azure.keyvault.secrets import SecretClient
+class KeyVaultClient:
+    def __init__(self, tenant_id: str=None, client_id: str=None, client_secret: str=None, key_vault_name: str='kv-pi-001'):
+        """
+        Initialize the KeyVaultClient with Azure credentials and Key Vault URL.
+
+        :param tenant_id: Azure Tenant ID
+        :param client_id: Service Principal Client ID
+        :param client_secret: Service Principal Client Secret
+        :param key_vault_url: The URL of the Azure Key Vault
+        """
+        # If user didn't pass any of the credentials, retrieve from environment
+        if not all([tenant_id, client_id, client_secret]):
+            tenant_id = tenant_id or os.getenv("AZURE_SPN_PI01_TENANT_ID")
+            client_id = client_id or os.getenv("AZURE_SPN_PI01_CLIENT_ID")
+            client_secret = client_secret or os.getenv("AZURE_SPN_PI01_CLIENT_SECRET")
+            if not all([tenant_id, client_id, client_secret]):
+                raise ValueError("Azure credentials must be provided.")
+
+        self.tenant_id = tenant_id
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+        self.credential = ClientSecretCredential(
+            tenant_id=tenant_id,
+            client_id=client_id,
+            client_secret=client_secret
+        )
+        self.key_vault_url = key_vault_name
+        self.secret_client = SecretClient(vault_url=f'https://{key_vault_name}.vault.azure.net/', credential=self.credential)
+
+    def get_secret(self, secret_name: str):
+        """
+        Fetch a secret from the Azure Key Vault.
+
+        :param secret_name: Name of the secret to fetch
+        :return: The value of the secret
+        """
+        try:
+            secret = self.secret_client.get_secret(secret_name)
+            return secret.value
+        except Exception as e:
+            print(f"Error fetching secret '{secret_name}': {e}")
+            return None

pmdb_utils-0.3.0.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: pmdb_utils
+Version: 0.3.0
+Summary: Add your description here
+Author-email: saebod <saebod@hotmail.com>
+Requires-Python: >=3.13
+Requires-Dist: azure-identity>=1.25.0
+Requires-Dist: azure-keyvault>=4.2.0
+Requires-Dist: minio>=7.2.16

pmdb_utils-0.3.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+pmdb_utils/__init__.py,sha256=SV0armqoGpYok01i371MKRiYO_TQAqFeAhVkmFyH12E,188
+pmdb_utils/api_related.py,sha256=I4VLFeVwWPnRtZazzSANMv42CvrMs4exzUsVuJecF0I,2306
+pmdb_utils/key_vault_client.py,sha256=EmoibCEyrKbbkpceoEwKCT7EotVROkGJZXH5gtqFjJ8,1974
+pmdb_utils/dataacceslayer/__init__.py,sha256=swMCCnFr67XvNKgeNAf4y_aGImpPXNdRByaijQpsNsI,69
+pmdb_utils/dataacceslayer/storageclient.py,sha256=576-huGqzKq_Lqv9P3vbvg-3rownxyojjJQ70IKXOKo,3450
+pmdb_utils-0.3.0.dist-info/METADATA,sha256=q_6GJ4REGFdQFuubjrsk0J9Ir_JsiOPuASyx9Qn07vY,259
+pmdb_utils-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pmdb_utils-0.3.0.dist-info/entry_points.txt,sha256=bpYoW85JVYVja20kUmbSTBbxepoQLSQ4BSvE2fznyz4,47
+pmdb_utils-0.3.0.dist-info/RECORD,,

pmdb_utils-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pmdb_utils-0.3.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pmdb_utils = pmdb_utils:main