bb-integrations-library 3.0.11__py3-none-any.whl

bb_integrations_lib/storage/README.md

@@ -0,0 +1,236 @@
+# File Storage
+
+A standard internal library to interact with file storage systems across integration processes.
+
+# Installation
+
+To install run:
+
+- `pip install git+https://github.com/gravitate-energy/bb_file_storage.git`
+
+# Requirements
+
+- run `setup.py` to create default service_accounts files:
+    - `ftp.credentials.json` for FTP, which must include:
+        - `host`: `str`
+        - `username`: `str`
+        - `password`: `str`
+        - `port`: `int` = 22
+        - `ftp_type`: `str`
+    - `aws.credentials.json` for AWS, must include:
+        - `bucket_name`: `str`
+        - `access_key_id`: `str`
+        - `secret_access_key`: `str`
+    - `google.credentials.json` for Google, must include:
+        - `type`: `str`
+        - `project_id`: `str`
+        - `private_key_id`: `str`
+        - `private_key`: `str`
+        - `client_email`: `str`
+        - `client_id`: `str`
+        - `auth_uri`: `str`
+        - `token_uri`: `str`
+        - `auth_provider_x509_cert_url`: `str`
+        - `client_x509_cert_url`: `str`
+        - `universe_domain`: `str`
+
+# Methods
+
+Integrations within the library will follow the same protocol:
+
+```python
+class Integration(Protocol):
+    def list_files(self, directory: str, credential_file_name: str = None) -> list[str]:
+        """Integration method to list files in a remote directory"""
+        pass
+
+    def get_raw_data(self, min_date: datetime, credential_file_name: str = None, check_date: bool = True) -> \
+            Iterable[RawData]:
+        """Integration method to download a file from a remote directory"""
+        pass
+
+    def rename_file(self, old_name: str, new_name: str, credential_file_name: str = None) -> None:
+        """Integration method to rename a remote file"""
+        pass
+
+    def delete_file(self, path: str, credential_file_name: str = None) -> None:
+        """Integration method to delete a remote file"""
+        pass
+
+    def upload_file(self, file: File, path: str, credential_file_name: str = None) -> None:
+        """Integration method to upload a file to a remote directory"""
+        pass
+
+
+
+```
+
+### *Raw Data:
+
+RawData is defined as follows:
+
+```python
+class RawData(BaseModel):
+    file_name: str
+    data: Any 
+```
+
+File is defined as follows:
+```python
+class File(BaseModel):
+    file_name: str | None = None
+    file_data: str
+    content_type: str = 'text/csv'
+    is_public: bool = False
+    file_extension: str = 'csv'
+    check_if_exists: bool = True
+```
+
+# Services
+
+### FTP - SFTP
+
+Support for both `FTP` and `SFTP` protocols.
+
+- `ftp_type` will determine which protocol to use
+
+*Examples:*
+
+- Import FTP Client:
+
+```python
+from storage.ftp.client import FTPClient
+```
+
+- List files:
+
+``` python
+ls = FTPClient.list_files(directory="/")
+```
+
+- Rename file:
+
+``` python
+FTPClient.rename_file(old_name="old_folder/old_name.csv", new_name="new_folder/new_name.csv")
+```
+
+- Upload file:
+
+``` python
+ df = pd.DataFrame({
+        'column1': [1, 2, 3],
+        'column2': ['a', 'b', 'c']
+    })
+    f = File(
+        file_name="TestFile",
+        file_data=df,
+    )
+ FTPClient.upload_file(file=f, path='Inbound')
+```
+
+- Get file:
+
+``` python
+FTPClient.get_file( path='folder/sub_folder', file_name='file_name.txt')
+```
+
+- Delete file:
+
+``` python
+FTPClient.delete_file( path='folder/file_name.txt')
+```
+
+### GCP - Cloud Storage
+
+Support for Google Cloud Storage.
+
+*Examples:*
+
+- Import Google Client:
+
+```python
+from storage.gcs.client import GCSClient
+```
+
+- List files:
+
+``` python
+ls = GCSClient.list_files(directory="erp-export-bkp/japan")
+```
+
+- Rename file:
+
+``` python
+GCSClient().rename_file(old_name='bboil-integration-bkp/test/testfile.csv', new_name='bboil-integration-bkp/archive/testfile.csv')
+```
+
+- Upload file:
+
+```python
+df = pd.DataFrame({
+    'column1': [1, 2, 3],
+    'column2': ['a', 'b', 'c']
+})
+f = File(
+    file_name="TestFile",
+    file_data=df.to_csv(index=False),
+)
+GCSClient.upload_file(file=f, path="bboil-integration-bkp/test")
+```
+
+- Get file:
+
+``` python
+ff = GCSClient().get_file(path='bboil-integration-bkp/test/TestingSomeStuff', file_name="Test")
+```
+
+- Delete file:
+
+``` python
+GCSClient().delete_file(path='bboil-integration-bkp/archive/testfile.csv')
+```
+
+### GCP - Secret Manager
+Support for Google Cloud Secret Manager.
+
+*Examples:*
+
+- Import Google Client:
+
+```python
+from storage.gcsmanager.client import GCSMClient
+```
+
+- List files:
+
+``` python
+ls = GCSMClient.list_files(directory="projects")
+```
+
+- Rename file:
+Not Implemented
+
+- Upload file:
+
+```python
+file_data = {'name': 'your-name'}
+file = File(file_name="test", file_data=file_data)
+GCSMClient.upload_file(file=file, path="/")
+```
+
+- Get file:
+
+``` python
+
+```
+
+- Delete file:
+
+``` python
+
+```
+
+### AWS - S3
+
+
+

bb_integrations_lib/storage/aws/s3.py

@@ -0,0 +1,8 @@
+import warnings
+
+from bb_integrations_lib.provider.aws.s3.client import S3Client
+
+class AWSIntegration(S3Client):
+    def __init__(self, *args, **kwargs):
+        warnings.warn("Use provider.aws.s3.client.S3Client", DeprecationWarning, stacklevel=2)
+        super().__init__(*args, **kwargs)

bb_integrations_lib/storage/defaults.py

@@ -0,0 +1,72 @@
+from pydantic import BaseModel
+from typing import Dict
+from bb_integrations_lib.shared.model import CredentialType
+
+
+class Default(BaseModel):
+    name: str
+    type: CredentialType
+    credential: dict
+
+    @property
+    def file_name(self):
+        return f'{self.type.value}.json'
+
+
+class DefaultCredentials(BaseModel):
+    credentials: Dict[str, Default]
+
+
+# Example default objects
+google_default = Default(
+    name="Google",
+    type=CredentialType.google,
+    credential={
+        "type": "",
+        "project_id": "",
+        "private_key_id": "",
+        "private_key": "",
+        "client_email": "",
+        "client_id": "",
+        "auth_uri": "",
+        "token_uri": "",
+        "auth_provider_x509_cert_url": "",
+        "client_x509_cert_url": "",
+        "universe_domain": "googleapis.com"
+    }
+)
+
+ftp_default = Default(
+    name="FTP",
+    type=CredentialType.ftp,
+    credential={
+        "username": "",
+        "password": "",
+        "host": "",
+        "port": 0,
+        "ftp_type": "sftp",
+        "file_base_name": "",
+        "base_dir": "",
+        "date_format": "%Y-%m-%d-%H-%M"
+    }
+)
+
+aws_default = Default(
+    name="AWS",
+    type=CredentialType.aws,
+    credential={
+        "bucket_name": "",
+        "access_key_id": "",
+        "secret_access_key": ""
+    }
+)
+
+default_credentials = DefaultCredentials(credentials={
+    "google": google_default,
+    "ftp": ftp_default,
+    "aws": aws_default
+})
+
+if __name__ == '__main__':
+    for key, default in default_credentials.credentials.items():
+        print(f"Key: {key}, File Name: {default.file_name}")

bb_integrations_lib/storage/gcs/client.py

@@ -0,0 +1,8 @@
+import warnings
+
+from bb_integrations_lib.provider.gcp.cloud_storage.client import CloudStorageClient
+
+class GCSClient(CloudStorageClient):
+    def __init__(self):
+        warnings.warn("Use provider.gcp.cloud_storage.client.CloudStorageClient", DeprecationWarning, stacklevel=2)
+        super().__init__()

bb_integrations_lib/storage/gcsmanager/client.py

@@ -0,0 +1,8 @@
+import warnings
+
+from bb_integrations_lib.provider.gcp.cloud_secrets.client import CloudSecretsClient
+
+class GCSMClient(CloudSecretsClient):
+    def __init__(self):
+        warnings.warn("Use provider.gcp.cloud_secrets.client.CloudSecretsClient", DeprecationWarning, stacklevel=2)
+        super().__init__()

bb_integrations_lib/storage/setup.py

@@ -0,0 +1,29 @@
+"""
+Script for initiating the package.
+    - Creates [service].credentials.json if not in project
+    - Tries to create secrets for storage services if not created
+"""
+import json
+import os
+
+from loguru import logger
+
+from bb_integrations_lib.storage.defaults import default_credentials
+
+
+def check_credentials() -> None:
+    for key, default in default_credentials.credentials.items():
+        if not os.path.exists(default.file_name):
+            logger.debug(f'Could not find {default.file_name},creating {key} default credentials')
+            with open(default.file_name, "w") as f:
+                credentials_data = default_credentials.credentials[key].credential
+                json.dump(credentials_data, f, indent=4)
+    return None
+
+
+def setup():
+    check_credentials()
+
+
+if __name__ == '__main__':
+    setup()

bb_integrations_lib/util/cache/custom_ttl_cache.py

@@ -0,0 +1,75 @@
+import time
+from loguru import logger
+from typing import TypeVar, Callable, Dict, Tuple, Any
+import asyncio
+
+T = TypeVar('T')
+
+class CustomTTLCache:
+    def __init__(self, verbose: bool = False):
+        self.caches: dict = {}
+        self.verbose = verbose
+
+    def _is_expired(self, timestamp: float) -> bool:
+        return time.time() > timestamp
+
+    def ttl_cache(self, seconds: int,  cache_key: str):
+        def decorator(func: Callable[..., T]) -> Callable[..., T]:
+            def wrapper(*args, **kwargs):
+                if cache_key in self.caches:
+                    value, expires_at = self.caches[cache_key]
+                    if time.time() < expires_at:
+                        if self.verbose:
+                            logger.info(f"Cache HIT for {cache_key}")
+                        return value
+                    else:
+                        if self.verbose:
+                            logger.info(f"Cache EXPIRED for {cache_key}")
+                        del self.caches[cache_key]
+                else:
+                    if self.verbose:
+                        logger.info(f"Cache MISS for {cache_key}")
+                result = func(*args, **kwargs)
+                expires_at = time.time() + seconds
+                self.caches[cache_key] = (result, expires_at)
+                if self.verbose:
+                    logger.info(
+                        f"Cached result for {cache_key} (expires at {time.strftime('%H:%M:%S', time.localtime(expires_at))})")
+                return result
+            return wrapper
+        return decorator
+
+
+class CustomAsyncTTLCache:
+    def __init__(self, verbose: bool = False):
+        self.caches: Dict[str, Tuple[Any, float]] = {}
+        self.verbose = verbose
+        self.locks: Dict[str, asyncio.Lock] = {}
+
+    def _is_expired(self, timestamp: float) -> bool:
+        return time.time() > timestamp
+
+    async def get_or_set(self, cache_key: str, seconds: int, fetch_func, *args, **kwargs):
+        if cache_key not in self.locks:
+            self.locks[cache_key] = asyncio.Lock()
+        async with self.locks[cache_key]:
+            if cache_key in self.caches:
+                value, expires_at = self.caches[cache_key]
+                if time.time() < expires_at:
+                    if self.verbose:
+                        logger.info(f"Cache HIT for {cache_key}")
+                    return value
+                else:
+                    if self.verbose:
+                        logger.info(f"Cache EXPIRED for {cache_key}")
+                    del self.caches[cache_key]
+            else:
+                if self.verbose:
+                    logger.info(f"Cache MISS for {cache_key}")
+            result = await fetch_func(*args, **kwargs)
+            expires_at = time.time() + seconds
+            self.caches[cache_key] = (result, expires_at)
+            if self.verbose:
+                logger.info(
+                    f"Cached result for {cache_key} (expires at {time.strftime('%H:%M:%S', time.localtime(expires_at))})")
+            return result

bb_integrations_lib/util/cache/protocol.py

@@ -0,0 +1,9 @@
+from typing import Protocol
+
+
+class CustomTTLCacheProtocol(Protocol):
+    def _is_expired(self, timestamp: float) -> bool:
+        """Checks if cache is expired"""
+
+    def ttl_cache(self, seconds: int, cache_key: str):
+        """Wrapper function to manage a custom ttl object"""