cit-pydata 0.0.1.dev0__tar.gz

cit-pydata-0.0.1.dev0/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2024 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cit-pydata-0.0.1.dev0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: cit-pydata
+Version: 0.0.1.dev0
+Summary: Python clients for commonly used api services including Salesforce, SQL, Marketo, Box, SFSync*, and JobLog*
+Author-email: Dionis Wang <dwang@eab.com>
+Project-URL: Homepage, https://github.com/EAB-IT-DEV/cit-pydata
+Project-URL: Issues, https://github.com/EAB-IT-DEV/cit-pydata/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Provides-Extra: aws
+Provides-Extra: box
+Provides-Extra: joblog
+Provides-Extra: salesforce
+Provides-Extra: sftp
+Provides-Extra: sql
+Provides-Extra: sql-pyodbc
+Provides-Extra: sql-pymssql
+Provides-Extra: sql-psycopg2
+Provides-Extra: sql-mssql
+Provides-Extra: sql-postgres
+Provides-Extra: local
+Provides-Extra: all
+License-File: LICENSE
+
+# cit-pydata 
+
+This repo provides Python clients for commonly used api services including Salesforce, SQL, Marketo, Box, SFSync*, and JobLog*.  
+
+*SFSync = custom Salesforce <> MSQ SQL integration  
+*JobLog = custom centralized logging for data integration jobs  

cit-pydata-0.0.1.dev0/README.md

@@ -0,0 +1,6 @@
+# cit-pydata 
+
+This repo provides Python clients for commonly used api services including Salesforce, SQL, Marketo, Box, SFSync*, and JobLog*.  
+
+*SFSync = custom Salesforce <> MSQ SQL integration  
+*JobLog = custom centralized logging for data integration jobs  

cit-pydata-0.0.1.dev0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cit-pydata"
+version = "0.0.1.dev0"
+authors = [
+  { name="Dionis Wang", email="dwang@eab.com" },
+]
+description = "Python clients for commonly used api services including Salesforce, SQL, Marketo, Box, SFSync*, and JobLog*"
+readme = "README.md"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: POSIX :: Linux",
+]
+[project.urls]
+Homepage = "https://github.com/EAB-IT-DEV/cit-pydata"
+Issues = "https://github.com/EAB-IT-DEV/cit-pydata/issues"
+
+[project.optional-dependencies]
+aws = ["aws-lambda-powertools"]
+box = ["boxsdk"]
+joblog = ["marketorestpython"]
+salesforce = ["simple_salesforce","pyforce"]
+sftp = ["paramiko"]
+sql = ["SQLAlchemy==1.4.25", "numpy", "pandas"]
+sql-pyodbc = ["SQLAlchemy==1.4.25", "pyodbc", "numpy", "pandas"]
+sql-pymssql = ["SQLAlchemy==1.4.25", "pymssql", "numpy", "pandas"]
+sql-psycopg2 = ["SQLAlchemy==1.4.25", "psycopg2-binary==2.9.6", "numpy", "pandas"]
+sql-mssql = ["SQLAlchemy==1.4.25", "pymssql", "numpy", "pandas"]
+sql-postgres = ["SQLAlchemy==1.4.25", "psycopg2-binary==2.9.6", "numpy", "pandas"]
+local = ["python-dotenv", "boto3"]
+all = ["aws-lambda-powertools","pandas","boxsdk","marketorestpython","simple_salesforce","pyforce","paramiko","SQLAlchemy==1.4.25", "pyodbc","pymssql", "numpy", "pandas","psycopg2-binary==2.9.6","python-dotenv", "boto3"]

cit-pydata-0.0.1.dev0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cit-pydata-0.0.1.dev0/src/cit_pydata/__init__.py

@@ -0,0 +1,9 @@
+import aws
+import box
+import joblog
+import marketo
+import salesforce
+import sfsync
+import sftp
+import sql
+import util

cit-pydata-0.0.1.dev0/src/cit_pydata/aws/__init__.py

@@ -0,0 +1 @@
+from .api import S3Client, SSMClient

cit-pydata-0.0.1.dev0/src/cit_pydata/aws/api.py

@@ -0,0 +1,204 @@
+import sys
+
+from cit_pydata.util import api as util_api
+from cit_pydata.util import ON_AWS
+
+from typing import Literal
+import boto3
+
+
+def _get_boto_session(
+    service: str = None,
+    environment: str = None,
+    iam_user: str = None,
+    path_to_env_file: str = None,
+    region: str = None,
+    logger=util_api.get_logger(__name__, "info"),
+):
+    if ON_AWS:
+        return boto3.Session()
+
+    _environment_variable = environment + "_" + iam_user + "_" + "aws_access_key_id"
+    aws_access_key_id = util_api.get_environment_variable(
+        logger=logger,
+        path_to_env_file=path_to_env_file,
+        variable_name=_environment_variable,
+    )
+
+    if aws_access_key_id is None:
+        logger.error(f"Access Key Id does not exist for environment {environment}")
+        return
+
+    _environment_variable = environment + "_" + iam_user + "_" + "aws_secret_access_key"
+    aws_secret_access_key = util_api.get_environment_variable(
+        logger=logger,
+        path_to_env_file=path_to_env_file,
+        variable_name=_environment_variable,
+    )
+
+    if aws_secret_access_key is None:
+        logger.error(f"Secret Key not specified for environment {environment}")
+        return
+
+    _aws_region = region
+    if _aws_region is None:
+        _environment_variable = environment + "_" + iam_user + "_" + "aws_region"
+        _aws_region = util_api.get_environment_variable(
+            logger=logger,
+            path_to_env_file=path_to_env_file,
+            variable_name=_environment_variable,
+        )
+
+    if _aws_region is None:
+        logger.error(f"AWS Region not specified for environment {environment}")
+        return
+
+    _session = boto3.Session(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        region_name=_aws_region,
+    )
+
+    return _session
+
+
+class S3Client:
+    def __init__(
+        self,
+        environment: str = None,
+        iam_user: str = None,
+        path_to_env_file: str = None,
+        log_level: Literal["info", "debug"] = "info",
+    ):
+        self.logger = util_api.get_logger(__name__, log_level)
+        environment = environment.lower()
+
+        self.session = _get_boto_session(
+            service="s3",
+            environment=environment,
+            iam_user=iam_user,
+            path_to_env_file=path_to_env_file,
+            logger=self.logger,
+        )
+        self.client = self.session.client("s3")
+
+    def get_object_metadata(self, **kwargs):
+        """
+        Returns pandas dataframe with S3 Objects
+
+        kwargs for method client.list_objects_v2
+        https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.list_objects_v2
+        """
+
+        # self.logger.info(f'Getting S3 objects: {**kwargs}')
+        object_list = []
+        next_page_token = "init"
+
+        # Object metadata is retrieved in batches of 1000
+        while next_page_token:
+            response: dict = None
+            if next_page_token == "init":
+                response = self.client.list_objects_v2(**kwargs)
+            else:
+                response = self.client.list_objects_v2(
+                    ContinuationToken=next_page_token, **kwargs
+                )
+
+            next_page_token = response.get("NextContinuationToken", None)
+            # self.logger.debug(next_page_token)
+            for object_item in response["Contents"]:
+                object_list.append(object_item)
+            response.pop("Contents")
+
+        if not pandas in sys.modules:
+            import pandas
+        df_s3_objects = pandas.DataFrame(object_list)
+        # self.logger.debug(response)
+        # self.logger.debug(df_s3_objects.info())
+
+        return response, df_s3_objects
+
+    def metadata_to_sql(self, sql_client, table_name: str, dataframe):
+        if "sqlalchemy" not in sys.modules:
+            import sqlalchemy
+
+        try:
+            sql_client.insert_df(
+                table_name=table_name,
+                df=dataframe,
+                index=False,
+                dtype={"LastModified": sqlalchemy.DateTime},
+            )
+            # dataframe.to_sql(name = table_name, con = sql_client.sql_engine, if_exists='replace', index=False, dtype={'LastModified':sqlalchemy.DateTime})
+        except Exception as e:
+            self.logger.error(f"Failed to send S3 Object metadata to SQL")
+            self.logger.exception(str(e))
+
+
+class SSMClient:
+    def __init__(
+        self,
+        environment: str = None,
+        iam_user: str = None,
+        path_to_env_file: str = None,
+        log_level: Literal["info", "debug"] = "info",
+        logger=None,
+    ):
+        self.logger = util_api.get_logger(__name__, log_level) if not logger else logger
+        self.session = _get_boto_session(
+            service="ssm",
+            environment=environment,
+            iam_user=iam_user,
+            path_to_env_file=path_to_env_file,
+            logger=self.logger,
+        )
+        self.client = self.session.client("ssm")
+
+    def get_parameter(
+        self, name: str, with_decryption: bool = False, is_verbose: bool = False
+    ):
+        value = None
+        try:
+            value = self.client.get_parameter(Name=name, WithDecryption=with_decryption)
+        except Exception as ce:
+            self.logger.error(ce)
+            return None
+
+        if value is None:
+            self.logger.error(f'SSM Parameter not found "{name}"')
+            return
+
+        if is_verbose:
+            return value
+        else:
+            return value["Parameter"]["Value"]
+
+        # for parameter in response['Parameters']:
+        #     return parameter['Value']
+
+    def describe_parameters(
+        self, filter_parameters: list[dict], is_verbose: bool = False
+    ):
+        """
+        filter_parameters=[
+        {
+            'Key': 'string',
+            'Option': 'string',
+            'Values': [
+                'string',
+            ]
+        },
+        """
+        max_results = 50
+        describe_param_dict: dict = self.client.describe_parameters(
+            ParameterFilters=filter_parameters, MaxResults=max_results
+        )
+
+        param_list = describe_param_dict.get("Parameters", None)
+        # return_list = []
+        # for parameter in param_list
+        #     param_keys_to_return_list = ['Name']
+        #     param_dict = parameter.get()
+        #     return_list.append()
+
+        return param_list

cit-pydata-0.0.1.dev0/src/cit_pydata/box/__init__.py

@@ -0,0 +1 @@
+from .api import BoxClient

cit-pydata-0.0.1.dev0/src/cit_pydata/box/api.py

@@ -0,0 +1,107 @@
+import os
+import sys
+
+from cit_pydata.util import api as util_api
+
+
+class BoxClient:
+    def __init__(self, box_jwt_auth_file_path, logger=None):
+        self.logger = util_api.get_logger(__name__, "INFO") if not logger else logger
+
+        self.box_jwt_auth_file_path = box_jwt_auth_file_path
+
+        # client is the boxsdk Client object
+        self.client = None
+
+    def _get_client(self):
+        if "boxsdk" not in sys.modules:
+            from boxsdk import Client, JWTAuth, OAuth2
+        # JWT Auth
+        auth = JWTAuth.from_settings_file(self.box_jwt_auth_file_path)
+
+        try:
+            self.client = Client(auth)
+        except Client.BoxAPIException as err:
+            self.logger.error("Failed to Authenticate to Box API:", err)
+
+    def get_folder(self, folder_id: str):
+        if not self.client:
+            self._get_client()
+
+        return self.client.folder(folder_id=folder_id)
+
+    def get_file(self):
+        pass
+
+    def get_folder_items(self, folder_id: str, filter_type: str = None):
+        if not self.client:
+            self._get_client()
+
+        item_id_dict = {}
+        items = self.client.folder(folder_id=folder_id).get_items()
+
+        assert filter_type in ["file", "folder", None]
+        if filter_type == "file":
+            for item in items:
+                if item.type == filter_type:
+                    item_id_dict[item.id] = item
+        elif filter_type == "folder":
+            for item in items:
+                if item.type == filter_type:
+                    item_id_dict[item.id] = item
+        else:
+            for item in items:
+                item_id_dict[item.id] = item
+
+        for item in items:
+            if item.type == filter_type:
+                item_id_dict[item.id] = {"name": item.name, "type": item.type}
+            else:
+                item_id_dict[item.id] = {"name": item.name, "type": item.type}
+
+        return item_id_dict
+
+    def get_shared_link(self, folder_id):
+        if not self.client:
+            self._get_client()
+
+        folder = self.get_folder(folder_id)
+        shared_link = folder.get().shared_link
+        return shared_link
+
+    def copy_file(self, source_file_id, target_folder_id):
+        """
+        Copies Box fild to another folder
+            source_file_id - Box file id of the file to copy
+            target_folder_id - Box folder id of where to copy the file too
+        """
+        if not self.client:
+            self._get_client()
+
+        file_to_copy = self.client.file(source_file_id)
+        target_folder = self.client.folder(target_folder_id)
+        try:
+            file_copy = file_to_copy.copy(target_folder)
+            self.logger.info(
+                'File "{0}" has been copied into folder "{1}"'.format(
+                    file_copy.name, file_copy.parent.name
+                )
+            )
+        except Exception as e:
+            self.logger.error(f"Failed to copy Box item {e}")
+
+    def upload_file(self, file_path, box_folder_id):
+        if not self.client:
+            self._get_client()
+
+        if os.path.exists(file_path):
+            new_file = self.client.folder(box_folder_id).upload(file_path)
+            self.logger.info(
+                'File "{0}" uploaded to Box with file ID {1}'.format(
+                    new_file.name, new_file.id
+                )
+            )
+            return True
+        else:
+            self.logger.error(f"File does not exist {file_path}")
+            return False

cit-pydata-0.0.1.dev0/src/cit_pydata/joblog/__init__.py

@@ -0,0 +1 @@
+from .api import JobLogClient

cit-pydata-0.0.1.dev0/src/cit_pydata/joblog/api.py

@@ -0,0 +1,93 @@
+import sys
+
+from cit_pydata.util import api as util_api
+
+
+class JobLogClient:
+    def __init__(self, sql_client, job_name, logger=None):
+        self.logger = util_api.get_logger(__name__, "INFO") if not logger else logger
+
+        self.sql_client = sql_client
+
+        self.job_name = job_name
+        self.log_guid = None
+
+    def start(self):
+        if "uuid" not in sys.modules:
+            import uuid
+        if not self.log_guid:
+            self.log_guid = str(uuid.uuid4())
+            param_list = [self.job_name, "initializing", self.log_guid]
+            self.sql_client.execute_stored_procedure("usp_Start_JobLog", param_list)
+            self.logger.info(
+                f"JOBLOG - Started Job {self.job_name} with GUID {self.log_guid}"
+            )
+            return self.log_guid
+        else:
+            self.logger.error(f"JOBLOG - Failed to start. Logging already started")
+
+    def finish(self, is_success: bool = True, message: str = None):
+        if is_success:
+            result = "Success"
+            self.logger.info(
+                f'JOBLOG - SUCCESS - "{self.job_name}" with GUID {self.log_guid}'
+            )
+        else:
+            result = "Failure"
+            self.logger.info(
+                f'JOBLOG - FAILED - "{self.job_name}" with GUID {self.log_guid}'
+            )
+
+        param_list = [self.log_guid, message, result]
+
+        # Update JobLog with result
+        self.sql_client.execute_stored_procedure("usp_Update_JobLog", param_list)
+
+    def update_status(self, message):
+        param_list = [self.log_guid, message, None]
+        self.sql_client.execute_stored_procedure("usp_Update_JobLog", param_list)
+
+    def update_joblog(sql_client, method, guid, status, logger=None):
+        # If no sql_client supplied quietly ignore
+        if sql_client:
+            assert method in ["Success", "Failure", "Status"]
+
+            if method in ["Success", "Failure"]:
+                result = method
+                param_list = [guid, status, result]
+            elif method == "Status":
+                param_list = [guid, status]
+
+            stored_procedure_name = "usp_Update_JobLog"
+            sql_client.execute_stored_procedure(stored_procedure_name, param_list)
+
+    def insert_message(
+        self,
+        message_name,
+        message_type,
+        value_string: str = None,
+        value_bit: bool = None,
+        value_int: int = None,
+        value_date=None,
+    ):
+        """
+        INSERT INTO tbl_JobLogAttribute
+                (JobLogGUID,AttributeName,AttributeType,AttributeLogType,ValueType,ValueString,ValueBit,ValueInteger,ValueDate)
+
+                sample usage:
+                job_log.insert_message('Job Parameter', 'Input Parameter', 'getActiveMembershipMainContact')
+        """
+        param_list = [
+            self.log_guid,
+            message_name,
+            message_type,
+            None,
+            None,
+            value_string,
+            value_bit,
+            value_int,
+            value_date,
+        ]
+        self.logger.debug(f"JOBLOG - insert attribute param_list: {param_list}")
+        # Update JobLog with result
+        self.sql_client.execute_stored_procedure("usp_LogJobAttribute", param_list)

cit-pydata-0.0.1.dev0/src/cit_pydata/marketo/__init__.py

@@ -0,0 +1 @@
+from .api import MarketoClient