cit-pydata 0.0.1.dev0__tar.gz → 0.0.1.dev2__tar.gz

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cit-pydata
-Version: 0.0.1.dev0
+Version: 0.0.1.dev2
 Summary: Python clients for commonly used api services including Salesforce, SQL, Marketo, Box, SFSync*, and JobLog*
 Author-email: Dionis Wang <dwang@eab.com>
 Project-URL: Homepage, https://github.com/EAB-IT-DEV/cit-pydata

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cit-pydata"
-version = "0.0.1.dev0"
+version = "0.0.1.dev2"
 authors = [
   { name="Dionis Wang", email="dwang@eab.com" },
 ]
@@ -26,11 +26,11 @@ box = ["boxsdk"]
 joblog = ["marketorestpython"]
 salesforce = ["simple_salesforce","pyforce"]
 sftp = ["paramiko"]
-sql = ["SQLAlchemy==1.4.25", "numpy", "pandas"]
-sql-pyodbc = ["SQLAlchemy==1.4.25", "pyodbc", "numpy", "pandas"]
-sql-pymssql = ["SQLAlchemy==1.4.25", "pymssql", "numpy", "pandas"]
-sql-psycopg2 = ["SQLAlchemy==1.4.25", "psycopg2-binary==2.9.6", "numpy", "pandas"]
-sql-mssql = ["SQLAlchemy==1.4.25", "pymssql", "numpy", "pandas"]
-sql-postgres = ["SQLAlchemy==1.4.25", "psycopg2-binary==2.9.6", "numpy", "pandas"]
+sql = ["SQLAlchemy<2.0", "numpy", "pandas"]
+sql-pyodbc = ["SQLAlchemy<2.0", "pyodbc", "numpy", "pandas"]
+sql-pymssql = ["SQLAlchemy<2.0", "pymssql", "numpy", "pandas"]
+sql-psycopg2 = ["SQLAlchemy<2.0", "psycopg2-binary", "numpy", "pandas"]
+sql-mssql = ["SQLAlchemy<2.0", "pymssql", "numpy", "pandas"]
+sql-postgres = ["SQLAlchemy<2.0", "psycopg2-binary", "numpy", "pandas"]
 local = ["python-dotenv", "boto3"]
-all = ["aws-lambda-powertools","pandas","boxsdk","marketorestpython","simple_salesforce","pyforce","paramiko","SQLAlchemy==1.4.25", "pyodbc","pymssql", "numpy", "pandas","psycopg2-binary==2.9.6","python-dotenv", "boto3"]
+all = ["aws-lambda-powertools","pandas","boxsdk","marketorestpython","simple_salesforce","pyforce","paramiko","SQLAlchemy<2.0", "pyodbc","pymssql", "numpy", "pandas","psycopg2-binary","python-dotenv", "boto3"]

cit-pydata-0.0.1.dev2/src/cit_pydata/__init__.py

@@ -0,0 +1,9 @@
+from . import aws
+from . import box
+from . import joblog
+from . import marketo
+from . import salesforce
+from . import sfsync
+from . import sftp
+from . import sql
+from . import util

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata/marketo/api.py

@@ -48,6 +48,8 @@ class MarketoClient:
         ]
 
     def _authenticate(self):
+        from marketorestpython.client import MarketoClient as marketorestpython
+
         if self.marketo is None:
             try:
                 aws_ssm_client = aws_api.SSMClient(
@@ -109,8 +111,6 @@ class MarketoClient:
 
             api_limit = None
             max_retry_time = None
-            if not "marketorestpython" in sys.modules:
-                from marketorestpython.client import MarketoClient as marketorestpython
             self.marketo = marketorestpython(
                 munchkin_id=munchkin_id,
                 client_id=client_id,
@@ -134,6 +134,8 @@ class MarketoClient:
         return response
 
     def execute_api_call(self, method, endpoint, *args, **kwargs):
+        import json
+
         self._authenticate()
         self.marketo.authenticate()
 
@@ -143,9 +145,6 @@ class MarketoClient:
 
         data = kwargs.get("body", None)
 
-        if "json" not in sys.modules:
-            import json
-
         self.logger.info(f"Sending API Call {str(method).upper()}: {endpoint} ")
         self.logger.info(f"\tParams: {json.dumps(api_args)} ")
         # self.logger.debug(f'\tBody: {json.dumps(data)} ')
@@ -168,8 +167,8 @@ class MarketoClient:
         return True, result
 
     def submit_form(self, form_payload: dict):
-        if not "requests" in sys.modules:
-            import requests
+        import requests
+
         self._authenticate()
         self.marketo.authenticate()
         self.logger.debug(self.marketo.token)
@@ -183,10 +182,9 @@ class MarketoClient:
         self.logger.debug(request.text)
 
     def get_program(self, program_id):
-        self._authenticate()
+        import json
 
-        if "json" not in sys.modules:
-            import json
+        self._authenticate()
 
         try:
             lead = self.client.execute(method="get_program_by_id", id=program_id)
@@ -208,12 +206,11 @@ class MarketoClient:
         return result["result"]
 
     def get_lead_fields2(self):
+        import json
+
         self._authenticate()
         lead = self.marketo.execute(method="describe2")
 
-        if "json" not in sys.modules:
-            import json
-
         file_path = "src/core/marketo/data/out/marketo_lead_describe2.json"
         util_api.remove_file(file_path)
         with open(file_path, "w") as json_file:
@@ -226,8 +223,8 @@ class MarketoClient:
         """
         Returns DataFrame of Custom Object records based on filter criteria
         """
-        if "pandas" not in sys.modules:
-            import pandas as pd
+        import pandas as pd
+
         batch_size = 300
         field_list = self.CUSTOM_OBJECT_STANDARD_FIELDS
         for field in custom_field_list:
@@ -346,8 +343,8 @@ class MarketoClient:
         """
         Deletes Marketo custom object records with filter_list and delete_by arguments.
         """
-        if "pandas" not in sys.modules:
-            import pandas as pd
+        import pandas as pd
+
         self.logger.info(
             f"Deleting {len(filter_list)} {custom_object_name} custom object records"
         )
@@ -409,6 +406,7 @@ class MarketoClient:
         In order to delete CO records in Marketo first we get all Marketo leads/contacts that shouldn't have any CO records and we delete those.
         Next for leads/contacts that have at least one CO records, we compare existing CO records to all CO records and filter out good records
         """
+        import pandas as pd
 
         # this dict contains the sql queries for fetching the lead counter ids in order to interrogate
         #   the Marketo custom object records and search for records to delete
@@ -448,8 +446,6 @@ class MarketoClient:
             f"Checking Custom Object {custom_object_name} for records to delete"
         )
 
-        if "pandas" not in sys.modules:
-            import pandas as pd
         # init DataFrame
         co_records_delete_df = pd.DataFrame()
 

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata/salesforce/api.py

@@ -65,6 +65,8 @@ class SalesforceClient:
             'signature': 'zzz'
         }
         """
+        import requests
+
         if self.sf_token is None:
             try:
                 aws_ssm_client = aws_api.SSMClient(
@@ -168,8 +170,6 @@ class SalesforceClient:
                 "username": sf_username,
                 "password": sf_password + sf_security_token,
             }
-            if "requests" not in sys.modules:
-                import requests
             # Get OAuth access token
             with requests.Session() as session:
                 # headers={'Content-Type': 'application/json'}
@@ -204,12 +204,11 @@ class SalesforceClient:
         """
         Creates and returns a simple-salesforce object
         """
+        from simple_salesforce import Salesforce
+
         if self.sf_token is None:
             self._authenticate()
 
-        if "simple_salesforce" not in sys.modules:
-            from simple_salesforce import Salesforce
-
         _instance_url = self.get_instance_url()
         _access_token = self.get_access_token()
 
@@ -262,8 +261,7 @@ class SalesforceClient:
         """Returns DataFrame from Object and Record Id query
         TODO finish dev
         """
-        if "pandas" not in sys.modules:
-            import pandas
+        import pandas
 
         self._get_simple_sf()
 
@@ -326,11 +324,10 @@ class SalesforceClient:
         return None
 
     def get_replicatable_objects(self, is_refresh_from_sf=None):
+        from collections import OrderedDict
+        import json
+
         self._authenticate()
-        if "collections" not in sys.modules:
-            from collections import OrderedDict
-        if "json" not in sys.modules:
-            import json
 
         if is_refresh_from_sf == True:
             # Get All Salesforce Object Metadata
@@ -393,14 +390,12 @@ class SalesforceClient:
         """
         Returns dataframe from SOQL query
         """
-        if "pandas" not in sys.modules:
-            import pandas
+        import pandas
+        import requests
 
         self._authenticate()
 
         self.logger.debug("SOQL Query: {}".format(soql_query))
-        if "requests" not in sys.modules:
-            import requests
         with requests.Session() as session:
             ssf = self._get_simple_sf(session)
             sf_result_dict = ssf.query_all(soql_query)
@@ -414,12 +409,12 @@ class SalesforceClient:
         return df
 
     def update_record(self, object_api_name, record_id, data_dict):
+        import requests
+
         self._authenticate()
         self.logger.info(
             f'Update - {object_api_name}. Id:{record_id}, data_dict: "{data_dict}"'
         )
-        if "requests" not in sys.modules:
-            import requests
         with requests.Session() as session:
             ssf = self._get_simple_sf(session)
             sf_load_object_result = getattr(ssf, object_api_name).update(
@@ -429,9 +424,9 @@ class SalesforceClient:
         return sf_load_object_result
 
     def execute_apex(self, operation, method, payload):
+        import requests
+
         self._authenticate()
-        if "requests" not in sys.modules:
-            import requests
         with requests.Session() as session:
             ssf = self._get_simple_sf(session)
             self.logger.info(f"Executing Apex: {operation} {method}")
@@ -494,8 +489,8 @@ class SalesforceSOAPClient:
             'signature': 'zzz'
         }
         """
-        if "pyforce" not in sys.modules:
-            import pyforce
+        import pyforce
+
         if self.sf_token is None:
             try:
                 aws_ssm_client = aws_api.SSMClient(

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata/sftp/api.py

@@ -38,8 +38,8 @@ class SFTPClient:
     def _connect(self):
         """Method for connecting to SFTP Server
         Called from all loading/downloading methods"""
-        if "paramiko" not in sys.modules:
-            import paramiko
+        import paramiko
+
         try:
             aws_ssm_client = aws_api.SSMClient(
                 environment=self.aws_environment,
@@ -152,8 +152,7 @@ class SFTPClient:
         with an optional archive_suffix in the filename.
         Returns True if the file transfer is successful, False otherwise.
         Default behavior is to automatically close the connection after load"""
-        if "datetime" not in sys.modules:
-            from datetime import datetime
+        from datetime import datetime
 
         if self.sftp_session is None:
             connected = self._connect()
@@ -247,6 +246,9 @@ class SFTPClient:
         to the archive folder with an optional archive_suffix in the filename.
         Returns True if the file download is successful, False otherwise.
         Default behavior is to automatically close the connection after download."""
+        import shutil
+        from datetime import datetime
+
         if self.sftp_session is None:
             connected = self._connect()
             if not connected:
@@ -307,10 +309,6 @@ class SFTPClient:
                             )
                             return True
                         else:
-                            if "shutil" not in sys.modules:
-                                import shutil
-                            if "datetime" not in sys.modules:
-                                from datetime import datetime
                             # Archive the existing file
                             archive_suffix = archive_suffix or datetime.now().strftime(
                                 "%Y%m%d"

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata/sql/api.py

@@ -55,10 +55,8 @@ class SQLClient:
 
         This method will use the AWS IAM credentials defined in the local .env file to get the AWS SSM parameters defining SQL authentication details
         """
-        if "re" not in sys.modules:
-            import re
-        if "urllib.parse" not in sys.modules:
-            from urllib.parse import quote
+        import re
+        from urllib.parse import quote
 
         try:
             _aws_environment = None
@@ -263,8 +261,8 @@ class SQLClient:
     def execute_stored_procedure(
         self, stored_procedure_name, param_list, skip_transaction: bool = False
     ):
-        if "re" not in sys.modules:
-            import re
+        import re
+
         _param_markers_list = ",".join(["?" for param in param_list])
 
         if self.dialect == "pymssql":
@@ -346,8 +344,8 @@ class SQLClient:
         return True
 
     def __split_df(self, df, chunksize):
-        if "math" not in sys.modules:
-            from math import ceil
+        from math import ceil
+
         chunk_count = int(ceil(df.size / chunksize))
         return np.array_split(df, chunk_count)
 
@@ -452,8 +450,7 @@ class SQLClient:
         return dataframe
 
     def get_dataframe_table(self, table_name, chunk_count=None, **kwargs):
-        if "itertools" not in sys.modules:
-            from itertools import islice
+        from itertools import islice
 
         s = time.time()
         if "chunksize" not in kwargs.keys():
@@ -666,11 +663,8 @@ class SQLClient:
             data_iter : Iterable that iterates the values to be inserted
         """
         # gets a DBAPI connection that can provide a cursor
-        if "csv" not in sys.modules:
-            import csv
-
-        if "io" not in sys.modules:
-            from io import StringIO
+        import csv
+        from io import StringIO
 
         dbapi_conn = conn.connection
         with dbapi_conn.cursor() as cur:
@@ -697,8 +691,8 @@ class SQLClient:
 
     #### HAS NOT BEEN TESTED YET  MAY NOT WORK####
     def postgres_quickexport_df(self, table_name):
-        if "io" not in sys.modules:
-            from io import StringIO
+        from io import StringIO
+
         if self.dialect != "psycopg2":
             self.logger.error(f"unsupported dialect for upsert {self.dialect}")
         dbapi_conn = self._get_sql_engine.connect()

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata/util/api.py

@@ -11,9 +11,10 @@ def encode_base64(value: str):
     """
     Returns base64 encoded value for the input string parameter
     """
+    import base64
+
     binary_value = value.encode("ascii")
-    if not "base64" in sys.modules:
-        import base64
+
     return base64.b64encode(binary_value)
 
 
@@ -31,12 +32,12 @@ def multireplace(string, replacements, ignore_case=False):
     """
     # Source: https://gist.github.com/bgusach/a967e0587d6e01e889fd1d776c5f3729
 
+    import re
+
     if not replacements:
         # Edge case that'd produce a funny regex and cause a KeyError
         return string
 
-    if not "re" in sys.modules:
-        import re
     # If case insensitive, we need to normalize the old string so that later a replacement
     # can be found. For instance with {"HEY": "lol"} we should match and find a replacement for "hey",
     # "HEY", "hEy", etc.
@@ -76,6 +77,8 @@ def list_dict_to_csv(file_path, file_name, data_list, column_list=None):
     dictionary should be column_name:value format
 
     """
+    import csv
+
     if type(data_list) != list:
         return False
 
@@ -93,8 +96,6 @@ def list_dict_to_csv(file_path, file_name, data_list, column_list=None):
     if not os.path.exists(file_path):
         os.makedirs(file_path)
 
-    if "csv" not in sys.modules:
-        import csv
     with open(full_file_path, "w+", newline="") as csv_file:
         csv_writer = csv.DictWriter(csv_file, fieldnames=column_list)
         csv_writer.writeheader()
@@ -104,9 +105,9 @@ def list_dict_to_csv(file_path, file_name, data_list, column_list=None):
 
 
 def json_to_file(json_object, file_path):
+    import json
+
     remove_file(file_path)
-    if "json" not in sys.modules:
-        import json
     with open(file_path, "w") as json_file:
         json.dump(json_object, json_file)
 
@@ -130,8 +131,7 @@ def get_environment_variable(
         )
         return None
 
-    if not "dotenv" in sys.modules:
-        from dotenv import dotenv_values
+    from dotenv import dotenv_values
 
     _path_to_env_file = None
 
@@ -195,6 +195,7 @@ def get_logger(
     """
     if not ON_AWS and log_type in ("console", "file"):
         import logging
+        import re
 
         logger = logging.getLogger(logger_name)
 
@@ -266,8 +267,6 @@ def get_logger(
                     and type(cleanup_policy) == dict
                     and "retain_last_n" in cleanup_policy.keys()
                 ):
-                    if not "re" in sys.modules:
-                        import re
                     log_file_regex = re.compile(f"^{base_file_name}.*\.log$")
                     # try catch to prevent failure from affecting actual job
                     try:
@@ -324,8 +323,8 @@ def cleanup_logs(
     Returns:
     number of files that were attempted to be deleted: int
     """
-    if "re" not in sys.modules:
-        import re
+    import re
+
     remove_counter = 0
 
     # determine cutoff_time
@@ -359,7 +358,7 @@ def get_log_level(path_to_env=PATH_TO_ENV):
             path_to_env_file=path_to_env, variable_name="log_level"
         )
     elif ON_AWS:
-        _log_level = os.getenv("POWERTOOLS_LOG_LEVEL")
+        _log_level = os.getenv("POWERTOOLS_LOG_LEVEL", _log_level)
     return _log_level
 
 
@@ -368,10 +367,9 @@ def find_text_in_files(root_path, text_regex):
     Searches .py files line by  line matching on regex pattern
     Files are recursively searched starting in root_path
     """
-    if not "re" in sys.modules:
-        import re
-    if not "pprint" in sys.modules:
-        import pprint
+    import re
+    import pprint
+
     os.listdir()
     file_list = []
     for dirpath, dirnames, filenames in os.walk(root_path):
@@ -408,10 +406,9 @@ def find_text_in_files(root_path, text_regex):
 
 def get_core_conn(folder_path) -> dict:
     """Retrieves the connection configuration to the core systems"""
+    import json
 
     file_path = os.path.join(folder_path, "core_conn.json")
-    if "json" not in sys.modules:
-        import json
     with open(file_path, "r") as f:
         conn_dict = json.load(f)
 

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cit-pydata
-Version: 0.0.1.dev0
+Version: 0.0.1.dev2
 Summary: Python clients for commonly used api services including Salesforce, SQL, Marketo, Box, SFSync*, and JobLog*
 Author-email: Dionis Wang <dwang@eab.com>
 Project-URL: Homepage, https://github.com/EAB-IT-DEV/cit-pydata

{cit-pydata-0.0.1.dev0 → cit-pydata-0.0.1.dev2}/src/cit_pydata.egg-info/requires.txt

@@ -7,11 +7,11 @@ marketorestpython
 simple_salesforce
 pyforce
 paramiko
-SQLAlchemy==1.4.25
+SQLAlchemy<2.0
 pyodbc
 pymssql
 numpy
-psycopg2-binary==2.9.6
+psycopg2-binary
 python-dotenv
 boto3
 
@@ -36,36 +36,36 @@ pyforce
 paramiko
 
 [sql]
-SQLAlchemy==1.4.25
+SQLAlchemy<2.0
 numpy
 pandas
 
 [sql-mssql]
-SQLAlchemy==1.4.25
+SQLAlchemy<2.0
 pymssql
 numpy
 pandas
 
 [sql-postgres]
-SQLAlchemy==1.4.25
-psycopg2-binary==2.9.6
+SQLAlchemy<2.0
+psycopg2-binary
 numpy
 pandas
 
 [sql-psycopg2]
-SQLAlchemy==1.4.25
-psycopg2-binary==2.9.6
+SQLAlchemy<2.0
+psycopg2-binary
 numpy
 pandas
 
 [sql-pymssql]
-SQLAlchemy==1.4.25
+SQLAlchemy<2.0
 pymssql
 numpy
 pandas
 
 [sql-pyodbc]
-SQLAlchemy==1.4.25
+SQLAlchemy<2.0
 pyodbc
 numpy
 pandas

cit-pydata-0.0.1.dev0/src/cit_pydata/__init__.py

@@ -1,9 +0,0 @@
-import aws
-import box
-import joblog
-import marketo
-import salesforce
-import sfsync
-import sftp
-import sql
-import util