berryworld 1.0.0.188349__py3-none-any.whl → 1.0.0.189012__py3-none-any.whl

berryworld/snowflake_conn.py

@@ -29,7 +29,7 @@ class SnowflakeConn:
             except KeyError:
                 raise KeyError('Please provide a valid db_name and server_type')
 
-        self.db = kwargs['db_name']
+        self.db = server_creds['db_name']
         self.user = server_creds['user_name']
         self.pw = server_creds['password']
         self.account = server_creds['account']

berryworld/sql_conn.py

@@ -243,7 +243,8 @@ class SQLConn:
             self.close_connection()
 
     def insert(self, data, schema, table, truncate=False, delete=False, identity=False, chunk=1000, print_sql=False,
-               commit_all_together=False, output=None, bools2bits=True, nullable=False, commit_as_transaction=True):
+               commit_all_together=False, output=None, bools2bits=True, nullable=False, commit_as_transaction=True,
+               infer_datetime_format=None):
         """ Insert data in a table in SQL truncating the table if needed
         -----------------------------
         df = pd.DataFrame({'col1': ['a', 'b'], 'col2': [1, 2]})
@@ -265,6 +266,8 @@ class SQLConn:
         :return: A DataFrame with the output columns requested if output is not None, else None
         :param nullable: Used within bools2bits function to indicate which boolean column values to convert
         :param commit_as_transaction: Indicate whether the connection will be done using the autocommit option or not
+        :param infer_datetime_format: Indicate whether the datetime columns should be converted to string and if so,
+        then the format to be used
         """
         if output is None:
             output = []
@@ -278,8 +281,9 @@ class SQLConn:
         # Mapping the date datatype columns for SQL
         data = self.date_mapping_data_types(data)
 
-        # # Infer integer datatype columns for SQL
-        # data = self.int_mapping_data_types(data)
+        # Infer datetime format if provided
+        if infer_datetime_format is not None:
+            data = self.infer_datetime(data, infer_datetime_format)
 
         # Mapping the boolean columns to bit
         if bools2bits:
@@ -376,9 +380,6 @@ class SQLConn:
         # Mapping the date datatype columns for SQL
         data = self.date_mapping_data_types(data)
 
-        # # Infer integer datatype columns for SQL
-        # data = self.int_mapping_data_types(data)
-
         # Mapping the boolean columns to bit
         if bools2bits:
             data = self.boolean_mapping_data_types(data, nullable)
@@ -490,9 +491,6 @@ class SQLConn:
         # Mapping date type for SQL
         data = self.date_mapping_data_types(data)
 
-        # # Infer integer datatype columns for SQL
-        # data = self.int_mapping_data_types(data)
-
         # create connection
         self.open_write_connection(commit_as_transaction)
 
@@ -632,9 +630,6 @@ class SQLConn:
         data = data[on_list + update_list]
         data = self.date_mapping_data_types(data)
 
-        # # Infer integer datatype columns for SQL
-        # data = self.int_mapping_data_types(data)
-
         # create connection
         self.open_write_connection(commit_as_transaction)
 
@@ -934,30 +929,6 @@ class SQLConn:
 
         return data
 
-    @staticmethod
-    def int_mapping_data_types(data):
-        """
-        Map integer variables so they can be inserted as integer to SQL
-        :param data: DataFrame containing the variables to map
-        :return: The mapped DataFrame
-        """
-        thres_ = 0.9
-        for col in data.columns:
-            try:
-                if (((~(data[col].isnull())) & (data[col].str.contains('.0', regex=False))).sum() / (
-                        ~data[col].isnull()).sum() > thres_) & (~data[col].str.contains('{', regex=False)):
-                    # Avoid converting datetime values
-                    try:
-                        pd.to_datetime(data[col])
-                    except Exception as e:
-                        print(e)
-                        data[col] = data[col].str.replace('.0', '')
-            except Exception as e:
-                print(e)
-                continue
-
-        return data
-
     @staticmethod
     def boolean_mapping_data_types(data, nullable=False):
         """
@@ -1008,3 +979,14 @@ class SQLConn:
         """
         string = re.sub("'\)(?!(,[ ]+\())(?=([^$]))", "", string)
         return re.sub("Decimal\('", "", string)
+
+    @staticmethod
+    def infer_datetime(data, infer_datetime_format):
+        """ Method to infer datetime columns and format them as string
+        :param data: DataFrame to parse
+        :param infer_datetime_format: format to be used for the datetime columns
+        """
+        for col in data.select_dtypes(include=['datetime64']).columns:
+            data[col] = pd.to_datetime(data[col]).dt.strftime(infer_datetime_format)
+
+        return data

{berryworld-1.0.0.188349.dist-info → berryworld-1.0.0.189012.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: berryworld
-Version: 1.0.0.188349
+Version: 1.0.0.189012
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.188349.dist-info → berryworld-1.0.0.189012.dist-info}/RECORD

@@ -17,20 +17,20 @@ berryworld/persistent_storage.py,sha256=KQA57ez8eVTUCtudYkHPg_S5lcOEa_E7xXcaN1DY
 berryworld/pickle_management.py,sha256=5o6UuXBpTj23Jgpz6sj9V-vdcdWBK1xMEckWxT-Whj4,2436
 berryworld/power_automate.py,sha256=Y11GoeDEwd3Y2RdvtPPhBSFK65APEceAQKtNo4gVLK4,26464
 berryworld/sharepoint_con.py,sha256=TuH-Vxk1VxjTi7x80KFssf_J8YPLRXpV27RBaFZi37U,22254
-berryworld/snowflake_conn.py,sha256=52ugfkyVCbR_Xr8YiZtNwPrwe93_1uE9uRy_VNPT6Gs,2428
-berryworld/sql_conn.py,sha256=3_uPrn3rJjVr7-9wcXkh0CSitFD2ncQGGZjttFl6ELM,47803
+berryworld/snowflake_conn.py,sha256=go5ZJjnhz5SkG83B0G0XZSwKgU6tg7AFTBso59oRG5M,2434
+berryworld/sql_conn.py,sha256=r2rAVRc3Mes0m6_62J4uFg5GLBNMr7Vkme9Ip6jT3aA,47283
 berryworld/teams_logging.py,sha256=8NwXyWr4fLj7W6GzAm2nRQCGFDxibQpAHDHHD24FrP8,6997
 berryworld/transportation_solver.py,sha256=tNc1JJk71azIBccdWVHbqcvXWhalOdKffv6HmBD6tG0,5014
 berryworld/verify_keys.py,sha256=X4Nuz3o0XbRDYofbJGvxIDeN5gfWj19PN7lhO6T3hR8,4356
 berryworld/vivantio.py,sha256=QfZo0UKqkzVRg_LyiwivNd3aEup4TH57x4KxLZkCJwc,10627
 berryworld/vivantio_logging.py,sha256=ciy7gA4u3FrgUIpEBnMgocbNPp6jcu9TPoy-kLcrTZU,5736
 berryworld/xml_parser.py,sha256=HWD71NaTN3DaIOGT6Wzxs4CEsroFhGQwe9iPLIL80Co,957
-berryworld-1.0.0.188349.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.189012.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.188349.dist-info/METADATA,sha256=IQNILK3na6R8NKy5LsS7k4mRTciBjh9mJjsdlVI-Sv0,1362
-berryworld-1.0.0.188349.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-berryworld-1.0.0.188349.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.188349.dist-info/RECORD,,
+berryworld-1.0.0.189012.dist-info/METADATA,sha256=G_C7noVeA6eQTLZN_aevjCibBaGJ0dj3dk6mawRqrlY,1362
+berryworld-1.0.0.189012.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+berryworld-1.0.0.189012.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.189012.dist-info/RECORD,,

{berryworld-1.0.0.188349.dist-info → berryworld-1.0.0.189012.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any