vavacars-data-utils 1.3.4__tar.gz → 1.3.6__tar.gz

{vavacars_data_utils-1.3.4 → vavacars_data_utils-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vavacars_data_utils
-Version: 1.3.4
+Version: 1.3.6
 Summary: Package with utils
 Home-page: https://dev.azure.com/vavacars/DataScience/_git/Data.Utils
 Author: Vavacars Data Science Team
@@ -13,13 +13,19 @@ Description-Content-Type: text/markdown
 Requires-Dist: pandas
 Requires-Dist: PyYAML>=6.0
 Requires-Dist: requests
-Requires-Dist: sqlalchemy<2.0
-Requires-Dist: redis
-Requires-Dist: msal
-Requires-Dist: pyodbc
 Requires-Dist: unidecode
-Requires-Dist: azure-communication-email
-Requires-Dist: psycopg2-binary
+Provides-Extra: bigquery
+Requires-Dist: google-cloud-bigquery; extra == "bigquery"
+Requires-Dist: google-api-core; extra == "bigquery"
+Provides-Extra: sql
+Requires-Dist: sqlalchemy<2.0; extra == "sql"
+Requires-Dist: psycopg2-binary; extra == "sql"
+Requires-Dist: pyodbc; extra == "sql"
+Requires-Dist: msal; extra == "sql"
+Provides-Extra: emai
+Requires-Dist: azure-communication-email; extra == "emai"
+Provides-Extra: azure
+Requires-Dist: azureml-core; extra == "azure"
 
 # Vavacars
 

{vavacars_data_utils-1.3.4 → vavacars_data_utils-1.3.6}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vavacars_data_utils
-version = 1.3.4
+version = 1.3.6
 author = Vavacars Data Science Team
 author_email = acg@vava.cars
 description = Package with utils
@@ -22,13 +22,7 @@ install_requires =
 	pandas
 	PyYAML>=6.0
 	requests
-	sqlalchemy<2.0
-	redis
-	msal
-	pyodbc
 	unidecode
-	azure-communication-email
-	psycopg2-binary
 
 [options.packages.find]
 where = .
@@ -37,6 +31,20 @@ exclude =
 	README.md
 	notebooks
 
+[options.extras_require]
+bigquery = 
+	google-cloud-bigquery
+	google-api-core
+sql = 
+	sqlalchemy<2.0
+	psycopg2-binary
+	pyodbc
+	msal
+emai = 
+	azure-communication-email
+azure = 
+	azureml-core
+
 [egg_info]
 tag_build = 
 tag_date = 0

{vavacars_data_utils-1.3.4 → vavacars_data_utils-1.3.6}/vava_utils/sql/sql_helper.py

@@ -26,9 +26,7 @@ def get_postgresql_helper(host, user, port, database=None, password=None, **kwar
     engine_url = URL.create(
         drivername="postgresql+psycopg2", host=host, username=user, database=database, password=password, port=port
     )
-    engine = create_engine(
-        engine_url, pool_pre_ping=True, **kwargs
-    )
+    engine = create_engine(engine_url, pool_pre_ping=True, **kwargs)
 
     return SQL_Helper(engine, engine_type=POSTGRESQL)
 
@@ -162,19 +160,29 @@ class SQL_Helper:
         """
         return pd.read_sql_query(text(query), self._engine, **kwargs)
 
-    def write_df(self, df: pd.DataFrame, table_name: str, chunksize: int = 1000, index: bool = False, **kwargs):
+    def write_df(
+        self, df: pd.DataFrame, table_name: str, chunksize: int = 1000, index: bool = False, if_exists="fail", **kwargs
+    ):
         """
         Store Pandas Dataframe into SQL table
 
         Args:
             df (pd.DataFrame): data to write
             table_name (str): output database table name
+            if_exists (str): action to take if table already exists
+                * fail,
+                * replace
+                * append
+                * truncate: NEW, truncate table before writing, like 'replace' but keeps structure
             **kwargs: additional keyword parameters passed to df.to_sql
         """
         with self._engine.connect() as conn:
             trans = conn.begin()
             try:
-                df.to_sql(table_name.lower(), conn, chunksize=chunksize, index=index, **kwargs)
+                if if_exists == "truncate":
+                    conn.execute(f"TRUNCATE TABLE {table_name}")
+                    if_exists = "append"
+                df.to_sql(table_name.lower(), conn, chunksize=chunksize, index=index, if_exists=if_exists, **kwargs)
                 trans.commit()
             except Exception as ex:
                 LOGGER.warning(str(ex))
@@ -213,8 +221,8 @@ class SQL_Helper:
                     WHERE {not_exists_part_cols}
                 );
             """
-            col_names = [f'[{x}]' for x in df.columns]
-            upsert_cols = [f'[{x}]' for x in upsert_cols]
+            col_names = [f"[{x}]" for x in df.columns]
+            upsert_cols = [f"[{x}]" for x in upsert_cols]
 
         elif self.engine_type == MYSQL:
             tmp_table_name = f"{table_name}_tmp_{random.randrange(0, 1000)}"
@@ -237,8 +245,8 @@ class SQL_Helper:
                     WHERE {not_exists_part_cols}
                 );
             """
-            col_names = [f'`{x}`' for x in df.columns]
-            upsert_cols = [f'`{x}`' for x in upsert_cols]
+            col_names = [f"`{x}`" for x in df.columns]
+            upsert_cols = [f"`{x}`" for x in upsert_cols]
 
         elif self.engine_type == POSTGRESQL:
             tmp_table_name = f"{table_name}_tmp_{random.randrange(0, 1000)}"
@@ -261,8 +269,8 @@ class SQL_Helper:
                     WHERE {not_exists_part_cols}
                 );
             """
-            col_names = [f'`{x}`' for x in df.columns]
-            upsert_cols = [f'`{x}`' for x in upsert_cols]
+            col_names = [f"`{x}`" for x in df.columns]
+            upsert_cols = [f"`{x}`" for x in upsert_cols]
 
         else:
             raise NotImplementedError(f"Method not implemented for {self.engine_type} engine")
@@ -309,10 +317,10 @@ class SQL_Helper:
                         not_exists_part_cols=not_exists_part_cols,
                     )
                     conn.execute(text(insert_sql))
-                    
+
                     # DROP tmp table
                     conn.execute(text(f"DROP TABLE {tmp_table_name};"))
-                    
+
                     trans.commit()
                 except Exception as ex:
                     LOGGER.warning(str(ex))
@@ -389,6 +397,5 @@ class SQL_Helper:
 
         return "UPDATE" if exists else "CREATE"
 
-
     def execute(self, query: str) -> LegacyCursorResult:
-        return self._engine.execute(query)
+        return self._engine.execute(query)

{vavacars_data_utils-1.3.4 → vavacars_data_utils-1.3.6}/vavacars_data_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vavacars_data_utils
-Version: 1.3.4
+Version: 1.3.6
 Summary: Package with utils
 Home-page: https://dev.azure.com/vavacars/DataScience/_git/Data.Utils
 Author: Vavacars Data Science Team
@@ -13,13 +13,19 @@ Description-Content-Type: text/markdown
 Requires-Dist: pandas
 Requires-Dist: PyYAML>=6.0
 Requires-Dist: requests
-Requires-Dist: sqlalchemy<2.0
-Requires-Dist: redis
-Requires-Dist: msal
-Requires-Dist: pyodbc
 Requires-Dist: unidecode
-Requires-Dist: azure-communication-email
-Requires-Dist: psycopg2-binary
+Provides-Extra: bigquery
+Requires-Dist: google-cloud-bigquery; extra == "bigquery"
+Requires-Dist: google-api-core; extra == "bigquery"
+Provides-Extra: sql
+Requires-Dist: sqlalchemy<2.0; extra == "sql"
+Requires-Dist: psycopg2-binary; extra == "sql"
+Requires-Dist: pyodbc; extra == "sql"
+Requires-Dist: msal; extra == "sql"
+Provides-Extra: emai
+Requires-Dist: azure-communication-email; extra == "emai"
+Provides-Extra: azure
+Requires-Dist: azureml-core; extra == "azure"
 
 # Vavacars
 

{vavacars_data_utils-1.3.4 → vavacars_data_utils-1.3.6}/vavacars_data_utils.egg-info/requires.txt

@@ -1,10 +1,20 @@
 pandas
 PyYAML>=6.0
 requests
-sqlalchemy<2.0
-redis
-msal
-pyodbc
 unidecode
+
+[azure]
+azureml-core
+
+[bigquery]
+google-cloud-bigquery
+google-api-core
+
+[emai]
 azure-communication-email
+
+[sql]
+sqlalchemy<2.0
 psycopg2-binary
+pyodbc
+msal