databricks-modules-vy 1.2__tar.gz → 1.2.2__tar.gz

{databricks_modules_vy-1.2 → databricks_modules_vy-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks_modules_vy
-Version: 1.2
+Version: 1.2.2
 Summary: Databricks modules for Vy
 Author-email: Sindre Kjetså <sindre.kjetsa@vy.no>
 Project-URL: Homepage, https://github.com/nsbno/

{databricks_modules_vy-1.2 → databricks_modules_vy-1.2.2}/databricks_modules_vy/sharepoint.py

@@ -283,6 +283,27 @@ def df_from_sharepoint_csv_utf8(
     return df
 
 
+def df_from_sharepoint_excel(item_path, site_name="VyMasterdata", **kwargs) -> DataFrame:
+    """Gets the bytes from an excel file in sharepoint and converts it to a spark dataframe.
+
+    Requires the 'openpyxl' package, install it with '%pip install openpyxl'.
+
+    Args:
+        item_path (str) : path relative to the site the file is located. Initiate the path with '/'
+        site_name (str) : name of site to fetch the file from
+        **kwargs         : additional keyword arguments passed directly to pd.read_excel
+    Returns:
+        (spark.DataFrame) dataframe of the excel sheet
+    """
+    file_bytesio = _get_file_bytes_from_sharepoint(item_path, site_name)
+    dp = pd.read_excel(io=file_bytesio, **kwargs)
+    dp = dp.replace([np.nan], [None])
+
+    df = spark.createDataFrame(dp)
+
+    return df
+
+
 # # Functions for traversing and extracting files and folders from a site # #
 def get_files_in_folder(relative_path_to_folder, site_name="VyMasterdata"):
     """

{databricks_modules_vy-1.2 → databricks_modules_vy-1.2.2}/databricks_modules_vy/unity_catalog.py

@@ -9,6 +9,24 @@ import re
 spark = SparkSession.builder.appName("module").getOrCreate()
 
 
+def _sql_string_literal(value: object) -> str:
+    """
+    Converts a Python value into a safe SQL string literal.
+
+    Escapes single quotes and removes null bytes that can break SQL parsing.
+    """
+    text = "" if value is None else str(value)
+    text = text.replace("\x00", "").replace("'", "''")
+    return f"'{text}'"
+
+
+def _quote_identifier(identifier: str) -> str:
+    """
+    Quotes a SQL identifier using backticks and escapes embedded backticks.
+    """
+    return f"`{identifier.replace('`', '``')}`"
+
+
 def write_to_uc(
     df: DataFrame,
     target: str,
@@ -81,7 +99,9 @@ def write_to_uc(
 
         if table_description:
             lp("Commenting table ...")
-            spark.sql(f"COMMENT ON TABLE {target} IS '{table_description}'")
+            spark.sql(
+                f"COMMENT ON TABLE {target} IS {_sql_string_literal(table_description)}"
+            )
 
         if comments:
             if autoformat_columns:
@@ -207,7 +227,9 @@ def upsert_to_uc(
 
         if table_description:
             lp("Commenting table ...")
-            spark.sql(f"COMMENT ON TABLE {target} IS '{table_description}'")
+            spark.sql(
+                f"COMMENT ON TABLE {target} IS {_sql_string_literal(table_description)}"
+            )
 
         if comments:
             if autoformat_columns:
@@ -308,7 +330,9 @@ def autoloader_batch_to_uc(
 
         if table_description:
             lp("Commenting table ...")
-            spark.sql(f"COMMENT ON TABLE {target} IS '{table_description}'")
+            spark.sql(
+                f"COMMENT ON TABLE {target} IS {_sql_string_literal(table_description)}"
+            )
 
         if comments:
             if autoformat_columns:
@@ -455,12 +479,16 @@ def _apply_comments(
     if overwrite_schema or not current_comments:
         # If schema is overwritten or there are no comments currently, reapply all comments
         for col, comment in comments.items():
-            spark.sql(f"ALTER TABLE {target} CHANGE {col} COMMENT '{comment}'")
+            spark.sql(
+                f"ALTER TABLE {target} CHANGE {_quote_identifier(col)} COMMENT {_sql_string_literal(comment)}"
+            )
     else:
         # If schema is not overwritten, only apply comments that have changed or are new
         for col, comment in comments.items():
             if col not in current_comments or comment != current_comments[col]:
-                spark.sql(f"ALTER TABLE {target} CHANGE {col} COMMENT '{comment}'")
+                spark.sql(
+                    f"ALTER TABLE {target} CHANGE {_quote_identifier(col)} COMMENT {_sql_string_literal(comment)}"
+                )
 
 
 def _format_column_name(column_name: str) -> str:

{databricks_modules_vy-1.2 → databricks_modules_vy-1.2.2}/databricks_modules_vy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks_modules_vy
-Version: 1.2
+Version: 1.2.2
 Summary: Databricks modules for Vy
 Author-email: Sindre Kjetså <sindre.kjetsa@vy.no>
 Project-URL: Homepage, https://github.com/nsbno/

{databricks_modules_vy-1.2 → databricks_modules_vy-1.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "databricks_modules_vy"
-version = "1.2"
+version = "1.2.2"
 authors = [
   { name="Sindre Kjetså", email="sindre.kjetsa@vy.no" },
 ]