fabricks 3.0.17__tar.gz → 3.0.18__tar.gz

{fabricks-3.0.17 → fabricks-3.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.17
+Version: 3.0.18
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.17 → fabricks-3.0.18}/fabricks/context/_types.py

@@ -16,6 +16,9 @@ class RuntimeTimeoutOptions(TypedDict):
     pre_run: int
     post_run: int
 
+class UDFOptions(TypedDict):
+    prefix: Optional[str]
+    schema: Optional[str]
 
 class StepTimeoutOptions(TypedDict):
     step: Optional[int]
@@ -123,6 +126,7 @@ class Conf(TypedDict):
     options: RuntimeOptions
     path_options: RuntimePathOptions
     extender_options: Optional[ExtenderOptions]
+    udf_options: Optional[UDFOptions]
     spark_options: SparkOptions
     bronze: Optional[List[Bronze]]
     silver: Optional[List[Silver]]

{fabricks-3.0.17 → fabricks-3.0.18}/fabricks/core/jobs/gold.py

@@ -10,7 +10,7 @@ from fabricks.cdc.nocdc import NoCDC
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core.jobs.base._types import JobDependency, TGold
 from fabricks.core.jobs.base.job import BaseJob
-from fabricks.core.udfs import is_registered, register_udf
+from fabricks.core.udfs import is_registered, register_udf, udf_prefix
 from fabricks.metastore.view import create_or_replace_global_temp_view
 from fabricks.utils.path import Path
 from fabricks.utils.sqlglot import fix, get_tables
@@ -90,8 +90,8 @@ class Gold(BaseJob):
 
         else:
             matches = []
-            if "udf_" in self.sql:
-                r = re.compile(r"(?<=udf_)\w*(?=\()")
+            if f"{udf_prefix}" in self.sql:
+                r = re.compile(rf"(?<={udf_prefix})\w*(?=\()")
                 matches = re.findall(r, self.sql)
                 matches = set(matches)
                 matches = list(matches)

{fabricks-3.0.17 → fabricks-3.0.18}/fabricks/core/udfs.py

@@ -5,11 +5,13 @@ from typing import Callable, List, Optional
 
 from pyspark.sql import SparkSession
 
-from fabricks.context import CATALOG, IS_UNITY_CATALOG, PATH_UDFS, SPARK
+from fabricks.context import CATALOG, IS_UNITY_CATALOG, PATH_UDFS, SPARK, CONF_RUNTIME
 from fabricks.context.log import DEFAULT_LOGGER
 
 UDFS: dict[str, Callable] = {}
 
+udf_schema = CONF_RUNTIME.get("udf_options", {}).get("schema", "default")
+udf_prefix = CONF_RUNTIME.get("udf_options", {}).get("prefix", "udf_")
 
 def register_all_udfs(extension: Optional[str] = None, override: bool = False):
     """
@@ -47,12 +49,12 @@ def is_registered(udf: str, spark: Optional[SparkSession] = None) -> bool:
         spark = SPARK
     assert spark is not None
 
-    df = spark.sql("show user functions in default")
+    df = spark.sql(f"show user functions in {udf_schema}")
 
     if CATALOG:
-        df = df.where(f"function == '{CATALOG}.default.udf_{udf}'")
+        df = df.where(f"function == '{CATALOG}.{udf_schema}.{udf_prefix}{udf}'")
     else:
-        df = df.where(f"function == 'spark_catalog.default.udf_{udf}'")
+        df = df.where(f"function == 'spark_catalog.{udf_schema}.{udf_prefix}{udf}'")
 
     return not df.isEmpty()
 
@@ -72,9 +74,9 @@ def register_udf(
 
     if not is_registered(udf, spark) or override:
         if override:
-            DEFAULT_LOGGER.debug(f"override udf {udf}")
+            DEFAULT_LOGGER.debug(f"override udf {udf}", extra={"label": "fabricks"})
         else:
-            DEFAULT_LOGGER.debug(f"register udf {udf}")
+            DEFAULT_LOGGER.debug(f"register udf {udf}", extra={"label": "fabricks"})
 
         if extension is None:
             extension = get_extension(udf)
@@ -89,8 +91,6 @@ def register_udf(
         elif extension == "py":
             if not IS_UNITY_CATALOG:
                 assert path.exists(), f"udf not found ({path.string})"
-            else:
-                DEFAULT_LOGGER.debug(f"could not check if udf exists ({path.string})")
 
             spec = importlib.util.spec_from_file_location(udf, path.string)
             assert spec, f"no valid udf found ({path.string})"

{fabricks-3.0.17 → fabricks-3.0.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.17"
+version = "3.0.18"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.17 → fabricks-3.0.18}/uv.lock

@@ -645,7 +645,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.17"
+version = "3.0.18"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },