fabricks 3.0.17__py3-none-any.whl → 3.0.19__py3-none-any.whl

fabricks/context/_types.py

@@ -17,6 +17,11 @@ class RuntimeTimeoutOptions(TypedDict):
     post_run: int
 
 
+class UDFOptions(TypedDict):
+    prefix: Optional[str]
+    schema: Optional[str]
+
+
 class StepTimeoutOptions(TypedDict):
     step: Optional[int]
     job: Optional[int]
@@ -123,6 +128,7 @@ class Conf(TypedDict):
     options: RuntimeOptions
     path_options: RuntimePathOptions
     extender_options: Optional[ExtenderOptions]
+    udf_options: Optional[UDFOptions]
     spark_options: SparkOptions
     bronze: Optional[List[Bronze]]
     silver: Optional[List[Silver]]

fabricks/core/jobs/gold.py

@@ -10,7 +10,7 @@ from fabricks.cdc.nocdc import NoCDC
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core.jobs.base._types import JobDependency, TGold
 from fabricks.core.jobs.base.job import BaseJob
-from fabricks.core.udfs import is_registered, register_udf
+from fabricks.core.udfs import is_registered, register_udf, udf_prefix
 from fabricks.metastore.view import create_or_replace_global_temp_view
 from fabricks.utils.path import Path
 from fabricks.utils.sqlglot import fix, get_tables
@@ -90,8 +90,8 @@ class Gold(BaseJob):
 
         else:
             matches = []
-            if "udf_" in self.sql:
-                r = re.compile(r"(?<=udf_)\w*(?=\()")
+            if f"{udf_prefix}" in self.sql:
+                r = re.compile(rf"(?<={udf_prefix})\w*(?=\()")
                 matches = re.findall(r, self.sql)
                 matches = set(matches)
                 matches = list(matches)
@@ -126,7 +126,18 @@ class Gold(BaseJob):
             invokers = self.options.invokers.get_list("run")
             assert len(invokers) <= 1, "at most one invoker allowed when notebook is true"
 
-            global_temp_view = self.invoke(path=self.paths.runtime, schema_only=schema_only, **kwargs)
+            path = None
+            if invokers:
+                notebook = invokers[0].get("notebook")
+                if notebook:
+                    from fabricks.context import PATH_RUNTIME
+
+                    path = PATH_RUNTIME.joinpath(notebook)
+
+            if path is None:
+                path = self.paths.runtime
+
+            global_temp_view = self.invoke(path=path, schema_only=schema_only, **kwargs)
             assert global_temp_view is not None, "global_temp_view not found"
 
             df = self.spark.sql(f"select * from global_temp.{global_temp_view}")

fabricks/core/udfs.py

@@ -5,11 +5,14 @@ from typing import Callable, List, Optional
 
 from pyspark.sql import SparkSession
 
-from fabricks.context import CATALOG, IS_UNITY_CATALOG, PATH_UDFS, SPARK
+from fabricks.context import CATALOG, CONF_RUNTIME, IS_UNITY_CATALOG, PATH_UDFS, SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 
 UDFS: dict[str, Callable] = {}
 
+udf_schema = CONF_RUNTIME.get("udf_options", {}).get("schema", "default")
+udf_prefix = CONF_RUNTIME.get("udf_options", {}).get("prefix", "udf_")
+
 
 def register_all_udfs(extension: Optional[str] = None, override: bool = False):
     """
@@ -47,12 +50,12 @@ def is_registered(udf: str, spark: Optional[SparkSession] = None) -> bool:
         spark = SPARK
     assert spark is not None
 
-    df = spark.sql("show user functions in default")
+    df = spark.sql(f"show user functions in {udf_schema}")
 
     if CATALOG:
-        df = df.where(f"function == '{CATALOG}.default.udf_{udf}'")
+        df = df.where(f"function == '{CATALOG}.{udf_schema}.{udf_prefix}{udf}'")
     else:
-        df = df.where(f"function == 'spark_catalog.default.udf_{udf}'")
+        df = df.where(f"function == 'spark_catalog.{udf_schema}.{udf_prefix}{udf}'")
 
     return not df.isEmpty()
 
@@ -72,9 +75,9 @@ def register_udf(
 
     if not is_registered(udf, spark) or override:
         if override:
-            DEFAULT_LOGGER.debug(f"override udf {udf}")
+            DEFAULT_LOGGER.debug(f"override udf {udf}", extra={"label": "fabricks"})
         else:
-            DEFAULT_LOGGER.debug(f"register udf {udf}")
+            DEFAULT_LOGGER.debug(f"register udf {udf}", extra={"label": "fabricks"})
 
         if extension is None:
             extension = get_extension(udf)
@@ -89,8 +92,6 @@ def register_udf(
         elif extension == "py":
             if not IS_UNITY_CATALOG:
                 assert path.exists(), f"udf not found ({path.string})"
-            else:
-                DEFAULT_LOGGER.debug(f"could not check if udf exists ({path.string})")
 
             spec = importlib.util.spec_from_file_location(udf, path.string)
             assert spec, f"no valid udf found ({path.string})"

fabricks/utils/path.py

@@ -246,3 +246,13 @@ class Path:
 
     def __str__(self) -> str:
         return self.string
+
+
+class GitPath(Path):
+    def __init__(self, path: Union[str, PathlibPath]):
+        super().__init__(path=path, assume_git=True)
+
+
+class FileSharePath(Path):
+    def __init__(self, path: Union[str, PathlibPath]):
+        super().__init__(path=path, assume_git=False)

{fabricks-3.0.17.dist-info → fabricks-3.0.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.17
+Version: 3.0.19
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.17.dist-info → fabricks-3.0.19.dist-info}/RECORD

@@ -68,7 +68,7 @@ fabricks/cdc/templates/queries/scd2.sql.jinja,sha256=Nn0wUs9N-_QviZqUKRWAFdD17RR
 fabricks/cdc/templates/queries/nocdc/complete.sql.jinja,sha256=cVKKCSbiuuw1K7BOzfusX6KvzQNHU3YNUgoXgsu-c6k,267
 fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=mjNUwGVhZ08yUkdv9sCTkqyW60p0YavtWTqvSUVrwjA,1283
 fabricks/context/__init__.py,sha256=qfntJ9O6omzY_t6AhDP6Ndu9C5LMiVdWbo6ikhtoe7o,1446
-fabricks/context/_types.py,sha256=FzQJ35vp0uc6pAq18bc-VHwMVEWtd0VDdm8xQmNr2Sg,2681
+fabricks/context/_types.py,sha256=B3CvR_RR0wXCbBADh7Z9S8_7AXRRtvPCnFh7OyQJboI,2802
 fabricks/context/config.py,sha256=EmLUnswuWfrncaNJMDjvdMg-1lD8aneKAY8IDna7VPE,4814
 fabricks/context/helpers.py,sha256=igY8LwLIxzfOWKCg23XMsJoY7Bw0welpdNry2mKHjF0,1600
 fabricks/context/log.py,sha256=CadrRf8iL6iXlGIGIhEIswa7wGqC-E-oLwWcGTyJ10s,2074
@@ -80,7 +80,7 @@ fabricks/core/__init__.py,sha256=LaqDi4xuyHAoLOvS44PQdZdRfq9SmVr7mB6BDHyxYpc,209
 fabricks/core/extenders.py,sha256=oJzfv0hWxusnGmrjMwbrGyKfot8xzA4XtNquPWfFgPo,727
 fabricks/core/job_schema.py,sha256=6-70oy0ZJd3V9AiXfc0Q8b8NVEynxQza_h7mB13uB-s,853
 fabricks/core/masks.py,sha256=0ARgXE6stazRlfjE2v2sOdQWjAH2TbCOgtuD33BeZqE,1531
-fabricks/core/udfs.py,sha256=QlFOgN-Ceiv601-2O2sPXfSMqYYZiL29GvUkTfx_P0k,3163
+fabricks/core/udfs.py,sha256=nYG5MrTziDB3skZOCgpuxIOvLzs8I5vY8DC0Zdmkysk,3308
 fabricks/core/views.py,sha256=52tekqeP0Xk5EPYO220YdfFbzItX6NnObROb-ye9COQ,1181
 fabricks/core/dags/__init__.py,sha256=0DUKzVcXcROvxkN19P_kaOJ7da5BAM7Vt8EGQbp2KSY,240
 fabricks/core/dags/base.py,sha256=tFj27SqeZUZ7pB_LOWkpdowZz5gj30JUANI4gWK3Pl8,3139
@@ -98,7 +98,7 @@ fabricks/core/jobs/get_job_id.py,sha256=6dLyzxGHlRvJZVJSwZkCk3iXzWkIhePC_6FhoP0g
 fabricks/core/jobs/get_jobs.py,sha256=nJ-8DPFq1GyzWo9Mxlwq2dEeAqwg1jeQg-CHietAb1Q,3341
 fabricks/core/jobs/get_schedule.py,sha256=46pJR5LWZfuxUtLBmtB-RP6ng_W-K-ahJmD29KNmcGw,259
 fabricks/core/jobs/get_schedules.py,sha256=kryDUBrBrtAaMp8Ou5YqMOCOMKvg1GmbbOQBtiiRleM,794
-fabricks/core/jobs/gold.py,sha256=lwzUG-z0ACWUiR7EpDNbrrh7X7XwnixgrMoIptzK7uE,14552
+fabricks/core/jobs/gold.py,sha256=6YOhuMpCAuqfRtottikIemLQ-L62Av0ZXFcOb9uh4yE,14896
 fabricks/core/jobs/silver.py,sha256=kdrCBfh1jkhWJUFubGUV4kxan5eRUZl-LI-iSJxyJE4,13093
 fabricks/core/jobs/base/__init__.py,sha256=_AdWtyL7yZG2TOZ9e8WyNPrOjmm6EDkI_TNym5cLDws,208
 fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbew8AU,8405
@@ -155,7 +155,7 @@ fabricks/utils/fdict.py,sha256=cdnvNBSXKJIDKSdhQGJA4CGv0qLn5IVYKQ111l7nM9I,7978
 fabricks/utils/helpers.py,sha256=fKv6mpT-428xTSjdLfm7TnN1Xo9FadrSIY1qzYgWCzs,7909
 fabricks/utils/log.py,sha256=LCQEM81PhdojiyLrtEzv1QM__bWbaEhGddyd0IqyGXM,7985
 fabricks/utils/mermaid.py,sha256=XoiVxPaUJS4TC_ybA-e78qFzQkQ46uPf055JiiNDdSg,986
-fabricks/utils/path.py,sha256=ToTTS8QKGsWq8cR8SDE2ocWKx2GsuZ5psPTuAMq9R4s,6813
+fabricks/utils/path.py,sha256=H2jU2ixJ57eAd7Aq3Upln1wT9rJzy7od7AwdEnNvCgo,7082
 fabricks/utils/pip.py,sha256=UHo7NTjFGJNghWBuuDow28xUkZYg2YrlbAP49IxZyXY,1522
 fabricks/utils/pydantic.py,sha256=W0fiDLVMFrrInfQw2s5YPeSEvkN-4k864u3UyPoHaz4,2516
 fabricks/utils/spark.py,sha256=QWVpbGwOvURIVBlR7ygt6NQ9MHUsIDvlquJ65iI8UBI,2007
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.17.dist-info/METADATA,sha256=JTkqiUkvRKtEQKIWDsQhsNcmvMJ81vCx9kQhyxfznwM,798
-fabricks-3.0.17.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-fabricks-3.0.17.dist-info/RECORD,,
+fabricks-3.0.19.dist-info/METADATA,sha256=LVAn4EIa8k57SNEaZvYu7kFJ5LsQ6M64MpfYap63VYY,798
+fabricks-3.0.19.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+fabricks-3.0.19.dist-info/RECORD,,