fabricks 3.0.18__tar.gz → 3.0.19__tar.gz

{fabricks-3.0.18 → fabricks-3.0.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.18
+Version: 3.0.19
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.18 → fabricks-3.0.19}/fabricks/context/_types.py

@@ -16,10 +16,12 @@ class RuntimeTimeoutOptions(TypedDict):
     pre_run: int
     post_run: int
 
+
 class UDFOptions(TypedDict):
     prefix: Optional[str]
     schema: Optional[str]
 
+
 class StepTimeoutOptions(TypedDict):
     step: Optional[int]
     job: Optional[int]

{fabricks-3.0.18 → fabricks-3.0.19}/fabricks/core/jobs/gold.py

@@ -126,7 +126,18 @@ class Gold(BaseJob):
             invokers = self.options.invokers.get_list("run")
             assert len(invokers) <= 1, "at most one invoker allowed when notebook is true"
 
-            global_temp_view = self.invoke(path=self.paths.runtime, schema_only=schema_only, **kwargs)
+            path = None
+            if invokers:
+                notebook = invokers[0].get("notebook")
+                if notebook:
+                    from fabricks.context import PATH_RUNTIME
+
+                    path = PATH_RUNTIME.joinpath(notebook)
+
+            if path is None:
+                path = self.paths.runtime
+
+            global_temp_view = self.invoke(path=path, schema_only=schema_only, **kwargs)
             assert global_temp_view is not None, "global_temp_view not found"
 
             df = self.spark.sql(f"select * from global_temp.{global_temp_view}")

{fabricks-3.0.18 → fabricks-3.0.19}/fabricks/core/udfs.py

@@ -5,7 +5,7 @@ from typing import Callable, List, Optional
 
 from pyspark.sql import SparkSession
 
-from fabricks.context import CATALOG, IS_UNITY_CATALOG, PATH_UDFS, SPARK, CONF_RUNTIME
+from fabricks.context import CATALOG, CONF_RUNTIME, IS_UNITY_CATALOG, PATH_UDFS, SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 
 UDFS: dict[str, Callable] = {}
@@ -13,6 +13,7 @@ UDFS: dict[str, Callable] = {}
 udf_schema = CONF_RUNTIME.get("udf_options", {}).get("schema", "default")
 udf_prefix = CONF_RUNTIME.get("udf_options", {}).get("prefix", "udf_")
 
+
 def register_all_udfs(extension: Optional[str] = None, override: bool = False):
     """
     Register all user-defined functions (UDFs).

{fabricks-3.0.18 → fabricks-3.0.19}/fabricks/utils/path.py

@@ -246,3 +246,13 @@ class Path:
 
     def __str__(self) -> str:
         return self.string
+
+
+class GitPath(Path):
+    def __init__(self, path: Union[str, PathlibPath]):
+        super().__init__(path=path, assume_git=True)
+
+
+class FileSharePath(Path):
+    def __init__(self, path: Union[str, PathlibPath]):
+        super().__init__(path=path, assume_git=False)

{fabricks-3.0.18 → fabricks-3.0.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.18"
+version = "3.0.19"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.18 → fabricks-3.0.19}/tests/integration/jobs/job5/test_step.py

@@ -57,19 +57,19 @@ def test_update_configurations():
     step.update_configurations()
 
     df = SPARK.sql("select * from fabricks.gold_jobs")
-    assert df.count() == 51, f"{df.count()} job(s) <> 51"
+    assert df.count() == 53, f"{df.count()} job(s) <> 53"
 
     SPARK.sql("update fabricks.gold_jobs set options = null")
 
     df = SPARK.sql("select * from fabricks.gold_jobs where options is null")
-    assert df.count() == 51, f"{df.count()} job(s) without options <> 51"
+    assert df.count() == 53, f"{df.count()} job(s) without options <> 53"
 
     SPARK.sql("delete from fabricks.gold_jobs where topic == 'scd1'")
 
     step.update_configurations()
 
     df = SPARK.sql("select * from fabricks.gold_jobs")
-    assert df.count() == 51, f"{df.count()} job(s) <> 51"
+    assert df.count() == 53, f"{df.count()} job(s) <> 53"
 
     df = SPARK.sql("select * from fabricks.gold_jobs where options is null")
     assert df.count() == 0, f"{df.count()} job(s) without options <> 0"

{fabricks-3.0.18 → fabricks-3.0.19}/tests/integration/runtime/gold/gold/invoke/_config.invoke.yml

@@ -109,4 +109,29 @@
           arguments:
             arg1: 1
             arg2: 2
-            arg3: 3
+            arg3: 3
+- job:
+    step: gold
+    topic: invoke
+    item: complete_invoker
+    tags: [test]
+    options:
+      mode: complete
+      notebook: true
+    invoker_options:
+      run:
+        - notebook: gold/gold/invoke/complete_notebook
+          arguments:
+            arg1: 1
+- job:
+    step: gold
+    topic: invoke
+    item: complete_notebook
+    tags: [test]
+    options:
+      mode: complete
+      notebook: true
+    invoker_options:
+      run:
+        - arguments:
+            arg1: 1

fabricks-3.0.19/tests/integration/runtime/gold/gold/invoke/add_missing_modules.py

@@ -0,0 +1,24 @@
+# Databricks notebook source
+import os
+import sys
+from pathlib import Path
+
+# COMMAND ----------
+
+# https://docs.databricks.com/aws/en/files/workspace-modules
+
+# COMMAND ----------
+
+p = Path(os.getcwd())
+while not (p / "pyproject.toml").exists():
+    p = p.parent
+
+# COMMAND ----------
+
+root = p.absolute()
+
+# COMMAND ----------
+
+if str(root) not in sys.path:
+    print(f"adding {root} to sys.path")
+    sys.path.insert(0, str(root))

fabricks-3.0.19/tests/integration/runtime/gold/gold/invoke/complete_notebook.py

@@ -0,0 +1,51 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+from databricks.sdk.runtime import dbutils, spark
+
+from fabricks.metastore.view import View
+
+# COMMAND ----------
+
+
+# COMMAND ----------
+
+dbutils.widgets.text("step", "--")
+dbutils.widgets.text("topic", "--")
+dbutils.widgets.text("item", "--")
+
+# COMMAND ----------
+
+dbutils.widgets.text("arg1", "--")
+
+# COMMAND ----------
+
+step = dbutils.widgets.get("step")
+topic = dbutils.widgets.get("topic")
+item = dbutils.widgets.get("item")
+
+# COMMAND ----------
+
+arg1 = dbutils.widgets.get("arg1")
+
+# COMMAND ----------
+
+print(step, topic, item)
+
+# COMMAND ----------
+
+assert arg1 == "1"
+
+# COMMAND ----------
+
+df = spark.sql("select 1 as dummy")
+
+# COMMAND ----------
+
+uuid = View.create_or_replace(df)
+
+# COMMAND ----------
+
+dbutils.notebook.exit(uuid)  # type: ignore

fabricks-3.0.18/tests/integration/runtime/gold/gold/invoke/pre_run/exe_2.py → fabricks-3.0.19/tests/integration/runtime/gold/transf/post_run.py

@@ -27,4 +27,8 @@ print(step, topic, item)
 
 # COMMAND ----------
 
+assert arg1 == "1"
+
+# COMMAND ----------
+
 dbutils.notebook.exit(value="exit (0)")  # type: ignore

{fabricks-3.0.18 → fabricks-3.0.19}/uv.lock

@@ -645,7 +645,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.18"
+version = "3.0.19"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },