fabricks 3.0.6__tar.gz → 3.0.8__tar.gz

{fabricks-3.0.6 → fabricks-3.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.6
+Version: 3.0.8
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.6 → fabricks-3.0.8}/fabricks/cdc/templates/queries/nocdc/update.sql.jinja

@@ -16,8 +16,7 @@
                 {% for output in outputs %} c.`{{ output }}`, {% endfor %}
             from __current c
             left anti join
-                {{ parent_cdc }} s on s.__key == c.__key and s.__hash == c.__hash
-                {% if has_source %} and s.__source = c.__source {% endif %}
+                {{ parent_cdc }} s on s.__key == c.__key {% if has_source %} and s.__source = c.__source {% endif %}
         {% endif %}
     ),
 {% else %}

{fabricks-3.0.6 → fabricks-3.0.8}/fabricks/context/utils.py

@@ -4,7 +4,7 @@ import fabricks.context.config as c
 import fabricks.context.runtime as r
 
 
-def pprint_runtime():
+def pprint_runtime(extended: bool = False) -> None:
     print("=" * 60)
     print("FABRICKS RUNTIME CONFIGURATION")
     print("=" * 60)
@@ -50,27 +50,31 @@ def pprint_runtime():
     print("\n🔐 SECURITY:")
     print(f"   Secret Scope: {r.SECRET_SCOPE}")
 
-    # Component Paths Section
-    print("\n🛠️ COMPONENT PATHS:")
-    components = [
-        ("UDFs", r.PATH_UDFS),
-        ("Parsers", r.PATH_PARSERS),
-        ("Extenders", r.PATH_EXTENDERS),
-        ("Views", r.PATH_VIEWS),
-        ("Schedules", r.PATH_SCHEDULES),
-    ]
+    print("\n🌐 ADDITIONAL SETTINGS:")
+    print(f"   Timezone: {r.TIMEZONE}")
 
-    for name, path in components:
-        print(f"   {name}: {path.string}")
+    if extended:
+        # Component Paths Section
+        print("\n🛠️ COMPONENT PATHS:")
+        components = [
+            ("UDFs", r.PATH_UDFS),
+            ("Parsers", r.PATH_PARSERS),
+            ("Extenders", r.PATH_EXTENDERS),
+            ("Views", r.PATH_VIEWS),
+            ("Schedules", r.PATH_SCHEDULES),
+        ]
 
-    # Storage Paths Section
-    print("\n📦 STORAGE PATHS:")
-    for name, path in sorted(r.PATHS_STORAGE.items()):
-        icon = "🏭" if name == "fabricks" else "📊"
-        print(f"   {icon} {name}: {path.string}")
+        for name, path in components:
+            print(f"   {name}: {path.string}")
 
-    # Runtime Paths Section
-    if r.PATHS_RUNTIME:
-        print("\n⚡ RUNTIME PATHS:")
-        for name, path in sorted(r.PATHS_RUNTIME.items()):
-            print(f"   📂 {name}: {path.string}")
+        # Storage Paths Section
+        print("\n📦 STORAGE PATHS:")
+        for name, path in sorted(r.PATHS_STORAGE.items()):
+            icon = "🏭" if name == "fabricks" else "📊"
+            print(f"   {icon} {name}: {path.string}")
+
+        # Runtime Paths Section
+        if r.PATHS_RUNTIME:
+            print("\n⚡ RUNTIME PATHS:")
+            for name, path in sorted(r.PATHS_RUNTIME.items()):
+                print(f"   📂 {name}: {path.string}")

{fabricks-3.0.6 → fabricks-3.0.8}/fabricks/core/jobs/gold.py

@@ -13,7 +13,7 @@ from fabricks.core.jobs.base.job import BaseJob
 from fabricks.core.udfs import is_registered, register_udf
 from fabricks.metastore.view import create_or_replace_global_temp_view
 from fabricks.utils.path import Path
-from fabricks.utils.sqlglot import get_tables
+from fabricks.utils.sqlglot import fix, get_tables
 
 
 class Gold(BaseJob):
@@ -68,7 +68,8 @@ class Gold(BaseJob):
 
     @property
     def sql(self) -> str:
-        return self.paths.runtime.get_sql()
+        sql = self.paths.runtime.get_sql()
+        return fix(sql)
 
     @deprecated("use sql instead")
     def get_sql(self) -> str:

fabricks-3.0.8/fabricks/core/schedules/diagrams.py

@@ -0,0 +1,21 @@
+from pyspark.sql import DataFrame
+
+
+def get_dependencies(name: str) -> DataFrame:
+    from fabricks.core.dags import DagGenerator
+
+    g = DagGenerator(schedule=name)
+    return g.get_dependencies()
+
+
+def get_mermaid_diagram(name: str) -> str:
+    from fabricks.utils.mermaid import get_mermaid_diagram as get_diagram
+
+    df = get_dependencies(name)
+
+    df = df.withColumnRenamed("ParentId", "parent_id")
+    df = df.withColumnRenamed("Parent", "parent")
+    df = df.withColumnRenamed("JobId", "job_id")
+    df = df.withColumnRenamed("Job", "job")
+
+    return get_diagram(df)

fabricks-3.0.6/fabricks/core/schedules/diagrams.py → fabricks-3.0.8/fabricks/utils/mermaid.py

@@ -1,21 +1,7 @@
 from pyspark.sql import DataFrame
 
 
-def get_dependencies(name: str) -> DataFrame:
-    from fabricks.core.dags import DagGenerator
-
-    g = DagGenerator(schedule=name)
-    return g.get_dependencies()
-
-
-def get_mermaid_diagram(name: str) -> str:
-    df = get_dependencies(name)
-
-    df = df.withColumnRenamed("ParentId", "parent_id")
-    df = df.withColumnRenamed("Parent", "parent")
-    df = df.withColumnRenamed("JobId", "job_id")
-    df = df.withColumnRenamed("Job", "job")
-
+def get_mermaid_diagram(df: DataFrame) -> str:
     dependencies = df.select("parent_id", "parent", "job_id", "job").collect()
 
     out = "flowchart TD\n"

{fabricks-3.0.6 → fabricks-3.0.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.6"
+version = "3.0.8"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.6 → fabricks-3.0.8}/tests/integration/fabricksconfig.json

@@ -3,6 +3,7 @@
     "notebooks": "runtime/fabricks/notebooks",
     "job_config_from_yaml": true,
     "loglevel": "info",
+    "debugmode": true,
     "config": "runtime/fabricks/conf.uc.fabricks.yml",
     "devmode": true
 }

{fabricks-3.0.6 → fabricks-3.0.8}/tests/integration/jobs/job1/test_gold.py

@@ -142,3 +142,14 @@ def test_gold_fact_no_drop():
         assert False, "drop is allowed while no_drop is set"
     except ValueError:
         assert True, "drop not allowed while no_drop is set"
+
+
+@pytest.mark.order(131)
+def test_gold_fact_masker_and_commenter():
+    df = SPARK.sql("select dummy from gold.fact_masker_and_commenter order by 1 asc")
+
+    masked_value = df.select("dummy").collect()[0][0]
+    assert masked_value == "***", f"dummy {masked_value} <> ***"
+
+    masked_value = df.select("dummy").collect()[1][0]
+    assert masked_value == "2", f"dummy {masked_value} <> 2"

{fabricks-3.0.6 → fabricks-3.0.8}/tests/integration/jobs/job5/test_step.py

@@ -10,7 +10,7 @@ DEFAULT_LOGGER.setLevel(ERROR)
 
 
 @pytest.mark.order(501)
-def test_step_gold():
+def test_update_dependencies():
     step = get_step("gold")
 
     deps, error = step.get_dependencies(loglevel=ERROR)
@@ -35,7 +35,7 @@ def test_step_gold():
 
 
 @pytest.mark.order(502)
-def test_step_transf():
+def test_create_db_objects():
     step = get_step("transf")
 
     SPARK.sql("drop view if exists transf.fact_memory")
@@ -48,3 +48,28 @@ def test_step_transf():
 
     df = SPARK.sql("select * from fabricks.transf_views")
     assert df.count() == 1, f"{df.count()} view(s) <> 1"
+
+
+@pytest.mark.order(503)
+def test_update_configurations():
+    step = get_step("gold")
+
+    step.update_configurations()
+
+    df = SPARK.sql("select * from fabricks.gold_jobs")
+    assert df.count() == 51, f"{df.count()} job(s) <> 51"
+
+    SPARK.sql("update fabricks.gold_jobs set options = null")
+
+    df = SPARK.sql("select * from fabricks.gold_jobs where options is null")
+    assert df.count() == 51, f"{df.count()} job(s) without options <> 51"
+
+    SPARK.sql("delete from fabricks.gold_jobs where topic == 'scd1'")
+
+    step.update_configurations()
+
+    df = SPARK.sql("select * from fabricks.gold_jobs")
+    assert df.count() == 51, f"{df.count()} job(s) <> 51"
+
+    df = SPARK.sql("select * from fabricks.gold_jobs where options is null")
+    assert df.count() == 0, f"{df.count()} job(s) without options <> 0"

{fabricks-3.0.6 → fabricks-3.0.8}/tests/integration/runtime/gold/gold/fact/_config.fact.yml

@@ -113,6 +113,7 @@
     step: gold
     topic: fact
     item: masker_and_commenter
+    tags: [test]
     options:
       mode: complete
     table_options:

fabricks-3.0.8/tests/integration/runtime/semantic/dim/_config.semantic.yml

@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=../../../job-schema.json
+- job:
+    step: semantic
+    topic: dim
+    item: complex_query
+    options:
+      mode: memory

fabricks-3.0.8/tests/integration/runtime/semantic/dim/complex_query.sql

@@ -0,0 +1,122 @@
+select *
+from
+values
+    /*
+    |0|Not Royal Stock|
+    |1|Royal Stock|
+    |2|Rarely Visited|
+    |3|Forgotten in Tower|
+    |4|Royal Clearance|
+  */
+    -- Prince 02 (The Charming Prince)
+    ('02', 0, 2),  -- Not in royal stock -> 2 - available at court
+    ('02', 1, 3),  -- In royal stock -> 3 - available / 1st priority at royal ball
+    ('02', 2, 2),  -- Rarely visited -> 2 - available at court
+    ('02', 3, 2),  -- Forgotten in tower -> 2 - available at court
+    ('02', 4, 1),  -- Royal clearance -> 1 - available (lowest priority, no dancing allowed)
+    ('02', null, 1),
+    -- Princess 03 (The Graceful Princess)
+    ('03', 0, 2),  -- Not in royal stock -> 2 - available at court
+    ('03', 1, 3),  -- In royal stock -> 3 - available
+    ('03', 2, 2),  -- Rarely visited -> 2 - available at court
+    ('03', 3, 2),  -- Forgotten in tower -> 2 - available at court
+    ('03', 4, 1),  -- Royal clearance -> 1 - available (lowest priority, no dancing allowed)
+    ('03', null, 1),
+    -- Prince 04 (The Banished Prince)
+    ('04', 0, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 1, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 2, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 3, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 4, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', null, 0),
+    -- Princess 05 (The Sleeping Princess)
+    ('05', 0, 0),  -- n/a -> 0 - fast asleep
+    ('05', 1, 0),  -- n/a -> 0 - fast asleep
+    ('05', 2, 0),  -- n/a -> 0 - fast asleep
+    ('05', 3, 0),  -- n/a -> 0 - fast asleep
+    ('05', 4, 0),  -- n/a -> 0 - fast asleep
+    ('05', null, 0),
+    -- Prince 06 (The Retired Prince)
+    ('06', 0, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 1, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 2, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 3, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 4, 1),  -- Royal clearance (Replaced by younger prince) -> 1 - available (lowest priority, no dancing allowed)
+    ('06', null, 0),
+    -- Princess 07 (The Former Court Princess)
+    ('07', 0, 0),  -- Not available -> 0 - left the court
+    ('07', 1, 0),  -- Not available -> 0 - left the court
+    ('07', 2, 0),  -- Not available -> 0 - left the court
+    ('07', 3, 0),  -- Not available -> 0 - left the court
+    ('07', 4, 1),  -- Royal clearance (Duties discontinued) -> 1 - available (lowest priority, no dancing allowed)
+    ('07', null, 0),
+    -- Prince 08 (The Liquidating Prince)
+    ('08', 0, 0),  -- Not available -> 0 - selling the castle
+    ('08', 1, 0),  -- Not available -> 0 - selling the castle
+    ('08', 2, 0),  -- Not available -> 0 - selling the castle
+    ('08', 3, 0),  -- Not available -> 0 - selling the castle
+    ('08', 4, 1),  -- Royal clearance (Estate sale) -> 1 - available (lowest priority, no dancing allowed)
+    ('08', null, 0),
+    -- Princess 09-14 (The Lost Princesses)
+    ('09', 0, 0),  -- n/a -> 0 - lost in the enchanted forest
+    ('09', 1, 0),
+    ('09', 2, 0),
+    ('09', 3, 0),
+    ('09', 4, 0),
+    ('09', null, 0),
+    ('10', 0, 0),
+    ('10', 1, 0),
+    ('10', 2, 0),
+    ('10', 3, 0),
+    ('10', 4, 0),
+    ('10', null, 0),
+    ('11', 0, 0),
+    ('11', 1, 0),
+    ('11', 2, 0),
+    ('11', 3, 0),
+    ('11', 4, 0),
+    ('11', null, 0),
+    ('12', 0, 0),
+    ('12', 1, 0),
+    ('12', 2, 0),
+    ('12', 3, 0),
+    ('12', 4, 0),
+    ('12', null, 0),
+    ('13', 0, 0),
+    ('13', 1, 0),
+    ('13', 2, 0),
+    ('13', 3, 0),
+    ('13', 4, 0),
+    ('13', null, 0),
+    ('14', 0, 0),
+    ('14', 1, 0),
+    ('14', 2, 0),
+    ('14', 3, 0),
+    ('14', 4, 0),
+    ('14', null, 0),
+    -- Prince 15 (The Service Prince)
+    ('15', 0, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 1, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 2, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 3, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 4, 1),  -- Royal clearance (Special service) -> 1 - available (lowest priority, no dancing allowed)
+    ('15', null, 0),
+    -- Princes 20, 30, 40 (The Distant Princes)
+    ('20', 0, 0),
+    ('20', 1, 0),
+    ('20', 2, 0),
+    ('20', 3, 0),
+    ('20', 4, 0),
+    ('20', null, 0),
+    ('30', 0, 0),
+    ('30', 1, 0),
+    ('30', 2, 0),
+    ('30', 3, 0),
+    ('30', 4, 0),
+    ('30', null, 0),
+    ('40', 0, 0),
+    ('40', 1, 0),
+    ('40', 2, 0),
+    ('40', 3, 0),
+    ('40', 4, 0),
+    ('40', null, 0) as royal_court_status(royal_id, crown_jewels_status_id, royal_availability)