fabricks 3.0.7__tar.gz → 3.0.9__tar.gz

{fabricks-3.0.7 → fabricks-3.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.7
+Version: 3.0.9
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.7 → fabricks-3.0.9}/fabricks/cdc/templates/queries/nocdc/update.sql.jinja

@@ -16,8 +16,7 @@
                 {% for output in outputs %} c.`{{ output }}`, {% endfor %}
             from __current c
             left anti join
-                {{ parent_cdc }} s on s.__key == c.__key
-                {% if has_source %} and s.__source = c.__source {% endif %}
+                {{ parent_cdc }} s on s.__key == c.__key {% if has_source %} and s.__source = c.__source {% endif %}
         {% endif %}
     ),
 {% else %}

{fabricks-3.0.7 → fabricks-3.0.9}/fabricks/core/jobs/gold.py

@@ -13,7 +13,7 @@ from fabricks.core.jobs.base.job import BaseJob
 from fabricks.core.udfs import is_registered, register_udf
 from fabricks.metastore.view import create_or_replace_global_temp_view
 from fabricks.utils.path import Path
-from fabricks.utils.sqlglot import get_tables
+from fabricks.utils.sqlglot import fix, get_tables
 
 
 class Gold(BaseJob):
@@ -68,7 +68,8 @@ class Gold(BaseJob):
 
     @property
     def sql(self) -> str:
-        return self.paths.runtime.get_sql()
+        sql = self.paths.runtime.get_sql()
+        return fix(sql, keep_comments=False)
 
     @deprecated("use sql instead")
     def get_sql(self) -> str:

fabricks-3.0.9/fabricks/core/schedules/diagrams.py

@@ -0,0 +1,21 @@
+from pyspark.sql import DataFrame
+
+
+def get_dependencies(name: str) -> DataFrame:
+    from fabricks.core.dags import DagGenerator
+
+    g = DagGenerator(schedule=name)
+    return g.get_dependencies()
+
+
+def get_mermaid_diagram(name: str) -> str:
+    from fabricks.utils.mermaid import get_mermaid_diagram as get_diagram
+
+    df = get_dependencies(name)
+
+    df = df.withColumnRenamed("ParentId", "parent_id")
+    df = df.withColumnRenamed("Parent", "parent")
+    df = df.withColumnRenamed("JobId", "job_id")
+    df = df.withColumnRenamed("Job", "job")
+
+    return get_diagram(df)

fabricks-3.0.7/fabricks/core/schedules/diagrams.py → fabricks-3.0.9/fabricks/utils/mermaid.py

@@ -1,21 +1,7 @@
 from pyspark.sql import DataFrame
 
 
-def get_dependencies(name: str) -> DataFrame:
-    from fabricks.core.dags import DagGenerator
-
-    g = DagGenerator(schedule=name)
-    return g.get_dependencies()
-
-
-def get_mermaid_diagram(name: str) -> str:
-    df = get_dependencies(name)
-
-    df = df.withColumnRenamed("ParentId", "parent_id")
-    df = df.withColumnRenamed("Parent", "parent")
-    df = df.withColumnRenamed("JobId", "job_id")
-    df = df.withColumnRenamed("Job", "job")
-
+def get_mermaid_diagram(df: DataFrame) -> str:
     dependencies = df.select("parent_id", "parent", "job_id", "job").collect()
 
     out = "flowchart TD\n"

{fabricks-3.0.7 → fabricks-3.0.9}/fabricks/utils/sqlglot.py

@@ -12,7 +12,7 @@ class Fabricks(Databricks):
         }
 
 
-def fix(sql: str):
+def fix(sql: str, keep_comments: bool = True):
     sql = transpile(
         sql,
         "fabricks",
@@ -22,6 +22,7 @@ def fix(sql: str):
         normalize_functions="lower",
         leading_comma=True,
         max_text_width=119,
+        comments=keep_comments
     )[0]
     return sql
 

{fabricks-3.0.7 → fabricks-3.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.7"
+version = "3.0.9"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.7 → fabricks-3.0.9}/tests/integration/fabricksconfig.json

@@ -3,6 +3,7 @@
     "notebooks": "runtime/fabricks/notebooks",
     "job_config_from_yaml": true,
     "loglevel": "info",
+    "debugmode": true,
     "config": "runtime/fabricks/conf.uc.fabricks.yml",
     "devmode": true
 }

fabricks-3.0.9/tests/integration/runtime/semantic/dim/_config.semantic.yml

@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=../../../job-schema.json
+- job:
+    step: semantic
+    topic: dim
+    item: complex_query
+    options:
+      mode: memory

fabricks-3.0.9/tests/integration/runtime/semantic/dim/complex_query.sql

@@ -0,0 +1,129 @@
+/*
+|0|Not Royal Stock|
+|1|Royal Stock|
+|2|Rarely Visited|
+|3|Forgotten in Tower|
+|4|Royal Clearance|
+*/
+select *
+from
+values
+    /*
+    |0|Not Royal Stock|
+    |1|Royal Stock|
+    |2|Rarely Visited|
+    |3|Forgotten in Tower|
+    |4|Royal Clearance|
+  */
+    -- Prince 02 (The Charming Prince)
+    ('02', 0, 2),  -- Not in royal stock -> 2 - available at court
+    ('02', 1, 3),  -- In royal stock -> 3 - available / 1st priority at royal ball
+    ('02', 2, 2),  -- Rarely visited -> 2 - available at court
+    ('02', 3, 2),  -- Forgotten in tower -> 2 - available at court
+    ('02', 4, 1),  -- Royal clearance -> 1 - available (lowest priority, no dancing allowed)
+    ('02', null, 1),
+    -- Princess 03 (The Graceful Princess)
+    ('03', 0, 2),  -- Not in royal stock -> 2 - available at court
+    ('03', 1, 3),  -- In royal stock -> 3 - available
+    ('03', 2, 2),  -- Rarely visited -> 2 - available at court
+    ('03', 3, 2),  -- Forgotten in tower -> 2 - available at court
+    ('03', 4, 1),  -- Royal clearance -> 1 - available (lowest priority, no dancing allowed)
+    ('03', null, 1),
+    -- Prince 04 (The Banished Prince)
+    ('04', 0, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 1, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 2, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 3, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', 4, 0),  -- n/a -> 0 - banished from kingdom
+    ('04', null, 0),
+    -- Princess 05 (The Sleeping Princess)
+    ('05', 0, 0),  -- n/a -> 0 - fast asleep
+    ('05', 1, 0),  -- n/a -> 0 - fast asleep
+    ('05', 2, 0),  -- n/a -> 0 - fast asleep
+    ('05', 3, 0),  -- n/a -> 0 - fast asleep
+    ('05', 4, 0),  -- n/a -> 0 - fast asleep
+    ('05', null, 0),
+    -- Prince 06 (The Retired Prince)
+    ('06', 0, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 1, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 2, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 3, 0),  -- Not available -> 0 - retired to countryside
+    ('06', 4, 1),  -- Royal clearance (Replaced by younger prince) -> 1 - available (lowest priority, no dancing allowed)
+    ('06', null, 0),
+    -- Princess 07 (The Former Court Princess)
+    ('07', 0, 0),  -- Not available -> 0 - left the court
+    ('07', 1, 0),  -- Not available -> 0 - left the court
+    ('07', 2, 0),  -- Not available -> 0 - left the court
+    ('07', 3, 0),  -- Not available -> 0 - left the court
+    ('07', 4, 1),  -- Royal clearance (Duties discontinued) -> 1 - available (lowest priority, no dancing allowed)
+    ('07', null, 0),
+    -- Prince 08 (The Liquidating Prince)
+    ('08', 0, 0),  -- Not available -> 0 - selling the castle
+    ('08', 1, 0),  -- Not available -> 0 - selling the castle
+    ('08', 2, 0),  -- Not available -> 0 - selling the castle
+    ('08', 3, 0),  -- Not available -> 0 - selling the castle
+    ('08', 4, 1),  -- Royal clearance (Estate sale) -> 1 - available (lowest priority, no dancing allowed)
+    ('08', null, 0),
+    -- Princess 09-14 (The Lost Princesses)
+    ('09', 0, 0),  -- n/a -> 0 - lost in the enchanted forest
+    ('09', 1, 0),
+    ('09', 2, 0),
+    ('09', 3, 0),
+    ('09', 4, 0),
+    ('09', null, 0),
+    ('10', 0, 0),
+    ('10', 1, 0),
+    ('10', 2, 0),
+    ('10', 3, 0),
+    ('10', 4, 0),
+    ('10', null, 0),
+    ('11', 0, 0),
+    ('11', 1, 0),
+    ('11', 2, 0),
+    ('11', 3, 0),
+    ('11', 4, 0),
+    ('11', null, 0),
+    ('12', 0, 0),
+    ('12', 1, 0),
+    ('12', 2, 0),
+    ('12', 3, 0),
+    ('12', 4, 0),
+    ('12', null, 0),
+    ('13', 0, 0),
+    ('13', 1, 0),
+    ('13', 2, 0),
+    ('13', 3, 0),
+    ('13', 4, 0),
+    ('13', null, 0),
+    ('14', 0, 0),
+    ('14', 1, 0),
+    ('14', 2, 0),
+    ('14', 3, 0),
+    ('14', 4, 0),
+    ('14', null, 0),
+    -- Prince 15 (The Service Prince)
+    ('15', 0, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 1, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 2, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 3, 0),  -- Not available -> 0 - in royal service elsewhere
+    ('15', 4, 1),  -- Royal clearance (Special service) -> 1 - available (lowest priority, no dancing allowed)
+    ('15', null, 0),
+    -- Princes 20, 30, 40 (The Distant Princes)
+    ('20', 0, 0),
+    ('20', 1, 0),
+    ('20', 2, 0),
+    ('20', 3, 0),
+    ('20', 4, 0),
+    ('20', null, 0),
+    ('30', 0, 0),
+    ('30', 1, 0),
+    ('30', 2, 0),
+    ('30', 3, 0),
+    ('30', 4, 0),
+    ('30', null, 0),
+    ('40', 0, 0),
+    ('40', 1, 0),
+    ('40', 2, 0),
+    ('40', 3, 0),
+    ('40', 4, 0),
+    ('40', null, 0) as royal_court_status(royal_id, crown_jewels_status_id, royal_availability)

{fabricks-3.0.7 → fabricks-3.0.9}/uv.lock

@@ -645,7 +645,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.7"
+version = "3.0.9"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },