fabricks 3.0.7__py3-none-any.whl → 3.0.9__py3-none-any.whl

fabricks/cdc/templates/queries/nocdc/update.sql.jinja

@@ -16,8 +16,7 @@
                 {% for output in outputs %} c.`{{ output }}`, {% endfor %}
             from __current c
             left anti join
-                {{ parent_cdc }} s on s.__key == c.__key
-                {% if has_source %} and s.__source = c.__source {% endif %}
+                {{ parent_cdc }} s on s.__key == c.__key {% if has_source %} and s.__source = c.__source {% endif %}
         {% endif %}
     ),
 {% else %}

fabricks/core/jobs/gold.py

@@ -13,7 +13,7 @@ from fabricks.core.jobs.base.job import BaseJob
 from fabricks.core.udfs import is_registered, register_udf
 from fabricks.metastore.view import create_or_replace_global_temp_view
 from fabricks.utils.path import Path
-from fabricks.utils.sqlglot import get_tables
+from fabricks.utils.sqlglot import fix, get_tables
 
 
 class Gold(BaseJob):
@@ -68,7 +68,8 @@ class Gold(BaseJob):
 
     @property
     def sql(self) -> str:
-        return self.paths.runtime.get_sql()
+        sql = self.paths.runtime.get_sql()
+        return fix(sql, keep_comments=False)
 
     @deprecated("use sql instead")
     def get_sql(self) -> str:

fabricks/core/schedules/diagrams.py

@@ -9,6 +9,8 @@ def get_dependencies(name: str) -> DataFrame:
 
 
 def get_mermaid_diagram(name: str) -> str:
+    from fabricks.utils.mermaid import get_mermaid_diagram as get_diagram
+
     df = get_dependencies(name)
 
     df = df.withColumnRenamed("ParentId", "parent_id")
@@ -16,31 +18,4 @@ def get_mermaid_diagram(name: str) -> str:
     df = df.withColumnRenamed("JobId", "job_id")
     df = df.withColumnRenamed("Job", "job")
 
-    dependencies = df.select("parent_id", "parent", "job_id", "job").collect()
-
-    out = "flowchart TD\n"
-
-    unique_nodes = set()
-
-    for row in dependencies:
-        parent_id = str(row["parent_id"])
-        parent_name = str(row["parent"])
-        child_id = str(row["job_id"])
-        child_name = str(row["job"])
-
-        if parent_id != "0" and parent_id is not None:
-            if parent_id not in unique_nodes:
-                out += f"    {parent_id}[{parent_name}]\n"
-                unique_nodes.add(parent_id)
-
-            if child_id not in unique_nodes:
-                out += f"    {child_id}[{child_name}]\n"
-                unique_nodes.add(child_id)
-
-            out += f"    {parent_id} --> {child_id}\n"
-        else:
-            if child_id not in unique_nodes:
-                out += f"    {child_id}[{child_name}]\n"
-                unique_nodes.add(child_id)
-
-    return out
+    return get_diagram(df)

fabricks/utils/mermaid.py

@@ -0,0 +1,32 @@
+from pyspark.sql import DataFrame
+
+
+def get_mermaid_diagram(df: DataFrame) -> str:
+    dependencies = df.select("parent_id", "parent", "job_id", "job").collect()
+
+    out = "flowchart TD\n"
+
+    unique_nodes = set()
+
+    for row in dependencies:
+        parent_id = str(row["parent_id"])
+        parent_name = str(row["parent"])
+        child_id = str(row["job_id"])
+        child_name = str(row["job"])
+
+        if parent_id != "0" and parent_id is not None:
+            if parent_id not in unique_nodes:
+                out += f"    {parent_id}[{parent_name}]\n"
+                unique_nodes.add(parent_id)
+
+            if child_id not in unique_nodes:
+                out += f"    {child_id}[{child_name}]\n"
+                unique_nodes.add(child_id)
+
+            out += f"    {parent_id} --> {child_id}\n"
+        else:
+            if child_id not in unique_nodes:
+                out += f"    {child_id}[{child_name}]\n"
+                unique_nodes.add(child_id)
+
+    return out

fabricks/utils/sqlglot.py

@@ -12,7 +12,7 @@ class Fabricks(Databricks):
         }
 
 
-def fix(sql: str):
+def fix(sql: str, keep_comments: bool = True):
     sql = transpile(
         sql,
         "fabricks",
@@ -22,6 +22,7 @@ def fix(sql: str):
         normalize_functions="lower",
         leading_comma=True,
         max_text_width=119,
+        comments=keep_comments
     )[0]
     return sql
 

{fabricks-3.0.7.dist-info → fabricks-3.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.7
+Version: 3.0.9
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.7.dist-info → fabricks-3.0.9.dist-info}/RECORD

@@ -66,7 +66,7 @@ fabricks/cdc/templates/queries/final.sql.jinja,sha256=vxH434CO5k8Ia7tugaH8LC1co7
 fabricks/cdc/templates/queries/scd1.sql.jinja,sha256=siHULgKE3uRBGQYZFUR_eHNqFuGgO9xUCRVV2jnAXAI,3019
 fabricks/cdc/templates/queries/scd2.sql.jinja,sha256=Nn0wUs9N-_QviZqUKRWAFdD17RR3EFBTMs9BpBu6z7E,3877
 fabricks/cdc/templates/queries/nocdc/complete.sql.jinja,sha256=cVKKCSbiuuw1K7BOzfusX6KvzQNHU3YNUgoXgsu-c6k,267
-fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=SDJFo7rWiU58DqzOG-Pl76Dto82PIHWDdKlX62M1j58,1299
+fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=mjNUwGVhZ08yUkdv9sCTkqyW60p0YavtWTqvSUVrwjA,1283
 fabricks/context/__init__.py,sha256=qfntJ9O6omzY_t6AhDP6Ndu9C5LMiVdWbo6ikhtoe7o,1446
 fabricks/context/_types.py,sha256=FzQJ35vp0uc6pAq18bc-VHwMVEWtd0VDdm8xQmNr2Sg,2681
 fabricks/context/log.py,sha256=CadrRf8iL6iXlGIGIhEIswa7wGqC-E-oLwWcGTyJ10s,2074
@@ -98,7 +98,7 @@ fabricks/core/jobs/get_job_id.py,sha256=6dLyzxGHlRvJZVJSwZkCk3iXzWkIhePC_6FhoP0g
 fabricks/core/jobs/get_jobs.py,sha256=nJ-8DPFq1GyzWo9Mxlwq2dEeAqwg1jeQg-CHietAb1Q,3341
 fabricks/core/jobs/get_schedule.py,sha256=46pJR5LWZfuxUtLBmtB-RP6ng_W-K-ahJmD29KNmcGw,259
 fabricks/core/jobs/get_schedules.py,sha256=kryDUBrBrtAaMp8Ou5YqMOCOMKvg1GmbbOQBtiiRleM,794
-fabricks/core/jobs/gold.py,sha256=HzeuuOtsjr3lsasBJ1ODzLQzcoanhbzZiP6a9acNsnA,14503
+fabricks/core/jobs/gold.py,sha256=lwzUG-z0ACWUiR7EpDNbrrh7X7XwnixgrMoIptzK7uE,14552
 fabricks/core/jobs/silver.py,sha256=kdrCBfh1jkhWJUFubGUV4kxan5eRUZl-LI-iSJxyJE4,13093
 fabricks/core/jobs/base/__init__.py,sha256=_AdWtyL7yZG2TOZ9e8WyNPrOjmm6EDkI_TNym5cLDws,208
 fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbew8AU,8405
@@ -116,7 +116,7 @@ fabricks/core/parsers/decorator.py,sha256=kn_Mj-JLWTFaRiciZ3KavmSUcWFPY3ve-buMru
 fabricks/core/parsers/get_parser.py,sha256=TTnVPwKqKpFu6jJJnXEuiEctWGtimk8w2p1jF2U7ibg,909
 fabricks/core/parsers/utils.py,sha256=qdn2ElpqBgDsW55-tACWZaFOT0ebrBYg2fenqSgd6YI,2456
 fabricks/core/schedules/__init__.py,sha256=bDjNMcm7itimAo4gun0W4W9bZKwZmWUjkMqAQIcqI2Y,431
-fabricks/core/schedules/diagrams.py,sha256=DoQR80DLs0CQpUzxscBeO1mWNjgx8btBWaqcj2EFOBc,1379
+fabricks/core/schedules/diagrams.py,sha256=YA4T7Etl_UPfW-3IGFq5Xj9OlXZGQ27Aot6RVa3ZUgg,578
 fabricks/core/schedules/generate.py,sha256=aKnAe7ZCafAczLa4ka9Er_oltOxgXyNoS63_OZEktcE,623
 fabricks/core/schedules/get_schedule.py,sha256=PJcEq4enlsRJunS-MjXi-VFIczbeuBStP2giZ_-EaRc,116
 fabricks/core/schedules/get_schedules.py,sha256=b6KSl-QmiNgih2l6dESB0va9yDVxaGOJ_ZB96Wc3NC8,174
@@ -154,11 +154,12 @@ fabricks/utils/console.py,sha256=X4lLgL_UxCjoFRx-ZRCwzdBveRGPKlFYZDi6vl7uevQ,101
 fabricks/utils/fdict.py,sha256=cdnvNBSXKJIDKSdhQGJA4CGv0qLn5IVYKQ111l7nM9I,7978
 fabricks/utils/helpers.py,sha256=h7SuOVpBP5qcgX1nM1suvkXG9BhiK5-257EBepCvrO8,7452
 fabricks/utils/log.py,sha256=LCQEM81PhdojiyLrtEzv1QM__bWbaEhGddyd0IqyGXM,7985
+fabricks/utils/mermaid.py,sha256=XoiVxPaUJS4TC_ybA-e78qFzQkQ46uPf055JiiNDdSg,986
 fabricks/utils/path.py,sha256=Bs3PayWtg62-mrsDbvu8kh0VLZZhX7tU9YiyHFiYNhs,6698
 fabricks/utils/pip.py,sha256=UHo7NTjFGJNghWBuuDow28xUkZYg2YrlbAP49IxZyXY,1522
 fabricks/utils/pydantic.py,sha256=W0fiDLVMFrrInfQw2s5YPeSEvkN-4k864u3UyPoHaz4,2516
 fabricks/utils/spark.py,sha256=QWVpbGwOvURIVBlR7ygt6NQ9MHUsIDvlquJ65iI8UBI,2007
-fabricks/utils/sqlglot.py,sha256=Q3SOc4CJfGBIe4vr9laipJaPIknz_hnKGrJgLc7DumE,1439
+fabricks/utils/sqlglot.py,sha256=Wohw61ntA0J8eieQ5MYCMdDUMiN8S8do2V0dKlRM8Mg,1498
 fabricks/utils/read/__init__.py,sha256=a_5l60m1AyzQUII170bfRRuXR_ynC3EwfysidRy44GE,272
 fabricks/utils/read/_types.py,sha256=_YY0Yp8IPNYvcRFNa9WMcIl20cN93Cdcgh3jddypnuk,76
 fabricks/utils/read/read.py,sha256=el6GvynJrCl2Ap_bM7xY750fSXjrr9xmp14P92h-40g,8401
@@ -170,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.7.dist-info/METADATA,sha256=mxCXO3itBIJ1MORFhE0uEBoXIlEmEN5bsvGS53_8Gfk,761
-fabricks-3.0.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.7.dist-info/RECORD,,
+fabricks-3.0.9.dist-info/METADATA,sha256=E5DsWLV3UwSrgce4HALc7E51JBKKcYrWTh9SpVvKIos,761
+fabricks-3.0.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.9.dist-info/RECORD,,