fabricks 3.0.5.2__py3-none-any.whl → 3.0.6__py3-none-any.whl

fabricks/{core/deploy → deploy}/tables.py

@@ -7,7 +7,7 @@ from fabricks.metastore.table import Table
 
 
 def deploy_tables(drop: bool = False):
-    DEFAULT_LOGGER.info("🌟 (create or replace tables)")
+    DEFAULT_LOGGER.info("create or replace fabricks (default) tables")
 
     create_table_log(drop)
     create_table_dummy(drop)
@@ -64,16 +64,19 @@ def create_table_log(drop: bool = False):
 
 
 def create_table_dummy(drop: bool = False):
-    table = NoCDC("fabricks", "dummy")
-    df = SPARK.sql(
-        """
-          select
-          1 as __key,
-          md5('1') as __hash,
-          cast('1900-01-01' as timestamp) as __valid_from,
-          cast('9999-12-31' as timestamp) as __valid_to
-        """
-    )
+    cdc = NoCDC("fabricks", "dummy")
+
     if drop:
-        table.drop()
-    table.overwrite(df)
+        cdc.drop()
+
+    if not cdc.table.exists():
+        df = SPARK.sql(
+            """
+            select
+            1 as __key,
+            md5('1') as __hash,
+            cast('1900-01-01' as timestamp) as __valid_from,
+            cast('9999-12-31' as timestamp) as __valid_to
+            """
+        )
+        cdc.overwrite(df)

fabricks/{core/deploy → deploy}/udfs.py

@@ -1,11 +1,13 @@
 from fabricks.context import SPARK
 from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.core.udfs import register_all_udfs
 from fabricks.utils.sqlglot import fix as fix_sql
 
 
 def deploy_udfs():
-    DEFAULT_LOGGER.info("🌟 (create or replace udfs)")
+    DEFAULT_LOGGER.info("create or replace udfs")
 
+    register_all_udfs(extension="sql")
     create_or_replace_udf_job_id()
 
 

fabricks/deploy/utils.py

@@ -0,0 +1,36 @@
+import time
+
+
+def print_atomic_bomb(nowait: bool = False):
+    def print_and_wait(message: str):
+        if not nowait:
+            time.sleep(0.5)
+        print(message)
+
+    print("")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⡤⠤⠴⠾⠋⠉⠛⢾⡏⠙⠿⠦⠤⢤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡤⢶⣿⠉⢀⣀⡠⠆⠀⠀⠀⠀⠀⠀⠀⢤⣀⣀⠈⢹⣦⢤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠁⢋⡙⠁⠀⡝⠀⠀⠀⠀⣀⡸⠋⠁⠀⠀⠹⡀⠀⠈⠈⠆⢹⢦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⢀⣠⣤⣿⣁⡡⣴⡏⠀⠀⠀⢀⠀⢧⣀⠄⠀⠀⠀⣀⣰⠆⢀⠁⠀⠀⢈⣶⡤⣀⢹⣦⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀                  ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⣠⢴⠟⢁⡝⠀⠁⠀⠃⠉⠀⠀⠘⣯⠀⡀⠾⣤⣄⣠⢤⠾⠄⠀⣸⠖⠀⠀⠈⠀⠃⠀⠀⠹⡄⠙⣶⢤⡀⠀⠀⠀⠀⠀                 ")
+    print_and_wait("        ⠀⠀⠀⣠⠾⡇⠈⣀⡞⠀⠀⠀⠀⡀⠀⢀⣠⣄⣇⠀⣳⠴⠃⠀⠀⠀⠣⢴⠉⣰⣇⣀⣀⠀⠀⡄⠀⠀⠀⢹⣄⡘⠈⡷⣦⠀⠀⠀⠀                 ")
+    print_and_wait("        ⢠⠞⠉⢻⡄⠀⠀⠈⠙⠀⠀⠀⠀⠙⣶⣏⣤⣤⠟⠉⠁⠀⠀⠀⠀⠀⠀⠀⠉⠙⢦⣱⣌⣷⠊⠀⠀⠀⠀⠈⠁⠀⠀⠀⡝⠉⠻⣄⠀                 ")
+    print_and_wait("        ⠛⢀⡠⢼⡇⠀⠀⢀⡄⠀⢀⣀⡽⠚⠁⠀⠀⠀⢠⡀⢠⣀⠠⣔⢁⡀⠀⣄⠀⡄⠀⠀⠀⠈⠑⠺⣄⡀⠀⠠⡀⠀⠀⢠⡧⠄⠀⠘⢧                ")
+    print_and_wait("        ⡶⠋⠀⠀⠈⣠⣈⣩⠗⠒⠋⠀⠀⠀⠀⣀⣠⣆⡼⣷⣞⠛⠻⡉⠉⡟⠒⡛⣶⠧⣀⣀⣀⠀⠀⠀⠀⠈⠓⠺⢏⣉⣠⠋⠀⠀⠀⢢⣸                ")
+    print_and_wait("        ⠇⠐⠤⠤⠖⠁⣿⣀⣀⠀⠀⠀⠀⠀⠉⠁⠈⠉⠙⠛⢿⣷⡄⢣⡼⠀⣾⣿⠧⠒⠓⠚⠛⠉⠀⠀⠀⠀⠀⢀⣀⣾⡉⠓⠤⡤⠄⠸⢿                ")
+    print_and_wait("        ⣆⣤⠀⠀⠠⠀⠈⠓⠈⠓⠤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⣿⢸⠀⢸⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⢀⡤⠒⠁⠰⠃⠀⠠⠀⠀⢀⣀⠞                  ")
+    print_and_wait("        ⠀⠉⠓⢲⣄⡈⢀⣠⠀⠀⠀⡸⠶⠂⠀⠀⢀⠀⠀⠤⠞⢻⡇⠀⠀⢘⡟⠑⠤⠄⠀⢀⠀⠀⠐⠲⢿⡀⠀⠀⢤⣀⢈⣀⡴⠖⠋⠀⠀                 ")
+    print_and_wait("        ⠀⠀⠀⠀⠈⠉⠉⠙⠓⠒⣾⣁⣀⣴⠀⣀⠙⢧⠂⢀⣆⣀⣷⣤⣀⣾⣇⣀⡆⠀⢢⠛⢁⠀⢰⣀⣀⣹⠒⠒⠛⠉⠉⠉⠀⠀⠀⠀⠀                ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠁⠈⠉⠉⠛⠉⠙⠉⠀⠀⣿⡟⣿⣿⠀⠀⠈⠉⠉⠙⠋⠉⠉⠀⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⡇⢻⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                      ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣤⣶⣾⣿⣿⠁⠀⢹⡛⣟⡶⢤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣴⠛⢯⣽⡟⢿⣿⠛⠿⠳⠞⠻⣿⠻⣆⢽⠟⣶⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⠃⠲⠯⠴⣦⣼⣷⣤⣤⣶⣤⣩⡧⠽⠷⠐⠛⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⡇⠀⣿⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                     ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣄⡀⢀⣀⣠⡾⡿⢡⢐⠻⣿⣄⣀⡀⠀⣀⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                  ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣤⢴⡏⠁⠀⠝⠉⣡⠟⣰⠃⢸⣿⠀⣷⠙⢧⡉⠻⡅⠀⠙⡷⢤⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                 ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⡟⠀⠈⣿⢄⡴⠞⠻⣄⣰⣡⠤⣞⣸⡤⢬⣧⣀⡿⠛⠦⣤⣶⡃⠀⢹⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                 ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠀⢀⣴⣶⡿⠃⠉⢺⠁⠙⠒⠀⠀⣠⡉⠀⠉⠚⠉⠉⠑⠈⠀⠈⣧⠀⠀⠒⠋⠀⡹⠋⠀⢻⡶⠶⡄⠀⠀⠀⠀⠀⠀⠀                 ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⣠⣾⣿⣇⠁⢈⡦⠀⡍⠋⠁⡀⠸⡋⠀⠀⠀⢘⠏⠉⡏⠀⠀⠀⢉⡷⠀⡌⠉⠋⡇⠠⣏⠈⢁⣦⣿⣦⠀⠀⠀⠀⠀⠀                 ")
+    print_and_wait("        ⠀⠀⠀⠀⠀⠉⣁⠀⠉⠉⠉⠙⠛⠛⠒⠚⠳⠤⢼⣤⣠⠤⣮⣠⣤⣼⠦⢤⣤⣿⠤⠾⠓⠒⠛⢓⠛⠉⠉⠉⠀⠈⠉⠀⠀⠀⠀⠀⠀                ")
+    print("")

fabricks/{core/deploy → deploy}/views.py

@@ -5,24 +5,20 @@ from fabricks.utils.sqlglot import fix as fix_sql
 
 
 def deploy_views():
-    DEFAULT_LOGGER.info("🌟 (create or replace views)")
+    DEFAULT_LOGGER.info("create or replace fabricks (default) views")
 
     create_or_replace_jobs_view()
     create_or_replace_tables_view()
     create_or_replace_views_view()
-
     create_or_replace_logs_pivot_view()
     create_or_replace_last_schedule_view()
     create_or_replace_last_status_view()
     create_or_replace_previous_schedule_view()
-
     create_or_replace_schedules_view()
-
     create_or_replace_dependencies_view()
     create_or_replace_dependencies_flat_view()
     create_or_replace_dependencies_unpivot_view()
     create_or_replace_dependencies_circular_view()
-
     create_or_replace_jobs_to_be_updated_view()
 
 
@@ -73,7 +69,7 @@ def create_or_replace_jobs_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.warning(f"fabricks.{table} not found")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{table}")
 
     sql = f"""create or replace view fabricks.jobs with schema evolution as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -100,7 +96,7 @@ def create_or_replace_tables_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.warning(f"fabricks.{step}_tables not found")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_tables")
 
     sql = f"""create or replace view fabricks.tables with schema evolution as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -127,7 +123,7 @@ def create_or_replace_views_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.warning(f"fabricks.{step}_views not found")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_views")
 
     sql = f"""create or replace view fabricks.views with schema evolution as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -157,7 +153,7 @@ def create_or_replace_dependencies_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.warning(f"fabricks.{step}_dependencies not found")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_dependencies")
 
     sql = f"""create or replace view fabricks.dependencies with schema evolution  as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)

fabricks/metastore/database.py

@@ -33,17 +33,17 @@ class Database:
         return self.storage.joinpath("delta")
 
     def create(self):
-        DEFAULT_LOGGER.info("🌟 (create database)", extra={"step": self})
+        DEFAULT_LOGGER.info("create database", extra={"label": self})
         self.spark.sql(f"create database if not exists {self.name};")
 
     def drop(self, rm: Optional[bool] = True):
         if self.exists():
-            DEFAULT_LOGGER.warning("💣 (drop database)", extra={"step": self})
+            DEFAULT_LOGGER.warning("drop database", extra={"label": self})
             self.spark.sql(f"drop database if exists {self.name} cascade;")
 
         if rm:
             if self.delta_path.exists():
-                DEFAULT_LOGGER.debug("🧹 (remove delta files)", extra={"step": self})
+                DEFAULT_LOGGER.debug("remove delta files", extra={"label": self})
                 self.delta_path.rm()
 
     def exists(self) -> bool:

fabricks/metastore/dbobject.py

@@ -29,7 +29,7 @@ class DbObject:
         return f"{self.database.name}.{self.name}"
 
     @property
-    def is_registered(self) -> bool:
+    def registered(self) -> bool:
         try:
             df = self.spark.sql(f"show tables in {self.database}").where(f"tableName == '{self.name}'")
             return not df.isEmpty()
@@ -51,15 +51,15 @@ class DbObject:
         if self.is_view:
             return False
         else:
-            return self.is_registered
+            return self.registered
 
     def drop(self):
         if self.is_view:
-            DEFAULT_LOGGER.warning("drop view from metastore", extra={"job": self})
+            DEFAULT_LOGGER.warning("drop view from metastore", extra={"label": self})
             self.spark.sql(f"drop view if exists {self}")
 
         elif self.is_table:
-            DEFAULT_LOGGER.warning("drop table from metastore", extra={"job": self})
+            DEFAULT_LOGGER.warning("drop table from metastore", extra={"label": self})
             self.spark.sql(f"drop table if exists {self}")
 
     def __str__(self):