fabricks 3.0.5.2__py3-none-any.whl → 3.0.7__py3-none-any.whl

fabricks/metastore/view.py

@@ -1,13 +1,13 @@
-from typing import Optional, Union
+from typing import Any, Optional, Union
 from uuid import uuid4
 
 import pandas as pd
 from pyspark.sql import DataFrame, SparkSession
-from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
 
 from fabricks.context import SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.metastore.dbobject import DbObject
+from fabricks.utils._types import DataFrameLike
 
 
 class View(DbObject):
@@ -18,7 +18,7 @@ class View(DbObject):
         spark: Optional[SparkSession] = None,
     ) -> str:
         if spark is None:
-            if isinstance(df, (DataFrame, CDataFrame)):
+            if isinstance(df, DataFrameLike):
                 spark = df.sparkSession
             else:
                 spark = SPARK
@@ -35,12 +35,19 @@ class View(DbObject):
         return uuid
 
 
-def create_or_replace_global_temp_view(name: str, df: DataFrame, uuid: Optional[bool] = False) -> str:
+def create_or_replace_global_temp_view(
+    name: str,
+    df: DataFrame,
+    uuid: Optional[bool] = False,
+    job: Optional[Any] = None,
+) -> str:
     if uuid:
         name = f"{name}__{str(uuid4().hex)}"
 
-    job = name.split("__")[0]
-    DEFAULT_LOGGER.debug(f"create global temp view {name}", extra={"job": job})
+    if job is None:
+        job = name.split("__")[0]
+
+    DEFAULT_LOGGER.debug(f"create global temp view {name}", extra={"label": job})
     df.createOrReplaceGlobalTempView(name)
 
     return f"global_temp.{name}"

fabricks/utils/_types.py

@@ -0,0 +1,6 @@
+from typing import Union
+
+from pyspark.sql import DataFrame
+from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
+
+DataFrameLike = Union[DataFrame, ConnectDataFrame]

fabricks/utils/azure_table.py

@@ -3,9 +3,10 @@ from typing import TYPE_CHECKING, List, Optional, Union
 
 from azure.data.tables import TableClient, TableServiceClient
 from pyspark.sql import DataFrame
-from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 
+from fabricks.utils._types import DataFrameLike
+
 if TYPE_CHECKING:
     from azure.core.credentials import TokenCredential
 
@@ -121,7 +122,7 @@ class AzureTable:
                 raise e
 
     def delete(self, data: Union[List, DataFrame, dict]):
-        if isinstance(data, (DataFrame, CDataFrame)):
+        if isinstance(data, DataFrameLike):
             data = [row.asDict() for row in data.collect()]
         elif not isinstance(data, List):
             data = [data]
@@ -130,7 +131,7 @@ class AzureTable:
         self.submit(operations)
 
     def upsert(self, data: Union[List, DataFrame, dict]):
-        if isinstance(data, (DataFrame, CDataFrame)):
+        if isinstance(data, DataFrameLike):
             data = [row.asDict() for row in data.collect()]
         elif not isinstance(data, List):
             data = [data]

fabricks/utils/helpers.py

@@ -1,11 +1,12 @@
-from concurrent.futures import ThreadPoolExecutor
+import logging
 from functools import reduce
-from typing import Any, Callable, Iterable, List, Optional, Union
+from queue import Queue
+from typing import Any, Callable, Iterable, List, Literal, Optional, Union
 
 from pyspark.sql import DataFrame
-from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
 from typing_extensions import deprecated
 
+from fabricks.utils._types import DataFrameLike
 from fabricks.utils.path import Path
 from fabricks.utils.spark import spark
 
@@ -34,34 +35,163 @@ def run_threads(func: Callable, iter: Union[List, DataFrame, range, set], worker
     return run_in_parallel(func, iter, workers)
 
 
+def _process_queue_item(func: Callable, task_queue: Queue, result_queue: Queue, stop_signal: Any):
+    """Worker function that processes items from a queue."""
+    while True:
+        try:
+            item = task_queue.get(timeout=1)
+
+            if item is stop_signal:
+                task_queue.put(stop_signal)  # Put it back for other workers
+                break
+
+            result = func(item)
+            result_queue.put(result)
+        except Exception:
+            continue
+
+
+def _run_in_parallel_legacy(
+    func: Callable,
+    iterable: Union[List, DataFrame, range, set],
+    workers: int = 8,
+    progress_bar: Optional[bool] = False,
+    position: Optional[int] = None,
+) -> List[Any]:
+    from concurrent.futures import ThreadPoolExecutor
+
+    iterable = iterable.collect() if isinstance(iterable, DataFrameLike) else iterable  # type: ignore
+
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        if progress_bar:
+            from tqdm import tqdm
+
+            results = list(tqdm(executor.map(func, iterable), total=len(iterable), position=position))
+        else:
+            results = list(executor.map(func, iterable))
+
+    return results
+
+
 def run_in_parallel(
     func: Callable,
     iterable: Union[List, DataFrame, range, set],
     workers: int = 8,
     progress_bar: Optional[bool] = False,
     position: Optional[int] = None,
+    loglevel: int = logging.CRITICAL,
+    logger: Optional[logging.Logger] = None,
+    run_as: Optional[Literal["ThreadPool", "ProcessPool", "Pool", "Queue", "Legacy"]] = "Legacy",
 ) -> List[Any]:
     """
-    Runs the given function in parallel on the elements of the iterable using multiple threads.
+    Runs the given function in parallel on the elements of the iterable using multiple threads or processes.
 
     Args:
         func (Callable): The function to be executed in parallel.
         iterable (Union[List, DataFrame, range, set]): The iterable containing the elements on which the function will be executed.
-        workers (int, optional): The number of worker threads to use. Defaults to 8.
+        workers (int, optional): The number of worker threads/processes to use. Defaults to 8.
+        progress_bar (Optional[bool], optional): Whether to display a progress bar. Defaults to False.
+        position (Optional[int], optional): Position for the progress bar. Defaults to None.
+        loglevel (int, optional): Log level to set during execution. Defaults to logging.CRITICAL.
+        logger (Optional[logging.Logger], optional): Logger instance to use. Defaults to None.
+        run_as (Optional[Literal["ThreadPool", "ProcessPool", "Pool", "Queue"]], optional): Type of run as to use.
 
     Returns:
         List[Any]: A list containing the results of the function calls.
 
     """
-    iterable = iterable.collect() if isinstance(iterable, (DataFrame, CDataFrame)) else iterable  # type: ignore
+    if logger is None:
+        logger = logging.getLogger()
 
-    with ThreadPoolExecutor(max_workers=workers) as executor:
-        if progress_bar:
-            from tqdm import tqdm
+    current_loglevel = logger.getEffectiveLevel()
+    logger.setLevel(loglevel)
+
+    if run_as == "Legacy":
+        results = _run_in_parallel_legacy(
+            func=func,
+            iterable=iterable,
+            workers=workers,
+            progress_bar=progress_bar,
+            position=position,
+        )
+
+    else:
+        iterables = iterable.collect() if isinstance(iterable, DataFrameLike) else iterable  # type: ignore
+        results = []
+
+        if run_as == "Queue":
+            import threading
+
+            task_queue = Queue()
+            result_queue = Queue()
+            stop_signal = object()
+
+            for item in iterables:
+                task_queue.put(item)
+
+            task_queue.put(stop_signal)
+
+            threads = []
+            for _ in range(workers):
+                t = threading.Thread(target=_process_queue_item, args=(func, task_queue, result_queue, stop_signal))
+                t.start()
+
+                threads.append(t)
+
+            if progress_bar:
+                from tqdm import tqdm
+
+                with tqdm(total=len(iterables), position=position) as t:
+                    for _ in range(len(iterables)):
+                        result = result_queue.get()
+                        results.append(result)
+
+                        t.update()
+                        t.refresh()
+
+            else:
+                for _ in range(len(iterables)):
+                    results.append(result_queue.get())
+
+            for t in threads:
+                t.join()
+
+        elif run_as == "Pool":
+            from multiprocessing import Pool
+
+            with Pool(processes=workers) as p:
+                if progress_bar:
+                    from tqdm import tqdm
+
+                    with tqdm(total=len(iterables), position=position) as t:
+                        for result in p.map(func, iterables):
+                            results.append(result)
+
+                            t.update()
+                            t.refresh()
+
+                else:
+                    results = list(p.map(func, iterables))
 
-            results = list(tqdm(executor.map(func, iterable), total=len(iterable), position=position))
         else:
-            results = list(executor.map(func, iterable))
+            from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+
+            Executor = ProcessPoolExecutor if run_as == "ProcessPool" else ThreadPoolExecutor
+            with Executor(max_workers=workers) as exe:
+                if progress_bar:
+                    from tqdm import tqdm
+
+                    with tqdm(total=len(iterables), position=position) as t:
+                        for result in exe.map(func, iterables):
+                            results.append(result)
+
+                            t.update()
+                            t.refresh()
+
+                else:
+                    results = list(exe.map(func, iterables))
+
+    logger.setLevel(current_loglevel)
 
     return results
 

fabricks/utils/log.py

@@ -3,7 +3,9 @@ import json
 import logging
 import sys
 from datetime import datetime
+from datetime import timezone as tz
 from typing import Optional, Tuple
+from zoneinfo import ZoneInfo
 
 from pyspark.sql import DataFrame
 
@@ -11,12 +13,11 @@ from fabricks.utils.azure_table import AzureTable
 
 
 class LogFormatter(logging.Formatter):
-    def __init__(self, debugmode: Optional[bool] = False):
+    def __init__(self, debugmode: Optional[bool] = False, timezone: Optional[str] = None):
         super().__init__(fmt="%(levelname)s%(prefix)s%(message)s [%(timestamp)s]%(extra)s")
 
-        if debugmode is None:
-            debugmode = False
-        self.debugmode = debugmode
+        self.debugmode = False if debugmode is None else debugmode
+        self.timezone = ZoneInfo(timezone) if timezone else tz.utc
 
     COLORS = {
         logging.DEBUG: "\033[36m",
@@ -37,8 +38,8 @@ class LogFormatter(logging.Formatter):
         "CRITICAL": "",
     }
 
-    def formatTime(self, record):
-        ct = datetime.fromtimestamp(record.created)
+    def formatTime(self, record) -> str:
+        ct = datetime.fromtimestamp(record.created, tz=tz.utc).astimezone(self.timezone)
         s = ct.strftime("%d/%m/%y %H:%M:%S")
         return f"{self.COLORS[logging.DEBUG]}{s}{self.RESET}"
 
@@ -48,8 +49,11 @@ class LogFormatter(logging.Formatter):
         levelname_formatted = f"{self.COLORS[record.levelno]}{levelname}:{padding}{self.RESET}"
 
         prefix = ""
-        if hasattr(record, "job"):
-            prefix = f"{record.__dict__.get('job')} - "
+
+        if hasattr(record, "label"):
+            prefix = f"{record.__dict__.get('label')} - "
+        elif hasattr(record, "job"):
+            prefix = f"{record.__dict__.get('job')} - "  # keep for backward compatibility
         elif hasattr(record, "step"):
             prefix = f"{self.BRIGHT}{record.__dict__.get('step')}{self.RESET} - "
 
@@ -65,6 +69,9 @@ class LogFormatter(logging.Formatter):
             if hasattr(record, "content"):
                 extra += f"\n---\n{record.__dict__.get('content')}\n---"
 
+            if hasattr(record, "context"):
+                extra += f"\n---\n{json.dumps(record.__dict__.get('context'), indent=2, default=str)}\n---"
+
             if hasattr(record, "df"):
                 df = record.__dict__.get("df")
                 if isinstance(df, DataFrame):
@@ -79,15 +86,19 @@ class LogFormatter(logging.Formatter):
 
 
 class AzureTableLogHandler(logging.Handler):
-    def __init__(self, table: AzureTable, debugmode: Optional[bool] = False):
+    def __init__(self, table: AzureTable, debugmode: Optional[bool] = False, timezone: Optional[str] = None):
         super().__init__()
 
         self.buffer = []
         self.table = table
 
-        if debugmode is None:
-            debugmode = False
-        self.debugmode = debugmode
+        self.debugmode = False if debugmode is None else debugmode
+        self.timezone = ZoneInfo(timezone) if timezone else tz.utc
+
+    def formatTime(self, record) -> str:
+        ct = datetime.fromtimestamp(record.created, tz=tz.utc).astimezone(self.timezone)
+        s = ct.strftime("%d/%m/%y %H:%M:%S")
+        return s
 
     def emit(self, record):
         if hasattr(record, "target"):
@@ -108,9 +119,7 @@ class AzureTableLogHandler(logging.Handler):
                 level = "INFO"
 
             r = {
-                "Created": str(
-                    datetime.fromtimestamp(record.created).strftime("%d/%m/%y %H:%M:%S")
-                ),  # timestamp not present when querying Azure Table
+                "Created": self.formatTime(record),  # timestamp not present when querying Azure Table
                 "Level": level,
                 "Message": record.message,
             }
@@ -178,7 +187,8 @@ class AzureTableLogHandler(logging.Handler):
 class CustomConsoleHandler(logging.StreamHandler):
     def __init__(self, stream=None, debugmode: Optional[bool] = False):
         super().__init__(stream or sys.stderr)
-        self.debugmode = debugmode if debugmode is not None else False
+
+        self.debugmode = False if debugmode is None else debugmode
 
     def emit(self, record):
         if hasattr(record, "sql"):
@@ -193,6 +203,7 @@ def get_logger(
     level: int,
     table: Optional[AzureTable] = None,
     debugmode: Optional[bool] = False,
+    timezone: Optional[str] = None,
 ) -> Tuple[logging.Logger, Optional[AzureTableLogHandler]]:
     logger = logging.getLogger(name)
     if logger.hasHandlers():
@@ -208,12 +219,12 @@ def get_logger(
     # Console handler
     console_handler = CustomConsoleHandler(debugmode=debugmode)
     console_handler.setLevel(level)
-    console_format = LogFormatter(debugmode=debugmode)
+    console_format = LogFormatter(debugmode=debugmode, timezone=timezone)
     console_handler.setFormatter(console_format)
 
     if table is not None:
         # Azure Table handler
-        azure_table_handler = AzureTableLogHandler(table=table, debugmode=debugmode)
+        azure_table_handler = AzureTableLogHandler(table=table, debugmode=debugmode, timezone=timezone)
         azure_table_handler.setLevel(level)
     else:
         azure_table_handler = None

fabricks/utils/read/_types.py

@@ -1,3 +1,3 @@
 from typing import Literal
 
-IOModes = Literal["overwrite", "append"]
+AllowedIOModes = Literal["overwrite", "append"]

fabricks/utils/schema/get_schema_for_type.py

@@ -90,4 +90,10 @@ def get_schema_for_type(proptype: Type) -> DataType:  # type: ignore
         fields = [StructField(f.name, get_schema_for_type(f.type)) for f in dataclasses.fields(proptype)]
         return StructType(fields=fields)
 
+    if hasattr(proptype, "__origin__") and proptype.__origin__ == dict:  # noqa E721
+        if len(proptype.__args__) == 2:
+            value_type = proptype.__args__[1]
+            value_schema = get_schema_for_type(value_type)
+            return MapType(StringType(), value_schema)
+
     raise NotImplementedError()

fabricks/utils/write/delta.py

@@ -3,17 +3,17 @@ from typing import List, Optional, Union, get_args
 from pyspark.sql import DataFrame
 
 from fabricks.utils.path import Path
-from fabricks.utils.read._types import IOModes
+from fabricks.utils.read._types import AllowedIOModes
 
 
 def write_delta(
     df: DataFrame,
     path: Path,
-    mode: IOModes,
+    mode: AllowedIOModes,
     options: Optional[dict[str, str]] = None,
     partition_by: Union[Optional[List[str]], str] = None,
 ):
-    assert mode in list(get_args(IOModes))
+    assert mode in list(get_args(AllowedIOModes))
 
     if isinstance(partition_by, str):
         partition_by = [partition_by]

{fabricks-3.0.5.2.dist-info → fabricks-3.0.7.dist-info}/METADATA

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.5.2
+Version: 3.0.7
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0
 Requires-Dist: azure-identity>=1.10.0
 Requires-Dist: azure-storage-blob>=12.14.1
 Requires-Dist: azure-storage-queue<13,>=12.10.0
+Requires-Dist: databricks-cli>=0.18.0
 Requires-Dist: databricks-sdk>=0.20.0
 Requires-Dist: importlib-metadata>=8.6.1
 Requires-Dist: jinja2>=2.11.3

fabricks-3.0.7.dist-info/RECORD

@@ -0,0 +1,175 @@
+fabricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fabricks/api/__init__.py,sha256=4dnYubkrmdep64e7_X8Wsk4Kc2N487oOct8H4qPlbjs,251
+fabricks/api/context.py,sha256=tfGkdJNTJTc0meReV7y-9eL0_imF3nM2OIJOJRPBWso,513
+fabricks/api/core.py,sha256=5ctTW5N3tPnuh69OrDb7eS32Ol19dIICAWQ5VMDYY7A,157
+fabricks/api/deploy.py,sha256=4sZiYogkS3kZ2RZNxWtrB-vnI0SVoocxwwZrH4x1CZc,57
+fabricks/api/exceptions.py,sha256=F_V1hy0rnICdR_nFacRldoI3KbgA3oG02KffHWm3yjA,409
+fabricks/api/extenders.py,sha256=Qk4ZDgEkXe-dtPINmePkbtWlhkC209X6YKkqG-8lHAo,69
+fabricks/api/job_schema.py,sha256=Gtovg2RXFMpbfjD-wCc2H0GvRT1nTuMJHcuy7G42qU4,120
+fabricks/api/log.py,sha256=WKthYM8n_E1ng9x_XCVm7cJkfRvtd6Y470tgwz3vd2Y,130
+fabricks/api/masks.py,sha256=M1w5WsoNhM1Z0vIu3FSfM_VcNNPR269UuqKXvA5MbwM,117
+fabricks/api/parsers.py,sha256=nPUDzQ_Hz0fVmnBfGCqqHo7X7R6M-oGsXWDYSikjB54,121
+fabricks/api/schedules.py,sha256=gdYrujsUiCrVMMJaKMDkt7McncO-uZPl_QNgp10gZZk,215
+fabricks/api/udfs.py,sha256=3JTX4OWkoW7_AP9pUKHVS0C6zIBVdOJoAn8MpmB6R48,124
+fabricks/api/utils.py,sha256=a-YrCXkDFzMmcNN8QOSDs_-YQtSePaDP4C4WYMX2AEg,196
+fabricks/api/version.py,sha256=FukX94EbtmJMeajxyOwka8fMfFeaVc0cuM3I5CVIuK8,85
+fabricks/api/views.py,sha256=dPqsGgDs9QOYc-5_QG_i4F_VoaFO5hGZQnIPZ31h5Ps,156
+fabricks/api/cdc/__init__.py,sha256=Cl3LhLbQrA42IvNLqoV7CCbjQEYQMJfO6cAZv1l1aas,196
+fabricks/api/cdc/nocdc.py,sha256=3E1Cn6cPHfEszGMaHEknrLqEvVKS-5-hk8s_GRu6TYY,58
+fabricks/api/cdc/scd1.py,sha256=3mlWD50g-8SLMV51BLhtEz4ETcnOWgkOdaNIZCl75WY,55
+fabricks/api/cdc/scd2.py,sha256=2XNDDvY0xaaqcq8o7V3Z2L0oTEUCPNaB-MxN4YDWcvQ,55
+fabricks/api/metastore/__init__.py,sha256=ikM4mnzZRpcJ_CZcknYAq1qkWvKG2la-O453va8w1uo,255
+fabricks/api/metastore/database.py,sha256=e1MMgY312aChjhVpHDuqZatoyp4ucp0_-6HIK-ah-hI,64
+fabricks/api/metastore/table.py,sha256=2jsbBk90jzxZbHXm3MizwNGRruVrjIRMwD0_nK7WhiM,58
+fabricks/api/metastore/view.py,sha256=MBUSN0YW8CcLoVJdkV5hFmKfS8CmqEMJkbG0pdToAF8,132
+fabricks/api/notebooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fabricks/api/notebooks/cluster.py,sha256=86AX2pS98XJ9RPxPS5hHJBoQw-rlQ3i4hpRoWeg8DPc,151
+fabricks/api/notebooks/initialize.py,sha256=5M8VWECIdJgpB2Xf_RYOp90Cuil4KTCYLMUEd3gZQV8,803
+fabricks/api/notebooks/process.py,sha256=9qwIVyBgufBdVFW1yZCxxeRpwBJQsIxVBXFd2HRubIo,1157
+fabricks/api/notebooks/run.py,sha256=L7kZtti8D20jTQmAe6_a-8f92qHYFh3Td3x0qFKoFzo,1271
+fabricks/api/notebooks/schedule.py,sha256=N15KaRQWW1KEjvi_NEw4Aa_t0Vr_C5tclTDgliVopXo,1571
+fabricks/api/notebooks/terminate.py,sha256=ef6I0Pj3rlE_BlMiEXR6AkLxOuYS9tyWoj9ZJ-B0Y60,741
+fabricks/cdc/__init__.py,sha256=bkrlLlhXThIvtABuOkNYJaQudICGw2Q8TbyZUVYORaw,316
+fabricks/cdc/cdc.py,sha256=2CjPUtogWjnvyLjwiyVllcyDV1gpJ0QoRP0yUsiHXuc,69
+fabricks/cdc/nocdc.py,sha256=ObWojMINqwzNSipIKMj57wvQy3on_EIh65X-7MAspIs,490
+fabricks/cdc/scd.py,sha256=HzC9ifEu45B4P2aOSgi97AGB-C56l6sKTLqdVinnHKo,670
+fabricks/cdc/scd1.py,sha256=WsOVRsp55WEw4-7nEtb3dfv310icExrj-zEJSEehyz8,334
+fabricks/cdc/scd2.py,sha256=4vZkhc8pJAUlgiBmIw9j_2RsWuAFMcgCkU3WMVt0A-A,334
+fabricks/cdc/base/__init__.py,sha256=kU4LmQ7x1rekCt8T3r83MmAQac6n2ov-Gh8mBbxIC48,157
+fabricks/cdc/base/_types.py,sha256=WloCDC3ATrn0aZJ6E8BRYKZx19N3EE56r6qlBYhcuvQ,257
+fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
+fabricks/cdc/base/configurator.py,sha256=w6Ywif87iv1WG-5OM3XkzIRrsns-_QQ6XlADpk0YLlw,6434
+fabricks/cdc/base/generator.py,sha256=pa_GJn7Pdi5vMnXN8zExmOPMpCqdZ3QoxHEB0wv0lsk,5933
+fabricks/cdc/base/merger.py,sha256=1_j-hKnKKEcbogyXX0Cm2IdyB-tpDJAIMIz1MwWnXX0,4158
+fabricks/cdc/base/processor.py,sha256=b8ATjVX-dW8JCrt4n3v8HlteTi1498jrItEla0BqynU,17689
+fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fabricks/cdc/templates/filter.sql.jinja,sha256=AQcOj4KRwKscVG3zepnEAN9Yxb03AM7068hqW7dtVI8,236
+fabricks/cdc/templates/merge.sql.jinja,sha256=YS9wWckCVsUI1pUYiRSFBIuQ16WU3EPWSkhZVy2niBA,221
+fabricks/cdc/templates/query.sql.jinja,sha256=Z0kSm9sRKJTLQ2Lb3NS7yu93GBxNls9HL7uAjTdirjk,868
+fabricks/cdc/templates/ctes/base.sql.jinja,sha256=gO0dEQ00_NXOeZKRxfWvoUjpks0nHK-Is5H7xdI_J6s,1637
+fabricks/cdc/templates/ctes/current.sql.jinja,sha256=bb72XdaJVce5k57dx_N6T4OfyQDUrFLulr3VM6irdn0,1278
+fabricks/cdc/templates/ctes/deduplicate_hash.sql.jinja,sha256=DHcSNhAvoFZY0gSv3Zylat9ysGXKR22OGXs49KTNCjA,1095
+fabricks/cdc/templates/ctes/deduplicate_key.sql.jinja,sha256=4-K4nk2Bfr_8c_x3YDYBP7JUx4c-b3Ef-7HFx0s2jrc,1122
+fabricks/cdc/templates/ctes/rectify.sql.jinja,sha256=OfZmEmlI-0gvkD7Fa1sncWU9SJkcrR4mnlTIvX1Ysp0,4727
+fabricks/cdc/templates/ctes/slice.sql.jinja,sha256=250I2rqd4wFgmmVDEaeJ3GdKofHcRZ3eBPSo-BOxIV0,80
+fabricks/cdc/templates/filters/final.sql.jinja,sha256=x7pWNxxkw39FNt7KU5Se9_v-2tziItdL_rgzStSRv8U,173
+fabricks/cdc/templates/filters/latest.sql.jinja,sha256=UIkxx5XRVbXFVaEuPXbToMCTycsomU27g9Y16gw31Q4,584
+fabricks/cdc/templates/filters/update.sql.jinja,sha256=O_pqUljzUuyPGMtXR7Ia9dRj468h4L-oqVmwAKseL1I,1206
+fabricks/cdc/templates/macros/bactick.sql.jinja,sha256=VYRlBRRvfRID83naDuAAWp5JmR9f9-46xTW2-8lr8DY,59
+fabricks/cdc/templates/macros/hash.sql.jinja,sha256=0auoKcV9idL4Y6b4JP6ODaTFHK7L5nmY3MEL9YeKd5o,594
+fabricks/cdc/templates/merges/nocdc.sql.jinja,sha256=lA4-PXogC_u6RqummOQX_WBFCNq_GMEXdO4aL3_Eg0o,825
+fabricks/cdc/templates/merges/scd1.sql.jinja,sha256=GimfwgEobGgCzPce_FJdvQY9jmRJXFUZ4_CVhRgTTqY,1600
+fabricks/cdc/templates/merges/scd2.sql.jinja,sha256=5qBO_1lr7xa-Ep8RqvJGCRISOv4uo_tiAtVOybp4tUU,1210
+fabricks/cdc/templates/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fabricks/cdc/templates/queries/context.sql.jinja,sha256=19mAAAM9Gh2PAzuupCzc68zxEHH3spTpHZHGU9vke8o,3883
+fabricks/cdc/templates/queries/final.sql.jinja,sha256=vxH434CO5k8Ia7tugaH8LC1co7Epaj7Z1M7Y9BdqzaI,111
+fabricks/cdc/templates/queries/scd1.sql.jinja,sha256=siHULgKE3uRBGQYZFUR_eHNqFuGgO9xUCRVV2jnAXAI,3019
+fabricks/cdc/templates/queries/scd2.sql.jinja,sha256=Nn0wUs9N-_QviZqUKRWAFdD17RR3EFBTMs9BpBu6z7E,3877
+fabricks/cdc/templates/queries/nocdc/complete.sql.jinja,sha256=cVKKCSbiuuw1K7BOzfusX6KvzQNHU3YNUgoXgsu-c6k,267
+fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=SDJFo7rWiU58DqzOG-Pl76Dto82PIHWDdKlX62M1j58,1299
+fabricks/context/__init__.py,sha256=qfntJ9O6omzY_t6AhDP6Ndu9C5LMiVdWbo6ikhtoe7o,1446
+fabricks/context/_types.py,sha256=FzQJ35vp0uc6pAq18bc-VHwMVEWtd0VDdm8xQmNr2Sg,2681
+fabricks/context/log.py,sha256=CadrRf8iL6iXlGIGIhEIswa7wGqC-E-oLwWcGTyJ10s,2074
+fabricks/context/runtime.py,sha256=7pXU_5gPyvChwiyxP4ch59eAgyoNOH9jMUlGWxwXWn8,3915
+fabricks/context/secret.py,sha256=LEx7MXSFm29wvsBffNSIQ6p73wqqmFj2jmU2y64h-Fc,3191
+fabricks/context/spark_session.py,sha256=BPaxKJXHZDI5oQiOPhmua_xjXnrVgluh--AVpvUgbck,2553
+fabricks/context/utils.py,sha256=EQRscdUhdjwk2htZu8gCgNZ9PfRzzrR6e1kRrIbVlBM,2786
+fabricks/context/config/__init__.py,sha256=pFEsGXBQkX5_FP0cwQMX427j6dQuTx81NR9snMxc8cU,3127
+fabricks/context/config/utils.py,sha256=7KCTUiSbqQnDD5mbCO9_o1KbUgD-Xbei_UGgpMQi9nE,1371
+fabricks/core/__init__.py,sha256=LaqDi4xuyHAoLOvS44PQdZdRfq9SmVr7mB6BDHyxYpc,209
+fabricks/core/extenders.py,sha256=39bSm9QiW4vBAyT659joE-5p_EZiNM4gi8KA3-OgX3E,917
+fabricks/core/job_schema.py,sha256=6-70oy0ZJd3V9AiXfc0Q8b8NVEynxQza_h7mB13uB-s,853
+fabricks/core/masks.py,sha256=3UCxcCi-TgFHB7xT5ZvmEa8RMKev23X_JLE70Pr3rpY,1347
+fabricks/core/udfs.py,sha256=gu7K-ohxcO0TdgA7IjzVMOZatZQYhFTklHo60a6k_Yc,2960
+fabricks/core/views.py,sha256=52tekqeP0Xk5EPYO220YdfFbzItX6NnObROb-ye9COQ,1181
+fabricks/core/dags/__init__.py,sha256=0DUKzVcXcROvxkN19P_kaOJ7da5BAM7Vt8EGQbp2KSY,240
+fabricks/core/dags/base.py,sha256=tFj27SqeZUZ7pB_LOWkpdowZz5gj30JUANI4gWK3Pl8,3139
+fabricks/core/dags/generator.py,sha256=4fp_CRsWnl_UauM9Jx-E4UCaxnm2_Q5103J58fRws2U,4832
+fabricks/core/dags/log.py,sha256=v1xfpQGfddHDz9lflvXOWTXMde3CdERo9jzeSmNDRhY,402
+fabricks/core/dags/processor.py,sha256=dcEWk0y2fuNt7RfxkeCny5axdg5aE4UpD7QwEijcQWM,7864
+fabricks/core/dags/run.py,sha256=RIDULb9WakObSyYzmkglh8RwFRwC8-NFC-1yPDMkBC0,1074
+fabricks/core/dags/terminator.py,sha256=Y6pV2UnSyrCIx2AQWJXoHk5Roq12gZqpennHx_Lbnzs,793
+fabricks/core/dags/utils.py,sha256=4kyobLGl4tO0Flo6LxNzYjCU_G42vns1LrkxTO5_KLY,1585
+fabricks/core/jobs/__init__.py,sha256=W_1m6LoGiXBml_8cucedju_hllSSnZGKsZjyFh-2oJw,531
+fabricks/core/jobs/bronze.py,sha256=eDH2YLHbOgBoJoGZTFLJS9igqnqkeJtM56nahQK6zJ4,13815
+fabricks/core/jobs/get_job.py,sha256=35zay3Z_WoJIylzEQlGle6UvrE1EClfRbFEVGvszof0,3675
+fabricks/core/jobs/get_job_conf.py,sha256=3vAj_usCbNqejMUKOF85LPaHBYAwxgrDG7LYgY-vBUw,4812
+fabricks/core/jobs/get_job_id.py,sha256=6dLyzxGHlRvJZVJSwZkCk3iXzWkIhePC_6FhoP0gEN4,607
+fabricks/core/jobs/get_jobs.py,sha256=nJ-8DPFq1GyzWo9Mxlwq2dEeAqwg1jeQg-CHietAb1Q,3341
+fabricks/core/jobs/get_schedule.py,sha256=46pJR5LWZfuxUtLBmtB-RP6ng_W-K-ahJmD29KNmcGw,259
+fabricks/core/jobs/get_schedules.py,sha256=kryDUBrBrtAaMp8Ou5YqMOCOMKvg1GmbbOQBtiiRleM,794
+fabricks/core/jobs/gold.py,sha256=HzeuuOtsjr3lsasBJ1ODzLQzcoanhbzZiP6a9acNsnA,14503
+fabricks/core/jobs/silver.py,sha256=kdrCBfh1jkhWJUFubGUV4kxan5eRUZl-LI-iSJxyJE4,13093
+fabricks/core/jobs/base/__init__.py,sha256=_AdWtyL7yZG2TOZ9e8WyNPrOjmm6EDkI_TNym5cLDws,208
+fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbew8AU,8405
+fabricks/core/jobs/base/checker.py,sha256=Cdfh8rQYy4MvMFl0HyC3alGUWm8zrFXk08m2t2JMu6Y,5477
+fabricks/core/jobs/base/configurator.py,sha256=9G5F7Qg5FWHPbHgdh8Qxc85OoSX0rnjD4c9itwU5KKc,10415
+fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
+fabricks/core/jobs/base/generator.py,sha256=NlJgR1461rx4bOBfvpmBrS6PvNeHD6QH6FOqAZiXDvM,16987
+fabricks/core/jobs/base/invoker.py,sha256=FvjfpNqi542slxC2yLu1BIu5EklNUWySxDF8cD_SqKQ,7602
+fabricks/core/jobs/base/job.py,sha256=dWmk2PpQH2NETaaDS6KoiefRnDHfDMdCyhmogkdcSFI,93
+fabricks/core/jobs/base/processor.py,sha256=qkNiJSSLaEnivKGBcd9UZyIVFexnv-n1p_5mCZIy1rA,9076
+fabricks/core/parsers/__init__.py,sha256=TGjyUeiiTkJrAxIpu2D_c2rQcbe5YRpmBW9oh0F9AqU,322
+fabricks/core/parsers/_types.py,sha256=JC2Oh-wUvaX8SBzeuf5owPgRaj-Q3-7MXxyIYPQ7QwA,147
+fabricks/core/parsers/base.py,sha256=P8IrLQKGakwaAQ-4gf4vElVwWoSpkixYd9kNthu1VDM,3292
+fabricks/core/parsers/decorator.py,sha256=kn_Mj-JLWTFaRiciZ3KavmSUcWFPY3ve-buMruHrX_Q,307
+fabricks/core/parsers/get_parser.py,sha256=TTnVPwKqKpFu6jJJnXEuiEctWGtimk8w2p1jF2U7ibg,909
+fabricks/core/parsers/utils.py,sha256=qdn2ElpqBgDsW55-tACWZaFOT0ebrBYg2fenqSgd6YI,2456
+fabricks/core/schedules/__init__.py,sha256=bDjNMcm7itimAo4gun0W4W9bZKwZmWUjkMqAQIcqI2Y,431
+fabricks/core/schedules/diagrams.py,sha256=DoQR80DLs0CQpUzxscBeO1mWNjgx8btBWaqcj2EFOBc,1379
+fabricks/core/schedules/generate.py,sha256=aKnAe7ZCafAczLa4ka9Er_oltOxgXyNoS63_OZEktcE,623
+fabricks/core/schedules/get_schedule.py,sha256=PJcEq4enlsRJunS-MjXi-VFIczbeuBStP2giZ_-EaRc,116
+fabricks/core/schedules/get_schedules.py,sha256=b6KSl-QmiNgih2l6dESB0va9yDVxaGOJ_ZB96Wc3NC8,174
+fabricks/core/schedules/process.py,sha256=6124f7AbCSFjseXQsLIX_4kl_-tesi_CtwzYZKOMNRE,305
+fabricks/core/schedules/run.py,sha256=AcRiyUmepAUzlGO_DxO-JhRUtieXfxE6sR8xfRWYlWI,58
+fabricks/core/schedules/terminate.py,sha256=-RvtOrxTOZl2sZQ6KfNHJL6H2LCAEMSVRyylA-wVl_w,167
+fabricks/core/schedules/views.py,sha256=8hYwPLCvvN-nem2lNAKvUY5hC71v88z4-y8j0poUApM,1949
+fabricks/core/steps/__init__.py,sha256=JP-kaDa890-9XqBSPp6YdssAexdxv-MqQ__WfVYdgeg,132
+fabricks/core/steps/_types.py,sha256=VxIrH3nFwmPlwG-UI8sDDP0AwK_9jlsy6yQp6YfgtqE,90
+fabricks/core/steps/base.py,sha256=UTzVqdWaho9zgMrloq8ndqcWQ9R5Z-O1SInIrm9byB4,14403
+fabricks/core/steps/get_step.py,sha256=8q4rEDdTTZNJsXB2l5XY-Ktoow8ZHsON_tx5yKMUIzg,284
+fabricks/core/steps/get_step_conf.py,sha256=UPT3gB1Sh5yzawZ9qiVQlvVAKaxPX82gaWBDzxx75EM,633
+fabricks/deploy/__init__.py,sha256=CD--5eD60OIUwKAIiay_L82rHWJz71KWmtSvCKq5D80,2648
+fabricks/deploy/masks.py,sha256=umDRGoeyGnJ6_-Ug7926oDEtkdp9Ja5TtgznpQBUzBY,198
+fabricks/deploy/notebooks.py,sha256=pfmD1K-TCOuuxttEK45Ti1RDc-_nIOmWlb2EWr1Vp28,1545
+fabricks/deploy/schedules.py,sha256=rlT1PotBR0cig-k99Gi7a3DLjWoNujmnFPQn3iU0yLU,220
+fabricks/deploy/tables.py,sha256=IF822oxOCy12r08Dz54YUK5luud6dtTPxJ4TUIHE-No,2621
+fabricks/deploy/udfs.py,sha256=7fw3O5LgOOxDEhuS3s1yFdqybgFh65r_1IdfZUYeejs,597
+fabricks/deploy/utils.py,sha256=V41r1zVT9KcsICqTLAzpb4ixRk2q2ybJMrGhkPOtG6k,5099
+fabricks/deploy/views.py,sha256=Ket511q0v8QHgkNArb4qVPFDuuQnQn3v-fvr7KO6oxM,14360
+fabricks/metastore/README.md,sha256=utPUGAxmjyNMGe43GfL0Gup4MjeTKKwyiUoNVSfMquI,51
+fabricks/metastore/__init__.py,sha256=RhjY2CuqtZBg8fEizzzvW8qszqCM-vSCL1tQGuzoato,174
+fabricks/metastore/_types.py,sha256=NXYxwQHP0sCllM0N6QBbaK4CdtM_m_rHFDxRNRfBcLU,1919
+fabricks/metastore/database.py,sha256=23VAKKzjrwlEaj28DNNmiOhcfdKRzYk8eEfq-PzINbg,1924
+fabricks/metastore/dbobject.py,sha256=ve8p48OqEpJYsqWNhgesGSE0emM--uY8QrvBRoR3j3g,1881
+fabricks/metastore/pyproject.toml,sha256=6RZM9RMKMDF_EAequhORZ7TD0BQNk7aBCTWAv-sRcp0,519
+fabricks/metastore/table.py,sha256=mos701lEU_EwjdwwGb2ey-RSebkg8Ubms2VcOJvIARA,27937
+fabricks/metastore/utils.py,sha256=8SxhjDkz_aSH4IGUusel7hqOQxP9U8PNBCY0M7GH00Y,1355
+fabricks/metastore/view.py,sha256=f7hKJWtnH1KmZym8dkoucKOTndntzai_f2YqferxHLs,1431
+fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fabricks/utils/_types.py,sha256=AuOhknlozqx5QdAdvZSA6xAWhU8k4nxG1vxIkOVgHeY,184
+fabricks/utils/azure_queue.py,sha256=wtKAq_MD5QLxelerDO475dzL-SySIrxt9d5KGi-8vvw,3102
+fabricks/utils/azure_table.py,sha256=7tCD1iM7UWREaSQVVmtgHCAebLtWPP9ZmuU5zDALyo0,5305
+fabricks/utils/console.py,sha256=X4lLgL_UxCjoFRx-ZRCwzdBveRGPKlFYZDi6vl7uevQ,1017
+fabricks/utils/fdict.py,sha256=cdnvNBSXKJIDKSdhQGJA4CGv0qLn5IVYKQ111l7nM9I,7978
+fabricks/utils/helpers.py,sha256=h7SuOVpBP5qcgX1nM1suvkXG9BhiK5-257EBepCvrO8,7452
+fabricks/utils/log.py,sha256=LCQEM81PhdojiyLrtEzv1QM__bWbaEhGddyd0IqyGXM,7985
+fabricks/utils/path.py,sha256=Bs3PayWtg62-mrsDbvu8kh0VLZZhX7tU9YiyHFiYNhs,6698
+fabricks/utils/pip.py,sha256=UHo7NTjFGJNghWBuuDow28xUkZYg2YrlbAP49IxZyXY,1522
+fabricks/utils/pydantic.py,sha256=W0fiDLVMFrrInfQw2s5YPeSEvkN-4k864u3UyPoHaz4,2516
+fabricks/utils/spark.py,sha256=QWVpbGwOvURIVBlR7ygt6NQ9MHUsIDvlquJ65iI8UBI,2007
+fabricks/utils/sqlglot.py,sha256=Q3SOc4CJfGBIe4vr9laipJaPIknz_hnKGrJgLc7DumE,1439
+fabricks/utils/read/__init__.py,sha256=a_5l60m1AyzQUII170bfRRuXR_ynC3EwfysidRy44GE,272
+fabricks/utils/read/_types.py,sha256=_YY0Yp8IPNYvcRFNa9WMcIl20cN93Cdcgh3jddypnuk,76
+fabricks/utils/read/read.py,sha256=el6GvynJrCl2Ap_bM7xY750fSXjrr9xmp14P92h-40g,8401
+fabricks/utils/read/read_excel.py,sha256=TnirdvaVk3gFtEpTD20_U2v3KRnTAc_T85JzwqOHUwI,108
+fabricks/utils/read/read_yaml.py,sha256=PEMpMaVCfqfQ792rg1mt33WDpy2lOFR3DBLt4q7unPg,869
+fabricks/utils/schema/__init__.py,sha256=jWGhKohpxbeKE260n3GktiCnk11MnlVcbkkzjN-6ZJw,232
+fabricks/utils/schema/get_json_schema_for_type.py,sha256=ePIxRkYvsxhQiN59Dln4ONtSVd1cK_KsEoAxdCHbb5k,5846
+fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b7DN9fjrwYqmCw0Q,3716
+fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
+fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
+fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
+fabricks-3.0.7.dist-info/METADATA,sha256=mxCXO3itBIJ1MORFhE0uEBoXIlEmEN5bsvGS53_8Gfk,761
+fabricks-3.0.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.7.dist-info/RECORD,,

fabricks/api/notebooks/add_fabricks.py

@@ -1,13 +0,0 @@
-# Databricks notebook source
-import os
-import sys
-
-# COMMAND ----------
-
-root = os.path.abspath("../../..")
-
-# COMMAND ----------
-
-sys.path.append(root)
-
-# COMMAND ----------