fabricks 3.0.11__py3-none-any.whl

fabricks/context/runtime.py

@@ -0,0 +1,117 @@
+from typing import Final, List, Optional
+
+import yaml
+
+from fabricks.context.config import path_config, path_runtime
+from fabricks.utils.path import Path
+
+with open(str(path_config)) as f:
+    data = yaml.safe_load(f)
+
+conf: dict = [d["conf"] for d in data][0]
+assert conf, "conf mandatory"
+CONF_RUNTIME: Final[dict] = conf
+
+BRONZE = CONF_RUNTIME.get("bronze", [{}])
+SILVER = CONF_RUNTIME.get("silver", [{}])
+GOLD = CONF_RUNTIME.get("gold", [{}])
+STEPS = BRONZE + SILVER + GOLD
+
+databases = CONF_RUNTIME.get("databases", [{}])
+credentials = CONF_RUNTIME.get("credentials", {})
+variables = CONF_RUNTIME.get("variables", {})
+VARIABLES: dict = variables
+
+conf_options = CONF_RUNTIME.get("options", {})
+assert conf_options, "options mandatory"
+
+IS_UNITY_CATALOG: Final[bool] = str(conf_options.get("unity_catalog", "False")).lower() in ("true", "1", "yes")
+CATALOG: Optional[str] = conf_options.get("catalog")
+
+if IS_UNITY_CATALOG and not CATALOG:
+    raise ValueError("catalog mandatory in options when unity_catalog is enabled")
+
+secret_scope = conf_options.get("secret_scope")
+assert secret_scope, "secret_scope mandatory in options"
+SECRET_SCOPE: Final[str] = secret_scope
+
+timezone = conf_options.get("timezone")
+TIMEZONE: Final[str] = timezone
+
+IS_TYPE_WIDENING: Final[bool] = str(conf_options.get("type_widening", "True")).lower() in ("true", "1", "yes")
+
+path_options = CONF_RUNTIME.get("path_options", {})
+assert path_options, "options mandatory"
+
+fabricks_uri = path_options.get("storage")
+assert fabricks_uri, "storage mandatory in path options"
+FABRICKS_STORAGE: Final[Path] = Path.from_uri(fabricks_uri, regex=variables)
+
+FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_credential")
+
+path_udfs = path_options.get("udfs", "fabricks/udfs")
+assert path_udfs, "path to udfs mandatory"
+PATH_UDFS: Final[Path] = path_runtime.joinpath(path_udfs)
+
+path_parsers = path_options.get("parsers", "fabricks/parsers")
+assert path_parsers, "path to parsers mandatory"
+PATH_PARSERS: Final[Path] = path_runtime.joinpath(path_parsers)
+
+path_extenders = path_options.get("extenders", "fabricks/extenders")
+assert path_extenders, "path to extenders mandatory"
+PATH_EXTENDERS: Final[Path] = path_runtime.joinpath(path_extenders)
+
+path_views = path_options.get("views", "fabricks/views")
+assert path_views, "path to views mandatory"
+PATH_VIEWS: Final[Path] = path_runtime.joinpath(path_views)
+
+path_schedules = path_options.get("schedules", "fabricks/schedules")
+assert path_schedules, "path to schedules mandatory"
+PATH_SCHEDULES: Final[Path] = path_runtime.joinpath(path_schedules)
+
+path_requirements = path_options.get("requirements", "fabricks/requirements")
+assert path_requirements, "path to requirements mandatory"
+PATH_REQUIREMENTS: Final[Path] = path_runtime.joinpath(path_requirements)
+
+path_masks = path_options.get("masks", "fabricks/masks")
+assert path_masks, "path to masks mandatory"
+PATH_MASKS: Final[Path] = path_runtime.joinpath(path_masks)
+
+
+def _get_storage_paths(objects: List[dict]) -> dict:
+    d = {}
+    for o in objects:
+        if o:
+            name = o.get("name")
+            assert name
+            uri = o.get("path_options", {}).get("storage")
+            assert uri
+            d[name] = Path.from_uri(uri, regex=variables)
+    return d
+
+
+PATHS_STORAGE: Final[dict[str, Path]] = {
+    "fabricks": FABRICKS_STORAGE,
+    **_get_storage_paths(BRONZE),
+    **_get_storage_paths(SILVER),
+    **_get_storage_paths(GOLD),
+    **_get_storage_paths(databases),
+}
+
+
+def _get_runtime_path(objects: List[dict]) -> dict:
+    d = {}
+    for o in objects:
+        name = o.get("name")
+        assert name
+        uri = o.get("path_options", {}).get("runtime")
+        assert uri
+        d[name] = path_runtime.joinpath(uri)
+    return d
+
+
+PATHS_RUNTIME: Final[dict[str, Path]] = {
+    **_get_runtime_path(BRONZE),
+    **_get_runtime_path(SILVER),
+    **_get_runtime_path(GOLD),
+}

fabricks/context/secret.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Optional
+
+from pyspark.sql import SparkSession
+
+from fabricks.context import IS_UNITY_CATALOG
+from fabricks.utils.spark import spark as _spark
+
+
+@dataclass
+class Secret:
+    pass
+
+
+@dataclass
+class ApplicationRegistration(Secret):
+    secret: str
+    application_id: str
+    directory_id: str
+
+
+@dataclass
+class AccessKey(Secret):
+    key: str
+
+
+_scopes = None
+
+
+@lru_cache(maxsize=None)
+def _get_secret_from_secret_scope(secret_scope: str, name: str) -> str:
+    from databricks.sdk.runtime import dbutils
+
+    global _scopes
+
+    if not _scopes or secret_scope not in _scopes:  # we get the scopes only once, unless you search for something new
+        _scopes = [s.name for s in dbutils.secrets.listScopes()]
+
+    assert secret_scope in _scopes, f"scope {secret_scope} not found"
+
+    return dbutils.secrets.get(scope=secret_scope, key=name)
+
+
+def get_secret_from_secret_scope(secret_scope: str, name: str) -> Secret:
+    secret = _get_secret_from_secret_scope(secret_scope=secret_scope, name=name)
+
+    if name.endswith("application-registration"):
+        s = json.loads(secret)
+        assert s.get("secret"), f"no secret found in {name}"
+        assert s.get("application_id"), f"no application_id found in {name}"
+        assert s.get("directory_id"), f"no directory_id found in {name}"
+
+        return ApplicationRegistration(
+            secret=s.get("secret"),
+            application_id=s.get("application_id"),
+            directory_id=s.get("directory_id"),
+        )
+
+    elif name.endswith("access-key"):
+        return AccessKey(key=secret)
+
+    else:
+        raise ValueError(f"{name} is not valid")
+
+
+def _add_secret_to_spark(key: str, value: str, spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = _spark
+
+    spark.conf.set(key, value)  # needed for check (invalid configuration value detected for fs.azure.account.key)
+
+    if not IS_UNITY_CATALOG:
+        spark._jsc.hadoopConfiguration().set(key, value)  # type: ignore
+
+
+def add_secret_to_spark(secret: Secret, uri: str, spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = _spark
+
+    if isinstance(secret, ApplicationRegistration):
+        _add_secret_to_spark(f"fs.azure.account.auth.type.{uri}", "OAuth", spark=spark)
+        _add_secret_to_spark(
+            f"fs.azure.account.oauth.provider.type.{uri}",
+            "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+            spark=spark,
+        )
+        _add_secret_to_spark(f"fs.azure.account.oauth2.client.id.{uri}", secret.application_id, spark=spark)
+        _add_secret_to_spark(f"fs.azure.account.oauth2.client.secret.{uri}", secret.secret, spark=spark)
+        _add_secret_to_spark(
+            f"fs.azure.account.oauth2.client.endpoint.{uri}",
+            f"https://login.microsoftonline.com/{secret.directory_id}/oauth2/token",
+            spark=spark,
+        )
+
+    elif isinstance(secret, AccessKey):
+        _add_secret_to_spark(f"fs.azure.account.key.{uri}", secret.key, spark=spark)
+
+    else:
+        raise ValueError("secret is not valid")

fabricks/context/spark_session.py

@@ -0,0 +1,82 @@
+from typing import Optional
+
+from pyspark.sql import SparkSession
+from typing_extensions import deprecated
+
+from fabricks.context import CATALOG, CONF_RUNTIME, IS_UNITY_CATALOG, SECRET_SCOPE
+from fabricks.context.secret import add_secret_to_spark, get_secret_from_secret_scope
+from fabricks.utils.spark import get_dbutils, get_spark
+
+
+def add_catalog_to_spark(spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = get_spark()
+
+    if CATALOG is not None:
+        spark.sql(f"use catalog {CATALOG};")
+
+
+def add_credentials_to_spark(spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = get_spark()
+
+    credentials = CONF_RUNTIME.get("credentials", {})
+    for uri, secret in credentials.items():
+        s = get_secret_from_secret_scope(secret_scope=SECRET_SCOPE, name=secret)
+        add_secret_to_spark(secret=s, uri=uri, spark=spark)
+
+
+def add_spark_options_to_spark(spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = get_spark()
+
+    # delta default options
+    spark.sql("set spark.databricks.delta.schema.autoMerge.enabled = True;")
+    spark.sql("set spark.databricks.delta.resolveMergeUpdateStructsByName.enabled = True;")
+
+    # runtime options
+    spark_options = CONF_RUNTIME.get("spark_options", {})
+    if spark_options:
+        sql_options = spark_options.get("sql", {})
+        for key, value in sql_options.items():
+            spark.sql(f"set {key} = {value};")
+
+        conf_options = spark_options.get("conf", {})
+        for key, value in conf_options.items():
+            spark.conf.set(key, value)
+
+
+def build_spark_session(spark: Optional[SparkSession] = None, app_name: Optional[str] = "default") -> SparkSession:
+    if app_name is None:
+        app_name = "default"
+
+    if spark is not None:
+        _spark = spark
+        _spark.builder.appName(app_name)
+
+    else:
+        _spark = (
+            SparkSession.builder.appName(app_name)  # type: ignore
+            .config("spark.driver.allowMultipleContexts", "true")
+            .enableHiveSupport()
+            .getOrCreate()
+        )
+
+    add_catalog_to_spark(spark=_spark)
+    if not IS_UNITY_CATALOG:
+        add_credentials_to_spark(spark=_spark)
+    add_spark_options_to_spark(spark=_spark)
+
+    return _spark
+
+
+@deprecated("use build_spark_session instead")
+def init_spark_session(spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = get_spark()
+
+    return build_spark_session(spark=spark)
+
+
+SPARK = build_spark_session(app_name="default")
+DBUTILS = get_dbutils(SPARK)

fabricks/context/utils.py

@@ -0,0 +1,80 @@
+import logging
+
+import fabricks.context.config as c
+import fabricks.context.runtime as r
+
+
+def pprint_runtime(extended: bool = False) -> None:
+    print("=" * 60)
+    print("FABRICKS RUNTIME CONFIGURATION")
+    print("=" * 60)
+
+    # Core Paths Section
+    print("\n📁 CORE CONFIG:")
+    print(f"   Runtime: {c.PATH_RUNTIME.string}")
+    print(f"   Notebooks: {c.PATH_NOTEBOOKS.string}")
+    print(f"   Config: {c.PATH_CONFIG.string}")
+    print(f"   Log Level: {logging.getLevelName(c.LOGLEVEL)}")
+    print(f"   Debug Mode: {'✓' if c.IS_DEBUGMODE else '✗'}")
+    print(f"   Job Config from YAML: {'✓' if c.IS_JOB_CONFIG_FROM_YAML else '✗'}")
+
+    print("\n⚙️ RUNTIME SETTINGS:")
+    print("\n🔄 PIPELINE STEPS:")
+
+    def _print_steps(steps_list, layer_name, icon):
+        if steps_list and any(step for step in steps_list if step):
+            print(f"   {icon} {layer_name}:")
+            for step in steps_list:
+                if step:
+                    step_name = step.get("name", "Unnamed")
+                    print(f"      • {step_name}")
+        else:
+            print(f"   {icon} {layer_name}: No steps")
+
+    _print_steps(r.BRONZE, "Bronze", "🥉")
+    _print_steps(r.SILVER, "Silver", "🥈")
+    _print_steps(r.GOLD, "Gold", "🥇")
+
+    # Storage Configuration Section
+    print("\n💾 STORAGE CONFIGURATION:")
+    print(f"   Storage URI: {r.FABRICKS_STORAGE.string}")
+    print(f"   Storage Credential: {r.FABRICKS_STORAGE_CREDENTIAL or 'Not configured'}")
+
+    # Unity Catalog Section
+    print("\n🏛️ UNITY CATALOG:")
+    print(f"   Enabled:  {'✓' if r.IS_UNITY_CATALOG else '✗'}")
+    if r.IS_UNITY_CATALOG and r.CATALOG:
+        print(f"   Catalog: {r.CATALOG}")
+
+    # Security Section
+    print("\n🔐 SECURITY:")
+    print(f"   Secret Scope: {r.SECRET_SCOPE}")
+
+    print("\n🌐 ADDITIONAL SETTINGS:")
+    print(f"   Timezone: {r.TIMEZONE}")
+
+    if extended:
+        # Component Paths Section
+        print("\n🛠️ COMPONENT PATHS:")
+        components = [
+            ("UDFs", r.PATH_UDFS),
+            ("Parsers", r.PATH_PARSERS),
+            ("Extenders", r.PATH_EXTENDERS),
+            ("Views", r.PATH_VIEWS),
+            ("Schedules", r.PATH_SCHEDULES),
+        ]
+
+        for name, path in components:
+            print(f"   {name}: {path.string}")
+
+        # Storage Paths Section
+        print("\n📦 STORAGE PATHS:")
+        for name, path in sorted(r.PATHS_STORAGE.items()):
+            icon = "🏭" if name == "fabricks" else "📊"
+            print(f"   {icon} {name}: {path.string}")
+
+        # Runtime Paths Section
+        if r.PATHS_RUNTIME:
+            print("\n⚡ RUNTIME PATHS:")
+            for name, path in sorted(r.PATHS_RUNTIME.items()):
+                print(f"   📂 {name}: {path.string}")

fabricks/core/__init__.py

@@ -0,0 +1,4 @@
+from fabricks.core.jobs import Bronzes, Golds, Silvers, Steps, get_job, get_jobs
+from fabricks.core.steps import get_step
+
+__all__ = ["get_job", "get_jobs", "get_step", "Bronzes", "Silvers", "Golds", "Steps"]

fabricks/core/dags/__init__.py

@@ -0,0 +1,9 @@
+from fabricks.core.dags.generator import DagGenerator
+from fabricks.core.dags.processor import DagProcessor
+from fabricks.core.dags.terminator import DagTerminator
+
+__all__ = [
+    "DagGenerator",
+    "DagProcessor",
+    "DagTerminator",
+]

fabricks/core/dags/base.py

@@ -0,0 +1,99 @@
+import re
+from typing import Optional
+
+from azure.core.exceptions import AzureError
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import expr
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from fabricks.context import FABRICKS_STORAGE, SPARK
+from fabricks.core.dags.log import TABLE_LOG_HANDLER
+from fabricks.core.dags.utils import get_connection_info
+from fabricks.metastore.table import Table
+from fabricks.utils.azure_table import AzureTable
+
+
+class BaseDags:
+    def __init__(self, schedule_id: str):
+        self.schedule_id = schedule_id
+        self._connection_info = None
+        self._table = None
+
+    @property
+    def storage_account(self) -> str:
+        return FABRICKS_STORAGE.get_storage_account()
+
+    def get_connection_info(self) -> dict:
+        if not self._connection_info:
+            self._connection_info = get_connection_info(self.storage_account)
+        return self._connection_info
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        retry=retry_if_exception_type((Exception, AzureError)),
+        reraise=True,
+    )
+    def get_table(self) -> AzureTable:
+        if not self._table:
+            cs = self.get_connection_info()
+            self._table = AzureTable(f"t{self.schedule_id}", **dict(cs))  # type: ignore
+
+        if self._table is None:
+            raise ValueError("Azure table for logs not found")
+
+        return self._table
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        if self._table is not None:
+            self._table.__exit__()
+
+    def get_logs(self, step: Optional[str] = None) -> DataFrame:
+        q = f"PartitionKey eq '{self.schedule_id}'"
+        if step:
+            q += f" and Step eq '{step}'"
+
+        d = TABLE_LOG_HANDLER.table.query(q)
+        df = SPARK.createDataFrame(d)
+
+        if "Exception" not in df.columns:
+            df = df.withColumn("Exception", expr("null"))
+        if "NotebookId" not in df.columns:
+            df = df.withColumn("NotebookId", expr("null"))
+
+        df = SPARK.sql(
+            """
+            select
+              ScheduleId as schedule_id,
+              Schedule as schedule,
+              Step as step,
+              JobId as job_id,
+              Job as job,
+              NotebookId as notebook_id,
+              `Level` as `level`,
+              `Message` as `status`,
+              to_timestamp(`Created`, 'dd/MM/yy HH:mm:ss') as `timestamp`,
+              from_json(Exception, 'type STRING, message STRING, traceback STRING') as exception
+            from
+              {df}
+            """,
+            df=df,
+        )
+
+        return df
+
+    def write_logs(self, df: DataFrame):
+        (
+            df.write.format("delta")
+            .mode("overwrite")
+            .option("mergeSchema", "true")
+            .option("partitionOverwriteMode", "dynamic")
+            .save(Table("fabricks", "logs").delta_path.string)
+        )
+
+    def remove_invalid_characters(self, s: str) -> str:
+        out = re.sub("[^a-zA-Z0-9]", "", s)
+        return out

fabricks/core/dags/generator.py

@@ -0,0 +1,157 @@
+import time
+from typing import Optional, Tuple
+from uuid import uuid4
+
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import lit
+
+from fabricks.context import SPARK
+from fabricks.core.dags.base import BaseDags
+from fabricks.core.dags.log import TABLE_LOG_HANDLER
+from fabricks.utils.azure_queue import AzureQueue
+
+
+class DagGenerator(BaseDags):
+    def __init__(self, schedule: str):
+        self.schedule = schedule
+        schedule_id = str(uuid4().hex)
+        super().__init__(schedule_id=schedule_id)
+
+    def get_jobs(self) -> DataFrame:
+        return SPARK.sql(
+            f"""
+            with logs as (
+              select 
+                l.job_id,
+                median(l.duration) as median_duration
+              from
+                fabricks.logs_pivot l
+              where
+                true
+                and duration is not null
+                and date_diff(day, l.start_time , current_date) < 10
+              group by
+                l.job_id
+            )
+            select
+              'statuses' as PartitionKey,
+              '{self.schedule_id}' as ScheduleId,
+              '{self.schedule}' as Schedule,
+              j.job_id::string as RowKey,
+              j.step as Step,
+              j.job_id as JobId,
+              j.job as Job,
+              'scheduled' as `Status`,
+              max(median_duration) as `MedianDuration`, 
+              dense_rank() over (order by max(median_duration) desc) as Rank
+            from
+              fabricks.jobs j
+              inner join fabricks.{self.schedule}_schedule v on j.job_id = v.job_id
+              left join logs l on j.job_id = l.job_id
+            group by all
+            """
+        )
+
+    def get_dependencies(self, job_df: Optional[DataFrame] = None) -> DataFrame:
+        if job_df is None:
+            job_df = self.get_jobs()
+
+        df = SPARK.sql(
+            """
+            select
+              'dependencies' as PartitionKey, 
+              d.dependency_id :: string as RowKey,
+              d.dependency_id as DependencyId,
+              j.Step as Step,
+              j.Job as Job,
+              j.JobId as JobId,
+              p.Step as ParentStep,
+              p.Job as Parent,
+              p.JobId as ParentId
+            from
+              fabricks.dependencies d
+              inner join {job} j on d.job_id = j.JobId
+              inner join {job} p on d.parent_id = p.JobId
+            where
+              true
+              and d.parent_id is not null
+              and not d.job_id = d.parent_id
+              and not exists (
+                select 1
+                from 
+                  fabricks.dependencies_circular dc
+                where
+                  true 
+                  and d.job_id = dc.job_id 
+                  and d.parent_id = dc.parent_id
+
+              )
+            group by all
+            """,
+            job=job_df,
+        )
+        df = df.withColumn("ScheduleId", lit(self.schedule_id))
+        return df.withColumn("Schedule", lit(self.schedule))
+
+    def get_steps(self, job_df: Optional[DataFrame] = None) -> DataFrame:
+        if job_df is None:
+            job_df = self.get_jobs()
+
+        return SPARK.sql(
+            """
+            select
+              Step
+            from
+              {job}
+            group by
+              Step
+            """,
+            job=job_df,
+        )
+
+    def generate(self) -> Tuple[str, DataFrame, DataFrame]:
+        job_df = self.get_jobs()
+        deps_df = self.get_dependencies(job_df)
+        step_df = self.get_steps(job_df)
+
+        table = self.get_table()
+
+        table.create_if_not_exists()
+        table.truncate_all_partitions()
+
+        table.upsert(job_df)
+        table.upsert(deps_df)
+
+        df = SPARK.sql(
+            """
+            select
+              ScheduleId as PartitionKey,
+              ScheduleId,
+              `Schedule`,
+              Step,
+              Job,
+              JobId,
+              date_format(current_timestamp(), 'dd/MM/yy HH:mm:ss') as Created,
+              'INFO' as `Level`,
+              `Status` as `Message`,
+              from_json(null, 'type STRING, message STRING, traceback STRING') as Exception,
+              md5(array_join(array(ScheduleId, `Schedule`, Step, Job, JobId, Created,  `Level`, `Message`, '-1'), "*")) as RowKey
+            from
+              {df}        
+            """,
+            df=job_df,
+        )
+
+        TABLE_LOG_HANDLER.table.upsert(df)
+
+        cs = self.get_connection_info()
+        for row in step_df.collect():
+            step = self.remove_invalid_characters(row.Step)
+
+            with AzureQueue(f"q{step}{self.schedule_id}", **dict(cs)) as queue:  # type: ignore
+                queue.create_if_not_exists()
+                queue.clear()
+
+        time.sleep(60)
+
+        return self.schedule_id, job_df, deps_df

fabricks/core/dags/log.py

@@ -0,0 +1,12 @@
+import logging
+from typing import Final
+
+from fabricks.core.dags.utils import get_table
+from fabricks.utils.log import AzureTableLogHandler, get_logger
+
+table = get_table()
+Logger, TableLogHandler = get_logger("dags", logging.INFO, table=table, debugmode=False)
+
+LOGGER: Final[logging.Logger] = Logger
+assert TableLogHandler is not None
+TABLE_LOG_HANDLER: Final[AzureTableLogHandler] = TableLogHandler