fabricks 3.0.18__py3-none-any.whl → 4.0.0__py3-none-any.whl

fabricks/context/runtime.py

@@ -2,92 +2,58 @@ from typing import Final, Optional
 
 import yaml
 
-from fabricks.context.config import PATH_CONFIG, PATH_RUNTIME
-from fabricks.context.helpers import get_runtime_path, get_storage_paths
-from fabricks.utils.path import Path
+from fabricks.context.config import PATH_CONFIG
+from fabricks.models import Database, RuntimeConf, StepBronzeConf, StepGoldConf, StepSilverConf
+from fabricks.utils.path import FileSharePath, GitPath
 
 with open(str(PATH_CONFIG)) as f:
     data = yaml.safe_load(f)
 
-conf: dict = [d["conf"] for d in data][0]
-assert conf, "conf mandatory"
-CONF_RUNTIME: Final[dict] = conf
+conf_data = [d["conf"] for d in data][0]
+assert conf_data, "conf mandatory"
+CONF_RUNTIME: Final[RuntimeConf] = RuntimeConf.model_validate(conf_data)
 
-BRONZE = CONF_RUNTIME.get("bronze", [{}])
-SILVER = CONF_RUNTIME.get("silver", [{}])
-GOLD = CONF_RUNTIME.get("gold", [{}])
+BRONZE: list[StepBronzeConf] = CONF_RUNTIME.bronze or []
+SILVER: list[StepSilverConf] = CONF_RUNTIME.silver or []
+GOLD: list[StepGoldConf] = CONF_RUNTIME.gold or []
 STEPS = BRONZE + SILVER + GOLD
 
-databases = CONF_RUNTIME.get("databases", [{}])
-credentials = CONF_RUNTIME.get("credentials", {})
-variables = CONF_RUNTIME.get("variables", {})
+databases: list[Database] = CONF_RUNTIME.databases or []
+credentials = CONF_RUNTIME.credentials or []
+variables = CONF_RUNTIME.variables or {}
 VARIABLES: dict = variables
 
-conf_options = CONF_RUNTIME.get("options", {})
-assert conf_options, "options mandatory"
 
-IS_UNITY_CATALOG: Final[bool] = str(conf_options.get("unity_catalog", "False")).lower() in ("true", "1", "yes")
-CATALOG: Optional[str] = conf_options.get("catalog")
+IS_UNITY_CATALOG: Final[bool] = CONF_RUNTIME.options.unity_catalog or False
+CATALOG: Optional[str] = CONF_RUNTIME.options.catalog
 
 if IS_UNITY_CATALOG and not CATALOG:
-    raise ValueError("catalog mandatory in options when unity_catalog is enabled")
+    raise ValueError("catalog mandatory in options if unity catalog is enabled")
 
-secret_scope = conf_options.get("secret_scope")
-assert secret_scope, "secret_scope mandatory in options"
-SECRET_SCOPE: Final[str] = secret_scope
+SECRET_SCOPE: Final[str] = CONF_RUNTIME.options.secret_scope
 
-timezone = conf_options.get("timezone")
-TIMEZONE: Final[str] = timezone
+TIMEZONE: Final[Optional[str]] = CONF_RUNTIME.options.timezone
 
-IS_TYPE_WIDENING: Final[bool] = str(conf_options.get("type_widening", "True")).lower() in ("true", "1", "yes")
+IS_TYPE_WIDENING: Final[bool] = CONF_RUNTIME.options.type_widening or False
 
-path_options = CONF_RUNTIME.get("path_options", {})
-assert path_options, "options mandatory"
+# Resolve all paths at once
+PATHS_RESOLVED = CONF_RUNTIME.resolved_path_options
 
-fabricks_uri = path_options.get("storage")
-assert fabricks_uri, "storage mandatory in path options"
-FABRICKS_STORAGE: Final[Path] = Path.from_uri(fabricks_uri, regex=variables)
+FABRICKS_STORAGE: Final[FileSharePath] = PATHS_RESOLVED.storage
 
-FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_credential")
+FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = CONF_RUNTIME.path_options.storage_credential
 
-path_udfs = path_options.get("udfs", "fabricks/udfs")
-assert path_udfs, "path to udfs mandatory"
-PATH_UDFS: Final[Path] = PATH_RUNTIME.joinpath(path_udfs)
+PATH_UDFS: Final[GitPath] = PATHS_RESOLVED.udfs
+PATH_PARSERS: Final[GitPath] = PATHS_RESOLVED.parsers
+PATH_EXTENDERS: Final[GitPath] = PATHS_RESOLVED.extenders
+PATH_VIEWS: Final[GitPath] = PATHS_RESOLVED.views
+PATH_SCHEDULES: Final[GitPath] = PATHS_RESOLVED.schedules
+PATH_REQUIREMENTS: Final[GitPath] = PATHS_RESOLVED.requirements
+PATH_MASKS: Final[GitPath] = PATHS_RESOLVED.masks
+PATHS_STORAGE: Final[dict[str, FileSharePath]] = PATHS_RESOLVED.storages
+PATHS_RUNTIME: Final[dict[str, GitPath]] = PATHS_RESOLVED.runtimes
 
-path_parsers = path_options.get("parsers", "fabricks/parsers")
-assert path_parsers, "path to parsers mandatory"
-PATH_PARSERS: Final[Path] = PATH_RUNTIME.joinpath(path_parsers)
-
-path_extenders = path_options.get("extenders", "fabricks/extenders")
-assert path_extenders, "path to extenders mandatory"
-PATH_EXTENDERS: Final[Path] = PATH_RUNTIME.joinpath(path_extenders)
-
-path_views = path_options.get("views", "fabricks/views")
-assert path_views, "path to views mandatory"
-PATH_VIEWS: Final[Path] = PATH_RUNTIME.joinpath(path_views)
-
-path_schedules = path_options.get("schedules", "fabricks/schedules")
-assert path_schedules, "path to schedules mandatory"
-PATH_SCHEDULES: Final[Path] = PATH_RUNTIME.joinpath(path_schedules)
-
-path_requirements = path_options.get("requirements", "fabricks/requirements")
-assert path_requirements, "path to requirements mandatory"
-PATH_REQUIREMENTS: Final[Path] = PATH_RUNTIME.joinpath(path_requirements)
-
-path_masks = path_options.get("masks", "fabricks/masks")
-assert path_masks, "path to masks mandatory"
-PATH_MASKS: Final[Path] = PATH_RUNTIME.joinpath(path_masks)
-
-PATHS_STORAGE: Final[dict[str, Path]] = {
-    "fabricks": FABRICKS_STORAGE,
-    **get_storage_paths(BRONZE, variables),
-    **get_storage_paths(SILVER, variables),
-    **get_storage_paths(GOLD, variables),
-    **get_storage_paths(databases, variables),
-}
-
-PATHS_RUNTIME: Final[dict[str, Path]] = {
-    **get_runtime_path(BRONZE, PATH_RUNTIME),
-    **get_runtime_path(SILVER, PATH_RUNTIME),
-    **get_runtime_path(GOLD, PATH_RUNTIME),
-}
+Bronzes = [b.name for b in BRONZE]
+Silvers = [s.name for s in SILVER]
+Golds = [g.name for g in GOLD]
+Steps = Bronzes + Silvers + Golds

fabricks/context/spark_session.py

@@ -20,10 +20,11 @@ def add_credentials_to_spark(spark: Optional[SparkSession] = None):
     if spark is None:
         spark = get_spark()
 
-    credentials = CONF_RUNTIME.get("credentials", {})
-    for uri, secret in credentials.items():
-        s = get_secret_from_secret_scope(secret_scope=SECRET_SCOPE, name=secret)
-        add_secret_to_spark(secret=s, uri=uri, spark=spark)
+    credentials = CONF_RUNTIME.credentials or []
+    for cred in credentials:
+        for uri, secret in cred.items():
+            s = get_secret_from_secret_scope(secret_scope=SECRET_SCOPE, name=secret)
+            add_secret_to_spark(secret=s, uri=uri, spark=spark)
 
 
 def add_spark_options_to_spark(spark: Optional[SparkSession] = None):
@@ -35,13 +36,13 @@ def add_spark_options_to_spark(spark: Optional[SparkSession] = None):
     spark.sql("set spark.databricks.delta.resolveMergeUpdateStructsByName.enabled = True;")
 
     # runtime options
-    spark_options = CONF_RUNTIME.get("spark_options", {})
+    spark_options = CONF_RUNTIME.spark_options
     if spark_options:
-        sql_options = spark_options.get("sql", {})
+        sql_options = spark_options.sql or {}
         for key, value in sql_options.items():
             spark.sql(f"set {key} = {value};")
 
-        conf_options = spark_options.get("conf", {})
+        conf_options = spark_options.conf or {}
         for key, value in conf_options.items():
             spark.conf.set(key, value)
 

fabricks/context/utils.py

@@ -4,54 +4,53 @@ import fabricks.context.config as c
 import fabricks.context.runtime as r
 
 
-def pprint_runtime(extended: bool = False) -> None:
+def pprint_runtime(extended: bool = True) -> None:
     print("=" * 60)
     print("FABRICKS RUNTIME CONFIGURATION")
     print("=" * 60)
 
     # Core Paths Section
-    print("\n📁 CORE CONFIG:")
-    print(f"   Runtime: {c.PATH_RUNTIME.string}")
-    print(f"   Notebooks: {c.PATH_NOTEBOOKS.string}")
-    print(f"   Config: {c.PATH_CONFIG.string}")
-    print(f"   Log Level: {logging.getLevelName(c.LOGLEVEL)}")
-    print(f"   Debug Mode: {'✓' if c.IS_DEBUGMODE else '✗'}")
-    print(f"   Job Config from YAML: {'✓' if c.IS_JOB_CONFIG_FROM_YAML else '✗'}")
+    print("\n📁 CONFIG:")
+    print(f"    • Runtime: {c.PATH_RUNTIME.string}")
+    print(f"    • Notebooks: {c.PATH_NOTEBOOKS.string}")
+    print(f"    • Config: {c.PATH_CONFIG.string}")
+    print(f"    • Log Level: {logging.getLevelName(c.LOGLEVEL)}")
+    print(f"    • Debug Mode: {'✅' if c.IS_DEBUGMODE else '❌'}")
+    print(f"    • Job Config from YAML: {'✅' if c.IS_JOB_CONFIG_FROM_YAML else '❌'}")
 
-    print("\n⚙️ RUNTIME SETTINGS:")
-    print("\n🔄 PIPELINE STEPS:")
+    print("\n⚙️ STEPS:")
 
-    def _print_steps(steps_list, layer_name, icon):
-        if steps_list and any(step for step in steps_list if step):
-            print(f"   {icon} {layer_name}:")
-            for step in steps_list:
-                if step:
-                    step_name = step.get("name", "Unnamed")
-                    print(f"      • {step_name}")
+    def _print_steps(steps: list[r.StepBronzeConf] | list[r.StepSilverConf] | list[r.StepGoldConf], layer, icon):
+        if steps:
+            print(f"   {icon} {layer}:")
+            for step in steps:
+                print(f"      • {step.name}")
+                if extended:
+                    print(f"         - 📖 {r.PATHS_RUNTIME.get(step.name)}")
+                    print(f"         - 💾 {r.PATHS_STORAGE.get(step.name)}")
         else:
-            print(f"   {icon} {layer_name}: No steps")
+            print(f"   {icon} {layer}: No steps")
 
     _print_steps(r.BRONZE, "Bronze", "🥉")
     _print_steps(r.SILVER, "Silver", "🥈")
     _print_steps(r.GOLD, "Gold", "🥇")
 
     # Storage Configuration Section
-    print("\n💾 STORAGE CONFIGURATION:")
-    print(f"   Storage URI: {r.FABRICKS_STORAGE.string}")
-    print(f"   Storage Credential: {r.FABRICKS_STORAGE_CREDENTIAL or 'Not configured'}")
+    print("\n💾 FABRICKS STORAGE:")
+    print(f"    • Storage URI: {r.FABRICKS_STORAGE.string}")
+    print(f"    • Storage Credential: {r.FABRICKS_STORAGE_CREDENTIAL or 'Not configured'}")
 
     # Unity Catalog Section
     print("\n🏛️ UNITY CATALOG:")
-    print(f"   Enabled:  {'✓' if r.IS_UNITY_CATALOG else '✗'}")
+    print(f"    • Enabled:  {'✅' if r.IS_UNITY_CATALOG else '❌'}")
     if r.IS_UNITY_CATALOG and r.CATALOG:
-        print(f"   Catalog: {r.CATALOG}")
+        print(f"    • Catalog: {r.CATALOG}")
 
     # Security Section
     print("\n🔐 SECURITY:")
-    print(f"   Secret Scope: {r.SECRET_SCOPE}")
-
+    print(f"    • Secret Scope: {r.SECRET_SCOPE}")
     print("\n🌐 ADDITIONAL SETTINGS:")
-    print(f"   Timezone: {r.TIMEZONE}")
+    print(f"    • Timezone: {r.TIMEZONE}")
 
     if extended:
         # Component Paths Section
@@ -65,16 +64,4 @@ def pprint_runtime(extended: bool = False) -> None:
         ]
 
         for name, path in components:
-            print(f"   {name}: {path.string}")
-
-        # Storage Paths Section
-        print("\n📦 STORAGE PATHS:")
-        for name, path in sorted(r.PATHS_STORAGE.items()):
-            icon = "🏭" if name == "fabricks" else "📊"
-            print(f"   {icon} {name}: {path.string}")
-
-        # Runtime Paths Section
-        if r.PATHS_RUNTIME:
-            print("\n⚡ RUNTIME PATHS:")
-            for name, path in sorted(r.PATHS_RUNTIME.items()):
-                print(f"   📂 {name}: {path.string}")
+            print(f"    • {name}: {path.string}")

fabricks/core/__init__.py

@@ -1,4 +1,4 @@
-from fabricks.core.jobs import Bronzes, Golds, Silvers, Steps, get_job, get_jobs
+from fabricks.core.jobs import get_job, get_jobs
 from fabricks.core.steps import get_step
 
-__all__ = ["get_job", "get_jobs", "get_step", "Bronzes", "Silvers", "Golds", "Steps"]
+__all__ = ["get_job", "get_jobs", "get_step"]

fabricks/core/dags/base.py

@@ -59,10 +59,9 @@ class BaseDags:
         d = TABLE_LOG_HANDLER.table.query(q)
         df = SPARK.createDataFrame(d)
 
-        if "Exception" not in df.columns:
-            df = df.withColumn("Exception", expr("null"))
-        if "NotebookId" not in df.columns:
-            df = df.withColumn("NotebookId", expr("null"))
+        for column in ["Exception", "NotebookId", "Json"]:
+            if column not in df.columns:
+                df = df.withColumn(column, expr("null"))
 
         df = SPARK.sql(
             """
@@ -76,7 +75,8 @@ class BaseDags:
               `Level` as `level`,
               `Message` as `status`,
               to_timestamp(`Created`, 'dd/MM/yy HH:mm:ss') as `timestamp`,
-              from_json(Exception, 'type STRING, message STRING, traceback STRING') as exception
+              from_json(Exception, 'type STRING, message STRING, traceback STRING') as exception,
+              Json as json
             from
               {df}
             """,

fabricks/core/dags/processor.py

@@ -2,7 +2,7 @@ import json
 import threading
 import time
 from multiprocessing import Process
-from typing import Any, List, Union
+from typing import Any, List
 
 from azure.core.exceptions import AzureError
 from databricks.sdk.runtime import dbutils, spark
@@ -12,14 +12,13 @@ from fabricks.context import PATH_NOTEBOOKS
 from fabricks.core.dags.base import BaseDags
 from fabricks.core.dags.log import LOGGER, TABLE_LOG_HANDLER
 from fabricks.core.dags.run import run
-from fabricks.core.jobs.base._types import TStep
 from fabricks.core.steps.get_step import get_step
 from fabricks.utils.azure_queue import AzureQueue
 from fabricks.utils.azure_table import AzureTable
 
 
 class DagProcessor(BaseDags):
-    def __init__(self, schedule_id: str, schedule: str, step: Union[TStep, str], notebook: bool = True):
+    def __init__(self, schedule_id: str, schedule: str, step: str, notebook: bool = True):
         self.step = get_step(step=step)
         self.schedule = schedule
 

fabricks/core/extenders.py

@@ -12,7 +12,7 @@ def get_extender(name: str) -> Callable:
     if not IS_UNITY_CATALOG:
         assert path.exists(), "no valid extender found in {path.string}"
     else:
-        DEFAULT_LOGGER.debug(f"could not check if extender exists ({path.string})")
+        DEFAULT_LOGGER.debug(f"could not check if extender exists ({path.string})", extra={"label": "fabricks"})
 
     load_module_from_path(name, path)
     e = EXTENDERS[name]

fabricks/core/job_schema.py

@@ -1,31 +1,41 @@
-from dataclasses import dataclass
 from typing import List
 
-from fabricks.core.jobs.base._types import JobConf
-from fabricks.utils.schema import get_json_schema_for_type
+from pydantic import BaseModel
+
+from fabricks.models import JobConf
+
+
+class JobWrapper(BaseModel):
+    """Wrapper for JobConf to generate array schema."""
+
+    job: JobConf
 
 
 def get_job_schema() -> str:
     import json
 
-    @dataclass
-    class JobWrapper:
-        job: JobConf
+    # Generate JSON schema using Pydantic's built-in method
+    # Use List[JobWrapper] to create the array schema
+    from pydantic import TypeAdapter
 
-    sc = get_json_schema_for_type(List[JobWrapper])
-    defs: dict[str, dict] = sc["$defs"]
+    adapter = TypeAdapter(List[JobWrapper])
+    sc = adapter.json_schema()
+
+    # Remove properties that are not defined in YAML
+    defs: dict[str, dict] = sc.get("$defs", {})
     removals = [("Job", "job_id"), ("Job", "table")]
 
     for key, defi in defs.items():
         for ent, prop in removals:
-            if key.startswith(ent) and prop in defi["properties"]:
-                req: List[str] = defi["required"]
-                req.remove(prop)  # not defined in yaml
-                jobprops: dict = defi["properties"]
-                jobprops.pop(prop)
-
-    j = json.dumps(sc, indent=4)
-    return j
+            if key.startswith(ent) and prop in defi.get("properties", {}):
+                req: List[str] = defi.get("required", [])
+                if prop in req:
+                    req.remove(prop)  # not defined in yaml
+
+                jobprops: dict = defi.get("properties", {})
+                jobprops.pop(prop, None)
+
+    return json.dumps(sc, indent=4)
 
 
 def print_job_schema():

fabricks/core/jobs/__init__.py

@@ -1,7 +1,6 @@
-from fabricks.core.jobs.base import BaseJob, Bronzes, Golds, Silvers, Steps
+from fabricks.core.jobs.base import BaseJob
 from fabricks.core.jobs.bronze import Bronze
 from fabricks.core.jobs.get_job import get_job
-from fabricks.core.jobs.get_job_id import get_job_id
 from fabricks.core.jobs.get_jobs import get_jobs
 from fabricks.core.jobs.gold import Gold
 from fabricks.core.jobs.silver import Silver
@@ -9,13 +8,8 @@ from fabricks.core.jobs.silver import Silver
 __all__ = [
     "BaseJob",
     "Bronze",
-    "Bronzes",
-    "get_job_id",
     "get_job",
     "get_jobs",
     "Gold",
-    "Golds",
     "Silver",
-    "Silvers",
-    "Steps",
 ]