fabricks 3.0.5.2__py3-none-any.whl → 3.0.6__py3-none-any.whl

fabricks/context/__init__.py

@@ -1,3 +1,12 @@
+from fabricks.context.config import (
+    IS_DEBUGMODE,
+    IS_DEVMODE,
+    IS_JOB_CONFIG_FROM_YAML,
+    LOGLEVEL,
+    PATH_CONFIG,
+    PATH_NOTEBOOKS,
+    PATH_RUNTIME,
+)
 from fabricks.context.runtime import (
     BRONZE,
     CATALOG,
@@ -5,15 +14,12 @@ from fabricks.context.runtime import (
     FABRICKS_STORAGE,
     FABRICKS_STORAGE_CREDENTIAL,
     GOLD,
-    IS_JOB_CONFIG_FROM_YAML,
     IS_TYPE_WIDENING,
     IS_UNITY_CATALOG,
-    LOGLEVEL,
-    PATH_CONFIG,
     PATH_EXTENDERS,
+    PATH_MASKS,
     PATH_PARSERS,
     PATH_REQUIREMENTS,
-    PATH_RUNTIME,
     PATH_SCHEDULES,
     PATH_UDFS,
     PATH_VIEWS,
@@ -22,9 +28,11 @@ from fabricks.context.runtime import (
     SECRET_SCOPE,
     SILVER,
     STEPS,
+    TIMEZONE,
     VARIABLES,
 )
 from fabricks.context.spark_session import DBUTILS, SPARK, build_spark_session, init_spark_session
+from fabricks.context.utils import pprint_runtime
 
 __all__ = [
     "BRONZE",
@@ -36,12 +44,16 @@ __all__ = [
     "FABRICKS_STORAGE",
     "GOLD",
     "init_spark_session",
+    "IS_DEBUGMODE",
+    "IS_DEVMODE",
     "IS_JOB_CONFIG_FROM_YAML",
     "IS_TYPE_WIDENING",
     "IS_UNITY_CATALOG",
     "LOGLEVEL",
     "PATH_CONFIG",
     "PATH_EXTENDERS",
+    "PATH_MASKS",
+    "PATH_NOTEBOOKS",
     "PATH_PARSERS",
     "PATH_REQUIREMENTS",
     "PATH_RUNTIME",
@@ -49,10 +61,12 @@ __all__ = [
     "PATH_UDFS",
     "PATH_VIEWS",
     "PATHS_RUNTIME",
+    "pprint_runtime",
     "PATHS_STORAGE",
     "SECRET_SCOPE",
     "SILVER",
     "SPARK",
     "STEPS",
+    "TIMEZONE",
     "VARIABLES",
 ]

fabricks/context/_types.py

@@ -32,6 +32,7 @@ class RuntimeOptions(TypedDict):
     workers: int
     timeouts: RuntimeTimeoutOptions
     retention_days: int
+    timezone: Optional[str]
 
 
 class SparkOptions(TypedDict):
@@ -83,6 +84,7 @@ class TableOptions(TypedDict):
     liquid_clustering: Optional[bool]
     properties: Optional[dict[str, str]]
     retention_days: Optional[int]
+    masks: Optional[dict[str, str]]
 
 
 class Bronze(Step):

fabricks/context/config/__init__.py

@@ -0,0 +1,92 @@
+import logging
+import os
+from typing import Final
+
+from fabricks.context.config.utils import get_config_from_file
+from fabricks.utils.path import Path
+from fabricks.utils.spark import spark
+
+file_path, file_config = get_config_from_file()
+
+runtime = os.environ.get("FABRICKS_RUNTIME", "none")
+runtime = None if runtime.lower() == "none" else runtime
+if runtime is None:
+    if runtime := file_config.get("runtime"):
+        assert file_path is not None
+        runtime = file_path.joinpath(runtime)
+
+if runtime is None:
+    if file_path is not None:
+        runtime = file_path
+    else:
+        raise ValueError(
+            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
+        )
+
+path_runtime = Path(runtime, assume_git=True)
+PATH_RUNTIME: Final[Path] = path_runtime
+
+notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
+notebooks = None if notebooks.lower() == "none" else notebooks
+if notebooks is None:
+    if notebooks := file_config.get("notebooks"):
+        assert file_path is not None
+        notebooks = file_path.joinpath(notebooks)
+
+notebooks = notebooks if notebooks else path_runtime.joinpath("notebooks")
+PATH_NOTEBOOKS: Final[Path] = Path(str(notebooks), assume_git=True)
+
+is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
+if is_job_config_from_yaml is None:
+    assert file_path is not None
+    is_job_config_from_yaml = file_config.get("job_config_from_yaml")
+
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+
+is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
+if is_debugmode is None:
+    is_debugmode = file_config.get("debugmode")
+
+IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+
+is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
+if is_devmode is None:
+    is_devmode = file_config.get("devmode")
+
+IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
+
+loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
+if loglevel is None:
+    loglevel = file_config.get("loglevel")
+
+loglevel = loglevel.upper() if loglevel else "INFO"
+if loglevel == "DEBUG":
+    _loglevel = logging.DEBUG
+elif loglevel == "INFO":
+    _loglevel = logging.INFO
+elif loglevel == "WARNING":
+    _loglevel = logging.WARNING
+elif loglevel == "ERROR":
+    _loglevel = logging.ERROR
+elif loglevel == "CRITICAL":
+    _loglevel = logging.CRITICAL
+else:
+    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+
+LOGLEVEL = _loglevel
+
+path_config = os.environ.get("FABRICKS_CONFIG")
+if path_config is None:
+    if path_config := file_config.get("config"):
+        assert file_path is not None
+        path_config = file_path.joinpath(path_config)
+else:
+    path_config = PATH_RUNTIME.joinpath(path_config).string if path_config else None
+
+if not path_config:
+    path_config = PATH_RUNTIME.joinpath(
+        "fabricks",
+        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
+    ).string
+
+PATH_CONFIG: Final[Path] = Path(path_config, assume_git=True)

fabricks/context/config/utils.py

@@ -0,0 +1,53 @@
+def get_config_from_toml():
+    import os
+    import pathlib
+    import sys
+
+    if sys.version_info >= (3, 11):
+        import tomllib
+    else:
+        import tomli as tomllib  # type: ignore
+
+    path = pathlib.Path(os.getcwd())
+    while path is not None and not (path / "pyproject.toml").exists():
+        if path == path.parent:
+            break
+        path = path.parent
+
+    if (path / "pyproject.toml").exists():
+        with open((path / "pyproject.toml"), "rb") as f:
+            config = tomllib.load(f)
+            return path, config.get("tool", {}).get("fabricks", {})
+
+    return None, {}
+
+
+def get_config_from_json():
+    import json
+    import os
+    import pathlib
+
+    path = pathlib.Path(os.getcwd())
+    while path is not None and not (path / "fabricksconfig.json").exists():
+        if path == path.parent:
+            break
+        path = path.parent
+
+    if (path / "fabricksconfig.json").exists():
+        with open((path / "fabricksconfig.json"), "r") as f:
+            config = json.load(f)
+            return path, config
+
+    return None, {}
+
+
+def get_config_from_file():
+    json_path, json_config = get_config_from_json()
+    if json_config:
+        return json_path, json_config
+
+    pyproject_path, pyproject_config = get_config_from_toml()
+    if pyproject_config:
+        return pyproject_path, pyproject_config
+
+    return None, {}

fabricks/context/log.py

@@ -4,10 +4,16 @@ from typing import Final, Literal, Optional
 
 import requests
 
-from fabricks.context.runtime import IS_DEBUGMODE, LOGLEVEL, SECRET_SCOPE
+from fabricks.context import IS_DEBUGMODE, LOGLEVEL, SECRET_SCOPE, TIMEZONE
 from fabricks.utils.log import get_logger
 
-logger, _ = get_logger("logs", LOGLEVEL, table=None, debugmode=IS_DEBUGMODE)
+logger, _ = get_logger(
+    "logs",
+    LOGLEVEL,
+    table=None,
+    debugmode=IS_DEBUGMODE,
+    timezone=TIMEZONE,
+)
 logging.getLogger("SQLQueryContextLogger").setLevel(logging.CRITICAL)
 
 DEFAULT_LOGGER: Final[logging.Logger] = logger

fabricks/context/runtime.py

@@ -1,293 +1,117 @@
-import logging
-import os
 from typing import Final, List, Optional
 
 import yaml
 
+from fabricks.context.config import path_config, path_runtime
 from fabricks.utils.path import Path
-from fabricks.utils.spark import spark
 
+with open(str(path_config)) as f:
+    data = yaml.safe_load(f)
 
-def get_config_from_toml():
-    import os
-    import pathlib
-    import sys
+conf: dict = [d["conf"] for d in data][0]
+assert conf, "conf mandatory"
+CONF_RUNTIME: Final[dict] = conf
 
-    if sys.version_info >= (3, 11):
-        import tomllib
-    else:
-        import tomli as tomllib  # type: ignore
+BRONZE = CONF_RUNTIME.get("bronze", [{}])
+SILVER = CONF_RUNTIME.get("silver", [{}])
+GOLD = CONF_RUNTIME.get("gold", [{}])
+STEPS = BRONZE + SILVER + GOLD
 
-    path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "pyproject.toml").exists():
-        if path == path.parent:
-            break
-        path = path.parent
+databases = CONF_RUNTIME.get("databases", [{}])
+credentials = CONF_RUNTIME.get("credentials", {})
+variables = CONF_RUNTIME.get("variables", {})
+VARIABLES: dict = variables
 
-    if (path / "pyproject.toml").exists():
-        with open((path / "pyproject.toml"), "rb") as f:
-            config = tomllib.load(f)
-            return path, config.get("tool", {}).get("fabricks", {})
+conf_options = CONF_RUNTIME.get("options", {})
+assert conf_options, "options mandatory"
 
-    return None, {}
+IS_UNITY_CATALOG: Final[bool] = str(conf_options.get("unity_catalog", "False")).lower() in ("true", "1", "yes")
+CATALOG: Optional[str] = conf_options.get("catalog")
 
+if IS_UNITY_CATALOG and not CATALOG:
+    raise ValueError("catalog mandatory in options when unity_catalog is enabled")
 
-try:
-    pyproject_path, pyproject_config = get_config_from_toml()
+secret_scope = conf_options.get("secret_scope")
+assert secret_scope, "secret_scope mandatory in options"
+SECRET_SCOPE: Final[str] = secret_scope
 
-    runtime = os.environ.get("FABRICKS_RUNTIME", "none")
-    runtime = None if runtime.lower() == "none" else runtime
-    if runtime is None:
-        if runtime := pyproject_config.get("runtime"):
-            assert pyproject_path is not None  # Cannot be null since we got the config from it
-            runtime = pyproject_path.joinpath(runtime)  # Must resolve relative to pyproject.toml
+timezone = conf_options.get("timezone")
+TIMEZONE: Final[str] = timezone
 
-    if runtime is None and pyproject_path is not None:
-        runtime = pyproject_path
-    elif runtime is None:
-        raise ValueError("Must have at least a pyproject.toml or set FABRICKS_RUNTIME")
+IS_TYPE_WIDENING: Final[bool] = str(conf_options.get("type_widening", "True")).lower() in ("true", "1", "yes")
 
-    path_runtime = Path(runtime, assume_git=True)
-    assert path_runtime, "runtime mandatory in cluster config"
-    PATH_RUNTIME: Final[Path] = path_runtime
+path_options = CONF_RUNTIME.get("path_options", {})
+assert path_options, "options mandatory"
 
-    notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
-    notebooks = None if notebooks.lower() == "none" else notebooks
-    if notebooks is None:
-        if notebooks := pyproject_config.get("notebooks"):
-            assert pyproject_path is not None
-            notebooks = pyproject_path.joinpath(notebooks)
+fabricks_uri = path_options.get("storage")
+assert fabricks_uri, "storage mandatory in path options"
+FABRICKS_STORAGE: Final[Path] = Path.from_uri(fabricks_uri, regex=variables)
 
-    notebooks = notebooks if notebooks else path_runtime.joinpath("notebooks")
-    assert notebooks, "notebooks mandatory"
-    PATH_NOTEBOOKS: Final[Path] = Path(str(notebooks), assume_git=True)
+FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_credential")
 
-    is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
-    if is_job_config_from_yaml is None:
-        is_job_config_from_yaml = pyproject_config.get("job_config_from_yaml")
+path_udfs = path_options.get("udfs", "fabricks/udfs")
+assert path_udfs, "path to udfs mandatory"
+PATH_UDFS: Final[Path] = path_runtime.joinpath(path_udfs)
 
-    IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+path_parsers = path_options.get("parsers", "fabricks/parsers")
+assert path_parsers, "path to parsers mandatory"
+PATH_PARSERS: Final[Path] = path_runtime.joinpath(path_parsers)
 
-    is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
-    if is_debugmode is None:
-        is_debugmode = pyproject_config.get("debugmode")
+path_extenders = path_options.get("extenders", "fabricks/extenders")
+assert path_extenders, "path to extenders mandatory"
+PATH_EXTENDERS: Final[Path] = path_runtime.joinpath(path_extenders)
 
-    IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+path_views = path_options.get("views", "fabricks/views")
+assert path_views, "path to views mandatory"
+PATH_VIEWS: Final[Path] = path_runtime.joinpath(path_views)
 
-    loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
-    if loglevel is None:
-        loglevel = pyproject_config.get("loglevel")
+path_schedules = path_options.get("schedules", "fabricks/schedules")
+assert path_schedules, "path to schedules mandatory"
+PATH_SCHEDULES: Final[Path] = path_runtime.joinpath(path_schedules)
 
-    loglevel = loglevel.upper() if loglevel else "INFO"
-    if loglevel == "DEBUG":
-        _loglevel = logging.DEBUG
-    elif loglevel == "INFO":
-        _loglevel = logging.INFO
-    elif loglevel == "WARNING":
-        _loglevel = logging.WARNING
-    elif loglevel == "ERROR":
-        _loglevel = logging.ERROR
-    elif loglevel == "CRITICAL":
-        _loglevel = logging.CRITICAL
-    else:
-        raise ValueError(f"{loglevel} not allowed. Use DEBUG, INFO, WARNING, ERROR or CRITICAL")
+path_requirements = path_options.get("requirements", "fabricks/requirements")
+assert path_requirements, "path to requirements mandatory"
+PATH_REQUIREMENTS: Final[Path] = path_runtime.joinpath(path_requirements)
 
-    LOGLEVEL = _loglevel
+path_masks = path_options.get("masks", "fabricks/masks")
+assert path_masks, "path to masks mandatory"
+PATH_MASKS: Final[Path] = path_runtime.joinpath(path_masks)
 
-    config_path = os.environ.get("FABRICKS_CONFIG")
-    if config_path is None:
-        if config_path := pyproject_config.get("config"):
-            assert pyproject_path is not None  # Cannot be null since we got the config from it
-            config_path = pyproject_path.joinpath(config_path)
-    else:
-        config_path = PATH_RUNTIME.joinpath(config_path).string if config_path else None
 
-    if not config_path:
-        config_path = PATH_RUNTIME.joinpath(
-            "fabricks",
-            f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
-        ).string
-
-    PATH_CONFIG: Final[Path] = Path(config_path, assume_git=True)
-
-    with open(config_path) as f:
-        data = yaml.safe_load(f)
-
-    conf: dict = [d["conf"] for d in data][0]
-    assert conf, "conf mandatory"
-    CONF_RUNTIME: Final[dict] = conf
-
-    BRONZE = CONF_RUNTIME.get("bronze", [{}])
-    SILVER = CONF_RUNTIME.get("silver", [{}])
-    GOLD = CONF_RUNTIME.get("gold", [{}])
-    STEPS = BRONZE + SILVER + GOLD
-
-    databases = CONF_RUNTIME.get("databases", [{}])
-    credentials = CONF_RUNTIME.get("credentials", {})
-    variables = CONF_RUNTIME.get("variables", {})
-    VARIABLES: dict = variables
-
-    conf_options = CONF_RUNTIME.get("options", {})
-    assert conf_options, "options mandatory"
-
-    IS_UNITY_CATALOG: Final[bool] = str(conf_options.get("unity_catalog", "False")).lower() in ("true", "1", "yes")
-    CATALOG: Optional[str] = conf_options.get("catalog")
-
-    if IS_UNITY_CATALOG and not CATALOG:
-        raise ValueError("catalog mandatory in options when unity_catalog is enabled")
-
-    secret_scope = conf_options.get("secret_scope")
-    assert secret_scope, "secret_scope mandatory in options"
-    SECRET_SCOPE: Final[str] = secret_scope
-
-    IS_TYPE_WIDENING: Final[bool] = str(conf_options.get("type_widening", "True")).lower() in ("true", "1", "yes")
-
-    path_options = CONF_RUNTIME.get("path_options", {})
-    assert path_options, "options mandatory"
-
-    fabricks_uri = path_options.get("storage")
-    assert fabricks_uri, "storage mandatory in path options"
-    FABRICKS_STORAGE: Final[Path] = Path.from_uri(fabricks_uri, regex=variables)
-
-    FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_credential")
-
-    path_udfs = path_options.get("udfs")
-    assert path_udfs, "udfs mandatory in path options"
-    PATH_UDFS: Final[Path] = PATH_RUNTIME.joinpath(path_udfs)
-
-    path_parsers = path_options.get("parsers")
-    assert path_parsers, "parsers mandatory in path options"
-    PATH_PARSERS: Final[Path] = PATH_RUNTIME.joinpath(path_parsers)
-
-    path_extenders = path_options.get("extenders")
-    assert path_extenders, "extenders mandatory in path options"
-    PATH_EXTENDERS: Final[Path] = PATH_RUNTIME.joinpath(path_extenders)
-
-    path_views = path_options.get("views")
-    assert path_views, "views mandatory in path options"
-    PATH_VIEWS: Final[Path] = PATH_RUNTIME.joinpath(path_views)
-
-    path_schedules = path_options.get("schedules")
-    assert path_schedules, "schedules mandatory in path options"
-    PATH_SCHEDULES: Final[Path] = PATH_RUNTIME.joinpath(path_schedules)
-
-    path_requirements = path_options.get("requirements")
-    assert path_requirements, "requirements mandatory in path options"
-    PATH_REQUIREMENTS: Final[Path] = PATH_RUNTIME.joinpath(path_requirements)
-
-    def _get_storage_paths(objects: List[dict]) -> dict:
-        d = {}
-        for o in objects:
-            if o:
-                name = o.get("name")
-                assert name
-                uri = o.get("path_options", {}).get("storage")
-                assert uri
-                d[name] = Path.from_uri(uri, regex=variables)
-        return d
-
-    PATHS_STORAGE: Final[dict[str, Path]] = {
-        "fabricks": FABRICKS_STORAGE,
-        **_get_storage_paths(BRONZE),
-        **_get_storage_paths(SILVER),
-        **_get_storage_paths(GOLD),
-        **_get_storage_paths(databases),
-    }
-
-    def _get_runtime_path(objects: List[dict]) -> dict:
-        d = {}
-        for o in objects:
+def _get_storage_paths(objects: List[dict]) -> dict:
+    d = {}
+    for o in objects:
+        if o:
             name = o.get("name")
             assert name
-            uri = o.get("path_options", {}).get("runtime")
+            uri = o.get("path_options", {}).get("storage")
             assert uri
-            d[name] = PATH_RUNTIME.joinpath(uri)
-        return d
-
-    PATHS_RUNTIME: Final[dict[str, Path]] = {
-        **_get_runtime_path(BRONZE),
-        **_get_runtime_path(SILVER),
-        **_get_runtime_path(GOLD),
-    }
-
-except KeyError as e:
-    raise e
-
-except AssertionError as e:
-    raise e
-
-
-def pprint_runtime():
-    print("=" * 60)
-    print("FABRICKS RUNTIME CONFIGURATION")
-    print("=" * 60)
-
-    # Core Paths Section
-    print("\n📁 CORE PATHS:")
-    print(f"   Runtime: {PATH_RUNTIME.string}")
-    print(f"   Notebooks: {PATH_NOTEBOOKS.string}")
-    print(f"   Config: {PATH_CONFIG.string}")
-
-    # Runtime Settings Section
-    print("\n⚙️ RUNTIME SETTINGS:")
-    print(f"   Log Level: {logging.getLevelName(LOGLEVEL)}")
-    print(f"   Debug Mode: {'✓' if IS_DEBUGMODE else '✗'}")
-    print(f"   Job Config from YAML: {'✓' if IS_JOB_CONFIG_FROM_YAML else '✗'}")
-    print(f"   Type Widening: {'✓' if IS_TYPE_WIDENING else '✗'}")
-
-    print("\n🔄 PIPELINE STEPS:")
-
-    def _print_steps(steps_list, layer_name, icon):
-        if steps_list and any(step for step in steps_list if step):
-            print(f"   {icon} {layer_name}:")
-            for step in steps_list:
-                if step:
-                    step_name = step.get("name", "Unnamed")
-                    print(f"      • {step_name}")
-        else:
-            print(f"   {icon} {layer_name}: No steps")
-
-    _print_steps(BRONZE, "Bronze", "🥉")
-    _print_steps(SILVER, "Silver", "🥈")
-    _print_steps(GOLD, "Gold", "🥇")
-
-    # Storage Configuration Section
-    print("\n💾 STORAGE CONFIGURATION:")
-    print(f"   Storage URI: {FABRICKS_STORAGE.string}")
-    print(f"   Storage Credential: {FABRICKS_STORAGE_CREDENTIAL or 'Not configured'}")
-
-    # Unity Catalog Section
-    print("\n🏛️ UNITY CATALOG:")
-    print(f"   Enabled:  {'✓' if IS_UNITY_CATALOG else '✗'}")
-    if IS_UNITY_CATALOG and CATALOG:
-        print(f"   Catalog: {CATALOG}")
-
-    # Security Section
-    print("\n🔐 SECURITY:")
-    print(f"   Secret Scope: {SECRET_SCOPE}")
-
-    # Component Paths Section
-    print("\n🛠️ COMPONENT PATHS:")
-    components = [
-        ("UDFs", PATH_UDFS),
-        ("Parsers", PATH_PARSERS),
-        ("Extenders", PATH_EXTENDERS),
-        ("Views", PATH_VIEWS),
-        ("Schedules", PATH_SCHEDULES),
-        ("Requirements", PATH_REQUIREMENTS),
-    ]
-
-    for name, path in components:
-        print(f"   {name}: {path.string}")
-
-    # Storage Paths Section
-    print("\n📦 STORAGE PATHS:")
-    for name, path in sorted(PATHS_STORAGE.items()):
-        icon = "🏭" if name == "fabricks" else "📊"
-        print(f"   {icon} {name}: {path.string}")
-
-    # Runtime Paths Section
-    if PATHS_RUNTIME:
-        print("\n⚡ RUNTIME PATHS:")
-        for name, path in sorted(PATHS_RUNTIME.items()):
-            print(f"   📂 {name}: {path.string}")
+            d[name] = Path.from_uri(uri, regex=variables)
+    return d
+
+
+PATHS_STORAGE: Final[dict[str, Path]] = {
+    "fabricks": FABRICKS_STORAGE,
+    **_get_storage_paths(BRONZE),
+    **_get_storage_paths(SILVER),
+    **_get_storage_paths(GOLD),
+    **_get_storage_paths(databases),
+}
+
+
+def _get_runtime_path(objects: List[dict]) -> dict:
+    d = {}
+    for o in objects:
+        name = o.get("name")
+        assert name
+        uri = o.get("path_options", {}).get("runtime")
+        assert uri
+        d[name] = path_runtime.joinpath(uri)
+    return d
+
+
+PATHS_RUNTIME: Final[dict[str, Path]] = {
+    **_get_runtime_path(BRONZE),
+    **_get_runtime_path(SILVER),
+    **_get_runtime_path(GOLD),
+}

fabricks/context/secret.py

@@ -7,7 +7,7 @@ from typing import Optional
 
 from pyspark.sql import SparkSession
 
-from fabricks.context.runtime import IS_UNITY_CATALOG
+from fabricks.context import IS_UNITY_CATALOG
 from fabricks.utils.spark import spark as _spark
 
 

fabricks/context/spark_session.py

@@ -3,7 +3,7 @@ from typing import Optional
 from pyspark.sql import SparkSession
 from typing_extensions import deprecated
 
-from fabricks.context.runtime import CATALOG, CONF_RUNTIME, IS_UNITY_CATALOG, SECRET_SCOPE
+from fabricks.context import CATALOG, CONF_RUNTIME, IS_UNITY_CATALOG, SECRET_SCOPE
 from fabricks.context.secret import add_secret_to_spark, get_secret_from_secret_scope
 from fabricks.utils.spark import get_dbutils, get_spark