fabricks 3.0.19__py3-none-any.whl → 4.0.1__py3-none-any.whl

fabricks/cdc/templates/queries/context.sql.jinja

@@ -1,189 +1,197 @@
 /*
-
-⚙️ BASE
+{%- if cdc or mode %}
+⚙️ Base Configuration
 {%- if cdc %}
-☐ cdc: {{ cdc }}
+  ▸ CDC Type: {{ cdc }}
 {%- endif %}
 {%- if mode %}
-☐ mode: {{ mode }}
+  ▸ Mode: {{ mode }}
 {%- endif %}
-
-🎯 SOURCE & TARTGET
+{%- endif %}
+{%- if format or src or tgt %}
+🎯 Source & Target
 {%- if format %}
-☐ format: {{ format }}
+  ▸ Format: {{ format }}
 {%- endif %}
 {%- if src %}
-☐ src: {{ src | truncate(100, killwords=True) }}
+  ▸ Source: {{ src | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if tgt %}
-☐ tgt: {{ tgt }}
-{%- endif %}
-
-📊 CTE's
-{%- if slice %}
-🗹 slice?
+  ▸ Target: {{ tgt }}
 {%- endif %}
-{%- if deduplicate %}
-🗹 deduplicate?
 {%- endif %}
-{%- if advanced_deduplication %}
-🗹 advanced deduplication?
+{%- if slice or deduplicate_key or deduplicate_hash or rectify or correct_valid_from or advanced_deduplication or order_duplicate_by %}
+🔄 Applied Transformations
+{%- if slice %}
+  ▸ Slice
 {%- endif %}
 {%- if deduplicate_key %}
-🗹 deduplicate key?
+  ▸ Deduplicate Key
 {%- endif %}
 {%- if deduplicate_hash %}
-🗹 deduplicate hash?
-{%- endif %}
-{%- if order_duplicate_by %}
-🗹 order duplicate by?
+  ▸ Deduplicate Hash
 {%- endif %}
 {%- if rectify %}
-🗹 rectify?
+  ▸ Rectify
 {%- endif %}
 {%- if correct_valid_from %}
-🗹 correct valid from?
+  ▸ Correct Valid From
+{%- endif %}
+{%- if advanced_deduplication %}
+  ▸ Advanced Deduplication
 {%- endif %}
-
-🔪 FILTERING
+{%- if order_duplicate_by %}
+  ▸ Order Duplicate By
+{%- endif %}
+{%- endif %}
+{%- if filter_where or update_where or slices or sources %}
+🔍 Filtering & Slicing
 {%- if filter_where %}
-☐ filter where: {{ filter_where }}
+  ▸ Filter Where: {{ filter_where }}
 {%- endif %}
 {%- if update_where %}
-☐ update where: {{ update_where }}
+  ▸ Update Where: {{ update_where }}
 {%- endif %}
 {%- if slices %}
-☐ slices: {{ slices }}
+  ▸ Slices: {{ slices }}
 {%- endif %}
 {%- if sources %}
-☐ sources: {{ sources }}
+  ▸ Sources: {{ sources }}
 {%- endif %}
-
-🗑️ DELETES
+{%- endif %}
+{%- if delete_missing or soft_delete %}
+🗑️ Deletes
 {%- if delete_missing %}
-🗹 delete missing?
+  ▸ Delete Missing ✓
 {%- endif %}
 {%- if soft_delete %}
-🗹 soft delete?
+  ▸ Soft Delete ✓
+{%- endif %}
 {%- endif %}
-
-✅ DATA VALIDATION
-{%- if has_no_data %}
-☒ has_data?
-{%- else %}
-🗹 has data?
+{%- if has_no_data is defined or has_rows is defined or has_source %}
+✅ Current State
+{%- if has_no_data is defined %}
+  ▸ Data: {% if has_no_data %}✗{% else %}✓{% endif %}
 {%- endif %}
-{%- if has_rows %}
-🗹 has rows?
-{%- else %}
-☒ has rows?
+{%- if has_rows is defined %}
+  ▸ Rows: {% if has_rows %}✓{% else %}✗{% endif %}
 {%- endif %}
 {%- if has_source %}
-🗹 has source?
+  ▸ Source: ✓
 {%- endif %}
-
-🏷️ HAS FIELDS
+{%- endif %}
+{%- if has_metadata or has_timestamp or has_last_updated or has_key or has_hash or has_operation or has_identity or has_rescued_data or has_order_by %}
+🏷️ Found System Fields
 {%- if has_metadata %}
-🗹 has metadata?
+  ▸ __metadata
 {%- endif %}
 {%- if has_timestamp %}
-🗹 has timestamp?
+  ▸ __timestamp
 {%- endif %}
-{%- if has_identity %}
-🗹 has identity?
+{%- if has_last_updated %}
+  ▸ __last_updated
 {%- endif %}
 {%- if has_key %}
-🗹 has key?
+  ▸ __key
 {%- endif %}
 {%- if has_hash %}
-🗹 has hash?
+  ▸ __hash
 {%- endif %}
 {%- if has_operation %}
-🗹 has operation?
+  ▸ __operation
 {%- endif %}
-{%- if has_order_by %}
-🗹 has order by?
+{%- if has_identity %}
+  ▸ __identity
 {%- endif %}
 {%- if has_rescued_data %}
-🗹 has rescued data?
+  ▸ __rescued_data
 {%- endif %}
-
-➕ ADD COLUMNS
+{%- if has_order_by %}
+  ▸ Order By
+{%- endif %}
+{%- endif %}
+{%- if add_metadata or add_timestamp or add_last_updated or add_key or add_hash or add_operation or add_source or add_calculated_columns %}
+➕ Added System Fields
 {%- if add_metadata %}
-🗹 add metadata?
+  ▸ __metadata
 {%- endif %}
 {%- if add_timestamp %}
-🗹 add timestamp?
+  ▸ __timestamp
+{%- endif %}
+{%- if add_last_updated %}
+  ▸ __last_updated
 {%- endif %}
 {%- if add_key %}
-🗹 add key?
+  ▸ __key
 {%- endif %}
 {%- if add_hash %}
-🗹 add hash?
+  ▸ __hash
 {%- endif %}
 {%- if add_operation %}
-☐ add_operation: {{ add_operation }}
+  ▸ __operation = {{ add_operation }}
 {%- endif %}
 {%- if add_source %}
-☐ add_source: {{ add_source }}
+  ▸ __source = {{ add_source }}
 {%- endif %}
 {%- if add_calculated_columns %}
-☐ add_calculated_columns: {{ add_calculated_columns }}
+  ▸ Calculated = {{ add_calculated_columns }}
+{%- endif %}
 {%- endif %}
-
-🔄 EXTRA COLUMN OPERATIONs
+{%- if all_except or all_overwrite or overwrite or cast %}
+🔧 Column Transformations
 {%- if all_except %}
-☐ all_except: {{ all_except | join(", ") | truncate(100, killwords=True) }}
+  ▸ Exclude All Except: {{ all_except | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if all_overwrite %}
-☐ all_overwrite: {{ all_overwrite | join(", ") | truncate(100, killwords=True) }}
+  ▸ Overwrite All: {{ all_overwrite | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if overwrite %}
-☐ overwrite: {{ overwrite | join(", ") | truncate(100, killwords=True) }}
+  ▸ Overwrite: {{ overwrite | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if cast %}
-☐ cast: {{ cast | join(", ") | truncate(100, killwords=True) }}
+  ▸ Cast: {{ cast | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
-
-👨‍👩‍👧 PARENTS
-{%- if parent_slice %}
-☐ parent_slice: {{ parent_slice }}
 {%- endif %}
-{%- if parent_rectify %}
-☐ parent_rectify: {{ parent_rectify }}
+{%- if parent_slice or parent_deduplicate_key or parent_deduplicate_hash or parent_rectify or parent_cdc or parent_final %}
+🔗 CTE Lineage
+{%- if parent_slice %}
+  ▸ parent_slice → {{ parent_slice }}
 {%- endif %}
 {%- if parent_deduplicate_key %}
-☐ parent_deduplicate_key: {{ parent_deduplicate_key }}
+  ▸ parent_deduplicate_key → {{ parent_deduplicate_key }}
 {%- endif %}
 {%- if parent_deduplicate_hash %}
-☐ parent_deduplicate_hash: {{ parent_deduplicate_hash }}
+  ▸ parent_deduplicate_hash → {{ parent_deduplicate_hash }}
+{%- endif %}
+{%- if parent_rectify %}
+  ▸ parent_rectify → {{ parent_rectify }}
 {%- endif %}
 {%- if parent_cdc %}
-☐ parent_cdc: {{ parent_cdc }}
+  ▸ parent_cdc → {{ parent_cdc }}
 {%- endif %}
 {%- if parent_final %}
-☐ parent_final: {{ parent_final }}
+  ▸ parent_final → {{ parent_final }}
 {%- endif %}
-
-📦 LAYOUT
-{%- if columns %}
-☐ columns: {{ columns | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if keys or hashes or inputs or intermediates or outputs or columns %}
+📦 Columns
+{%- if keys %}
+  ▸ Keys: {{ keys | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if hashes %}
+  ▸ Hashes: {{ hashes | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if inputs %}
-☐ inputs: {{ inputs | join(", ") | truncate(100, killwords=True) }}
+  ▸ Inputs: {{ inputs | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if intermediates %}
-☐ intermediates: {{ intermediates | join(", ") | truncate(100, killwords=True) }}
+  ▸ Intermediates: {{ intermediates | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
 {%- if outputs %}
-☐ outputs: {{ outputs | join(", ") | truncate(100, killwords=True) }}
+  ▸ Outputs: {{ outputs | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
-{%- if keys %}
-☐ keys: {{ keys | join(", ") | truncate(100, killwords=True) }}
+{%- if columns %}
+  ▸ All Columns: {{ columns | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
-{%- if hashes %}
-☐ hashes: {{ hashes | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
-
 */
-

fabricks/cdc/templates/query.sql.jinja

@@ -1,4 +1,4 @@
-{% include 'queries/context.sql.jinja' %}
+{% if debugmode %} {% include 'queries/context.sql.jinja' %} {% endif %}
 {% include 'ctes/base.sql.jinja' %}
 {% if slice %} {% include 'ctes/slice.sql.jinja' %} {% endif %}
 {% if deduplicate_key %} {% include 'ctes/deduplicate_key.sql.jinja' %} {% endif %}

fabricks/context/__init__.py

@@ -1,6 +1,8 @@
 from fabricks.context.config import (
+    CONFIG,
     IS_DEBUGMODE,
     IS_DEVMODE,
+    IS_FUNMODE,
     IS_JOB_CONFIG_FROM_YAML,
     LOGLEVEL,
     PATH_CONFIG,
@@ -30,22 +32,30 @@ from fabricks.context.runtime import (
     STEPS,
     TIMEZONE,
     VARIABLES,
+    Bronzes,
+    Golds,
+    Silvers,
+    Steps,
 )
 from fabricks.context.spark_session import DBUTILS, SPARK, build_spark_session, init_spark_session
 from fabricks.context.utils import pprint_runtime
 
 __all__ = [
     "BRONZE",
+    "Bronzes",
     "build_spark_session",
     "CATALOG",
     "CONF_RUNTIME",
+    "CONFIG",
     "DBUTILS",
     "FABRICKS_STORAGE_CREDENTIAL",
     "FABRICKS_STORAGE",
     "GOLD",
+    "Golds",
     "init_spark_session",
     "IS_DEBUGMODE",
     "IS_DEVMODE",
+    "IS_FUNMODE",
     "IS_JOB_CONFIG_FROM_YAML",
     "IS_TYPE_WIDENING",
     "IS_UNITY_CATALOG",
@@ -61,11 +71,13 @@ __all__ = [
     "PATH_UDFS",
     "PATH_VIEWS",
     "PATHS_RUNTIME",
-    "pprint_runtime",
     "PATHS_STORAGE",
+    "pprint_runtime",
     "SECRET_SCOPE",
     "SILVER",
+    "Silvers",
     "SPARK",
+    "Steps",
     "STEPS",
     "TIMEZONE",
     "VARIABLES",

fabricks/context/config.py

@@ -1,127 +1,18 @@
-import logging
-import os
 from typing import Final
 
-from fabricks.context.helpers import get_config_from_file
-from fabricks.utils.log import get_logger
-from fabricks.utils.path import Path
-from fabricks.utils.spark import spark
+from fabricks.models.config import ConfigOptions
+from fabricks.utils.path import GitPath
 
-logger, _ = get_logger("logs", level=logging.DEBUG)
-file_path, file_config, origin = get_config_from_file()
-
-if file_path:
-    logger.debug(f"found {origin} config ({file_path})", extra={"label": "config"})
-
-# path to runtime
-runtime = os.environ.get("FABRICKS_RUNTIME", "none")
-runtime = None if runtime.lower() == "none" else runtime
-if runtime is None:
-    if runtime := file_config.get("runtime"):
-        assert file_path is not None
-        runtime = file_path.joinpath(runtime)
-        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve runtime from env", extra={"label": "config"})
-
-if runtime is None:
-    if file_path is not None:
-        runtime = file_path
-        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
-    else:
-        raise ValueError(
-            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
-        )
-
-path_runtime = Path(runtime, assume_git=True)
-
-# path to config
-config = os.environ.get("FABRICKS_CONFIG")
-if config is None:
-    if config := file_config.get("config"):
-        assert file_path is not None
-        config = file_path.joinpath(config)
-        logger.debug(f"resolve config from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve config from env", extra={"label": "config"})
-
-if config is None:
-    logger.debug("resolve config from default path", extra={"label": "config"})
-    config = path_runtime.joinpath(
-        "fabricks",
-        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
-    ).string
-
-path_config = Path(config, assume_git=True)
-
-# path to notebooks
-notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
-notebooks = None if notebooks.lower() == "none" else notebooks
-if notebooks is None:
-    if notebooks := file_config.get("notebooks"):
-        assert file_path is not None
-        notebooks = file_path.joinpath(notebooks)
-        logger.debug(f"resolve notebooks from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve notebooks from env", extra={"label": "config"})
-
-if notebooks is None:
-    logger.debug("resolve notebooks from default path", extra={"label": "config"})
-    notebooks = path_runtime.joinpath("notebooks")
-
-path_notebooks = Path(str(notebooks), assume_git=True)
-
-# job config from yaml
-is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
-if is_job_config_from_yaml is None:
-    if is_job_config_from_yaml := file_config.get("job_config_from_yaml"):
-        logger.debug(f"resolve job_config_from_yaml from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve job_config_from_yaml from env", extra={"label": "config"})
-
-# debug mode
-is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
-if is_debugmode is None:
-    if is_debugmode := file_config.get("debugmode"):
-        logger.debug(f"resolve debugmode from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve debugmode from env", extra={"label": "config"})
-
-# dev mode
-is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
-if is_devmode is None:
-    if is_devmode := file_config.get("devmode"):
-        logger.debug(f"resolve devmode from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve devmode from env", extra={"label": "config"})
-
-# log level
-loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
-if loglevel is None:
-    if loglevel := file_config.get("loglevel"):
-        logger.debug(f"resolve loglevel from {origin} file", extra={"label": "config"})
-else:
-    logger.debug("resolve loglevel from env", extra={"label": "config"})
-
-loglevel = loglevel.upper() if loglevel else "INFO"
-if loglevel == "DEBUG":
-    _loglevel = logging.DEBUG
-elif loglevel == "INFO":
-    _loglevel = logging.INFO
-elif loglevel == "WARNING":
-    _loglevel = logging.WARNING
-elif loglevel == "ERROR":
-    _loglevel = logging.ERROR
-elif loglevel == "CRITICAL":
-    _loglevel = logging.CRITICAL
-else:
-    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+config = ConfigOptions()
 
 # Constants
-PATH_CONFIG: Final[Path] = path_config
-PATH_RUNTIME: Final[Path] = path_runtime
-PATH_NOTEBOOKS: Final[Path] = path_notebooks
-IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
-IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
-IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
-LOGLEVEL: Final[int] = _loglevel
+CONFIG: Final[ConfigOptions] = config
+
+PATH_CONFIG: Final[GitPath] = config.resolved_paths.config
+PATH_RUNTIME: Final[GitPath] = config.resolved_paths.runtime
+PATH_NOTEBOOKS: Final[GitPath] = config.resolved_paths.notebooks
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = config.job_config_from_yaml
+IS_DEBUGMODE: Final[bool] = config.debugmode
+IS_DEVMODE: Final[bool] = config.devmode
+IS_FUNMODE: Final[bool] = config.funmode
+LOGLEVEL: Final[int] = config.loglevel

fabricks/context/log.py

@@ -1,10 +1,11 @@
 import json
 import logging
+from datetime import datetime
 from typing import Final, Literal, Optional
 
 import requests
 
-from fabricks.context import IS_DEBUGMODE, LOGLEVEL, SECRET_SCOPE, TIMEZONE
+from fabricks.context import IS_DEBUGMODE, IS_FUNMODE, LOGLEVEL, SECRET_SCOPE, TIMEZONE
 from fabricks.utils.log import get_logger
 
 logger, _ = get_logger(
@@ -18,6 +19,96 @@ logging.getLogger("SQLQueryContextLogger").setLevel(logging.CRITICAL)
 
 DEFAULT_LOGGER: Final[logging.Logger] = logger
 
+if IS_FUNMODE:
+    # 🎄 Christmas Easter Egg 🎅
+    _now = datetime.now()
+    if _now.month == 12:
+        _day = _now.day
+        if _day <= 24:
+            _days_until = 25 - _day
+            if _days_until == 1:
+                DEFAULT_LOGGER.info("🎄 Ho ho ho! Only 1 day until Christmas! Happy data processing! 🎅")
+            elif _days_until <= 7:
+                DEFAULT_LOGGER.info(
+                    f"🎄 'Tis the season! {_days_until} days until Christmas! May your pipelines run smoothly! 🎁"
+                )
+            else:
+                DEFAULT_LOGGER.info("🎄 Merry December! Wishing you bug-free data pipelines this holiday season! ⛄")
+        elif _day == 25:
+            DEFAULT_LOGGER.info("🎄🎅 MERRY CHRISTMAS! May all your queries be optimized and your data be clean! 🎁✨")
+        else:
+            DEFAULT_LOGGER.info("🎄 Happy Holidays! Hope you're enjoying the festive season between data runs! 🎉")
+
+    if _now.month == 10 and _now.day == 31:
+        DEFAULT_LOGGER.info("🎃👻 Happy Halloween! May your data be spooky good and your bugs be few! 🕸️🦇")
+
+    if _now.month == 7 and _now.day == 4:
+        DEFAULT_LOGGER.info("🎆🇺🇸 Happy 4th of July! Celebrate freedom with flawless data processing! 🎇🍔")
+
+    if _now.month == 1 and _now.day == 1:
+        DEFAULT_LOGGER.info("🎉 Happy New Year! Wishing you a year of successful data projects and clean code! 🥳🎆")
+
+    if _now.month == 2 and _now.day == 14:
+        DEFAULT_LOGGER.info(
+            "💖 Happy Valentine's Day! May your data relationships be strong and your joins be perfect! 🌹📊"
+        )
+
+    if _now.month == 3 and _now.day == 17:
+        DEFAULT_LOGGER.info(
+            "🍀 Happy St. Patrick's Day! Wishing you the luck of the Irish in all your data endeavors! 🌈☘️"
+        )
+
+    if _now.month == 6 and _now.day == 19:
+        DEFAULT_LOGGER.info(
+            "🌞 Happy Juneteenth! Celebrating freedom and the power of data to enlighten and empower! ✊🏿📈"
+        )
+
+    if _now.month == 11 and _now.day == 11:
+        DEFAULT_LOGGER.info(
+            "🦃 Happy Veterans Day! Honoring those who served while we serve up great data insights! 🇺🇸📊"
+        )
+
+    if _now.month == 5 and _now.day == 1:
+        DEFAULT_LOGGER.info("🌸 Happy May Day! Celebrating spring and the blossoming of new data opportunities! 🌷📈")
+
+    if _now.month == 5 and _now.day == 5:
+        DEFAULT_LOGGER.info("🎉 Happy Cinco de Mayo! Celebrating culture and the fiesta of data analytics! 🌮📊")
+
+    if _now.month == 5 and _now.day == 4:
+        DEFAULT_LOGGER.info(
+            "🌌 May the 4th be with you! Harness the force of data to conquer your analytics challenges! 🚀📊"
+        )
+
+    if _now.month == 8 and _now.day == 1:
+        DEFAULT_LOGGER.info(
+            "🏖️ Happy Swiss National Day! Celebrating precision and excellence in data, just like Swiss craftsmanship! 🇨🇭📈"
+        )
+
+    if _now.month == 7 and _now.day == 14:
+        DEFAULT_LOGGER.info(
+            "🎉 Happy Bastille Day! Celebrating liberty, equality, and the power of data to transform societies! 🇫🇷📊"
+        )
+
+    if _now.month == 7 and _now.day == 21:
+        DEFAULT_LOGGER.info(
+            "🚴 Happy Belgian National Day! Celebrating unity and the strength of data-driven decisions! 🇧🇪📈"
+        )
+
+    if _now.day > 27 and _now.day < 31:
+        DEFAULT_LOGGER.warning(
+            "⚠️ Warning: End of month is near! Make sure to finalize your data reports and close out any pending tasks! 📅✅"
+        )
+
+    if _now.weekday() == 4:
+        DEFAULT_LOGGER.warning(
+            "📅🚫 Please do not deploy on Fridays! Avoid end-of-week surprises in your data pipelines! 🚫📅"
+        )
+
+    if _now.weekday() == 0:
+        DEFAULT_LOGGER.info(
+            "☕ Happy Monday! Kickstart your week with fresh data insights and a strong cup of coffee! 📊☕"
+        )
+
 
 def send_message_to_channel(
     channel: str,