sdmf 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

sdmf/cli/main.py

@@ -10,7 +10,10 @@ spark = (
     .config("spark.scheduler.mode", "FAIR")
     .config(
         "spark.jars.packages",
-        "io.delta:delta-spark_2.12:3.1.0"
+        ",".join([
+            "io.delta:delta-spark_2.12:3.1.0",
+            "com.databricks:spark-xml_2.12:0.17.0"
+        ])
     )
     .config(
         "spark.sql.extensions",

sdmf/cli/main2.py

@@ -8,9 +8,13 @@ spark = (
     SparkSession.builder
     .appName("sdmf")
     .enableHiveSupport()
+    .config("spark.scheduler.mode", "FAIR")
     .config(
         "spark.jars.packages",
-        "io.delta:delta-spark_2.12:3.1.0"
+        ",".join([
+            "io.delta:delta-spark_2.12:3.1.0",
+            "com.databricks:spark-xml_2.12:0.17.0"
+        ])
     )
     .config(
         "spark.sql.extensions",
@@ -43,3 +47,6 @@ spark = (
 # spark.sql('select count(*) from bronze.t_country_codes').show()
 
 
+
+spark.sql('select * from bronze.t_test2').show(truncate=False)
+

sdmf/data_movement_framework/BaseLoadStrategy.py

@@ -80,7 +80,6 @@ class BaseLoadStrategy(ABC):
         except Exception as e:
             raise DataLoadException(
                 message="Somethine went wrong while executing data load",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
             )
 
@@ -385,7 +384,6 @@ class BaseLoadStrategy(ABC):
         except Exception as e:
             raise DataLoadException(
                 message=f"Error in staging layer for {self.config.feed_specs['source_table_name']}",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
             )
 
@@ -416,7 +414,6 @@ class BaseLoadStrategy(ABC):
                                 f"Attempted: '{current_type}'. "
                                 f"Switching load types is not permitted."
                             ),
-                            load_type=self.config.master_specs["load_type"],
                             original_exception=None,
                         )
                     else:
@@ -438,6 +435,5 @@ class BaseLoadStrategy(ABC):
         except Exception as e:
             raise DataLoadException(
                 message="Something went wrong while enforcing load type consistency",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
             )

sdmf/data_movement_framework/LoadDispatcher.py

@@ -12,6 +12,7 @@ from sdmf.data_movement_framework.load_types.AppendLoad import AppendLoad
 from sdmf.data_movement_framework.load_types.IncrementalCDC import IncrementalCDC
 from sdmf.data_movement_framework.load_types.SCDType2 import SCDType2
 from sdmf.data_movement_framework.load_types.APIExtractor import APIExtractor
+from sdmf.data_movement_framework.load_types.StorageFetch import StorageFetch
 from sdmf.data_movement_framework.data_class.LoadConfig import LoadConfig
 from sdmf.data_movement_framework.data_class.LoadResult import LoadResult
 
@@ -56,7 +57,8 @@ class LoadDispatcher():
             "SCD_TYPE_2": SCDType2,
 
             # extraction
-            "API_EXTRACTOR": APIExtractor
+            "API_EXTRACTOR": APIExtractor,
+            "STORAGE_FETCH":StorageFetch
         }
 
         load_class = load_type_map.get(self.master_spec.get('load_type', ""))

sdmf/data_movement_framework/load_types/AppendLoad.py

@@ -71,7 +71,6 @@ class AppendLoad(BaseLoadStrategy):
 
         except Exception as e:
             raise DataLoadException(
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
                 message=f"Error during APPEND_LOAD for {self._current_target_table_name}: {str(e)}"           
             )

sdmf/data_movement_framework/load_types/FullLoad.py

@@ -81,6 +81,5 @@ class FullLoad(BaseLoadStrategy):
         except Exception as e:
             raise DataLoadException(
                 message=f"Feed ID: {self.config.master_specs['feed_id']}, Error during FULL LOAD for {self._current_target_table_name}: {str(e)}",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e
             )

sdmf/data_movement_framework/load_types/IncrementalCDC.py

@@ -134,7 +134,6 @@ class IncrementalCDC(BaseLoadStrategy):
             target_df = delta_target.toDF()
             if target_df.columns != incr_df.columns:
                 raise DataLoadException(
-                    load_type=self.config.feed_specs["load_type"],
                     original_exception=None,
                     message=f"Target table {target_table} schema [{target_df.columns}] does not match incremental data schema [{incr_df.columns}]."
                 )
@@ -191,7 +190,6 @@ class IncrementalCDC(BaseLoadStrategy):
             )
         except Exception as e:
             raise DataLoadException(
-                load_type=self.config.feed_specs["load_type"],
                 original_exception=e,
                 message=f"Error during Incremental CDC load for {self._current_target_table_name}: {str(e)}"           
             )

sdmf/data_movement_framework/load_types/SCDType2.py

@@ -162,7 +162,6 @@ class SCDType2(BaseLoadStrategy):
             )
         except Exception as e:
             raise DataLoadException(
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
                 message=f"Error during SCD_TYPE_2 for {self._current_target_table_name}: {str(e)}"           
             )

sdmf/data_movement_framework/load_types/StorageFetch.py

@@ -0,0 +1,144 @@
+# inbuilt
+import os
+import uuid
+import time
+import random
+import logging
+import requests
+from io import BytesIO
+from requests.exceptions import RequestException
+
+# external
+from pyspark.sql import SparkSession, DataFrame
+from pyspark.sql.types import StructType
+from pyspark.sql.functions import input_file_name
+
+# internal
+from sdmf.data_movement_framework.BaseLoadStrategy import BaseLoadStrategy
+from sdmf.data_movement_framework.data_class.LoadConfig import LoadConfig
+from sdmf.data_movement_framework.data_class.LoadResult import LoadResult
+from sdmf.exception.StorageFetchException import StorageFetchException
+
+
+class StorageFetch(BaseLoadStrategy):
+
+    def __init__(self, config: LoadConfig, spark: SparkSession) -> None:
+        super().__init__(config=config, spark=spark)
+        self.logger = logging.getLogger(__name__)
+        self.config = config
+        self.spark = spark
+        self.file_type = self.config.feed_specs['storage_config']['file_type']
+        self.lookup_directory = self.config.feed_specs['storage_config']['lookup_directory']
+        if self.config.target_unity_catalog == "testing":
+            self.__bronze_schema = f"bronze"
+        else:
+            self.__bronze_schema = f"{self.config.target_unity_catalog}.bronze"
+        self.logger.warning('Storage Fetch will always dump data in bronze schema as per medallion architecture.')
+
+    def load(self) -> LoadResult:
+        try:
+
+            results_df = self.__load_file_to_dataframe()
+            self.spark.sql(f"CREATE SCHEMA IF NOT EXISTS {self.__bronze_schema}")
+            feed_temp = (
+                f"{self.__bronze_schema}."
+                f"{self.config.master_specs['target_table_name']}"
+            )
+            self.logger.info(f"Creating bronze table: {feed_temp}")
+            
+
+            (
+                results_df.write.
+                format("delta")
+                .mode("overwrite")
+                .saveAsTable(feed_temp)
+            )
+            return LoadResult(
+                feed_id = self.config.master_specs['feed_id'],
+                success=True, 
+                total_rows_inserted=results_df.count(),
+                total_rows_updated=0,
+                total_rows_deleted=0
+            )
+        except Exception as e:
+            raise StorageFetchException(
+                message=f"Feed ID: {self.config.master_specs['feed_id']}, Error during FULL LOAD for {self._current_target_table_name}: {str(e)}",
+                original_exception=e
+            )
+    
+    def __iterate_over_latest_medallion_directory(self, base_path) -> str:
+        """
+        Returns the maximum integer directory under base_path.
+        Ignores files.
+        """
+        max_dir = float('-inf')
+        for item in os.listdir(base_path):
+            if max_dir < int(item):
+                max_dir = int(item)
+        return str(max_dir)
+    
+    def __load_file_to_dataframe(self) -> DataFrame:
+        file_path = self.__build_file_destination_directory(self.lookup_directory)
+        self.logger.info(f"Fetching data from path: {file_path}")
+
+        if self.file_type == 'xml':
+            df = (
+                self.spark.read
+                .format("xml")
+                .option(
+                    "rowTag",
+                    self.config.feed_specs['storage_config']['xml_row_tag']
+                )
+                .load(file_path)
+            )
+
+        elif self.file_type == 'json':
+            df = (
+                self.spark.read
+                .format("json")
+                .load(file_path)
+            )
+
+        elif self.file_type == 'parquet':
+            df = (
+                self.spark.read
+                .format("json")
+                .load(file_path)
+            )
+
+        else:
+            raise StorageFetchException(
+                "Invalid/missing value for [file_type] parameter in feed specs"
+            )
+        
+        schema = StructType.fromJson(self.config.feed_specs['selection_schema'])
+        df = self._enforce_schema(df, schema)
+        df = df.withColumn("_x_source_file", input_file_name())
+
+        return df
+        
+    def __build_file_destination_directory(self, base_path_prefix: str) -> str:
+        storage_type = self.config.feed_specs['storage_config']['storage_type']
+        is_multi_file = self.config.feed_specs['storage_config']['is_multi_file']
+        inside_timestamp_dir = self.config.feed_specs['storage_config']['inside_timestamp_dir']
+        file_name = self.config.feed_specs['storage_config']['file_name']
+
+        if storage_type == 'MEDALLION':
+            current_year = self.__iterate_over_latest_medallion_directory(base_path_prefix)
+            current_month = self.__iterate_over_latest_medallion_directory(os.path.join(base_path_prefix, current_year))
+            current_day = self.__iterate_over_latest_medallion_directory(os.path.join(base_path_prefix, current_year, current_month))
+            latest_timestamp = self.__iterate_over_latest_medallion_directory(os.path.join(base_path_prefix, current_year, current_month, current_day))
+            if is_multi_file == True:
+                return f"{base_path_prefix}/{current_year}/{current_month}/{current_day}/{latest_timestamp}/{inside_timestamp_dir}/*.{self.file_type}"
+            else:
+                return f"{base_path_prefix}/{current_year}/{current_month}/{current_day}/{latest_timestamp}/{inside_timestamp_dir}/{file_name}"
+
+        elif storage_type == 'STANDARD':
+            if is_multi_file == True:
+                return f"{base_path_prefix}/*.{self.file_type}"
+            else:
+                return f"{base_path_prefix}/{file_name}"
+        else:
+            raise StorageFetchException(
+                "Invalid/missing value for [storage_type] parameter in feed specs"
+            )

sdmf/data_quality/executors/ComprehensiveDQExecutor.py

@@ -2,7 +2,7 @@
 import logging
 
 # internal
-from sdmf.exception.DataQualityError import DataQualityError
+from sdmf.exception.DataQualityException import DataQualityException
 
 
 class ComprehensiveDQExecutor:
@@ -24,7 +24,7 @@ class ComprehensiveDQExecutor:
             dependency_ds = check.get("dependency_dataset", [])
             for dds in dependency_ds:
                 if self.spark.catalog.tableExists(dds) == False:
-                    raise DataQualityError
+                    raise DataQualityException
             query = check.get("query")
             severity = check.get("severity", "").upper()
             threshold = check.get("threshold", 0)

sdmf/exception/BaseException.py

@@ -0,0 +1,100 @@
+# inbuilt
+import sys
+import traceback
+import logging
+
+
+class BaseException(Exception):
+    """
+    Unified base exception for all SDMF pipeline errors.
+    Automatically logs a clean, human-readable error block.
+    """
+
+    def __init__(
+        self,
+        message=None,
+        details=None,
+        context=None,
+        original_exception=None,
+        log=True,
+    ):
+        super().__init__(message)
+
+        self.message = message or self.__class__.__name__
+        self.details = details
+        self.context = context or {}
+        self.original_exception = original_exception
+
+        # Capture traceback safely
+        exc_type, exc_value, exc_tb = sys.exc_info()
+        self.exc_type = exc_type.__name__ if exc_type else None
+        self.exc_value = str(exc_value) if exc_value else None
+        self.full_traceback = (
+            "".join(traceback.format_exception(exc_type, exc_value, exc_tb))
+            if exc_type
+            else None
+        )
+
+        self.logger = logging.getLogger(__name__)
+
+        # Log once, cleanly
+        if log:
+            self.logger.error(self.to_pretty_text())
+
+    # --------------------------------------------------
+    # Human-readable output (for logs / console)
+    # --------------------------------------------------
+    def __str__(self):
+        return self.to_pretty_text()
+
+    def to_pretty_text(self):
+        return f"""
+        ==================== SDMF ERROR ====================
+
+        Error Type:
+        {self.__class__.__name__}
+
+        Message:
+        {self.message}
+
+        -------------------- DETAILS --------------------
+        {self._format_block(self.details)}
+
+        -------------------- CONTEXT --------------------
+        {self._format_block(self.context)}
+
+        ------------- ORIGINAL EXCEPTION ---------------
+        {self._format_block(repr(self.original_exception) if self.original_exception else None)}
+
+        ------------------ STACK TRACE ------------------
+        {self._format_block(self.full_traceback)}
+
+        =================================================
+        """.strip()
+
+    # --------------------------------------------------
+    # Structured output (for MLflow / REST / JSON)
+    # --------------------------------------------------
+    def to_dict(self):
+        """Structured error payload for APIs, MLflow, or persistence."""
+        return {
+            "error_type": self.__class__.__name__,
+            "message": self.message,
+            "details": self.details,
+            "context": self.context,
+            "original_exception": repr(self.original_exception)
+            if self.original_exception
+            else None,
+            "exception_type": self.exc_type,
+            "exception_message": self.exc_value,
+            "traceback": self.full_traceback,
+        }
+
+    # --------------------------------------------------
+    # Helpers
+    # --------------------------------------------------
+    @staticmethod
+    def _format_block(value):
+        if value in (None, "", {}, []):
+            return "N/A"
+        return value

sdmf/exception/DataLoadException.py

@@ -1,8 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class DataLoadException(BasePipelineException):
-    def __init__(self, message: str, load_type: str, original_exception):
-        super().__init__(message)
-        self.message = message
-        self.load_type = load_type
-        self.original_exception = original_exception
+class DataLoadException(BaseException):
+    def __init__(self, message=None, details=None, original_exception=None):
+        super().__init__(
+            message or "Data Load Exception",
+            details=details,
+            original_exception=original_exception
+        )

sdmf/exception/{DataSpecValidationError.py → DataQualityException.py}

@@ -1,9 +1,10 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class DataSpecValidationError(BasePipelineException):
+class DataQualityException(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
-            message or "Failed to Validate.",
+            message or "Data Quality Exception",
             details=details,
             original_exception=original_exception
-        )
+        )
+        

sdmf/exception/ExtractionException.py

@@ -1,9 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class ExtractionException(BasePipelineException):
+class ExtractionException(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
-            message or "Extraction Error",
+            message or "Extraction Exception",
             details=details,
             original_exception=original_exception
         )

sdmf/exception/ResultGenerationException.py

@@ -1,8 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class ResultGenerationException(BasePipelineException):
-    def __init__(self, message: str,original_exception, details=None):
-        super().__init__(message)
-        self.message = message
-        self.details = details
-        self.original_exception = original_exception
+class ResultGenerationException(BaseException):
+    def __init__(self, message=None, details=None, original_exception=None):
+        super().__init__(
+            message or "Result Generation Exception",
+            details=details,
+            original_exception=original_exception
+        )

sdmf/exception/StorageFetchException.py

@@ -0,0 +1,9 @@
+from sdmf.exception.BaseException import BaseException
+
+class StorageFetchException(BaseException):
+    def __init__(self, message=None, details=None, original_exception=None):
+        super().__init__(
+            message or "Storage Fetch Exception",
+            details=details,
+            original_exception=original_exception
+        )

sdmf/exception/SystemError.py

@@ -1,6 +1,6 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class SystemError(BasePipelineException):
+class SystemError(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
             message or "System Error",

sdmf/exception/ValidationError.py

@@ -1,9 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class ValidationError(BasePipelineException):
+class ValidationError(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
-            message or "Extraction Error",
+            message or "Validation Error",
             details=details,
             original_exception=original_exception
         )