sdmf 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl

sdmf/cli/main.py

@@ -10,7 +10,10 @@ spark = (
     .config("spark.scheduler.mode", "FAIR")
     .config(
         "spark.jars.packages",
-        "io.delta:delta-spark_2.12:3.1.0"
+        ",".join([
+            "io.delta:delta-spark_2.12:3.1.0",
+            "com.databricks:spark-xml_2.12:0.17.0"
+        ])
     )
     .config(
         "spark.sql.extensions",

sdmf/cli/main2.py

@@ -8,9 +8,13 @@ spark = (
     SparkSession.builder
     .appName("sdmf")
     .enableHiveSupport()
+    .config("spark.scheduler.mode", "FAIR")
     .config(
         "spark.jars.packages",
-        "io.delta:delta-spark_2.12:3.1.0"
+        ",".join([
+            "io.delta:delta-spark_2.12:3.1.0",
+            "com.databricks:spark-xml_2.12:0.17.0"
+        ])
     )
     .config(
         "spark.sql.extensions",
@@ -43,3 +47,6 @@ spark = (
 # spark.sql('select count(*) from bronze.t_country_codes').show()
 
 
+
+spark.sql('select * from bronze.t_test2').show(truncate=False)
+

sdmf/data_movement_framework/BaseLoadStrategy.py

@@ -80,7 +80,6 @@ class BaseLoadStrategy(ABC):
         except Exception as e:
             raise DataLoadException(
                 message="Somethine went wrong while executing data load",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
             )
 
@@ -385,7 +384,6 @@ class BaseLoadStrategy(ABC):
         except Exception as e:
             raise DataLoadException(
                 message=f"Error in staging layer for {self.config.feed_specs['source_table_name']}",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
             )
 
@@ -416,7 +414,6 @@ class BaseLoadStrategy(ABC):
                                 f"Attempted: '{current_type}'. "
                                 f"Switching load types is not permitted."
                             ),
-                            load_type=self.config.master_specs["load_type"],
                             original_exception=None,
                         )
                     else:
@@ -438,6 +435,5 @@ class BaseLoadStrategy(ABC):
         except Exception as e:
             raise DataLoadException(
                 message="Something went wrong while enforcing load type consistency",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
             )

sdmf/data_movement_framework/LoadDispatcher.py

@@ -12,6 +12,7 @@ from sdmf.data_movement_framework.load_types.AppendLoad import AppendLoad
 from sdmf.data_movement_framework.load_types.IncrementalCDC import IncrementalCDC
 from sdmf.data_movement_framework.load_types.SCDType2 import SCDType2
 from sdmf.data_movement_framework.load_types.APIExtractor import APIExtractor
+from sdmf.data_movement_framework.load_types.StorageFetch import StorageFetch
 from sdmf.data_movement_framework.data_class.LoadConfig import LoadConfig
 from sdmf.data_movement_framework.data_class.LoadResult import LoadResult
 
@@ -56,7 +57,8 @@ class LoadDispatcher():
             "SCD_TYPE_2": SCDType2,
 
             # extraction
-            "API_EXTRACTOR": APIExtractor
+            "API_EXTRACTOR": APIExtractor,
+            "STORAGE_FETCH":StorageFetch
         }
 
         load_class = load_type_map.get(self.master_spec.get('load_type', ""))

sdmf/data_movement_framework/load_types/AppendLoad.py

@@ -71,7 +71,6 @@ class AppendLoad(BaseLoadStrategy):
 
         except Exception as e:
             raise DataLoadException(
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
                 message=f"Error during APPEND_LOAD for {self._current_target_table_name}: {str(e)}"           
             )

sdmf/data_movement_framework/load_types/FullLoad.py

@@ -81,6 +81,5 @@ class FullLoad(BaseLoadStrategy):
         except Exception as e:
             raise DataLoadException(
                 message=f"Feed ID: {self.config.master_specs['feed_id']}, Error during FULL LOAD for {self._current_target_table_name}: {str(e)}",
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e
             )

sdmf/data_movement_framework/load_types/IncrementalCDC.py

@@ -134,7 +134,6 @@ class IncrementalCDC(BaseLoadStrategy):
             target_df = delta_target.toDF()
             if target_df.columns != incr_df.columns:
                 raise DataLoadException(
-                    load_type=self.config.feed_specs["load_type"],
                     original_exception=None,
                     message=f"Target table {target_table} schema [{target_df.columns}] does not match incremental data schema [{incr_df.columns}]."
                 )
@@ -191,7 +190,6 @@ class IncrementalCDC(BaseLoadStrategy):
             )
         except Exception as e:
             raise DataLoadException(
-                load_type=self.config.feed_specs["load_type"],
                 original_exception=e,
                 message=f"Error during Incremental CDC load for {self._current_target_table_name}: {str(e)}"           
             )

sdmf/data_movement_framework/load_types/SCDType2.py

@@ -162,7 +162,6 @@ class SCDType2(BaseLoadStrategy):
             )
         except Exception as e:
             raise DataLoadException(
-                load_type=self.config.master_specs["load_type"],
                 original_exception=e,
                 message=f"Error during SCD_TYPE_2 for {self._current_target_table_name}: {str(e)}"           
             )

sdmf/data_movement_framework/load_types/StorageFetch.py

@@ -0,0 +1,144 @@
+# inbuilt
+import os
+import uuid
+import time
+import random
+import logging
+import requests
+from io import BytesIO
+from requests.exceptions import RequestException
+
+# external
+from pyspark.sql import SparkSession, DataFrame
+from pyspark.sql.types import StructType
+from pyspark.sql.functions import input_file_name
+
+# internal
+from sdmf.data_movement_framework.BaseLoadStrategy import BaseLoadStrategy
+from sdmf.data_movement_framework.data_class.LoadConfig import LoadConfig
+from sdmf.data_movement_framework.data_class.LoadResult import LoadResult
+from sdmf.exception.StorageFetchException import StorageFetchException
+
+
+class StorageFetch(BaseLoadStrategy):
+
+    def __init__(self, config: LoadConfig, spark: SparkSession) -> None:
+        super().__init__(config=config, spark=spark)
+        self.logger = logging.getLogger(__name__)
+        self.config = config
+        self.spark = spark
+        self.file_type = self.config.feed_specs['storage_config']['file_type']
+        self.lookup_directory = self.config.feed_specs['storage_config']['lookup_directory']
+        if self.config.target_unity_catalog == "testing":
+            self.__bronze_schema = f"bronze"
+        else:
+            self.__bronze_schema = f"{self.config.target_unity_catalog}.bronze"
+        self.logger.warning('Storage Fetch will always dump data in bronze schema as per medallion architecture.')
+
+    def load(self) -> LoadResult:
+        try:
+
+            results_df = self.__load_file_to_dataframe()
+            self.spark.sql(f"CREATE SCHEMA IF NOT EXISTS {self.__bronze_schema}")
+            feed_temp = (
+                f"{self.__bronze_schema}."
+                f"{self.config.master_specs['target_table_name']}"
+            )
+            self.logger.info(f"Creating bronze table: {feed_temp}")
+            
+
+            (
+                results_df.write.
+                format("delta")
+                .mode("overwrite")
+                .saveAsTable(feed_temp)
+            )
+            return LoadResult(
+                feed_id = self.config.master_specs['feed_id'],
+                success=True, 
+                total_rows_inserted=results_df.count(),
+                total_rows_updated=0,
+                total_rows_deleted=0
+            )
+        except Exception as e:
+            raise StorageFetchException(
+                message=f"Feed ID: {self.config.master_specs['feed_id']}, Error during FULL LOAD for {self._current_target_table_name}: {str(e)}",
+                original_exception=e
+            )
+    
+    def __iterate_over_latest_medallion_directory(self, base_path) -> str:
+        """
+        Returns the maximum integer directory under base_path.
+        Ignores files.
+        """
+        max_dir = float('-inf')
+        for item in os.listdir(base_path):
+            if max_dir < int(item):
+                max_dir = int(item)
+        return str(max_dir)
+    
+    def __load_file_to_dataframe(self) -> DataFrame:
+        file_path = self.__build_file_destination_directory(self.lookup_directory)
+        self.logger.info(f"Fetching data from path: {file_path}")
+
+        if self.file_type == 'xml':
+            df = (
+                self.spark.read
+                .format("xml")
+                .option(
+                    "rowTag",
+                    self.config.feed_specs['storage_config']['xml_row_tag']
+                )
+                .load(file_path)
+            )
+
+        elif self.file_type == 'json':
+            df = (
+                self.spark.read
+                .format("json")
+                .load(file_path)
+            )
+
+        elif self.file_type == 'parquet':
+            df = (
+                self.spark.read
+                .format("json")
+                .load(file_path)
+            )
+
+        else:
+            raise StorageFetchException(
+                "Invalid/missing value for [file_type] parameter in feed specs"
+            )
+        
+        schema = StructType.fromJson(self.config.feed_specs['selection_schema'])
+        df = self._enforce_schema(df, schema)
+        df = df.withColumn("_x_source_file", input_file_name())
+
+        return df
+        
+    def __build_file_destination_directory(self, base_path_prefix: str) -> str:
+        storage_type = self.config.feed_specs['storage_config']['storage_type']
+        is_multi_file = self.config.feed_specs['storage_config']['is_multi_file']
+        inside_timestamp_dir = self.config.feed_specs['storage_config']['inside_timestamp_dir']
+        file_name = self.config.feed_specs['storage_config']['file_name']
+
+        if storage_type == 'MEDALLION':
+            current_year = self.__iterate_over_latest_medallion_directory(base_path_prefix)
+            current_month = self.__iterate_over_latest_medallion_directory(os.path.join(base_path_prefix, current_year))
+            current_day = self.__iterate_over_latest_medallion_directory(os.path.join(base_path_prefix, current_year, current_month))
+            latest_timestamp = self.__iterate_over_latest_medallion_directory(os.path.join(base_path_prefix, current_year, current_month, current_day))
+            if is_multi_file == True:
+                return f"{base_path_prefix}/{current_year}/{current_month}/{current_day}/{latest_timestamp}/{inside_timestamp_dir}/*.{self.file_type}"
+            else:
+                return f"{base_path_prefix}/{current_year}/{current_month}/{current_day}/{latest_timestamp}/{inside_timestamp_dir}/{file_name}"
+
+        elif storage_type == 'STANDARD':
+            if is_multi_file == True:
+                return f"{base_path_prefix}/*.{self.file_type}"
+            else:
+                return f"{base_path_prefix}/{file_name}"
+        else:
+            raise StorageFetchException(
+                "Invalid/missing value for [storage_type] parameter in feed specs"
+            )

sdmf/data_quality/executors/ComprehensiveDQExecutor.py

@@ -2,7 +2,7 @@
 import logging
 
 # internal
-from sdmf.exception.DataQualityError import DataQualityError
+from sdmf.exception.DataQualityException import DataQualityException
 
 
 class ComprehensiveDQExecutor:
@@ -24,7 +24,7 @@ class ComprehensiveDQExecutor:
             dependency_ds = check.get("dependency_dataset", [])
             for dds in dependency_ds:
                 if self.spark.catalog.tableExists(dds) == False:
-                    raise DataQualityError
+                    raise DataQualityException
             query = check.get("query")
             severity = check.get("severity", "").upper()
             threshold = check.get("threshold", 0)

sdmf/exception/BaseException.py

@@ -0,0 +1,100 @@
+# inbuilt
+import sys
+import traceback
+import logging
+
+
+class BaseException(Exception):
+    """
+    Unified base exception for all SDMF pipeline errors.
+    Automatically logs a clean, human-readable error block.
+    """
+
+    def __init__(
+        self,
+        message=None,
+        details=None,
+        context=None,
+        original_exception=None,
+        log=True,
+    ):
+        super().__init__(message)
+
+        self.message = message or self.__class__.__name__
+        self.details = details
+        self.context = context or {}
+        self.original_exception = original_exception
+
+        # Capture traceback safely
+        exc_type, exc_value, exc_tb = sys.exc_info()
+        self.exc_type = exc_type.__name__ if exc_type else None
+        self.exc_value = str(exc_value) if exc_value else None
+        self.full_traceback = (
+            "".join(traceback.format_exception(exc_type, exc_value, exc_tb))
+            if exc_type
+            else None
+        )
+
+        self.logger = logging.getLogger(__name__)
+
+        # Log once, cleanly
+        if log:
+            self.logger.error(self.to_pretty_text())
+
+    # --------------------------------------------------
+    # Human-readable output (for logs / console)
+    # --------------------------------------------------
+    def __str__(self):
+        return self.to_pretty_text()
+
+    def to_pretty_text(self):
+        return f"""
+        ==================== SDMF ERROR ====================
+
+        Error Type:
+        {self.__class__.__name__}
+
+        Message:
+        {self.message}
+
+        -------------------- DETAILS --------------------
+        {self._format_block(self.details)}
+
+        -------------------- CONTEXT --------------------
+        {self._format_block(self.context)}
+
+        ------------- ORIGINAL EXCEPTION ---------------
+        {self._format_block(repr(self.original_exception) if self.original_exception else None)}
+
+        ------------------ STACK TRACE ------------------
+        {self._format_block(self.full_traceback)}
+
+        =================================================
+        """.strip()
+
+    # --------------------------------------------------
+    # Structured output (for MLflow / REST / JSON)
+    # --------------------------------------------------
+    def to_dict(self):
+        """Structured error payload for APIs, MLflow, or persistence."""
+        return {
+            "error_type": self.__class__.__name__,
+            "message": self.message,
+            "details": self.details,
+            "context": self.context,
+            "original_exception": repr(self.original_exception)
+            if self.original_exception
+            else None,
+            "exception_type": self.exc_type,
+            "exception_message": self.exc_value,
+            "traceback": self.full_traceback,
+        }
+
+    # --------------------------------------------------
+    # Helpers
+    # --------------------------------------------------
+    @staticmethod
+    def _format_block(value):
+        if value in (None, "", {}, []):
+            return "N/A"
+        return value

sdmf/exception/DataLoadException.py

@@ -1,8 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class DataLoadException(BasePipelineException):
-    def __init__(self, message: str, load_type: str, original_exception):
-        super().__init__(message)
-        self.message = message
-        self.load_type = load_type
-        self.original_exception = original_exception
+class DataLoadException(BaseException):
+    def __init__(self, message=None, details=None, original_exception=None):
+        super().__init__(
+            message or "Data Load Exception",
+            details=details,
+            original_exception=original_exception
+        )

sdmf/exception/{DataSpecValidationError.py → DataQualityException.py}

@@ -1,9 +1,10 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class DataSpecValidationError(BasePipelineException):
+class DataQualityException(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
-            message or "Failed to Validate.",
+            message or "Data Quality Exception",
             details=details,
             original_exception=original_exception
-        )
+        )
+        

sdmf/exception/ExtractionException.py

@@ -1,9 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class ExtractionException(BasePipelineException):
+class ExtractionException(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
-            message or "Extraction Error",
+            message or "Extraction Exception",
             details=details,
             original_exception=original_exception
         )

sdmf/exception/ResultGenerationException.py

@@ -1,8 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class ResultGenerationException(BasePipelineException):
-    def __init__(self, message: str,original_exception, details=None):
-        super().__init__(message)
-        self.message = message
-        self.details = details
-        self.original_exception = original_exception
+class ResultGenerationException(BaseException):
+    def __init__(self, message=None, details=None, original_exception=None):
+        super().__init__(
+            message or "Result Generation Exception",
+            details=details,
+            original_exception=original_exception
+        )

sdmf/exception/StorageFetchException.py

@@ -0,0 +1,9 @@
+from sdmf.exception.BaseException import BaseException
+
+class StorageFetchException(BaseException):
+    def __init__(self, message=None, details=None, original_exception=None):
+        super().__init__(
+            message or "Storage Fetch Exception",
+            details=details,
+            original_exception=original_exception
+        )

sdmf/exception/SystemError.py

@@ -1,6 +1,6 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class SystemError(BasePipelineException):
+class SystemError(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
             message or "System Error",

sdmf/exception/ValidationError.py

@@ -1,9 +1,9 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
+from sdmf.exception.BaseException import BaseException
 
-class ValidationError(BasePipelineException):
+class ValidationError(BaseException):
     def __init__(self, message=None, details=None, original_exception=None):
         super().__init__(
-            message or "Extraction Error",
+            message or "Validation Error",
             details=details,
             original_exception=original_exception
         )

{sdmf-0.1.6.dist-info → sdmf-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdmf
-Version: 0.1.6
+Version: 0.1.8
 Summary: SDMF - Standard Data Management Framework
 Author: Harsh Hando
 Author-email: Harsh Handoo <handoo.harsh@gmail.com>
@@ -206,12 +206,13 @@ License:                                  Apache License
            See the License for the specific language governing permissions and
            limitations under the License.
         
-Project-URL: License, https://github.com/yalsworldofficial/standard-data-management-framework/blob/main/LICENSE
+Project-URL: Homepage, https://github.com/hhandoo/sdmf-official
+Project-URL: Repository, https://github.com/hhandoo/sdmf-official
+Project-URL: Documentation, https://github.com/hhandoo/sdmf-official#readme
+Project-URL: Changelog, https://github.com/hhandoo/sdmf-official/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/hhandoo/sdmf-official/issues
+Project-URL: License, https://github.com/hhandoo/sdmf-official/blob/main/LICENSE
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 3 - Alpha
@@ -220,6 +221,8 @@ Description-Content-Type: text/markdown
 Requires-Dist: pandas
 Requires-Dist: openpyxl
 Requires-Dist: matplotlib
+Requires-Dist: delta-spark
+Requires-Dist: numpy
 
 # Standard Data Management Framework (SDMF)
 

{sdmf-0.1.6.dist-info → sdmf-0.1.8.dist-info}/RECORD

@@ -1,28 +1,28 @@
 sdmf/__init__.py,sha256=dsAMhbmYMsgekduahb91sqYr9I2hag3Ezhlzsjo3k5g,117
 sdmf/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdmf/cli/main.py,sha256=5UTZYQRMTwwZDiwyiu-oYs8bTletcn3KP9560wHOTMk,3400
-sdmf/cli/main2.py,sha256=jqt6cB-r5tNkBnLxKm0UG70plyS7-BWDVDb7VohynQ0,1409
+sdmf/cli/main.py,sha256=FiUoC8Zbr2oemaQKlHglJZjVNLrgDUGqCTa2TBmN_Go,3486
+sdmf/cli/main2.py,sha256=2RQ1YZvaTPMwEWz7ysF73uP-lCzF6wITjVds5gXa3w8,1604
 sdmf/cli/tt.py,sha256=ocxNSD93-bGWUk25cBJC8Zl3-mHCnvp45R3QLnwtNkI,128
 sdmf/config/LoggingConfig.py,sha256=-HeVRB12DNh5Lv8RTCxAY_jCDH-EKbcS2xV0tCgksLg,2984
 sdmf/config/LoggingPrettyFormatter.py,sha256=3UJBwmI_szxct1auy_YX9cM6qHD9EW1D_aSxlpJn6K8,1045
 sdmf/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/data_flow_diagram_generator/DataFlowDiagramGenerator.py,sha256=nerO6bH__g80VeqNQArFwEpdwJC9X1isLU3Q6hsAs6A,5852
-sdmf/data_movement_framework/BaseExtractor.py,sha256=1qmty27E5WPeHWJp3_yVJSvFAu7mrniKA4YZWmanIhU,667
-sdmf/data_movement_framework/BaseLoadStrategy.py,sha256=FQAMu60iUxoQRY9MM-sNK9jbEPjjBrbIIEn5xm9NFvc,19421
+sdmf/data_movement_framework/BaseLoadStrategy.py,sha256=0JTwnckFKtYwhWxIzQJ4N1XQ07yAyG70-21SUSZfSfk,19149
 sdmf/data_movement_framework/DataLoadController.py,sha256=e8NtvsK4gXQniym4DhjVSX9RT6NmF4klyaiXv2aYKx0,2797
-sdmf/data_movement_framework/LoadDispatcher.py,sha256=VZGJIpHMXlO7gi0g5gPr9o-Fhb3-m292wQkLGkLv-nw,3726
+sdmf/data_movement_framework/LoadDispatcher.py,sha256=UNbPnOXgc58nVWY-_dWKKInhxkgt11KQpyIvSKOc5GI,3846
 sdmf/data_movement_framework/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/data_movement_framework/data_class/LoadConfig.py,sha256=Jn0Un-Am-iJegtNpWBfo9NkXQRfErCf-EUzJA4oTe_A,262
 sdmf/data_movement_framework/data_class/LoadResult.py,sha256=XX5CUW50RS4n3igI3P6s6U2Oa4eGF66g_Zzh1cr1XSM,558
 sdmf/data_movement_framework/data_class/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/data_movement_framework/load_types/APIExtractor.py,sha256=g1dLjDtI27qdzyfSGrjtIC154TngInrbjZP1yyKD1DI,6927
-sdmf/data_movement_framework/load_types/AppendLoad.py,sha256=o3EPUHNNGfkWynKSaDyvG1yAGhdl7FghMK6d9nTw6kE,3325
-sdmf/data_movement_framework/load_types/FullLoad.py,sha256=roS6fjB8vcYf-7lsnqGuZHJi_QF5QUJ80qW-_m2pXPY,3686
-sdmf/data_movement_framework/load_types/IncrementalCDC.py,sha256=BQU8qUPWVQXP5XdEj7nzemeygQ4gMb1nIla7KX9e75A,8554
-sdmf/data_movement_framework/load_types/SCDType2.py,sha256=5gv8tAyE36-fXZFSN4lFg_5_ASgELPlcc6DiTWwXrqU,7771
+sdmf/data_movement_framework/load_types/AppendLoad.py,sha256=mez_zL6Snc5kyWZ8LZ0L1vW7l99FppDWW0WHbJfwZCI,3260
+sdmf/data_movement_framework/load_types/FullLoad.py,sha256=hqCYequ925kGNXqvZNIC8-CqCGIudaGDdzJrN9KQC0Y,3621
+sdmf/data_movement_framework/load_types/IncrementalCDC.py,sha256=QAj5Q3CngcMZ1EvLK3VTzuKLcaAYHUDo3yE-Vpi0mvo,8424
+sdmf/data_movement_framework/load_types/SCDType2.py,sha256=BfN47WBqrx2X2RIy_7PQupHtitzHXhgWqnZLBYh2wpk,7706
+sdmf/data_movement_framework/load_types/StorageFetch.py,sha256=VnPnK7tnKQv4C0etkwg5QgN59j9J31ODBdYzUU4-L0k,5785
 sdmf/data_movement_framework/load_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/data_quality/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdmf/data_quality/executors/ComprehensiveDQExecutor.py,sha256=ofgh_zd1Xdr3gdfY3SAd8q8zMv0S3OLLSW4y7eFsPy8,2225
+sdmf/data_quality/executors/ComprehensiveDQExecutor.py,sha256=REstqEtAMuyyi3G-qh9ju2OrX_Q1-cLDqG_36KQj49c,2237
 sdmf/data_quality/executors/StandardDQExecutor.py,sha256=z4aD8MYi6N1q-NrIsML8bLdU_fzioSVYvRA4PxqnixY,5612
 sdmf/data_quality/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/data_quality/model/FeedDQSummaryRow.py,sha256=pjZSYiqV-MAJ1jQGE77jFR5e2EvC1Z5CQUWmMt9YxXc,231
@@ -31,23 +31,15 @@ sdmf/data_quality/report/DQExcelReportWriter.py,sha256=5e7PSiivpFgDrNc5DOpHpRsMg
 sdmf/data_quality/report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/data_quality/runner/FeedDataQualityRunner.py,sha256=3F8mJG1js9A4KAGuiUYDYJF3mrV1BaA0B5gktdmIrPs,4638
 sdmf/data_quality/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdmf/exception/BasePipelineException.py,sha256=-npQ3BkZU-g0ltgTONMmWEiEm8HefoORmhcqOOX-gQc,2127
-sdmf/exception/DataLoadException.py,sha256=Zs3LTcvuMhdXAIC7RUuBiZaWWtQWatB92aMSMOHiskM,347
-sdmf/exception/DataQualityError.py,sha256=wn3QPZNwIxTI19djVsOxFxjzS8S6mgSXKPm2iZfQP64,344
-sdmf/exception/DataSpecRuleExecutionError.py,sha256=uzXJJWzMGSR49-IYXbsglPwGaazFdtyLQLpm734V3tI,373
-sdmf/exception/DataSpecValidationError.py,sha256=hiMdyuWpxvNS7ma_NUcDNIymsLSfBg4kQG2VZy5rzts,363
-sdmf/exception/EnvironmentPreparationError.py,sha256=w5A1UnPMmjOygPPwNjUeYbgrVrfKBhsucsi3_2Ko0go,371
-sdmf/exception/ExtractionException.py,sha256=lEYesSwV1XnkHpv9T5mtNrvGU54NsBVVK5arvOz5Jm4,356
-sdmf/exception/FeedSpecValidationError.py,sha256=XjLrCozHD2Ao4E6oW0jgRLD5K8zvwKDBYu7Lzbc2VxM,367
-sdmf/exception/ResultGenerationException.py,sha256=2ro3Fq3FiSPf3hn1BVBzLIzGzttlYRcaxi4imzk3Q0I,347
-sdmf/exception/SystemError.py,sha256=0FIykfTq39UNEm2lXiFT6w1Mype8Q82UD8xYKMZaeuU,344
-sdmf/exception/ValidationError.py,sha256=UN_stG-ySzDG9mZ3YMIn-XJddpBeoO2iE89IqEQcfgo,352
+sdmf/exception/BaseException.py,sha256=5WThiNLjkSe9rD5V37RPXR34lYWfVAqR5-EC9sD0Ujk,3080
+sdmf/exception/DataLoadException.py,sha256=kHJnWexMm3sOAwzRX88xod5IkxIuVZqUb8a6Pn84K0A,334
+sdmf/exception/DataQualityException.py,sha256=ww7apnNwikZ2nNk_Cas0Q7kalgCT4-vJgNdyzNs_yf8,348
+sdmf/exception/ExtractionException.py,sha256=0klZU0Jxl1ydARu-c-jjpy-D8ILmRKQFjwm4X_BkcbY,336
+sdmf/exception/ResultGenerationException.py,sha256=4wnGun3JTEpM6oIZFZ8mgtEhUdy2tvS5YuDa2m8y05k,349
+sdmf/exception/StorageFetchException.py,sha256=Zp-r2Xx5PnMIU1PMbvJaoRyDKc4bb34kvY0NU-6-CMk,341
+sdmf/exception/SystemError.py,sha256=qdSjWkxvwOb8dhmBQ-ek0s_5FgH1va7ILinOlwTNck8,320
+sdmf/exception/ValidationError.py,sha256=28YDbXd_1D-YfSUgekkyo1ze_FGI5kythpZ5CKv8fN0,328
 sdmf/exception/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdmf/extraction_toolkit/ExtractionController.py,sha256=q8Yfms1E0J4twH7Hx8jXN74GUR0XM4EtZv0HlQcoQsI,843
-sdmf/extraction_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdmf/extraction_toolkit/data_class/ExtractionConfig.py,sha256=dkJ_rc60RPuktw9MFdmneyxrQ7TgHUkzi83ATTIplxs,162
-sdmf/extraction_toolkit/data_class/ExtractionResult.py,sha256=ZzRKCK7vqJ5bk3CfYypXbAHV6tNLJA-asn_904Co1VU,467
-sdmf/extraction_toolkit/data_class/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/orchestrator/Orchestrator.py,sha256=UKkxnUmWITqx5nI1bsNsTSwdVNlGViN-fz9m5P_-aDI,6254
 sdmf/orchestrator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdmf/result_generator/ResultGenerator.py,sha256=PzmET2fJuhl8bjvUuXOFME2VlcNgKt95q0AD414u-qc,7228
@@ -71,7 +63,7 @@ sdmf/validation/validation_rules/StandardCheckStructureCheck.py,sha256=_5CrGlLsQ
 sdmf/validation/validation_rules/VacuumHoursCheck.py,sha256=FQI3RRpso2eQc_m6tX41KkOCAViNaKRTS42t8X4cfbQ,982
 sdmf/validation/validation_rules/ValidateFeedSpecsJSON.py,sha256=JRyYA1DaXHZT94oDWt1wm7Q5sghKe6OrjJydVII2ico,1046
 sdmf/validation/validation_rules/ValidateMasterSpecs.py,sha256=PJjL_goDrotit3D0bUWkcDqOgoNxPnFvUeGXfjluH54,666
-sdmf-0.1.6.dist-info/METADATA,sha256=2WpCdd6_zPHtTDg7pp2tF-nM7a2U76ZewscS1lx4ivs,20979
-sdmf-0.1.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-sdmf-0.1.6.dist-info/top_level.txt,sha256=a67a3_q-4a9HG0C80uz5kmOlzfO4AFoqiNag1KhTpUs,5
-sdmf-0.1.6.dist-info/RECORD,,
+sdmf-0.1.8.dist-info/METADATA,sha256=5s0O3IFtbREH_aseS-uQFIgv1h3m7eT0jA7GDxITZYs,21159
+sdmf-0.1.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+sdmf-0.1.8.dist-info/top_level.txt,sha256=a67a3_q-4a9HG0C80uz5kmOlzfO4AFoqiNag1KhTpUs,5
+sdmf-0.1.8.dist-info/RECORD,,

sdmf/data_movement_framework/BaseExtractor.py

@@ -1,23 +0,0 @@
-# inbuilt
-import logging
-from abc import ABC, abstractmethod
-
-# external
-from pyspark.sql import SparkSession, DataFrame
-
-# internal
-from sdmf.data_movement_framework.data_class.LoadConfig import LoadConfig
-from sdmf.data_movement_framework.data_class.LoadResult import LoadResult
-
-class BaseExtractor(ABC):
-    def __init__(self, config: LoadConfig, spark: SparkSession) -> None:
-        self.logger = logging.getLogger(__name__)
-        self.config = config
-        self.spark = spark
-
-    @abstractmethod
-    def extract(self) -> LoadResult:
-        """
-        Core load logic implemented by subclass.
-        Should return IngestionResult on success.
-        """

sdmf/exception/BasePipelineException.py

@@ -1,59 +0,0 @@
-import sys
-import traceback
-import logging
-
-class BasePipelineException(Exception):
-    """
-    Unified base exception for all pipeline errors.
-    Automatically logs in pretty console format.
-    """
-
-    def __init__(self, message=None, details=None, context=None, original_exception=None):
-        super().__init__(message)
-
-        self.message = message or self.__class__.__name__
-        self.details = details
-        self.context = context or {}
-        self.original_exception = original_exception
-        self.traceback = details or None
-        self.logger = logging.getLogger(__name__)
-        
-        # Capture exception info if available
-        exc_type, exc_value, _ = sys.exc_info()
-        self.exc_type = exc_type.__name__ if exc_type else None
-        self.exc_value = str(exc_value) if exc_value else None
-        self.full_error_info = ''.join(traceback.format_exception(*sys.exc_info())) if sys.exc_info()[0] else None
-
-
-        error_msg = self.__str__()
-        self.logger.error(f"{error_msg}, Full Message: {self.to_dict()}")
-
-    def __str__(self):
-        parts = [f"[{self.__class__.__name__}] {self.message}"]
-        
-        if self.full_error_info:
-            parts.append(f"\nStack Trace:\n{self.full_error_info}")
-        if self.details:
-            parts.append(f"Details: {self.details}")
-        if self.context:
-            parts.append(f"Context: {self.context}")
-        if self.original_exception:
-            parts.append(f"Caused by: {repr(self.original_exception)}")
-        if self.exc_type:
-            parts.append(f"Exception Type: {self.exc_type}")
-        if self.exc_value:
-            parts.append(f"Exception Message: {self.exc_value}")
-        
-        return " | ".join(parts)
-
-    def to_dict(self):
-        """Optional structured output if needed in MLflow or REST."""
-        return {
-            "error_type": self.__class__.__name__,
-            "message": self.message,
-            "details": self.details,
-            "context": self.context,
-            "original_exception": repr(self.original_exception),
-            "traceback": self.traceback,
-        }
-

sdmf/exception/DataQualityError.py

@@ -1,8 +0,0 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
-
-class DataQualityError(BasePipelineException):
-    def __init__(self, message=None, details=None, original_exception=None):
-        super().__init__(message)
-        self.message = message
-        self.original_exception = original_exception
-        self.details = details

sdmf/exception/DataSpecRuleExecutionError.py

@@ -1,9 +0,0 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
-
-class DataSpecRuleExecutionError(BasePipelineException):
-    def __init__(self, message=None, details=None, original_exception=None):
-        super().__init__(
-            message or "DataSpecRuleExecutionError",
-            details=details,
-            original_exception=original_exception
-        )

sdmf/exception/EnvironmentPreparationError.py

@@ -1,9 +0,0 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
-
-class EnvironmentPreparationError(BasePipelineException):
-    def __init__(self, message=None, details=None, original_exception=None):
-        super().__init__(
-            message or "FeedSpecValidationError",
-            details=details,
-            original_exception=original_exception
-        )

sdmf/exception/FeedSpecValidationError.py

@@ -1,9 +0,0 @@
-from sdmf.exception.BasePipelineException import BasePipelineException
-
-class FeedSpecValidationError(BasePipelineException):
-    def __init__(self, message=None, details=None, original_exception=None):
-        super().__init__(
-            message or "FeedSpecValidationError",
-            details=details,
-            original_exception=original_exception
-        )

sdmf/extraction_toolkit/ExtractionController.py

@@ -1,33 +0,0 @@
-# inbuilt
-import os
-import logging
-import configparser
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-
-# external
-import pandas as pd
-from pyspark.sql import SparkSession
-
-# internal
-from sdmf.extraction_toolkit.data_class.ExtractionConfig import ExtractionConfig
-from sdmf.extraction_toolkit.data_class.ExtractionResult import ExtractionResult
-
-
-class ExtractionController():
-
-    def __init__(
-            self,
-            spark: SparkSession,
-            allowed_df: pd.DataFrame,
-            config: configparser.ConfigParser
-        ) -> None:
-        self.logger = logging.getLogger(__name__)
-        self.logger.info("Extraction Controller has been initialized...")
-        self.master_specs_df = allowed_df
-        self.spark = spark
-        self.extraction_results_list = []
-        self.config = config
-
-
-        

sdmf/extraction_toolkit/data_class/ExtractionConfig.py

@@ -1,9 +0,0 @@
-# inbuilt
-from dataclasses import dataclass
-
-@dataclass
-class ExtractionConfig:
-    master_specs: dict
-    feed_specs: dict
-    config: dict
-    target_table: str

sdmf/extraction_toolkit/data_class/ExtractionResult.py

@@ -1,19 +0,0 @@
-# inbuilt
-from dataclasses import dataclass
-from typing import Optional
-
-# external
-from pyspark.sql import DataFrame
-
-@dataclass
-class ExtractionResult:
-    feed_id: int
-    success: bool
-    skipped: bool = False
-    start_epoch: float = 0.0
-    end_epoch: float = 0.0
-    total_human_readable_time: str = ""
-    target_table_path: str = ""
-    data_frame: Optional[DataFrame] = None
-    total_rows_inserted: int = 0
-    exception_if_any: Optional[Exception] = None