sdg-core-lib 0.1.9.dev2__tar.gz → 0.1.9.dev4__tar.gz

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg-core-lib
-Version: 0.1.9.dev2
+Version: 0.1.9.dev4
 Summary: Add your description here
 Author: emiliocimino
 Author-email: emiliocimino <emilio.cimino@outlook.it>

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdg-core-lib"
-version = "0.1.9.dev2"
+version = "0.1.9.dev4"
 description = "Add your description here"
 license = "AGPL-3.0"
 readme = "README.md"

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/src/sdg_core_lib/job.py

@@ -1,4 +1,5 @@
 from typing import Optional, Type
+from loguru import logger
 
 from sdg_core_lib.data_generator.models.UnspecializedModel import UnspecializedModel
 from sdg_core_lib.dataset.datasets import Dataset
@@ -108,7 +109,8 @@ class Job:
         )
         try:
             filtered_synthetic_data = function_generator.apply_all(synthetic_data)
-        except TypeError:
+        except (TypeError, ValueError) as e:
+            logger.error(f"Unable to apply functions to data: {e}")
             filtered_synthetic_data = synthetic_data
 
         report = {"available": "false"}

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/src/sdg_core_lib/post_process/TabularFunctionApplier.py

@@ -21,17 +21,36 @@ class TabularFunctionApplier:
 
     def _initialize(self):
         feature_function_mapping = {}
+        logger.info(
+            f"Initializing with function_feature_dict: {self.function_feature_dict}"
+        )
+
         for item in self.function_feature_dict:
             feature = item["feature"]
+            logger.debug(f"Processing function for feature: {feature}")
+
             if feature not in feature_function_mapping.keys():
                 feature_function_mapping[feature] = []
 
-            feature_function_mapping[feature].append(function_factory(item))
+            try:
+                function_instance = function_factory(item)
+                logger.debug(
+                    f"Successfully created function: {function_instance.__class__.__name__}"
+                )
+                feature_function_mapping[feature].append(function_instance)
+            except Exception as e:
+                logger.error(f"Failed to create function for feature {feature}: {e}")
+                logger.error(f"Function item: {item}")
+                raise
 
         for feature, functions in feature_function_mapping.items():
             functions.sort(key=lambda x: x.priority.value, reverse=True)
+            logger.debug(f"Sorted {len(functions)} functions for feature {feature}")
 
         self.function_feature_mapping = feature_function_mapping
+        logger.info(
+            f"Final function mapping: {list(self.function_feature_mapping.keys())}"
+        )
 
     def apply_all(self, dataset: Optional[Dataset] = None) -> Dataset:
         """
@@ -141,14 +160,32 @@ class TabularFunctionApplier:
         data_array = []
         unmapped_features = []
 
+        logger.info(
+            f"Available features in dataset: {[f['column_name'] for f in json_structure]}"
+        )
+        logger.info(f"Function mapping: {list(self.function_feature_mapping.keys())}")
+
         for feature in json_structure:
             feature_name = feature["column_name"]
+            logger.debug(f"Processing feature: {feature_name}")
 
             if feature_name in self.function_feature_mapping:
+                logger.info(f"Found functions for feature: {feature_name}")
                 functions = self.function_feature_mapping[feature_name]
-                self._validate_function_sequence(functions, from_scratch=False)
+                logger.debug(
+                    f"Functions to apply: {[f.__class__.__name__ for f in functions]}"
+                )
+
+                try:
+                    self._validate_function_sequence(functions, from_scratch=False)
+                except Exception as e:
+                    logger.error(f"Validation failed for {feature_name}: {e}")
+                    raise
 
                 feature_data = np.array(feature["column_data"])
+                logger.debug(
+                    f"Original data shape: {feature_data.shape}, dtype: {feature_data.dtype}"
+                )
                 original_shape = feature_data.shape
 
                 for function in functions:
@@ -159,9 +196,15 @@ class TabularFunctionApplier:
                         continue
 
                     try:
+                        logger.debug(
+                            f"Applying {function.__class__.__name__} to {feature_name}"
+                        )
                         feature_data, indexes, success = function.apply(
                             n_rows=self.n_rows, data=feature_data
                         )
+                        logger.debug(
+                            f"Function result: success={success}, indexes_sum={indexes.sum()}"
+                        )
                         if not success:
                             logger.warning(
                                 f"Function {function.__class__.__name__} failed to apply successfully"
@@ -182,6 +225,7 @@ class TabularFunctionApplier:
                 modified_features.add(feature_name)
             else:
                 # Preserve unmapped features
+                logger.debug(f"No function mapping for feature: {feature_name}")
                 data_array.append(np.array(feature["column_data"]))
                 unmapped_features.append(feature_name)
 

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/src/sdg_core_lib/post_process/functions/filter/implementation/InnerThreshold.py

@@ -7,7 +7,7 @@ from sdg_core_lib.post_process.functions.filter.IntervalThreshold import (
 
 
 class InnerThreshold(IntervalThreshold):
-    description = "Filters data between a given interval"
+    description = "Keeps data in a given interval"
 
     def __init__(self, parameters: list[Parameter]):
         super().__init__(parameters)
@@ -26,5 +26,6 @@ class InnerThreshold(IntervalThreshold):
             lower_indexes = np.less(data, self.upper_bound)
 
         final_indexes = lower_indexes & upper_indexes
-        data[final_indexes] = np.nan
-        return data, final_indexes, True
+        removed_indexes = ~final_indexes
+        data[removed_indexes] = np.nan
+        return data, removed_indexes, True

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/src/sdg_core_lib/post_process/functions/filter/implementation/LowerThreshold.py

@@ -18,5 +18,6 @@ class LowerThreshold(MonoThreshold):
         else:
             indexes = np.greater(data, self.value)
 
-        data[indexes] = np.nan
-        return data, indexes, True
+        removed_indexes = indexes
+        data[removed_indexes] = np.nan
+        return data, removed_indexes, True

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/src/sdg_core_lib/post_process/functions/filter/implementation/OuterThreshold.py

@@ -7,7 +7,7 @@ from sdg_core_lib.post_process.functions.filter.IntervalThreshold import (
 
 
 class OuterThreshold(IntervalThreshold):
-    description = "Filters data outside a given interval"
+    description = "Keeps data outside a given interval"
 
     def __init__(self, parameters: list[Parameter]):
         super().__init__(parameters)
@@ -25,6 +25,6 @@ class OuterThreshold(IntervalThreshold):
         else:
             lower_indexes = np.less(data, self.lower_bound)
         final_indexes = lower_indexes | upper_indexes
-
-        data[final_indexes] = np.nan
-        return data, final_indexes, True
+        removed_indexes = ~final_indexes
+        data[removed_indexes] = np.nan
+        return data, removed_indexes, True

{sdg_core_lib-0.1.9.dev2 → sdg_core_lib-0.1.9.dev4}/src/sdg_core_lib/post_process/functions/filter/implementation/UpperThreshold.py

@@ -18,5 +18,6 @@ class UpperThreshold(MonoThreshold):
         else:
             indexes = np.less(data, self.value)
 
-        data[indexes] = np.nan
-        return data, indexes, True
+        removed_indexes = indexes
+        data[removed_indexes] = np.nan
+        return data, removed_indexes, True