sinapsis-data-tools 0.2.26__tar.gz → 0.2.27__tar.gz

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sinapsis-data-tools
-Version: 0.2.26
+Version: 0.2.27
 Summary: Module that contains different packages to perform data-related operations through Sinapsis templates.
 Author-email: SinapsisAI <dev@sinapsis.tech>
 Project-URL: Homepage, https://sinapsis.tech

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/__init__.py

@@ -7,13 +7,15 @@ from sinapsis_core.template_base import Template
 _root_lib_path: str = "sinapsis_data_analysis.templates"
 
 _ADDITIONAL_TEMPLATE_MODULES = [
-    f"{_root_lib_path}.sklearn_manifold",
-    f"{_root_lib_path}.sklearn_train",
-    f"{_root_lib_path}.xgboost_train",
+    f"{_root_lib_path}.training.sklearn_manifold",
+    f"{_root_lib_path}.training.sklearn_train",
+    f"{_root_lib_path}.training.xgboost_train",
+
 ]
 _template_lookup: dict = {
-    "SKLearnInference": f"{_root_lib_path}.sklearn_inference",
-    "XGBoostInference": f"{_root_lib_path}.xgboost_inference",
+    "SKLearnInference": f"{_root_lib_path}.inference.sklearn_inference",
+    "XGBoostInference": f"{_root_lib_path}.inference.xgboost_inference",
+    "CategoricalToNumerical": f"{_root_lib_path}.transformation.categorical_to_numerical",
 }
 for t_module in _ADDITIONAL_TEMPLATE_MODULES:
     _template_lookup |= _import_template_package(t_module)

{sinapsis_data_tools-0.2.26/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates → sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference}/ml_base_inference.py

@@ -28,6 +28,7 @@ class MLBaseInference(Template):
         model_path: str
         root_dir : str = SINAPSIS_CACHE_DIR
         generic_field_key: str
+        target_key : str
 
     def __init__(self, attributes: TemplateAttributes) -> None:
         super().__init__(attributes)
@@ -68,7 +69,7 @@ class MLBaseInference(Template):
             Any: The preprocessed data
         """
         try:
-            data.pop("target")
+            data.pop(self.attributes.target_key)
         except (KeyError, IndexError):
             self.logger.info("No target column")
         return data

{sinapsis_data_tools-0.2.26/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates → sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference}/sklearn_inference.py

@@ -5,7 +5,7 @@ import joblib
 from sinapsis_core.template_base.base_models import UIPropertiesMetadata
 
 from sinapsis_data_analysis.helpers.tags import Tags
-from sinapsis_data_analysis.templates.ml_base_inference import MLBaseInference
+from sinapsis_data_analysis.templates.inference.ml_base_inference import MLBaseInference
 
 
 class SKLearnInference(MLBaseInference):

{sinapsis_data_tools-0.2.26/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates → sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference}/xgboost_inference.py

@@ -4,7 +4,7 @@
 from sinapsis_core.template_base.base_models import UIPropertiesMetadata
 
 from sinapsis_data_analysis.helpers.tags import Tags
-from sinapsis_data_analysis.templates.sklearn_inference import SKLearnInference
+from sinapsis_data_analysis.templates.inference.sklearn_inference import SKLearnInference
 
 
 class XGBoostInference(SKLearnInference):

{sinapsis_data_tools-0.2.26/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates → sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training}/ml_base_training.py

@@ -4,6 +4,7 @@ from abc import abstractmethod
 from typing import Any
 
 import numpy as np
+import pandas as pd
 from sinapsis_core.data_containers.data_packet import DataContainer
 from sinapsis_core.template_base.base_models import TemplateAttributes
 from sinapsis_core.template_base.dynamic_template import BaseDynamicWrapperTemplate
@@ -103,6 +104,7 @@ class MLBaseTraining(BaseDynamicWrapperTemplate):
             y_test = dataset.y_test
 
             return x_train, y_train, x_test, y_test
+
         except AttributeError:
             self.logger.warning("Dataset doesn't have the expected attributes")
             return None
@@ -114,6 +116,8 @@ class MLBaseTraining(BaseDynamicWrapperTemplate):
             x_train (Any): The training features
             y_train (Any): The training targets
         """
+
+
         self.trained_model = self.model.fit(x_train, y_train)
 
     @staticmethod
@@ -206,6 +210,7 @@ class MLBaseTraining(BaseDynamicWrapperTemplate):
         x_train, y_train, x_test, y_test = processed_data
 
         self.train_model(x_train, y_train)
+
         return self.generate_predictions(x_test, y_test)
 
     def save_model(self) -> None:

{sinapsis_data_tools-0.2.26/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates → sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training}/sklearn_train.py

@@ -14,7 +14,7 @@ from sinapsis_data_analysis.helpers.excluded_models import (
     excluded_tree_models,
 )
 from sinapsis_data_analysis.helpers.tags import Tags
-from sinapsis_data_analysis.templates.ml_base_training import MLBaseTraining
+from sinapsis_data_analysis.templates.training.ml_base_training import MLBaseTraining
 
 
 class SKLearnLinearModelsTrain(MLBaseTraining):

{sinapsis_data_tools-0.2.26/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates → sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training}/xgboost_train.py

@@ -7,7 +7,7 @@ from sinapsis_core.template_base.dynamic_template_factory import make_dynamic_te
 from sinapsis_core.utils.env_var_keys import SINAPSIS_BUILD_DOCS
 
 from sinapsis_data_analysis.helpers.tags import Tags
-from sinapsis_data_analysis.templates.sklearn_train import SKLearnLinearModelsTrain
+from sinapsis_data_analysis.templates.training.sklearn_train import SKLearnLinearModelsTrain
 
 INCLUDED_MODELS = [
     "XGBClassifier",

sinapsis_data_tools-0.2.27/packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/transformation/categorical_to_numerical.py

@@ -0,0 +1,45 @@
+import pandas as pd
+from sinapsis_core.data_containers.data_packet import DataContainer
+from sinapsis_core.template_base.template import Template
+from sinapsis_core.template_base.base_models import TemplateAttributes
+
+
+
+class CategoricalToNumerical(Template):
+	class AttributesBaseModel(TemplateAttributes):
+		generic_key: str
+
+	@staticmethod
+	def map_categorical_to_numerical(df: pd.DataFrame| pd.Series)->pd.DataFrame | pd.Series:
+
+		if isinstance(df, pd.Series):
+			categorical_cols =  [df.name] if df.dtype in ["object", "string", "category"] else []
+		else:
+			categorical_cols =  df.select_dtypes(include=["object", "string", "category"]).columns.tolist()
+
+
+		category_maps = {}
+
+		for col in categorical_cols:
+			categories = df[col].astype("string").unique()
+			category_maps[col] = {cat: idx for idx, cat in enumerate(categories)}
+		for col, mapping in category_maps.items():
+			df[col] = df[col].map(mapping)
+		return df, category_maps
+	@staticmethod
+	def unmap_numerical_to_categorical(df: pd.DataFrame| pd.Series, categories: dict):
+		inverse_maps = {col: {v: k for k, v in mapping.items()} for col, mapping in categories.items()}
+
+		for col in inverse_maps:
+			df[col] = df[col].map(inverse_maps[col])
+		return df
+
+
+	def execute(self, container: DataContainer) -> DataContainer:
+		dataset = self._get_generic_data(container)
+		transformed_dataset, labels = self.map_categorical_to_numerical(dataset)
+		self._set_generic_data(container, transformed_dataset)
+		print ('FINAL CONTAINER', container)
+		return container
+
+

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/packages/sinapsis_data_readers/src/sinapsis_data_readers/templates/datasets_readers/dataset_splitter.py

@@ -10,8 +10,8 @@ from sinapsis_core.template_base import Template
 from sinapsis_core.template_base.base_models import TemplateAttributes
 from sklearn.model_selection import train_test_split
 
-ArrayDataFrameType = Union[list[np.ndarray], pd.DataFrame]
-StringDataFrameType = Union[list[str | int], pd.DataFrame]
+ArrayDataFrameType = Union[list[np.ndarray], pd.DataFrame, pd.Series]
+StringDataFrameType = Union[list[str | int], pd.DataFrame, pd.Series]
 OptionalArrayDataFrameType = Union[ArrayDataFrameType, None]
 
 OptionalStringDataFrameType = Union[StringDataFrameType, None]
@@ -44,18 +44,18 @@ class TabularDatasetSplit(BaseModel):
     """BaseModel to store the content of the data packets as a list
     x_train (pd.DataFrame) :  Contains the x values for the train set.
         If there is no split, contains the x values for the whole set
-    y_train (pd.DataFrame) :  Contains the y values for the train set.
+    y_train (pd.DataFrame | pd.Series) :  Contains the y values for the train set.
         If there is no split, contains the x values for the whole set
     x_test (pd.DataFrame | None) :  Contains the x values for the test set.
         If there is no split, is set to None
-    y_test (pd.DataFrame | None) :  Contains the y values for the test set.
+    y_test (pd.DataFrame | None | pd.Series) :  Contains the y values for the test set.
         If there is no split, is set to None
     """
 
     x_train: pd.DataFrame
-    y_train: pd.DataFrame
+    y_train: pd.DataFrame  | pd.Series
     x_test: pd.DataFrame | None = None
-    y_test: pd.DataFrame | None = None
+    y_test: pd.DataFrame | pd.Series | None = None
 
     class Config:
         """allow arbitrary types"""
@@ -135,9 +135,9 @@ class DatasetSplitterBase(Template):
         if not packet:
             self.logger.debug("No data to be processed by dataset splitter")
             return container
-        if len(packet) == 1:
-            self.logger.debug("Not enough entries to divide dataset, returning original container")
-            return container
+        # if len(packet) == 1:
+        #     self.logger.debug("Not enough entries to divide dataset, returning original container")
+        #     return container
         x_data, y_data = self.extract_x_y_from_packet(packet)
 
         custom_dataset = self.store_data_in_data_splitter(x_data, y_data)
@@ -171,6 +171,7 @@ class ImageDatasetSplitter(DatasetSplitterBase):
 
     def extract_x_y_from_packet(self, packets: list[Packet] | dict) -> tuple[ArrayDataFrameType, StringDataFrameType]:
         packets = cast(list, packets)
+
         x, y = self.process_images_packet(packets)
         return x, y
 
@@ -221,25 +222,33 @@ class TabularDatasetSplitter(DatasetSplitterBase):
 
     class AttributesBaseModel(DatasetSplitterBase.AttributesBaseModel):
         generic_data_extract_key: str = "SKLearn-Datasets"
-        generic_data_target_key: str = "target"  # labels
-        generic_data_feature_key: str = "data"  # arrays
+        target_key: str = "target"  # labels
+        feature_key: str | None  = None  # arrays
+
 
     def extract_x_y_from_packet(self, packets: list[Packet] | dict) -> tuple[StringDataFrameType, ArrayDataFrameType]:
         packet = cast(dict, packets)
-        dataframe: pd.DataFrame | None = packet.get(self.attributes.generic_data_extract_key, None)
+        dataframe: pd.DataFrame | None = packet.get(self.attributes.generic_data_extract_key, {})
         target: pd.DataFrame = pd.DataFrame()
         feature: pd.DataFrame = pd.DataFrame()
         if isinstance(dataframe, pd.DataFrame):
-            target = dataframe.get(self.attributes.generic_data_target_key)
-            feature = dataframe.get(self.attributes.generic_data_feature_key)
+            target = dataframe.get(self.attributes.target_key)
+            feature =  dataframe.get(self.attributes.feature_key) \
+                if self.attributes.feature_key \
+                else dataframe.drop(columns=[self.attributes.target_key])
 
         return feature, target
 
-    @staticmethod
+
+
     def return_data_splitter_object(
+        self,
         x_train: ArrayDataFrameType,
         y_train: StringDataFrameType,
         x_test: OptionalArrayDataFrameType,
         y_test: OptionalStringDataFrameType,
     ) -> TabularDatasetSplit:
+
+
+
         return TabularDatasetSplit(x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test)

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/packages/sinapsis_data_readers/src/sinapsis_data_readers/templates/datasets_readers/sklearn_datasets.py

@@ -144,7 +144,6 @@ class SKLearnDatasets(BaseDynamicWrapperTemplate):
             X = results.iloc[:, :n_features]
             y = results.iloc[:, n_features:]
 
-
         x_train, x_test, y_train, y_test = train_test_split(X, y, train_size=split_size, random_state=0)
         split_data = TabularDatasetSplit(
             x_train=pd.DataFrame(x_train),

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/packages/sinapsis_data_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sinapsis-data-tools
-Version: 0.2.26
+Version: 0.2.27
 Summary: Module that contains different packages to perform data-related operations through Sinapsis templates.
 Author-email: SinapsisAI <dev@sinapsis.tech>
 Project-URL: Homepage, https://sinapsis.tech

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/packages/sinapsis_data_tools.egg-info/SOURCES.txt

@@ -1,19 +1,24 @@
 LICENSE
 README.md
 pyproject.toml
+packages/sinapsis_data_analysis/__init__.py
+packages/sinapsis_data_analysis/src/__init__.py
 packages/sinapsis_data_analysis/src/sinapsis_data_analysis/__init__.py
 packages/sinapsis_data_analysis/src/sinapsis_data_analysis/helpers/__init__.py
 packages/sinapsis_data_analysis/src/sinapsis_data_analysis/helpers/excluded_models.py
 packages/sinapsis_data_analysis/src/sinapsis_data_analysis/helpers/model_metrics.py
 packages/sinapsis_data_analysis/src/sinapsis_data_analysis/helpers/tags.py
 packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/__init__.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/ml_base_inference.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/ml_base_training.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/sklearn_inference.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/sklearn_manifold.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/sklearn_train.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/xgboost_inference.py
-packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/xgboost_train.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference/__init__.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference/ml_base_inference.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference/sklearn_inference.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/inference/xgboost_inference.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training/__init__.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training/ml_base_training.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training/sklearn_manifold.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training/sklearn_train.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/training/xgboost_train.py
+packages/sinapsis_data_analysis/src/sinapsis_data_analysis/templates/transformation/categorical_to_numerical.py
 packages/sinapsis_data_readers/src/sinapsis_data_readers/__init__.py
 packages/sinapsis_data_readers/src/sinapsis_data_readers/helpers/__init__.py
 packages/sinapsis_data_readers/src/sinapsis_data_readers/helpers/coco_dataclasses.py

{sinapsis_data_tools-0.2.26 → sinapsis_data_tools-0.2.27}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sinapsis-data-tools"
-version = "0.2.26"
+version = "0.2.27"
 description = "Module that contains different packages to perform data-related operations through Sinapsis templates."
 authors = [{ name = "SinapsisAI", email = "dev@sinapsis.tech" }]
 dependencies = [