ddi-fw 0.0.170__tar.gz → 0.0.172__tar.gz

{ddi_fw-0.0.170 → ddi_fw-0.0.172}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.170
+Version: 0.0.172
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.170 → ddi_fw-0.0.172}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.170"
+version = "0.0.172"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.170 → ddi_fw-0.0.172}/src/ddi_fw/datasets/core.py

@@ -1,4 +1,5 @@
 import glob
+import logging
 from typing import Any, Dict, List, Optional, Type
 import numpy as np
 import pandas as pd
@@ -20,7 +21,6 @@ except ImportError:
         "Failed to import langchain.embeddings module. ")
 
 
-
 def stack(df_column):
     return np.stack(df_column.values)
 
@@ -56,25 +56,20 @@ def generate_sim_matrices_new(df, generated_vectors, columns, key_column="id"):
 
 class BaseDataset(BaseModel):
     dataset_name: str
-    index_path: str
+    index_path: Optional[str] = None
     dataset_splitter_type: Type[DatasetSplitter]
     class_column: str = 'class'
     dataframe: Optional[pd.DataFrame] = None
-    X_train:	Optional[pd.DataFrame] = None
-    X_test:	Optional[pd.DataFrame] = None
-    y_train:	Optional[pd.Series] = None
-    y_test:	Optional[pd.Series] = None
-    train_indexes:	Optional[pd.Index] = None
-    test_indexes:	Optional[pd.Index] = None
-    train_idx_arr:	List|None = None
-    val_idx_arr:	List|None = None
-    # train_idx_arr:	Optional[List[np.ndarray]] = None
-    # val_idx_arr:	Optional[List[np.ndarray]] = None
+    X_train: Optional[pd.DataFrame | np.ndarray] = None
+    X_test: Optional[pd.DataFrame | np.ndarray] = None
+    y_train: Optional[pd.Series | np.ndarray] = None
+    y_test: Optional[pd.Series | np.ndarray] = None
+    train_indexes: Optional[pd.Index] = None
+    test_indexes: Optional[pd.Index] = None
+    train_idx_arr: Optional[List[np.ndarray]] = None
+    val_idx_arr: Optional[List[np.ndarray]] = None
     columns: List[str] = []
 
-    # feature_process: FeatureProcessor
-    # similarity_matrix_service: SimilarityMatrixService
-
     class Config:
         arbitrary_types_allowed = True
 
@@ -93,7 +88,7 @@ class BaseDataset(BaseModel):
             # items.append([f'{column}_embedding', train_data,
             #             y_train_label, test_data, y_test_label])
         return items
-    
+
     @computed_field
     @property
     def dataset_splitter(self) -> DatasetSplitter:
@@ -107,9 +102,22 @@ class BaseDataset(BaseModel):
         pass
 
     def load(self):
+        """
+        Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
+        skip deriving them. Otherwise, derive them from the dataframe and indices.
+        """
+        if self.X_train is not None and self.y_train is not None and self.X_test is not None and self.y_test is not None:
+            # Data is already provided, no need to calculate
+            logging.info(
+                "X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
+            return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
+
         if self.index_path is None:
             raise Exception(
-                "There is no index path, please call split function")
+                "There is no index path. Please call split_dataset or provide indices.")
+
+        if self.dataframe is None:
+            raise Exception("There is no dataframe to derive data from.")
 
         try:
             train_idx_all, test_idx_all, train_idx_arr, val_idx_arr = self.__get_indexes__(
@@ -119,9 +127,6 @@ class BaseDataset(BaseModel):
 
         self.prep()
 
-        if self.dataframe is None:
-            raise Exception("There is no dataframe")
-
         train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
         test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
 
@@ -135,7 +140,7 @@ class BaseDataset(BaseModel):
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
 
-        return self.X_train, self.X_test, self.y_train, self.y_test, self.X_train.index, self.X_test.index, train_idx_arr, val_idx_arr
+        return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
     def __get_indexes__(self, path):
         train_index_path = path+'/train_indexes.txt'
@@ -167,14 +172,21 @@ class BaseDataset(BaseModel):
             f.write('\n'.join(str_indexes))
 
     def split_dataset(self, save_indexes: bool = False):
-        # TODO class type should be parametric
+        """
+        Split the dataset into training and testing sets. This method is only available
+        if a dataframe exists. If X_train, y_train, X_test, and y_test are already present,
+        raise an error.
+        """
+        if self.X_train is not None or self.X_test is not None:
+            raise Exception(
+                "X_train and X_test are already present. Splitting is not allowed.")
+
+        if self.dataframe is None:
+            raise Exception("There is no dataframe to split.")
 
         save_path = self.index_path
         self.prep()
 
-        if self.dataframe is None:
-            raise Exception("There is no data")
-
         X = self.dataframe.drop(self.class_column, axis=1)
         y = self.dataframe[self.class_column]
 
@@ -206,8 +218,9 @@ class BaseDataset(BaseModel):
 
 
 class TextDatasetMixin(BaseDataset):
-    embedding_size: Optional[int] = None 
-    embedding_dict: Dict[str, Any] = Field(default_factory=dict, description="Dictionary for embeddings")
+    embedding_size: Optional[int] = None
+    embedding_dict: Dict[str, Any] = Field(
+        default_factory=dict, description="Dictionary for embeddings")
     embeddings_pooling_strategy: PoolingStrategy | None = None
 
     def process_text(self):

{ddi_fw-0.0.170 → ddi_fw-0.0.172}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.170
+Version: 0.0.172
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>