ddi-fw 0.0.170__py3-none-any.whl → 0.0.172__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -1,4 +1,5 @@
 import glob
+import logging
 from typing import Any, Dict, List, Optional, Type
 import numpy as np
 import pandas as pd
@@ -20,7 +21,6 @@ except ImportError:
         "Failed to import langchain.embeddings module. ")
 
 
-
 def stack(df_column):
     return np.stack(df_column.values)
 
@@ -56,25 +56,20 @@ def generate_sim_matrices_new(df, generated_vectors, columns, key_column="id"):
 
 class BaseDataset(BaseModel):
     dataset_name: str
-    index_path: str
+    index_path: Optional[str] = None
     dataset_splitter_type: Type[DatasetSplitter]
     class_column: str = 'class'
     dataframe: Optional[pd.DataFrame] = None
-    X_train:	Optional[pd.DataFrame] = None
-    X_test:	Optional[pd.DataFrame] = None
-    y_train:	Optional[pd.Series] = None
-    y_test:	Optional[pd.Series] = None
-    train_indexes:	Optional[pd.Index] = None
-    test_indexes:	Optional[pd.Index] = None
-    train_idx_arr:	List|None = None
-    val_idx_arr:	List|None = None
-    # train_idx_arr:	Optional[List[np.ndarray]] = None
-    # val_idx_arr:	Optional[List[np.ndarray]] = None
+    X_train: Optional[pd.DataFrame | np.ndarray] = None
+    X_test: Optional[pd.DataFrame | np.ndarray] = None
+    y_train: Optional[pd.Series | np.ndarray] = None
+    y_test: Optional[pd.Series | np.ndarray] = None
+    train_indexes: Optional[pd.Index] = None
+    test_indexes: Optional[pd.Index] = None
+    train_idx_arr: Optional[List[np.ndarray]] = None
+    val_idx_arr: Optional[List[np.ndarray]] = None
     columns: List[str] = []
 
-    # feature_process: FeatureProcessor
-    # similarity_matrix_service: SimilarityMatrixService
-
     class Config:
         arbitrary_types_allowed = True
 
@@ -93,7 +88,7 @@ class BaseDataset(BaseModel):
             # items.append([f'{column}_embedding', train_data,
             #             y_train_label, test_data, y_test_label])
         return items
-    
+
     @computed_field
     @property
     def dataset_splitter(self) -> DatasetSplitter:
@@ -107,9 +102,22 @@ class BaseDataset(BaseModel):
         pass
 
     def load(self):
+        """
+        Load the dataset. If X_train, y_train, X_test, and y_test are already provided,
+        skip deriving them. Otherwise, derive them from the dataframe and indices.
+        """
+        if self.X_train is not None and self.y_train is not None and self.X_test is not None and self.y_test is not None:
+            # Data is already provided, no need to calculate
+            logging.info(
+                "X_train, y_train, X_test, and y_test are already provided. Skipping calculation.")
+            return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
+
         if self.index_path is None:
             raise Exception(
-                "There is no index path, please call split function")
+                "There is no index path. Please call split_dataset or provide indices.")
+
+        if self.dataframe is None:
+            raise Exception("There is no dataframe to derive data from.")
 
         try:
             train_idx_all, test_idx_all, train_idx_arr, val_idx_arr = self.__get_indexes__(
@@ -119,9 +127,6 @@ class BaseDataset(BaseModel):
 
         self.prep()
 
-        if self.dataframe is None:
-            raise Exception("There is no dataframe")
-
         train = self.dataframe[self.dataframe.index.isin(train_idx_all)]
         test = self.dataframe[self.dataframe.index.isin(test_idx_all)]
 
@@ -135,7 +140,7 @@ class BaseDataset(BaseModel):
         self.train_idx_arr = train_idx_arr
         self.val_idx_arr = val_idx_arr
 
-        return self.X_train, self.X_test, self.y_train, self.y_test, self.X_train.index, self.X_test.index, train_idx_arr, val_idx_arr
+        return self.X_train, self.X_test, self.y_train, self.y_test, self.train_indexes, self.test_indexes, self.train_idx_arr, self.val_idx_arr
 
     def __get_indexes__(self, path):
         train_index_path = path+'/train_indexes.txt'
@@ -167,14 +172,21 @@ class BaseDataset(BaseModel):
             f.write('\n'.join(str_indexes))
 
     def split_dataset(self, save_indexes: bool = False):
-        # TODO class type should be parametric
+        """
+        Split the dataset into training and testing sets. This method is only available
+        if a dataframe exists. If X_train, y_train, X_test, and y_test are already present,
+        raise an error.
+        """
+        if self.X_train is not None or self.X_test is not None:
+            raise Exception(
+                "X_train and X_test are already present. Splitting is not allowed.")
+
+        if self.dataframe is None:
+            raise Exception("There is no dataframe to split.")
 
         save_path = self.index_path
         self.prep()
 
-        if self.dataframe is None:
-            raise Exception("There is no data")
-
         X = self.dataframe.drop(self.class_column, axis=1)
         y = self.dataframe[self.class_column]
 
@@ -206,8 +218,9 @@ class BaseDataset(BaseModel):
 
 
 class TextDatasetMixin(BaseDataset):
-    embedding_size: Optional[int] = None 
-    embedding_dict: Dict[str, Any] = Field(default_factory=dict, description="Dictionary for embeddings")
+    embedding_size: Optional[int] = None
+    embedding_dict: Dict[str, Any] = Field(
+        default_factory=dict, description="Dictionary for embeddings")
     embeddings_pooling_strategy: PoolingStrategy | None = None
 
     def process_text(self):

{ddi_fw-0.0.170.dist-info → ddi_fw-0.0.172.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.170
+Version: 0.0.172
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.170.dist-info → ddi_fw-0.0.172.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=gmasNdwohZ9Cd1qqhzijoTgX8VHQyzA0aBVtgjLQago,8344
+ddi_fw/datasets/core.py,sha256=jXPEMrlQ685qMEZ-Pj4izOVH7nkE62JtpMsDjfosBeQ,9350
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
@@ -97,7 +97,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.170.dist-info/METADATA,sha256=4DRR6BABhc8t4zXnswDItf6IB7jGRPpM0s4JPqYTiOE,2542
-ddi_fw-0.0.170.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.170.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.170.dist-info/RECORD,,
+ddi_fw-0.0.172.dist-info/METADATA,sha256=saohphdC9IZ8Fg4_QLDpyzEufhTWY_NBr2GzJqw5imU,2542
+ddi_fw-0.0.172.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.172.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.172.dist-info/RECORD,,