ddi-fw 0.0.153__py3-none-any.whl → 0.0.155__py3-none-any.whl

ddi_fw/datasets/dataset_splitter.py

@@ -14,11 +14,11 @@ class DatasetSplitter(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
-    def split(self, X: pd.DataFrame, y: pd.Series)-> Tuple[
-    pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.Index, pd.Index, List[np.ndarray], List[np.ndarray]]:
+    def split(self, X: pd.DataFrame, y: pd.Series) -> Tuple[
+            pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.Index, pd.Index, List[np.ndarray], List[np.ndarray]]:
         print(
             f"Splitting dataset into {self.fold_size} folds with shuffle={self.shuffle}...")
-        #TODO check it
+        # TODO check it
         if len(y.shape) == 1:
             y = pd.Series(np.expand_dims(y.to_numpy(), axis=1).flatten())
         stacked = np.vstack(tuple(y.to_numpy()))

ddi_fw/datasets/ddi_mdl/base.py

@@ -15,6 +15,9 @@ from abc import ABC, abstractmethod
 from sklearn.preprocessing import LabelBinarizer
 
 from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
+import logging
+
+logger = logging.getLogger(__name__)
 
 # Constants for embedding, chemical properties, and NER columns
 LIST_OF_EMBEDDING_COLUMNS = [
@@ -57,6 +60,8 @@ class DDIMDLDataset(TextDatasetMixin):
         return values
 
     def __init__(self, **kwargs):
+        logger.info(f'{self.dataset_name} is being initialized')
+
         super().__init__(**kwargs)
         self.class_column = 'event_category'
         _db_path = HERE.joinpath('data/event.db')
@@ -67,6 +72,7 @@ class DDIMDLDataset(TextDatasetMixin):
         # TODO with resource
         self._conn = create_connection(_db_path.absolute().as_posix())
         self.load_drugs_and_events()
+        logger.info(f'{self.dataset_name} is initialized')
 
     def load_drugs_and_events(self):
         self.drugs_df = self.__select_all_drugs_as_dataframe__()

ddi_fw/pipeline/multi_pipeline.py

@@ -68,6 +68,7 @@ class MultiPipeline():
         # Dynamically import the model and dataset classes
         # model_type = get_import(config.get("model_type"))
         dataset_type = get_import(config.get("dataset_type"))
+        dataset_splitter_type = get_import(config.get("dataset_splitter_type"))
 
         combination_type = None
         kwargs_combination_params=None
@@ -89,6 +90,7 @@ class MultiPipeline():
                 artifact_location=artifact_location,
                 tracking_uri=tracking_uri,
                 dataset_type=dataset_type,
+                dataset_splitter_type=dataset_splitter_type,
                 columns=columns,
                 column_embedding_configs=column_embedding_configs,
                 vector_db_persist_directory=vector_db_persist_directory,

ddi_fw/pipeline/pipeline.py

@@ -1,4 +1,5 @@
 from typing import Any, Dict, List, Optional, Type, Union
+from ddi_fw.datasets.dataset_splitter import DatasetSplitter
 import numpy as np
 import pandas as pd
 import chromadb
@@ -23,6 +24,7 @@ class Pipeline(BaseModel):
     artifact_location: Optional[str] = None
     tracking_uri: Optional[str] = None
     dataset_type: Type[BaseDataset]
+    dataset_splitter_type: Type[DatasetSplitter] = DatasetSplitter
     columns: Optional[List[str]] = None
     embedding_dict: Optional[Dict[str, Any]] = None
     column_embedding_configs: Optional[Dict] = None
@@ -181,6 +183,8 @@ class Pipeline(BaseModel):
         # self.ner_df = CTakesNER(df=None).load(
         #     filename=self.ner_data_file) if self.ner_data_file else None
 
+        dataset_splitter = self.dataset_splitter_type()
+        
         if issubclass(self.dataset_type, TextDatasetMixin):
             key, value = next(iter(embedding_dict.items()))
             embedding_size = value[next(iter(value))][0].shape[0]
@@ -190,6 +194,7 @@ class Pipeline(BaseModel):
                 embedding_dict=embedding_dict,
                 embedding_size=embedding_size,
                 embeddings_pooling_strategy=pooling_strategy,
+                dataset_splitter = dataset_splitter,
                 **kwargs)
         else:
             dataset = self.dataset_type(**kwargs)

{ddi_fw-0.0.153.dist-info → ddi_fw-0.0.155.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.153
+Version: 0.0.155
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.153.dist-info → ddi_fw-0.0.155.dist-info}/RECORD

@@ -1,12 +1,12 @@
 ddi_fw/datasets/__init__.py,sha256=yDsRQD_9Ijpm_Rl2wSDwdutG5Q_wca_UBPEvm7nBx04,444
 ddi_fw/datasets/core.py,sha256=vRMpUsIHIbOKi-6TuUPNu1Ve3ny3cS9hdsydQxFCNvE,8078
-ddi_fw/datasets/dataset_splitter.py,sha256=lLIelXv-8rCK0tbwLNgHBHYUO_65HT-_kErAlZhRQVE,1662
+ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/embedding_generator.py,sha256=jiDKwLaPMaQkloxQkuCrhl-A-2OdvocmkSzjWtUnk4g,2255
 ddi_fw/datasets/feature_vector_generation.py,sha256=gvjpEzkgVV8dp4V8NMMv59u0v-1tNAJ7v83R-keWGoA,4748
 ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=sj4WfwNmcTFznVxlND6FYoyqnL7VUrjn0TG24LDYk3w,9353
+ddi_fw/datasets/ddi_mdl/base.py,sha256=hk2DC4Xj-7vYu6m1qTYuBS0s3KP7jBrhAjQJofHP0WU,9537
 ddi_fw/datasets/ddi_mdl/debug.log,sha256=eWz05j8RFqZuHFDTCF7Rck5w4rvtTanFN21iZsgxO7Y,115
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
@@ -86,9 +86,9 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=D_BZ3ciHbVGuuB7m7cEmVQHESruh1gqhA-vxCMfNKj0,5407
+ddi_fw/pipeline/multi_pipeline.py,sha256=NfcH4Ze5U-JRiH3lrxEDWj-VPxYQYtp7tq6bLCImBzs,5550
 ddi_fw/pipeline/ner_pipeline.py,sha256=q1aKjb54Ra1HzZ7dARvBw6lB37je9R-POEf2h6QT_nU,6018
-ddi_fw/pipeline/pipeline.py,sha256=l-T-QnR_cVD590UovmsUlfjivaaZih7j_KMIJvGDAtA,11073
+ddi_fw/pipeline/pipeline.py,sha256=70lYsluAnTWDLTlf6rbecffw3Bl34L1_6ALfLUoSvtY,11324
 ddi_fw/utils/__init__.py,sha256=77563ikqAtdzjjgRlLp5OAsJBbpLA1Cao8iecGaVUXQ,354
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/json_helper.py,sha256=BVU6wmJgdXPxyqLPu3Ck_9Es5RrP1PDanKvE-OSj1D4,571
@@ -97,7 +97,7 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.153.dist-info/METADATA,sha256=Lj5ZaKAa6dL9CvHiGIo1KAUtbB8MA40dVlbpHNUcaHA,2082
-ddi_fw-0.0.153.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-ddi_fw-0.0.153.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.153.dist-info/RECORD,,
+ddi_fw-0.0.155.dist-info/METADATA,sha256=Eko-GWtTpFIvR_X4flADahxqTxPXYQ5Ei2TS8XuTEto,2082
+ddi_fw-0.0.155.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
+ddi_fw-0.0.155.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.155.dist-info/RECORD,,