ddi-fw 0.0.218__tar.gz → 0.0.219__tar.gz

{ddi_fw-0.0.218 → ddi_fw-0.0.219}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.218
+Version: 0.0.219
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.218 → ddi_fw-0.0.219}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.218"
+version = "0.0.219"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.218 → ddi_fw-0.0.219}/src/ddi_fw/datasets/ddi_mdl/base.py

@@ -1,5 +1,5 @@
 import pathlib
-from typing import List, Optional, Tuple
+from typing import Any, List, Optional, Tuple
 from ddi_fw.datasets.core import BaseDataset, TextDatasetMixin, generate_sim_matrices_new, generate_vectors
 from ddi_fw.datasets.db_utils import create_connection
 import numpy as np
@@ -51,6 +51,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
     tui_threshold: float | None = None
     cui_threshold: float | None = None
     entities_threshold: float | None = None
+    _ner_threshold: dict[str,Any] |None = None
 
     # @model_validator
 
@@ -68,14 +69,14 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
         # self.additional_config = kwargs.get('dataset_additional_config', {})
         if self.additional_config:
             ner = self.additional_config.get('ner', {})
-            self.ner_data_file = ner.get('data_file', None)
-            self.ner_threshold = ner.get('thresholds', None)
+            ner_data_file = ner.get('data_file', None)
+            self._ner_threshold = ner.get('thresholds', None)
             # if self.ner_threshold:
             #     for k, v in self.ner_threshold.items():
             #         kwargs[k] = v
         
             self.ner_df = CTakesNER(df=None).load(
-                filename=self.ner_data_file) if self.ner_data_file else None
+                filename=ner_data_file) if ner_data_file else None
         
         columns = kwargs['columns']
         if columns:
@@ -169,7 +170,7 @@ class DDIMDLDataset(BaseDataset,TextDatasetMixin):
 
             # for key in filtered_ner_df.keys():
             for key in self.ner_columns:
-                threshold = self.ner_threshold.get(key, 0)
+                threshold = self._ner_threshold.get(key, 0) if self._ner_threshold else 0
                 # threshold = 0
                 # if key.startswith('tui'):
                 #     threshold = self.tui_threshold

{ddi_fw-0.0.218 → ddi_fw-0.0.219}/src/ddi_fw/datasets/mdf_sa_ddi/base.py

@@ -1,6 +1,6 @@
 import os
 import pathlib
-from typing import List, Optional, Tuple
+from typing import Any, List, Optional, Tuple
 from ddi_fw.datasets.core import BaseDataset, TextDatasetMixin, generate_sim_matrices_new, generate_vectors
 from ddi_fw.datasets.db_utils import create_connection
 import numpy as np
@@ -105,6 +105,7 @@ class MDFSADDIDataset(BaseDataset,TextDatasetMixin):
     tui_threshold: float | None = None
     cui_threshold: float | None = None
     entities_threshold: float | None = None
+    _ner_threshold: dict[str,Any] |None= None
 
     # @model_validator
 
@@ -122,14 +123,14 @@ class MDFSADDIDataset(BaseDataset,TextDatasetMixin):
         # self.additional_config = kwargs.get('dataset_additional_config', {})
         if self.additional_config:
             ner = self.additional_config.get('ner', {})
-            self.ner_data_file = ner.get('data_file', None)
-            self.ner_threshold = ner.get('thresholds', None)
+            ner_data_file = ner.get('data_file', None)
+            self._ner_threshold = ner.get('thresholds', None)
             # if self.ner_threshold:
             #     for k, v in self.ner_threshold.items():
             #         kwargs[k] = v
         
             self.ner_df = CTakesNER(df=None).load(
-                filename=self.ner_data_file) if self.ner_data_file else None
+                filename=ner_data_file) if ner_data_file else None
         
         columns = kwargs['columns']
         if columns:
@@ -218,6 +219,120 @@ class MDFSADDIDataset(BaseDataset,TextDatasetMixin):
             self.ddis_df.to_sql('event', conn, if_exists='replace', index=False)
             ZipHelper().zip_single_file(
                 file_path=db_path, output_path=HERE, zip_name='mdf-sa-ddi')
+            
+    def prep(self):
+        # self.load_drugs_and_events()
+        if self.drugs_df is None or self.ddis_df is None:
+            raise Exception("There is no data")
+
+        drug_ids = self.drugs_df['id'].to_list()
+
+        filtered_df = self.drugs_df
+        combined_df = filtered_df.copy()
+
+        if self.ner_df is not None and not self.ner_df.empty:
+            filtered_ner_df = self.ner_df[self.ner_df['drugbank_id'].isin(
+                drug_ids)]
+            filtered_ner_df = self.ner_df.copy()
+
+            # TODO: eğer kullanılan veri setinde tui, cui veya entity bilgileri yoksa o veri setine bu sütunları eklemek için aşağısı gerekli
+
+            # idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
+            idf_calc = IDF(filtered_ner_df, self.ner_columns)
+            idf_calc.calculate()
+            idf_scores_df = idf_calc.to_dataframe()
+
+            # for key in filtered_ner_df.keys():
+            for key in self.ner_columns:
+                threshold = self._ner_threshold.get(key, 0) if self._ner_threshold else 0
+                # threshold = 0
+                # if key.startswith('tui'):
+                #     threshold = self.tui_threshold
+                # if key.startswith('cui'):
+                #     threshold = self.cui_threshold
+                # if key.startswith('entities'):
+                #     threshold = self.entities_threshold
+                combined_df[key] = filtered_ner_df[key]
+                valid_codes = idf_scores_df[idf_scores_df[key]
+                                            > threshold].index
+
+                # print(f'{key}: valid code size = {len(valid_codes)}')
+                combined_df[key] = combined_df[key].apply(lambda items:
+                                                          [item for item in items if item in valid_codes])
+
+        moved_columns = ['id']
+        moved_columns.extend(self.__similarity_related_columns__)
+        chemical_properties_df = combined_df[moved_columns]
+
+        chemical_properties_df = chemical_properties_df.fillna("").apply(list)
+
+        # generate vectors dictionary içinde ndarray dönecek
+        generated_vectors = generate_vectors(
+            chemical_properties_df, self.__similarity_related_columns__)
+
+        # TODO if necessary 
+        similarity_matrices = generate_sim_matrices_new(
+            chemical_properties_df, generated_vectors,  self.__similarity_related_columns__, key_column="id")
+
+        event_categories = self.ddis_df['event_category']
+        labels = event_categories.tolist()
+        lb = LabelBinarizer()
+        lb.fit(labels)
+        classes = lb.transform(labels)
+
+        def similarity_lambda_fnc(row, value):
+            if row['id1'] in value:
+                return value[row['id1']]
+
+        def lambda_fnc(row: pd.Series, value) -> Optional[np.float16]:
+            if row['id1'] in value and row['id2'] in value:
+                return np.float16(np.hstack(
+                    (value[row['id1']], value[row['id2']])))
+            return None
+            # return np.hstack(
+            #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
+
+        def x_fnc(row, embeddings_after_pooling):
+            if row['id1'] in embeddings_after_pooling:
+                v1 = embeddings_after_pooling[row['id1']]
+            else:
+                v1 = np.zeros(self.embedding_size)
+            if row['id2'] in embeddings_after_pooling:
+                v2 = embeddings_after_pooling[row['id2']]
+            else:
+                v2 = np.zeros(self.embedding_size)
+            return np.float16(np.hstack(
+                (v1, v2)))
+
+        for key, value in similarity_matrices.items():
+
+            print(f'sim matrix: {key}')
+            self.ddis_df[key] = self.ddis_df.apply(
+                lambda_fnc, args=(value,), axis=1)
+            self.columns.append(key)
+            print(self.ddis_df[key].head())
+        if isinstance(self, TextDatasetMixin):
+            if self.embedding_dict is not None:
+                for embedding_column in self.embedding_columns:
+                    print(f"concat {embedding_column} embeddings")
+                    embeddings_after_pooling = {k: self.pooling_strategy.apply(
+                        v) for k, v in self.embedding_dict[embedding_column].items()}
+                    # column_embeddings_dict = embedding_values[embedding_column]
+                    self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
+                        x_fnc, args=(embeddings_after_pooling,), axis=1)
+                    self.columns.append(embedding_column+'_embedding')
+
+        dataframe = self.ddis_df.copy()
+        if not isinstance(classes, (list, pd.Series, np.ndarray)):
+            raise TypeError(
+                "classes must be an iterable (list, Series, or ndarray)")
+
+        if len(classes) != len(dataframe):
+            raise ValueError(
+                "Length of classes must match the number of rows in the DataFrame")
+
+        dataframe[self.class_column] = list(classes)
+        self.set_dataframe(dataframe)
 
 
 def select_all_drugs(conn):

{ddi_fw-0.0.218 → ddi_fw-0.0.219}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.218
+Version: 0.0.219
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>