ddi-fw 0.0.53__tar.gz → 0.0.55__tar.gz

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.53
+Version: 0.0.55
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.53"
+version = "0.0.55"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/src/ddi_fw/datasets/core.py

@@ -298,11 +298,13 @@ class BaseDataset(ABC):
         combined_df = filtered_df.copy()
         # TODO: eğer kullanılan veri setinde tui, cui veya entity bilgileri yoksa o veri setine bu sütunları eklemek için aşağısı gerekli
 
-        idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
+        # idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
+        idf_calc = IDF(filtered_ner_df, self.ner_columns)
         idf_calc.calculate()
         idf_scores_df = idf_calc.to_dataframe()
 
-        for key in filtered_ner_df.keys():
+        # for key in filtered_ner_df.keys():
+        for key in self.ner_columns:
             threshold = 0
             if key.startswith('tui'):
                 threshold = self.tui_threshold

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/src/ddi_fw/experiments/__init__.py

@@ -1,3 +1,4 @@
 from .tensorflow_helper import TFMultiModal, TFSingleModal,Result
 from .evaluation_helper import evaluate, Metrics
-from .pipeline import Experiment
+from .pipeline import Experiment
+from .pipeline_ner import NerParameterSearch

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/src/ddi_fw/experiments/pipeline.py

@@ -61,21 +61,21 @@ class Experiment:
         kwargs = {"columns": self.columns}
         for k, v in self.ner_threshold.items():
             kwargs[k] = v
+        if self.vector_db_persist_directory:
+            self.vector_db = chromadb.PersistentClient(
+                path=self.vector_db_persist_directory)
+            self.collection = self.vector_db.get_collection(
+                self.vector_db_collection_name)
+            dictionary = self.collection.get(include=['embeddings', 'metadatas'])
 
-        self.vector_db = chromadb.PersistentClient(
-            path=self.vector_db_persist_directory)
-        self.collection = self.vector_db.get_collection(
-            self.vector_db_collection_name)
-        dictionary = self.collection.get(include=['embeddings', 'metadatas'])
+            embedding_dict = defaultdict(lambda: defaultdict(list))
 
-        embedding_dict = defaultdict(lambda: defaultdict(list))
+            for metadata, embedding in zip(dictionary['metadatas'], dictionary['embeddings']):
+                embedding_dict[metadata["type"]][metadata["id"]].append(embedding)
 
-        for metadata, embedding in zip(dictionary['metadatas'], dictionary['embeddings']):
-            embedding_dict[metadata["type"]][metadata["id"]].append(embedding)
+            embedding_size = dictionary['embeddings'].shape[1]
 
-        embedding_size = dictionary['embeddings'].shape[1]
-
-        pooling_strategy = self.embedding_pooling_strategy_type()
+            pooling_strategy = self.embedding_pooling_strategy_type()
 
         self.ner_df = CTakesNER().load(filename=self.ner_data_file)  if self.ner_data_file else None
 

ddi_fw-0.0.55/src/ddi_fw/experiments/pipeline_ner.py

@@ -0,0 +1,109 @@
+from collections import defaultdict
+from enum import Enum
+import numpy as np
+import pandas as pd
+from ddi_fw.datasets.core import BaseDataset
+from ddi_fw.experiments.tensorflow_helper import TFMultiModal
+from ddi_fw.experiments.pipeline import Experiment
+from typing import Dict, List
+from itertools import product
+
+from ddi_fw.utils.enums import DrugBankTextDataTypes, UMLSCodeTypes
+import mlflow
+from ddi_fw.ner.ner import CTakesNER
+
+def stack(df_column):
+    return np.stack(df_column.values)
+
+
+class NerParameterSearch:
+    def __init__(self,
+                 experiment_name,
+                 experiment_description,
+                 experiment_tags,
+                 tracking_uri,
+                 dataset_type: BaseDataset,
+                 umls_code_types: List[UMLSCodeTypes],
+                 text_types=List[DrugBankTextDataTypes],
+                 min_threshold_dict: Dict[str, float] = defaultdict(float),
+                 max_threshold_dict: Dict[str, float] = defaultdict(float),
+                 increase_step=0.5):
+        self.experiment_name = experiment_name
+        self.experiment_description = experiment_description
+        self.experiment_tags = experiment_tags
+        self.tracking_uri = tracking_uri
+
+        self.dataset_type = dataset_type
+        self.umls_code_types = umls_code_types
+        self.text_types = text_types
+        self.min_threshold_dict = min_threshold_dict
+        self.max_threshold_dict = max_threshold_dict
+        self.increase_step = increase_step
+
+    def build(self):
+        self.datasets = {}
+        self.items = []
+        columns = ['tui', 'cui', 'entities']
+        if self.umls_code_types is not None and self.text_types is not None:
+            # add checking statements
+            _umls_codes = [t.value[0] for t in self.umls_code_types]
+            _text_types = [t.value[0] for t in self.text_types]
+            _columns = [f'{item[0]}_{item[1]}' for item in product(
+                _umls_codes, _text_types)]
+            columns.extend(_columns)
+        print(f'Columns: {columns}')
+        self.ner_df = CTakesNER().load(filename=self.ner_data_file)  if self.ner_data_file else None
+        for column in columns:
+            min_threshold = self.min_threshold_dict[column]
+            max_threshold = self.max_threshold_dict[column]
+            kwargs = {}
+            kwargs['threshold_method'] = 'idf'
+            kwargs['tui_threshold'] = 0
+            kwargs['cui_threshold'] = 0
+            kwargs['entities_threshold'] = 0
+
+            for threshold in np.arange(min_threshold, max_threshold, self.increase_step):
+                print(threshold)
+                if column.startswith('tui'):
+                    kwargs['tui_threshold'] = threshold
+                if column.startswith('cui'):
+                    kwargs['cui_threshold'] = threshold
+                if column.startswith('entities'):
+                    kwargs['entities_threshold'] = threshold
+                dataset = self.dataset_type(
+                    # chemical_property_columns=[],
+                    # embedding_columns=[],
+                    # ner_columns=[column],
+                    columns=[column],
+                    ner_df= self.ner_df,
+                    **kwargs)
+
+                # train_idx_arr, val_idx_arr  bir kez hesaplanması yeterli aslında
+                X_train, X_test, y_train, y_test, X_train.index, X_test.index, train_idx_arr, val_idx_arr = dataset.load()
+                group_items = dataset.produce_inputs()
+                for item in group_items:
+                    # item[0] = f'threshold_{threshold}_{item[0]}'
+                    item[0] = f'threshold_{item[0]}_{threshold}'
+                self.datasets[item[0]] = dataset.ddis_df
+ 
+                self.items.extend(group_items)
+        self.y_test_label = self.items[0][4]
+        self.train_idx_arr = train_idx_arr
+        self.val_idx_arr = val_idx_arr
+
+
+    def run(self, model_func, batch_size=128, epochs=100):
+            mlflow.set_tracking_uri(self.tracking_uri)
+
+            if mlflow.get_experiment_by_name(self.experiment_name) == None:
+                mlflow.create_experiment(self.experiment_name)
+                mlflow.set_experiment_tags(self.experiment_tags)
+            mlflow.set_experiment(self.experiment_name)
+
+            y_test_label = self.items[0][4]
+            multi_modal = TFMultiModal(
+                model_func=model_func, batch_size=batch_size,  epochs=epochs)  # 100
+            multi_modal.set_data(
+                self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+            result = multi_modal.predict(self.combinations)
+            return result

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.53
+Version: 0.0.55
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.53 → ddi_fw-0.0.55}/src/ddi_fw.egg-info/SOURCES.txt

@@ -68,6 +68,7 @@ src/ddi_fw/experiments/custom_torch_model.py
 src/ddi_fw/experiments/evaluation_helper.py
 src/ddi_fw/experiments/pipeline.py
 src/ddi_fw/experiments/pipeline_builder_pattern.py
+src/ddi_fw/experiments/pipeline_ner.py
 src/ddi_fw/experiments/tensorflow_helper.py
 src/ddi_fw/experiments/test.py
 src/ddi_fw/ner/__init__.py