ddi-fw 0.0.193__py3-none-any.whl → 0.0.194__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -1,6 +1,9 @@
+from collections import defaultdict
 import glob
 import logging
 from typing import Any, Dict, List, Optional, Type
+import chromadb
+from chromadb.api.types import IncludeEnum
 import numpy as np
 import pandas as pd
 from pydantic import BaseModel, Field, computed_field
@@ -132,6 +135,9 @@ class BaseDataset(BaseModel):
         skip deriving them. Otherwise, derive them from the dataframe and indices.
         """
         self.prep()
+        
+        if isinstance(self, TextDatasetMixin):
+            self.process_text()
 
         if self.X_train is not None or self.y_train is not None or self.X_test is not None or self.y_test is not None:
             # Data is already provided, no need to calculate
@@ -254,12 +260,98 @@ class BaseDataset(BaseModel):
 
 class TextDatasetMixin(BaseDataset):
     embedding_size: Optional[int] = None
-    embedding_dict: Dict[str, Any] = Field(
+    embedding_dict: Dict[str, Any] | None = Field(
         default_factory=dict, description="Dictionary for embeddings")
-    embeddings_pooling_strategy: PoolingStrategy | None = None
+    pooling_strategy: PoolingStrategy | None = None
+    column_embedding_configs: Optional[Dict] = None
+    vector_db_persist_directory: Optional[str] = None
+    vector_db_collection_name: Optional[str] = None
 
+    def __create_or_update_embeddings__(self, embedding_dict, vector_db_persist_directory, vector_db_collection_name, column=None):
+        """
+        Fetch embeddings and metadata from a persistent Chroma vector database and update the provided embedding_dict.
+
+        Args:
+        - vector_db_persist_directory (str): The path to the directory where the Chroma vector database is stored.
+        - vector_db_collection_name (str): The name of the collection to query.
+        - embedding_dict (dict): The existing dictionary to update with embeddings.
+
+        """
+        if vector_db_persist_directory:
+            # Initialize the Chroma client and get the collection
+            vector_db = chromadb.PersistentClient(
+                path=vector_db_persist_directory)
+            collection = vector_db.get_collection(vector_db_collection_name)
+            include = [IncludeEnum.embeddings, IncludeEnum.metadatas]
+            dictionary: chromadb.GetResult
+            # Fetch the embeddings and metadata
+            if column == None:
+                dictionary = collection.get(
+                    include=include
+                    # include=['embeddings', 'metadatas']
+                )
+                print(
+                    f"Embeddings are calculated from {vector_db_collection_name}")
+            else:
+                dictionary = collection.get(
+                    include=include,
+                    # include=['embeddings', 'metadatas'],
+                    where={
+                        "type": {"$eq": f"{column}"}})
+                print(
+                    f"Embeddings of {column} are calculated from {vector_db_collection_name}")
+
+            # Populate the embedding dictionary with embeddings from the vector database
+            metadatas = dictionary["metadatas"]
+            embeddings = dictionary["embeddings"]
+            if metadatas is None or embeddings is None:
+                raise ValueError(
+                    "The collection does not contain embeddings or metadatas.")
+            for metadata, embedding in zip(metadatas, embeddings):
+                embedding_dict[metadata["type"]
+                               ][metadata["id"]].append(embedding)
+
+        else:
+            raise ValueError(
+                "Persistent directory for the vector DB is not specified.")
+    
     def process_text(self):
-        pass
+        # key, value = next(iter(embedding_dict.items()))
+        #     embedding_size = value[next(iter(value))][0].shape[0]
+        #     pooling_strategy = self.embedding_pooling_strategy_type(
+        #     ) if self.embedding_pooling_strategy_type else None   
+           
+           
+        # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
+        # kwargs = {"columns": self.columns}
+        # if self.ner_threshold:
+        #     for k, v in self.ner_threshold.items():
+        #         kwargs[k] = v
+        if self.embedding_dict == None:
+            embedding_dict = defaultdict(lambda: defaultdict(list))
+            # TODO find more effective solution
+
+            if self.column_embedding_configs:
+                for item in self.column_embedding_configs:
+                    col = item["column"]
+                    col_db_dir = item["vector_db_persist_directory"]
+                    col_db_collection = item["vector_db_collection_name"]
+                    self.__create_or_update_embeddings__(
+                        embedding_dict, col_db_dir, col_db_collection, col)
+
+            elif self.vector_db_persist_directory:
+                self.__create_or_update_embeddings__(
+                    embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
+
+            else:
+                print(
+                    f"There is no configuration of Embeddings")
+
+        else:
+            embedding_dict = self.embedding_dict
+            # TODO make generic
+            # embedding_size = list(embedding_dict['all_text'].values())[
+            #     0][0].shape
 
 
 # class ImageDatasetMixin(BaseModel):

ddi_fw/datasets/ddi_mdl/base.py

@@ -158,6 +158,7 @@ class DDIMDLDataset(TextDatasetMixin):
         generated_vectors = generate_vectors(
             chemical_properties_df, self.__similarity_related_columns__)
 
+        # TODO if necessary 
         similarity_matrices = generate_sim_matrices_new(
             chemical_properties_df, generated_vectors,  self.__similarity_related_columns__, key_column="id")
 

ddi_fw/pipeline/pipeline.py

@@ -61,54 +61,55 @@ class Pipeline(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
-    def __create_or_update_embeddings__(self, embedding_dict, vector_db_persist_directory, vector_db_collection_name, column=None):
-        """
-        Fetch embeddings and metadata from a persistent Chroma vector database and update the provided embedding_dict.
-
-        Args:
-        - vector_db_persist_directory (str): The path to the directory where the Chroma vector database is stored.
-        - vector_db_collection_name (str): The name of the collection to query.
-        - embedding_dict (dict): The existing dictionary to update with embeddings.
-
-        """
-        if vector_db_persist_directory:
-            # Initialize the Chroma client and get the collection
-            vector_db = chromadb.PersistentClient(
-                path=vector_db_persist_directory)
-            collection = vector_db.get_collection(vector_db_collection_name)
-            include = [IncludeEnum.embeddings, IncludeEnum.metadatas]
-            dictionary: chromadb.GetResult
-            # Fetch the embeddings and metadata
-            if column == None:
-                dictionary = collection.get(
-                    include=include
-                    # include=['embeddings', 'metadatas']
-                )
-                print(
-                    f"Embeddings are calculated from {vector_db_collection_name}")
-            else:
-                dictionary = collection.get(
-                    include=include,
-                    # include=['embeddings', 'metadatas'],
-                    where={
-                        "type": {"$eq": f"{column}"}})
-                print(
-                    f"Embeddings of {column} are calculated from {vector_db_collection_name}")
-
-            # Populate the embedding dictionary with embeddings from the vector database
-            metadatas = dictionary["metadatas"]
-            embeddings = dictionary["embeddings"]
-            if metadatas is None or embeddings is None:
-                raise ValueError(
-                    "The collection does not contain embeddings or metadatas.")
-            for metadata, embedding in zip(metadatas, embeddings):
-                embedding_dict[metadata["type"]
-                               ][metadata["id"]].append(embedding)
-
-        else:
-            raise ValueError(
-                "Persistent directory for the vector DB is not specified.")
-
+    # def __create_or_update_embeddings__(self, embedding_dict, vector_db_persist_directory, vector_db_collection_name, column=None):
+    #     """
+    #     Fetch embeddings and metadata from a persistent Chroma vector database and update the provided embedding_dict.
+
+    #     Args:
+    #     - vector_db_persist_directory (str): The path to the directory where the Chroma vector database is stored.
+    #     - vector_db_collection_name (str): The name of the collection to query.
+    #     - embedding_dict (dict): The existing dictionary to update with embeddings.
+
+    #     """
+    #     if vector_db_persist_directory:
+    #         # Initialize the Chroma client and get the collection
+    #         vector_db = chromadb.PersistentClient(
+    #             path=vector_db_persist_directory)
+    #         collection = vector_db.get_collection(vector_db_collection_name)
+    #         include = [IncludeEnum.embeddings, IncludeEnum.metadatas]
+    #         dictionary: chromadb.GetResult
+    #         # Fetch the embeddings and metadata
+    #         if column == None:
+    #             dictionary = collection.get(
+    #                 include=include
+    #                 # include=['embeddings', 'metadatas']
+    #             )
+    #             print(
+    #                 f"Embeddings are calculated from {vector_db_collection_name}")
+    #         else:
+    #             dictionary = collection.get(
+    #                 include=include,
+    #                 # include=['embeddings', 'metadatas'],
+    #                 where={
+    #                     "type": {"$eq": f"{column}"}})
+    #             print(
+    #                 f"Embeddings of {column} are calculated from {vector_db_collection_name}")
+
+    #         # Populate the embedding dictionary with embeddings from the vector database
+    #         metadatas = dictionary["metadatas"]
+    #         embeddings = dictionary["embeddings"]
+    #         if metadatas is None or embeddings is None:
+    #             raise ValueError(
+    #                 "The collection does not contain embeddings or metadatas.")
+    #         for metadata, embedding in zip(metadatas, embeddings):
+    #             embedding_dict[metadata["type"]
+    #                            ][metadata["id"]].append(embedding)
+
+    #     else:
+    #         raise ValueError(
+    #             "Persistent directory for the vector DB is not specified.")
+
+    #TODO embedding'leri set etme kimin görevi
     def build(self):
         if self.embedding_pooling_strategy_type is not None and not isinstance(self.embedding_pooling_strategy_type, type):
             raise TypeError(
@@ -122,49 +123,25 @@ class Pipeline(BaseModel):
         if self.ner_threshold:
             for k, v in self.ner_threshold.items():
                 kwargs[k] = v
-        if self.embedding_dict == None:
-            embedding_dict = defaultdict(lambda: defaultdict(list))
-            # TODO find more effective solution
-
-            if self.column_embedding_configs:
-                for item in self.column_embedding_configs:
-                    col = item["column"]
-                    col_db_dir = item["vector_db_persist_directory"]
-                    col_db_collection = item["vector_db_collection_name"]
-                    self.__create_or_update_embeddings__(
-                        embedding_dict, col_db_dir, col_db_collection, col)
-
-            elif self.vector_db_persist_directory:
-                self.__create_or_update_embeddings__(
-                    embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
-
-            else:
-                print(
-                    f"There is no configuration of Embeddings")
-
-        else:
-            embedding_dict = self.embedding_dict
-            # TODO make generic
-            # embedding_size = list(embedding_dict['all_text'].values())[
-            #     0][0].shape
+        
 
         # self.ner_df = CTakesNER(df=None).load(
         #     filename=self.ner_data_file) if self.ner_data_file else None
 
         dataset_splitter = self.dataset_splitter_type()
-
+        pooling_strategy = self.embedding_pooling_strategy_type(
+            ) if self.embedding_pooling_strategy_type else None   
         if issubclass(self.dataset_type, TextDatasetMixin):
-            key, value = next(iter(embedding_dict.items()))
-            embedding_size = value[next(iter(value))][0].shape[0]
-            pooling_strategy = self.embedding_pooling_strategy_type(
-            ) if self.embedding_pooling_strategy_type else None
 
             dataset = self.dataset_type(
-                embedding_dict=embedding_dict,
-                embedding_size=embedding_size,
-                embeddings_pooling_strategy=pooling_strategy,
+                embedding_dict=self.embedding_dict, 
+                pooling_strategy=pooling_strategy,
+                column_embedding_configs=self.column_embedding_configs,
+                vector_db_persist_directory=self.vector_db_persist_directory,
+                vector_db_collection_name=self.vector_db_collection_name,
                 dataset_splitter_type=self.dataset_splitter_type,
                 **kwargs)
+            
         elif self.dataset_type == BaseDataset:
             dataset = self.dataset_type(
                 dataset_splitter_type=self.dataset_splitter_type,
@@ -175,6 +152,7 @@ class Pipeline(BaseModel):
         # X_train, X_test, y_train, y_test, train_indexes, test_indexes, train_idx_arr, val_idx_arr = dataset.load()
         
         dataset.load()
+ 
         self._dataset = dataset
         
         dataframe = dataset.dataframe

{ddi_fw-0.0.193.dist-info → ddi_fw-0.0.194.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.193
+Version: 0.0.194
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.193.dist-info → ddi_fw-0.0.194.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
-ddi_fw/datasets/core.py,sha256=4705a94kKBueyWFXRJ3cnivAGKjrR89uBBKpxtMozOM,11080
+ddi_fw/datasets/core.py,sha256=HXU09CTbe3zpdBiUcE2w2Yxx_3yHfY_rqa31oS959jw,15531
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=bdcGmEbY_2Fe8fg0pKxfMuDopgaPUTUfQasCy8Bhcvc,9313
+ddi_fw/datasets/ddi_mdl/base.py,sha256=UevqzUUQozjRW9rnVaW2dogV_wahcEujH8c6MMvSQEo,9343
 ddi_fw/datasets/ddi_mdl/debug.log,sha256=eWz05j8RFqZuHFDTCF7Rck5w4rvtTanFN21iZsgxO7Y,115
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
@@ -85,7 +85,7 @@ ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,2
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
 ddi_fw/pipeline/multi_pipeline.py,sha256=fYyvwIOscUahjXd3QO5RSFrp1LliGR7RzOZyAXrXXz4,5637
 ddi_fw/pipeline/ner_pipeline.py,sha256=Bp6BA6nozfWFaMHH6jKlzesnCGO6qiMkzdGy_ed6nh0,5947
-ddi_fw/pipeline/pipeline.py,sha256=fRNUmKMrIiN_pX0aU57FGxaZ-1gdaI9IPBwAv3qgO7o,9961
+ddi_fw/pipeline/pipeline.py,sha256=CUHuy1nNgGD-eUcLnWFXcmSoTGssmg4ZFRAY1Cufey0,9047
 ddi_fw/utils/__init__.py,sha256=HC32XkYQTYH_9vt0eX6tqQngEFG-R70hGrYkT-BcHCk,519
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=gzb_vUDBrCMUhBxY1fBYTe8hmK72p0_uw3DTga8cqP8,1580
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
@@ -99,7 +99,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=EBf-XAiwQwr68az91erEYNegfeqssBR29kVgrliIyac,4765
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.193.dist-info/METADATA,sha256=onl9mPw_lsOyg_rdTxnYXNVGFGULp-SMmDGz82vPZE0,2542
-ddi_fw-0.0.193.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.193.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.193.dist-info/RECORD,,
+ddi_fw-0.0.194.dist-info/METADATA,sha256=Nv82MVq4n0p6vpkGa_mBn0kw0rRAKaYIauDLJhZVUkI,2542
+ddi_fw-0.0.194.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.194.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.194.dist-info/RECORD,,