ddi-fw 0.0.42__tar.gz → 0.0.44__tar.gz

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.42
+Version: 0.0.44
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
@@ -20,7 +20,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: python-stopwatch
 Requires-Dist: importlib-resources
@@ -41,5 +41,3 @@ Requires-Dist: tqdm
 Requires-Dist: xmlschema
 Requires-Dist: zipp
 Requires-Dist: py7zr
-Requires-Dist: tf2onnx
-Requires-Dist: tensorflow==2.15.0

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.42"
+version = "0.0.44"
 description = "Do not use :)"
 readme = "README.md"
 authors = [
@@ -43,7 +43,7 @@ keywords = [
 # See https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license
 license = { file = "LICENSE" }
 
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 dependencies = [
 "python-stopwatch"
 ,"importlib-resources"
@@ -64,8 +64,6 @@ dependencies = [
 ,"xmlschema"
 ,"zipp"
 ,"py7zr"
-,"tf2onnx"
-,"tensorflow==2.15.0"
 ]
  
 

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/src/ddi_fw/datasets/__init__.py

@@ -2,7 +2,7 @@ from .core import BaseDataset
 from .ddi_mdl.base import DDIMDLDataset 
 from .mdf_sa_ddi.base import MDFSADDIDataset
 from .embedding_generator import create_embeddings
-from .embedding_generator_new import EmbeddingGenerator,PretrainedEmbeddingGenerator,SBertEmbeddingGenerator,LLMEmbeddingGenerator,create_embeddings_new
+from .embedding_generator_new import PoolingStrategy,SumPoolingStrategy,MeanPoolingStrategy,SentenceTransformerDecorator,PretrainedEmbeddings,SBertEmbeddings
 from .idf_helper import IDF
 from .feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
 

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/src/ddi_fw/datasets/core.py

@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
 import numpy as np
 import pandas as pd
 import pathlib
+from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
 from ddi_fw.datasets.idf_helper import IDF
 
 from ddi_fw.utils.zip_helper import ZipHelper
@@ -21,10 +22,11 @@ def stack(df_column):
 
 
 class BaseDataset(ABC):
-    def __init__(self,embedding_size,embedding_dict, ner_df, chemical_property_columns, embedding_columns, ner_columns,
+    def __init__(self,embedding_size,embedding_dict, embeddings_pooling_strategy:PoolingStrategy, ner_df, chemical_property_columns, embedding_columns, ner_columns,
                  **kwargs):
         self.embedding_size = embedding_size
         self.embedding_dict = embedding_dict
+        self.embeddings_pooling_strategy = embeddings_pooling_strategy
         self.ner_df = ner_df
         self.__similarity_related_columns__ = []
         self.__similarity_related_columns__.extend(chemical_property_columns)
@@ -364,13 +366,13 @@ class BaseDataset(ABC):
                 # return np.hstack(
                 #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
 
-        def x_fnc(row, embedding_column):
+        def x_fnc(row, embedding_column,embeddings_after_pooling):
             if row['id1'] in self.embedding_dict[embedding_column]:
-                v1 = self.embedding_dict[embedding_column][row['id1']]
+                v1 = embeddings_after_pooling[embedding_column][row['id1']]
             else:
                 v1 = np.zeros(self.embedding_size)
             if row['id2'] in self.embedding_dict[embedding_column]:
-                v2 = self.embedding_dict[embedding_column][row['id2']]
+                v2 = embeddings_after_pooling[embedding_column][row['id2']]
             else:
                 v2 = np.zeros(self.embedding_size)
             return np.float16(np.hstack(
@@ -385,9 +387,10 @@ class BaseDataset(ABC):
 
         for embedding_column in self.embedding_columns:
             print(f"concat {embedding_column} embeddings")
+            embeddings_after_pooling = {k: self.embeddings_pooling_strategy.apply(v) for k,v in self.embedding_dict[embedding_column].items()}
             # column_embeddings_dict = embedding_values[embedding_column]
             self.ddis_df[embedding_column+'_embedding'] = self.ddis_df.apply(
-                x_fnc, args=(embedding_column,), axis=1)
+                x_fnc, args=(embedding_column,embeddings_after_pooling), axis=1)
 
         self.dataframe = self.ddis_df.copy()
         self.dataframe['class'] = list(classes)

ddi_fw-0.0.44/src/ddi_fw/datasets/embedding_generator_new.py

@@ -0,0 +1,186 @@
+# !pip install -U sentence-transformers
+
+# from transformers import BertTokenizer,BertForPreTraining,BertModel
+# from sentence_transformers import SentenceTransformer, util
+import pandas as pd
+import numpy as np
+from nltk import sent_tokenize
+import torch
+from tqdm import tqdm
+
+
+from collections import defaultdict
+from functools import partial
+from abc import ABC, abstractmethod
+from transformers import AutoModel, AutoTokenizer
+from sentence_transformers import SentenceTransformer, util
+
+from typing import Any, Dict, List, Optional
+from langchain_core.embeddings import Embeddings
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from langchain.embeddings import SentenceTransformerEmbeddings
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+import chromadb
+
+
+
+# def split_docs(documents, chunk_size=1000, chunk_overlap=20):
+#     text_splitter = RecursiveCharacterTextSplitter(
+#         chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+#     docs = text_splitter.split_documents(documents)
+#     return docs
+
+class PoolingStrategy():
+    def __init__(self):
+        pass
+
+    def apply(self, embeddings: List[List[float]]):
+        pass
+
+
+class MeanPoolingStrategy(PoolingStrategy):
+    def __init__(self):
+        pass
+
+    def apply(self, embeddings: List[List[float]]):
+        return np.mean(embeddings, axis=0)
+
+
+class SumPoolingStrategy(PoolingStrategy):
+    def __init__(self):
+        pass
+
+    def apply(self, embeddings: List[List[float]]):
+        return np.sum(embeddings, axis=0)
+
+
+class SentenceTransformerDecorator(BaseModel, Embeddings):
+    def __init__(self, model_name="all-MiniLM-L6-v2", **kwargs: Any):
+        self.embeddings = SentenceTransformerEmbeddings(model_name=model_name)
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        return self.embeddings.embed_documents(texts)
+
+    def embed_query(self, text: str) -> List[float]:
+        return self.embeddings.embed_query(text)
+
+
+class PretrainedEmbeddings(BaseModel, Embeddings):
+    def __init__(self, model_name):
+        self.mmodel_name = model_name
+        self.model = AutoModel.from_pretrained(model_name)
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.shape = self.model.get_input_embeddings().weight.shape
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        output_embeddings = []
+        texts = list(map(lambda x: x.replace("\n", " "), texts))
+        for text in texts:
+            input_ids = self.tokenizer.encode(
+                text, return_tensors='pt', padding=True)
+            output_embeddings.append(self.model(
+                input_ids).last_hidden_state.mean(dim=1))
+
+    def embed_query(self, text: str) -> List[float]:
+        return self.embed_documents([text])[0]
+
+
+class SBertEmbeddings(BaseModel, Embeddings):
+    def __init__(self, model_name):
+        self.model = SentenceTransformer(model_name)
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        return self.model.encode(texts)
+
+    def embed_query(self, text: str) -> List[float]:
+        return self.embed_documents([text])[0]
+
+# class EmbeddingGenerator(ABC):
+
+#     def __init__(self):
+#         self.shape = None
+
+#     @abstractmethod
+#     def generate(self, text):
+#         pass
+
+# # https://github.com/huggingface/transformers/issues/1791
+# class PretrainedEmbeddingGenerator(EmbeddingGenerator):
+#     def __init__(self, model_name, split_text=True):
+#         self.model_name = model_name
+#         self.model = AutoModel.from_pretrained(model_name)
+#         self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+#         self.shape = self.model.get_input_embeddings().weight.shape
+#         self.split_text = split_text
+
+#     def generate(self, text):
+#         if self.split_text:
+#             sentences = sent_tokenize(text)
+#             output_embeddings = None
+#             for sentence in sentences:
+#                 input_ids  = self.tokenizer.encode(sentence, return_tensors='pt', padding=True)
+#                 if output_embeddings == None:
+#                     output_embeddings = self.model(input_ids).last_hidden_state.mean(dim=1)
+#                 else:
+#                     output_embeddings += self.model(input_ids).last_hidden_state.mean(dim=1)
+#             if output_embeddings == None:
+#                 output_embeddings = torch.empty((1,self.model.get_input_embeddings().weight.shape[1]))
+#         else:
+#             encoded_input = self.tokenizer(text, return_tensors='pt')
+#             input_ids = self.tokenizer.encode(text, add_special_tokens=True, max_length=self.tokenizer.model_max_length, return_tensors='pt')
+#             # input_ids  = encoded_input.input_ids[:self.tokenizer.model_max_length]
+#             output_embeddings = self.model(input_ids)
+#             # output_embeddings = self.model(**encoded_input)
+#             # sentence embedding
+#             output_embeddings = output_embeddings.last_hidden_state.mean(dim=1)
+#         return torch.flatten(output_embeddings).detach().numpy()
+
+
+# class LLMEmbeddingGenerator(EmbeddingGenerator):
+#     pass
+
+
+# class SBertEmbeddingGenerator(PretrainedEmbeddingGenerator):
+#     def __init__(self, model_name, split_text=True):
+#         self.model = SentenceTransformer(model_name)
+#         self.shape = self.model._modules['0'].get_word_embedding_dimension()
+#         self.split_text = split_text
+
+#     def generate(self, text):
+#         if text == None or type(text) != str:
+#             embeddings = None
+#         else:
+#             if self.split_text:
+#                 sentences = sent_tokenize(text)
+#                 embeddings = self.model.encode(sentences)
+#             else:
+#                 embeddings = self.model.encode(text)
+#         return embeddings
+
+
+# # NOT modelden input size'ı anlama,
+# def create_embeddings_new(generator: EmbeddingGenerator, data, column, drop_column=True):
+#     column_embeddings_dict = defaultdict(lambda: np.zeros(generator.shape))
+#     for index, row in tqdm(data.iterrows()):
+#         # if index == 10:
+#         #   break
+#         text = data[column][index]
+#         embeddings = generator.generate(text)
+
+#     # TODO benzer olan ilacın embedding değerini vererek dene
+#         # embedding check none type
+#         if embeddings is None or len(embeddings) == 0:
+#             sum_of_embeddings = np.zeros(generator.shape)
+#         else:
+#             sum_of_embeddings = np.sum(embeddings, axis=0)
+#         # column_embeddings_dict[row['id']] = sum_of_embeddings.reshape(1, -1) # 2d
+#         column_embeddings_dict[row['id']] = sum_of_embeddings
+#         # data.iloc[index][column+'_embedding']=sum_of_embeddings
+
+#     data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
+#     if (drop_column):
+#         data.drop([column], axis=1, inplace=True)
+#     # data[column+'_embedding'] = [column_embeddings_dict[row['name']] for index, row in data.iterrows()]
+#     return column_embeddings_dict

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/src/ddi_fw/experiments/tensorflow_helper.py

@@ -14,8 +14,8 @@ import time
 from mlflow.models import infer_signature
 from ddi_fw.experiments.evaluation_helper import evaluate
 
-import tf2onnx
-import onnx
+# import tf2onnx
+# import onnx
 
 import itertools
 import ddi_fw.utils as utils
@@ -45,7 +45,7 @@ class TFMultiModal:
 
     def predict(self, combinations: list = [], generate_combinations=False):
         self.prefix = utils.utc_time_as_string()
-        self.date = utils.utc_time_as_string_simple_format
+        self.date = utils.utc_time_as_string_simple_format()
         sum = np.zeros(
             (self.y_test_label.shape[0], self.y_test_label.shape[1]))
         single_results = dict()
@@ -165,12 +165,13 @@ class TFSingleModal:
                 signature=signature,
             )
             print(run.info.artifact_uri)
-            onnx_model, _ = tf2onnx.convert.from_keras(
-                best_model, input_signature=None, opset=13)
-            onnx.save(onnx_model, run.info.artifact_uri +
-                      '/model/model.onnx')
+            # todo tf2onnx not compatible with keras > 2.15
+            # onnx_model, _ = tf2onnx.convert.from_keras(
+            #     best_model, input_signature=None, opset=13)
+            # onnx.save(onnx_model, run.info.artifact_uri +
+            #           '/model/model.onnx')
             utils.compress_and_save_data(
-                metrics.__dict__, run.info.artifact_uri, f'{self.date}metrics.gzip')
+                metrics.__dict__, run.info.artifact_uri, f'{self.date}_metrics.gzip')
             # mlflow.log_dict(metrics.__dict__, "metrics.json")
 
             # Plot Precision-Recall curves for each class and micro-average

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/src/ddi_fw/experiments/test.py

@@ -56,4 +56,6 @@
 #                           callbacks=[custom_callback])
 
 # loss, accuracy = model.evaluate(test_data, test_labels,callbacks=[custom_callback])
-# print('Test accuracy: %.2f' % (accuracy))
+# print('Test accuracy: %.2f' % (accuracy))
+
+from langchain.embeddings import SentenceTransformerEmbeddings

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.42
+Version: 0.0.44
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
@@ -20,7 +20,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: python-stopwatch
 Requires-Dist: importlib-resources
@@ -41,5 +41,3 @@ Requires-Dist: tqdm
 Requires-Dist: xmlschema
 Requires-Dist: zipp
 Requires-Dist: py7zr
-Requires-Dist: tf2onnx
-Requires-Dist: tensorflow==2.15.0

{ddi_fw-0.0.42 → ddi_fw-0.0.44}/src/ddi_fw.egg-info/requires.txt

@@ -17,5 +17,3 @@ tqdm
 xmlschema
 zipp
 py7zr
-tf2onnx
-tensorflow==2.15.0

ddi_fw-0.0.42/src/ddi_fw/datasets/embedding_generator_new.py

@@ -1,105 +0,0 @@
-# !pip install -U sentence-transformers
-
-# from transformers import BertTokenizer,BertForPreTraining,BertModel
-# from sentence_transformers import SentenceTransformer, util
-import pandas as pd
-import numpy as np
-from nltk import sent_tokenize
-import torch
-from tqdm import tqdm
-
-
-from collections import defaultdict
-from functools import partial
-from abc import ABC, abstractmethod
-from transformers import AutoModel, AutoTokenizer
-from sentence_transformers import SentenceTransformer, util
-
-
-class EmbeddingGenerator(ABC):
-
-    def __init__(self):
-        self.shape = None
-
-    @abstractmethod
-    def generate(self, text):
-        pass
-
-# https://github.com/huggingface/transformers/issues/1791
-class PretrainedEmbeddingGenerator(EmbeddingGenerator):
-    def __init__(self, model_name, split_text=True):
-        self.model_name = model_name
-        self.model = AutoModel.from_pretrained(model_name)
-        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
-        self.shape = self.model.get_input_embeddings().weight.shape
-        self.split_text = split_text
-
-    def generate(self, text):
-        if self.split_text:
-            sentences = sent_tokenize(text)
-            output_embeddings = None
-            for sentence in sentences:
-                input_ids  = self.tokenizer.encode(sentence, return_tensors='pt', padding=True)
-                if output_embeddings == None:
-                    output_embeddings = self.model(input_ids).last_hidden_state.mean(dim=1)
-                else:
-                    output_embeddings += self.model(input_ids).last_hidden_state.mean(dim=1)
-            if output_embeddings == None:
-                output_embeddings = torch.empty((1,self.model.get_input_embeddings().weight.shape[1]))
-        else:
-            encoded_input = self.tokenizer(text, return_tensors='pt')
-            input_ids = self.tokenizer.encode(text, add_special_tokens=True, max_length=self.tokenizer.model_max_length, return_tensors='pt')
-            # input_ids  = encoded_input.input_ids[:self.tokenizer.model_max_length]
-            output_embeddings = self.model(input_ids)
-            # output_embeddings = self.model(**encoded_input)
-            # sentence embedding
-            output_embeddings = output_embeddings.last_hidden_state.mean(dim=1)
-        return torch.flatten(output_embeddings).detach().numpy()
-
-
-class LLMEmbeddingGenerator(EmbeddingGenerator):
-    pass
-
-
-class SBertEmbeddingGenerator(PretrainedEmbeddingGenerator):
-    def __init__(self, model_name, split_text=True):
-        self.model = SentenceTransformer(model_name)
-        self.shape = self.model._modules['0'].get_word_embedding_dimension()
-        self.split_text = split_text
-
-    def generate(self, text):
-        if text == None or type(text) != str:
-            embeddings = None
-        else:
-            if self.split_text:
-                sentences = sent_tokenize(text)
-                embeddings = self.model.encode(sentences)
-            else:
-                embeddings = self.model.encode(text)
-        return embeddings
-
-
-# NOT modelden input size'ı anlama,
-def create_embeddings_new(generator: EmbeddingGenerator, data, column, drop_column=True):
-    column_embeddings_dict = defaultdict(lambda: np.zeros(generator.shape))
-    for index, row in tqdm(data.iterrows()):
-        # if index == 10:
-        #   break
-        text = data[column][index]
-        embeddings = generator.generate(text)
-
-    # TODO benzer olan ilacın embedding değerini vererek dene
-        # embedding check none type
-        if embeddings is None or len(embeddings) == 0:
-            sum_of_embeddings = np.zeros(generator.shape)
-        else:
-            sum_of_embeddings = np.sum(embeddings, axis=0)
-        # column_embeddings_dict[row['id']] = sum_of_embeddings.reshape(1, -1) # 2d
-        column_embeddings_dict[row['id']] = sum_of_embeddings
-        # data.iloc[index][column+'_embedding']=sum_of_embeddings
-
-    data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
-    if (drop_column):
-        data.drop([column], axis=1, inplace=True)
-    # data[column+'_embedding'] = [column_embeddings_dict[row['name']] for index, row in data.iterrows()]
-    return column_embeddings_dict