ddi-fw 0.0.157__py3-none-any.whl → 0.0.159__py3-none-any.whl

ddi_fw/datasets/__init__.py

@@ -2,9 +2,6 @@ from .core import BaseDataset
 from .ddi_mdl.base import DDIMDLDataset 
 from .ddi_mdl_text.base import DDIMDLDatasetV2
 from .mdf_sa_ddi.base import MDFSADDIDataset
-from .embedding_generator import create_embeddings
-from .idf_helper import IDF
-from .feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
 from .dataset_splitter import DatasetSplitter
 __all__  = ['BaseDataset','DDIMDLDataset','MDFSADDIDataset']
 

ddi_fw/datasets/core.py

@@ -4,11 +4,23 @@ import numpy as np
 import pandas as pd
 from pydantic import BaseModel, Field, computed_field
 from ddi_fw.datasets.dataset_splitter import DatasetSplitter
-from ddi_fw.datasets.feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
-from ddi_fw.langchain.embeddings import PoolingStrategy
 from ddi_fw.utils.utils import create_folder_if_not_exists
 
 
+try:
+    from ddi_fw.vectorization import SimilarityMatrixGenerator, VectorGenerator
+except ImportError:
+    raise ImportError(
+        "Failed to import vectorization module. Ensure that the module exists and is correctly installed. ")
+
+try:
+    from ddi_fw.langchain.embeddings import PoolingStrategy
+except ImportError:
+    raise ImportError(
+        "Failed to import langchain.embeddings module. ")
+
+
+
 def stack(df_column):
     return np.stack(df_column.values)
 

ddi_fw/datasets/ddi_mdl/base.py

@@ -1,22 +1,21 @@
-import glob
 import pathlib
 from typing import List, Optional, Tuple
-from ddi_fw.datasets.core import BaseDataset, TextDatasetMixin, generate_sim_matrices_new, generate_vectors
-from ddi_fw.datasets.dataset_splitter import DatasetSplitter
+from ddi_fw.datasets.core import TextDatasetMixin, generate_sim_matrices_new, generate_vectors
 from ddi_fw.datasets.db_utils import create_connection
-from ddi_fw.datasets.idf_helper import IDF
-from ddi_fw.utils.utils import create_folder_if_not_exists
 import numpy as np
 import pandas as pd
 from pydantic import BaseModel, Field, model_validator, root_validator
-from ddi_fw.datasets.feature_vector_generation import SimilarityMatrixGenerator,VectorGenerator
-from ddi_fw.langchain.embeddings import PoolingStrategy
 from abc import ABC, abstractmethod
 from sklearn.preprocessing import LabelBinarizer
-
-from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
 import logging
 
+
+try:
+    from ddi_fw.vectorization import IDF
+except ImportError:
+    raise ImportError(
+        "Failed to import vectorization module. Ensure that the module exists and is correctly installed. ")
+
 logger = logging.getLogger(__name__)
 
 # Constants for embedding, chemical properties, and NER columns
@@ -32,14 +31,15 @@ LIST_OF_NER_COLUMNS = ['tui', 'cui', 'entities']
 
 HERE = pathlib.Path(__file__).resolve().parent
 
+
 class DDIMDLDataset(TextDatasetMixin):
     dataset_name: str = "DDIMDLDataset"
     index_path: str = Field(default_factory=lambda: str(
         pathlib.Path(__file__).resolve().parent.joinpath('indexes')))
     # drugs_df: pd.DataFrame = Field(default_factory=pd.DataFrame)
     # ddis_df: pd.DataFrame = Field(default_factory=pd.DataFrame)
-    drugs_df: Optional[pd.DataFrame] = None 
-    ddis_df: Optional[pd.DataFrame] = None 
+    drugs_df: Optional[pd.DataFrame] = None
+    ddis_df: Optional[pd.DataFrame] = None
 
     chemical_property_columns: list[str] = Field(
         default_factory=lambda: LIST_OF_CHEMICAL_PROPERTY_COLUMNS)
@@ -50,8 +50,8 @@ class DDIMDLDataset(TextDatasetMixin):
     cui_threshold: float | None = None
     entities_threshold: float | None = None
 
-
     # @model_validator
+
     def validate_columns(self, values):
         if not set(values['chemical_property_columns']).issubset(LIST_OF_CHEMICAL_PROPERTY_COLUMNS):
             raise ValueError("Invalid chemical property columns")
@@ -64,9 +64,10 @@ class DDIMDLDataset(TextDatasetMixin):
         super().__init__(**kwargs)
         self.class_column = 'event_category'
         _db_path = HERE.joinpath('data/event.db')
-        
+
         self.__similarity_related_columns__ = []
-        self.__similarity_related_columns__.extend(self.chemical_property_columns)
+        self.__similarity_related_columns__.extend(
+            self.chemical_property_columns)
         self.__similarity_related_columns__.extend(self.ner_columns)
         # TODO with resource
         self._conn = create_connection(_db_path.absolute().as_posix())
@@ -112,9 +113,9 @@ class DDIMDLDataset(TextDatasetMixin):
     def prep(self):
         if self.drugs_df is None or self.ddis_df is None:
             raise Exception("There is no data")
-        
+
         drug_ids = self.drugs_df['id'].to_list()
-        
+
         filtered_df = self.drugs_df
         combined_df = filtered_df.copy()
 
@@ -140,11 +141,12 @@ class DDIMDLDataset(TextDatasetMixin):
                 if key.startswith('entities'):
                     threshold = self.entities_threshold
                 combined_df[key] = filtered_ner_df[key]
-                valid_codes = idf_scores_df[idf_scores_df[key] > threshold].index
+                valid_codes = idf_scores_df[idf_scores_df[key]
+                                            > threshold].index
 
                 # print(f'{key}: valid code size = {len(valid_codes)}')
                 combined_df[key] = combined_df[key].apply(lambda items:
-                                                        [item for item in items if item in valid_codes])
+                                                          [item for item in items if item in valid_codes])
 
         moved_columns = ['id']
         moved_columns.extend(self.__similarity_related_columns__)
@@ -153,28 +155,29 @@ class DDIMDLDataset(TextDatasetMixin):
         chemical_properties_df = chemical_properties_df.fillna("").apply(list)
 
         # generate vectors dictionary içinde ndarray dönecek
-        generated_vectors = generate_vectors(chemical_properties_df, self.__similarity_related_columns__)
+        generated_vectors = generate_vectors(
+            chemical_properties_df, self.__similarity_related_columns__)
 
         similarity_matrices = generate_sim_matrices_new(
-            chemical_properties_df,generated_vectors,  self.__similarity_related_columns__, key_column= "id")
+            chemical_properties_df, generated_vectors,  self.__similarity_related_columns__, key_column="id")
 
         event_categories = self.ddis_df['event_category']
         labels = event_categories.tolist()
         lb = LabelBinarizer()
         lb.fit(labels)
         classes = lb.transform(labels)
-      
+
         def similarity_lambda_fnc(row, value):
             if row['id1'] in value:
                 return value[row['id1']]
 
-        def lambda_fnc(row: pd.Series, value)-> Optional[np.float16]:
+        def lambda_fnc(row: pd.Series, value) -> Optional[np.float16]:
             if row['id1'] in value and row['id2'] in value:
                 return np.float16(np.hstack(
                     (value[row['id1']], value[row['id2']])))
             return None
-                # return np.hstack(
-                #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
+            # return np.hstack(
+            #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
 
         def x_fnc(row, embeddings_after_pooling):
             if row['id1'] in embeddings_after_pooling:
@@ -207,13 +210,12 @@ class DDIMDLDataset(TextDatasetMixin):
 
         dataframe = self.ddis_df.copy()
         if not isinstance(classes, (list, pd.Series, np.ndarray)):
-         raise TypeError("classes must be an iterable (list, Series, or ndarray)")
+            raise TypeError(
+                "classes must be an iterable (list, Series, or ndarray)")
 
         if len(classes) != len(dataframe):
-            raise ValueError("Length of classes must match the number of rows in the DataFrame")
+            raise ValueError(
+                "Length of classes must match the number of rows in the DataFrame")
 
         dataframe['class'] = list(classes)
         self.set_dataframe(dataframe)
-        
-
-    

ddi_fw/langchain/embeddings.py

@@ -1,36 +1,11 @@
-# !pip install -U sentence-transformers
-
-# from transformers import BertTokenizer,BertForPreTraining,BertModel
-# from sentence_transformers import SentenceTransformer, util
-import pandas as pd
 import numpy as np
-from nltk import sent_tokenize
-import torch
-from tqdm import tqdm
-
-
-from collections import defaultdict
-from functools import partial
-from abc import ABC, abstractmethod
 from transformers import AutoModel, AutoTokenizer
-from sentence_transformers import SentenceTransformer, util
-
-from typing import Any, Dict, List, Optional
+from sentence_transformers import SentenceTransformer
+from typing import Any, List
 from langchain_core.embeddings import Embeddings
-from pydantic import BaseModel, ConfigDict, Field, SecretStr, computed_field
+from pydantic import BaseModel, ConfigDict, computed_field
 from langchain.embeddings import SentenceTransformerEmbeddings
-
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.vectorstores import Chroma
-import chromadb
-
-
-
-# def split_docs(documents, chunk_size=1000, chunk_overlap=20):
-#     text_splitter = RecursiveCharacterTextSplitter(
-#         chunk_size=chunk_size, chunk_overlap=chunk_overlap)
-#     docs = text_splitter.split_documents(documents)
-#     return docs
+ 
 
 class PoolingStrategy():
     def __init__(self):

ddi_fw/pipeline/ner_pipeline.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 import numpy as np
 from ddi_fw.datasets.core import BaseDataset
-from ddi_fw.datasets.idf_helper import IDF
+from ddi_fw.vectorization.idf_helper import IDF
 from typing import Any, Dict, List, Optional
 from itertools import product
 

ddi_fw/vectorization/__init__.py

@@ -0,0 +1,2 @@
+from .idf_helper import IDF
+from .feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator

{ddi_fw-0.0.157.dist-info → ddi_fw-0.0.159.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.157
+Version: 0.0.159
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
@@ -8,7 +8,6 @@ Keywords: Machine Learning
 Classifier: Development Status :: 1 - Planning
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Framework :: Pytest
 Classifier: Framework :: tox
@@ -22,8 +21,10 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: pydantic==2.10.6
 Requires-Dist: tqdm>=4.66.6
 Requires-Dist: pandas>=2.2.0
+Requires-Dist: py7zr==0.22.0
 Provides-Extra: llm
 Requires-Dist: sentence-transformers<=3.3.1,>=3.0.1; extra == "llm"
 Requires-Dist: transformers>=4.42.4; extra == "llm"
@@ -32,6 +33,7 @@ Requires-Dist: tokenizers>=0.19.1; extra == "llm"
 Requires-Dist: openai>=1.52.2; extra == "llm"
 Requires-Dist: langchain>=0.3.4; extra == "llm"
 Requires-Dist: langchain_community==0.3.3; extra == "llm"
+Requires-Dist: chromadb>=0.5.15; extra == "llm"
 Provides-Extra: ml
 Requires-Dist: scikit-learn==1.5.2; extra == "ml"
 Requires-Dist: tensorflow<2.18.0,>=2.17.0; extra == "ml"
@@ -43,17 +45,15 @@ Requires-Dist: scipy==1.13.1; extra == "ml"
 Requires-Dist: pandas>=2.2.0; extra == "ml"
 Requires-Dist: plotly==5.24.1; extra == "ml"
 Requires-Dist: matplotlib==3.8.0; extra == "ml"
-Requires-Dist: rdkit==2023.3.3; extra == "ml"
 Requires-Dist: datasets==3.0.2; extra == "ml"
 Requires-Dist: tqdm>=4.66.6; extra == "ml"
 Provides-Extra: datasets
 Requires-Dist: datasets==3.0.2; extra == "datasets"
 Requires-Dist: unstructured==0.16.3; extra == "datasets"
-Requires-Dist: py7zr==0.22.0; extra == "datasets"
 Requires-Dist: xmlschema==3.4.2; extra == "datasets"
 Provides-Extra: drugbank
 Requires-Dist: rdkit==2023.3.3; extra == "drugbank"
-Requires-Dist: openai>=1.52.2; extra == "drugbank"
+Requires-Dist: xmlschema==3.4.2; extra == "drugbank"
 Provides-Extra: pipeline-and-ner
 Requires-Dist: nltk>=3.8.1; extra == "pipeline-and-ner"
 Requires-Dist: stanza==1.9.2; extra == "pipeline-and-ner"
@@ -61,16 +61,8 @@ Requires-Dist: transformers>=4.42.4; extra == "pipeline-and-ner"
 Requires-Dist: sentence-transformers<=3.3.1,>=3.0.1; extra == "pipeline-and-ner"
 Requires-Dist: mlflow==2.16.1; extra == "pipeline-and-ner"
 Provides-Extra: utils
-Requires-Dist: pydantic==2.10.6; extra == "utils"
 Requires-Dist: python-stopwatch==1.1.11; extra == "utils"
 Requires-Dist: importlib-resources==6.4.5; extra == "utils"
 Requires-Dist: lxml==5.3.0; extra == "utils"
 Requires-Dist: pyarrow==17.0.0; extra == "utils"
 Requires-Dist: pycryptodomex==3.22.0; extra == "utils"
-Requires-Dist: pydantic-settings-2.8.1; extra == "utils"
-Requires-Dist: python-dotenv-1.1.0; extra == "utils"
-Requires-Dist: python-iso639-2025.2.18; extra == "utils"
-Requires-Dist: python-magic-0.4.27; extra == "utils"
-Requires-Dist: pyzstd==0.16.2; extra == "utils"
-Requires-Dist: databricks-sdk-0.47.0; extra == "utils"
-Requires-Dist: python-tml-1.0.2; extra == "utils"

{ddi_fw-0.0.157.dist-info → ddi_fw-0.0.159.dist-info}/RECORD

@@ -1,12 +1,9 @@
-ddi_fw/datasets/__init__.py,sha256=yDsRQD_9Ijpm_Rl2wSDwdutG5Q_wca_UBPEvm7nBx04,444
-ddi_fw/datasets/core.py,sha256=vRMpUsIHIbOKi-6TuUPNu1Ve3ny3cS9hdsydQxFCNvE,8078
+ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
+ddi_fw/datasets/core.py,sha256=gmasNdwohZ9Cd1qqhzijoTgX8VHQyzA0aBVtgjLQago,8344
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
-ddi_fw/datasets/embedding_generator.py,sha256=jiDKwLaPMaQkloxQkuCrhl-A-2OdvocmkSzjWtUnk4g,2255
-ddi_fw/datasets/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
-ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=yLxNzDYjLekq1qE6mKh6WkzUU5Xvn3JBEET9Ed-7b_E,9471
+ddi_fw/datasets/ddi_mdl/base.py,sha256=bdcGmEbY_2Fe8fg0pKxfMuDopgaPUTUfQasCy8Bhcvc,9313
 ddi_fw/datasets/ddi_mdl/debug.log,sha256=eWz05j8RFqZuHFDTCF7Rck5w4rvtTanFN21iZsgxO7Y,115
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
@@ -72,7 +69,7 @@ ddi_fw/drugbank/drugbank_processor.py,sha256=vmkt68n9nFLevufgGyXhOSDtTo4G1XzwT9P
 ddi_fw/drugbank/drugbank_processor_org.py,sha256=eO5Yset50P91qkic79RUXPoEuxRxQKFkKW0l4G29Mas,13322
 ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHrBXsI8,4600
 ddi_fw/langchain/__init__.py,sha256=zS0CQrakWEP19biSRewFJGcBT8WBZq4899HrEKiMqUY,269
-ddi_fw/langchain/embeddings.py,sha256=XzIYgmqnAO93pnavKRDhYDoz0RhDn-RoC7CDc0yAvbM,7572
+ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
@@ -87,7 +84,7 @@ ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
 ddi_fw/pipeline/multi_pipeline.py,sha256=NfcH4Ze5U-JRiH3lrxEDWj-VPxYQYtp7tq6bLCImBzs,5550
-ddi_fw/pipeline/ner_pipeline.py,sha256=q1aKjb54Ra1HzZ7dARvBw6lB37je9R-POEf2h6QT_nU,6018
+ddi_fw/pipeline/ner_pipeline.py,sha256=kNGtkg5rNX5MDywzvRxmvyk-DxXAjEbYzZkp8pNlAZo,6023
 ddi_fw/pipeline/pipeline.py,sha256=70lYsluAnTWDLTlf6rbecffw3Bl34L1_6ALfLUoSvtY,11324
 ddi_fw/utils/__init__.py,sha256=77563ikqAtdzjjgRlLp5OAsJBbpLA1Cao8iecGaVUXQ,354
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
@@ -97,7 +94,10 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.157.dist-info/METADATA,sha256=BDNkvrnqN1gxAqUjpmyUqE-YGz86JZpVF0NM_q-oEJk,3612
-ddi_fw-0.0.157.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddi_fw-0.0.157.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.157.dist-info/RECORD,,
+ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
+ddi_fw/vectorization/feature_vector_generation.py,sha256=Z1A_DOBqDFPqLN4YB-3oYlOQWJK-X6Oes6UFjpzR47Q,4760
+ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
+ddi_fw-0.0.159.dist-info/METADATA,sha256=Ai8ONw45d5f5yEd3_SaNEKBew73TxPR0nrc75J4U0Ck,3145
+ddi_fw-0.0.159.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddi_fw-0.0.159.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.159.dist-info/RECORD,,

ddi_fw/datasets/embedding_generator.py

@@ -1,67 +0,0 @@
-# !pip install -U sentence-transformers
-
-# from transformers import BertTokenizer,BertForPreTraining,BertModel
-# from sentence_transformers import SentenceTransformer, util
-import pandas as pd
-import numpy as np
-import nltk
-from nltk import sent_tokenize
-from tqdm import tqdm
-
-
-nltk.download('punkt')
-
-import os
-def check_file_exists(path):
-  return os.path.isdir(path)
-
-def get_model_name_or_local_path(model_local_path, model_name):
-  if check_file_exists(model_local_path):
-    return model_local_path
-  return model_name
-
-import re
-def process_text(text):
-  text = re.sub("\[L\d*\]", "",text)
-  text = text.replace("[","")
-  text = text.replace("]","")
-  return text
-
-
-
-from collections import defaultdict
-from functools import partial
-
-# NOT modelden input size'ı anlama,
-def create_embeddings(model, data, column, drop_column=True):
-  # model._modules['1'].get_sentence_embedding_dimension()
-  # shape = (1,model._modules['0'].get_word_embedding_dimension())
-  shape = model._modules['0'].get_word_embedding_dimension()
-  column_embeddings_dict = defaultdict(lambda: np.zeros(shape))
-  for index, row in tqdm(data.iterrows()):
-    # if index == 10:
-    #   break
-    text = data[column][index]
-    # else'de zero
-    if text == None or type(text) != str:
-      embeddings = None
-    else:
-      sentences = sent_tokenize(text)
-      embeddings = model.encode(sentences)
-
-  #TODO benzer olan ilacın embedding değerini vererek dene
-    if embeddings is None or len(embeddings) == 0: #embedding check none type
-      sum_of_embeddings = np.zeros(shape)
-    else:
-      sum_of_embeddings = np.sum(embeddings, axis = 0)
-    # column_embeddings_dict[row['id']] = sum_of_embeddings.reshape(1, -1) # 2d
-    column_embeddings_dict[row['id']] = sum_of_embeddings
-    # data.iloc[index][column+'_embedding']=sum_of_embeddings
-  
-  # data[column+'_embedding'] = pd.Series(column_embeddings_dict.values())
-  data[column+'_embedding'] = pd.Series(list(column_embeddings_dict.values()))
-  if(drop_column):
-    data.drop([column], axis = 1, inplace = True)
-  # data[column+'_embedding'] = [column_embeddings_dict[row['name']] for index, row in data.iterrows()]
-  return column_embeddings_dict
-